7412 newproc() performs inadequate clean-up after failed lwp_create()

Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Dan McDonald <danmcd@omniti.com>
Reviewed by: Gordon Ross <gwr@nexenta.com>

1 files changed, 30 insertions, 22 deletions

diff --git a/usr/src/uts/common/os/fork.c b/usr/src/uts/common/os/fork.c
index fe3a362fa7..f835981610 100644
--- a/usr/src/uts/common/os/fork.c
+++ b/usr/src/uts/common/os/fork.c
@@ -125,13 +125,36 @@ forksys(int subcode, int flags)
 	}
 }
 
+/*
+ * Remove the associations of a child process from its parent and siblings.
+ */
+static void
+disown_proc(proc_t *pp, proc_t *cp)
+{
+	proc_t **orphpp;
+
+	ASSERT(MUTEX_HELD(&pidlock));
+
+	orphpp = &pp->p_orphan;
+	while (*orphpp != cp)
+		orphpp = &(*orphpp)->p_nextorph;
+	*orphpp = cp->p_nextorph;
+
+	if (pp->p_child == cp)
+		pp->p_child = cp->p_sibling;
+	if (cp->p_sibling)
+		cp->p_sibling->p_psibling = cp->p_psibling;
+	if (cp->p_psibling)
+		cp->p_psibling->p_sibling = cp->p_sibling;
+}
+
 /* ARGSUSED */
 static int64_t
 cfork(int isvfork, int isfork1, int flags)
 {
 	proc_t *p = ttoproc(curthread);
 	struct as *as;
-	proc_t *cp, **orphpp;
+	proc_t *cp;
 	klwp_t *clone;
 	kthread_t *t;
 	task_t *tk;
@@ -266,16 +289,7 @@ cfork(int isvfork, int isfork1, int flags)
 			sprunlock(p);
 			fork_fail(cp);
 			mutex_enter(&pidlock);
-			orphpp = &p->p_orphan;
-			while (*orphpp != cp)
-				orphpp = &(*orphpp)->p_nextorph;
-			*orphpp = cp->p_nextorph;
-			if (p->p_child == cp)
-				p->p_child = cp->p_sibling;
-			if (cp->p_sibling)
-				cp->p_sibling->p_psibling = cp->p_psibling;
-			if (cp->p_psibling)
-				cp->p_psibling->p_sibling = cp->p_sibling;
+			disown_proc(p, cp);
 			mutex_enter(&cp->p_lock);
 			tk = cp->p_task;
 			task_detach(cp);
@@ -640,16 +654,7 @@ forklwperr:
 	atomic_dec_32(&cp->p_pool->pool_ref);
 	mutex_exit(&cp->p_lock);
 
-	orphpp = &p->p_orphan;
-	while (*orphpp != cp)
-		orphpp = &(*orphpp)->p_nextorph;
-	*orphpp = cp->p_nextorph;
-	if (p->p_child == cp)
-		p->p_child = cp->p_sibling;
-	if (cp->p_sibling)
-		cp->p_sibling->p_psibling = cp->p_psibling;
-	if (cp->p_psibling)
-		cp->p_psibling->p_sibling = cp->p_sibling;
+	disown_proc(p, cp);
 	pid_exit(cp, tk);
 	mutex_exit(&pidlock);
 
@@ -886,18 +891,21 @@ newproc(void (*pc)(), caddr_t arg, id_t cid, int pri, struct contract **ct,
 		if ((lwp = lwp_create(pc, arg, 0, p, TS_STOPPED, pri,
 		    &curthread->t_hold, cid, 1)) == NULL) {
 			task_t *tk;
+
 			fork_fail(p);
 			mutex_enter(&pidlock);
+			disown_proc(p->p_parent, p);
+
 			mutex_enter(&p->p_lock);
 			tk = p->p_task;
 			task_detach(p);
 			ASSERT(p->p_pool->pool_ref > 0);
 			atomic_add_32(&p->p_pool->pool_ref, -1);
 			mutex_exit(&p->p_lock);
+
 			pid_exit(p, tk);
 			mutex_exit(&pidlock);
 			task_rele(tk);
-
 			return (EAGAIN);
 		}
 		t = lwptot(lwp);