OS-4937 lxbrand ptracer count updates can racerelease-20151112

Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Joshua M. Clulow <jmc@joyent.com>

11 files changed, 97 insertions, 29 deletions

diff --git a/usr/src/uts/common/brand/lx/os/lx_brand.c b/usr/src/uts/common/brand/lx/os/lx_brand.c
index 0bd0b63409..a7bb630669 100644
--- a/usr/src/uts/common/brand/lx/os/lx_brand.c
+++ b/usr/src/uts/common/brand/lx/os/lx_brand.c
@@ -257,6 +257,7 @@ struct brand_ops lx_brops = {
 	lx_lwpdata_alloc,		/* b_lwpdata_alloc */
 	lx_lwpdata_free,		/* b_lwpdata_free */
 	lx_initlwp,			/* b_initlwp */
+	lx_initlwp_post,		/* b_initlwp_post */
 	lx_forklwp,			/* b_forklwp */
 	lx_freelwp,			/* b_freelwp */
 	lx_exitlwp,			/* b_lwpexit */
@@ -333,6 +334,7 @@ lx_proc_exit(proc_t *p)
 
 	mutex_enter(&p->p_lock);
 	VERIFY(lxpd = ptolxproc(p));
+	VERIFY(lxpd->l_ptrace == 0);
 	if ((lxpd->l_flags & LX_PROC_CHILD_DEATHSIG) == 0) {
 		mutex_exit(&p->p_lock);
 		return;
@@ -1550,6 +1552,13 @@ lx_copy_procdata(proc_t *cp, proc_t *pp)
 	bcopy(ppd, cpd, sizeof (lx_proc_data_t));
 	mutex_exit(&pp->p_lock);
 
+	/*
+	 * The l_ptrace count is normally manipulated only while under holding
+	 * p_lock.  Since this is a freshly created process, it's safe to zero
+	 * out.  If it is to be inherited, the attach will occur later.
+	 */
+	cpd->l_ptrace = 0;
+
 	cpd->l_fake_limits[LX_RLFAKE_LOCKS].rlim_cur = LX_RLIM64_INFINITY;
 	cpd->l_fake_limits[LX_RLFAKE_LOCKS].rlim_max = LX_RLIM64_INFINITY;
 
diff --git a/usr/src/uts/common/brand/lx/os/lx_misc.c b/usr/src/uts/common/brand/lx/os/lx_misc.c
index 3c1e7a104a..84600e0c07 100644
--- a/usr/src/uts/common/brand/lx/os/lx_misc.c
+++ b/usr/src/uts/common/brand/lx/os/lx_misc.c
@@ -464,23 +464,30 @@ lx_initlwp(klwp_t *lwp, void *lwpbd)
 	lwp->lwp_brand_syscall = lx_syscall_enter;
 
 	/*
-	 * If the parent LWP has a ptrace(2) tracer, the new LWP may
-	 * need to inherit that same tracer.
-	 * In addition the new LPW inherits the parent LWP cgroup ID.
+	 * The new LWP inherits the parent LWP cgroup ID.
 	 */
 	if (plwpd != NULL) {
-		lx_ptrace_inherit_tracer(plwpd, lwpd);
 		lwpd->br_cgroupid = plwpd->br_cgroupid;
 	}
-
-	/* cgroup integration */
 	lxzdata = ztolxzd(p->p_zone);
 	if (lxzdata->lxzd_cgroup != NULL) {
 		ASSERT(lx_cgrp_initlwp != NULL);
 		(*lx_cgrp_initlwp)(lxzdata->lxzd_cgroup,
 		    lwpd->br_cgroupid, lwptot(lwp)->t_tid, lwpd->br_pid);
 	}
+}
 
+void
+lx_initlwp_post(klwp_t *lwp)
+{
+	lx_lwp_data_t *plwpd = ttolxlwp(curthread);
+	/*
+	 * If the parent LWP has a ptrace(2) tracer, the new LWP may
+	 * need to inherit that same tracer.
+	 */
+	if (plwpd != NULL) {
+		lx_ptrace_inherit_tracer(plwpd, lwptolxlwp(lwp));
+	}
 }
 
 /*
diff --git a/usr/src/uts/common/brand/lx/os/lx_ptrace.c b/usr/src/uts/common/brand/lx/os/lx_ptrace.c
index ae697c33d4..09389aa98f 100644
--- a/usr/src/uts/common/brand/lx/os/lx_ptrace.c
+++ b/usr/src/uts/common/brand/lx/os/lx_ptrace.c
@@ -918,6 +918,7 @@ lx_ptrace_detach(lx_ptrace_accord_t *accord, lx_lwp_data_t *remote, int signo,
 	/*
 	 * Decrement traced-lwp count for the process.
 	 */
+	ASSERT(MUTEX_HELD(&rlwp->lwp_procp->p_lock));
 	VERIFY(ptolxproc(rlwp->lwp_procp)->l_ptrace-- >= 1);
 
 	/*
@@ -1031,7 +1032,7 @@ lx_ptrace_attach(pid_t lx_pid)
 		VERIFY(rlwpd->br_ptrace_attach != LX_PTA_NONE);
 		error = EPERM;
 	} else {
-		lx_proc_data_t *rprocd;
+		lx_proc_data_t *rprocd = ptolxproc(rproc);
 
 		/*
 		 * Bond the tracee to the accord.
@@ -1049,6 +1050,10 @@ lx_ptrace_attach(pid_t lx_pid)
 		mutex_enter(&accord->lxpa_tracees_lock);
 		VERIFY(!list_link_active(&rlwpd->br_ptrace_linkage));
 		list_insert_tail(&accord->lxpa_tracees, rlwpd);
+		/*
+		 * Bump traced-lwp count for the remote process.
+		 */
+		rprocd->l_ptrace++;
 		mutex_exit(&accord->lxpa_tracees_lock);
 
 		/*
@@ -1056,11 +1061,6 @@ lx_ptrace_attach(pid_t lx_pid)
 		 */
 		sigtoproc(rproc, rthr, SIGSTOP);
 
-		/*
-		 * Bump traced-lwp count for the remote process.
-		 */
-		rprocd = ttolxproc(rthr);
-		rprocd->l_ptrace++;
 
 		error = 0;
 	}
@@ -1125,16 +1125,19 @@ lx_ptrace_inherit_tracer(lx_lwp_data_t *src, lx_lwp_data_t *dst)
 	proc_t *srcp = lwptoproc(src->br_lwp);
 	proc_t *dstp = lwptoproc(dst->br_lwp);
 	lx_ptrace_accord_t *accord;
-	boolean_t unlock = B_FALSE;
+	boolean_t is_fork = B_FALSE;
 
-	if (srcp == dstp) {
+	VERIFY(MUTEX_HELD(&dstp->p_lock));
+	if (srcp != dstp) {
 		/*
-		 * This is syslwp_create(), so the process p_lock is already
-		 * held.
+		 * In the case of being called via forklwp, some lock shuffling
+		 * is required.  The destination p_lock must be dropped to
+		 * avoid deadlocks when locking the source and manipulating
+		 * ptrace accord resources.
 		 */
-		VERIFY(MUTEX_HELD(&srcp->p_lock));
-	} else {
-		unlock = B_TRUE;
+		is_fork = B_TRUE;
+		sprlock_proc(dstp);
+		mutex_exit(&dstp->p_lock);
 		mutex_enter(&srcp->p_lock);
 	}
 
@@ -1226,6 +1229,11 @@ lx_ptrace_inherit_tracer(lx_lwp_data_t *src, lx_lwp_data_t *dst)
 	src->br_ptrace_flags &= ~LX_PTF_CLONING;
 
 	/*
+	 * Bump traced-lwp count for the process.
+	 */
+	ptolxproc(dstp)->l_ptrace++;
+
+	/*
 	 * If lx_ptrace_exit_tracer() is trying to detach our tracer, it will
 	 * be sleeping on this CV until LX_PTF_CLONING is clear.  Wake it
 	 * now.
@@ -1233,8 +1241,10 @@ lx_ptrace_inherit_tracer(lx_lwp_data_t *src, lx_lwp_data_t *dst)
 	cv_broadcast(&lx_ptrace_busy_cv);
 
 out:
-	if (unlock) {
+	if (is_fork) {
 		mutex_exit(&srcp->p_lock);
+		mutex_enter(&dstp->p_lock);
+		sprunprlock(dstp);
 	}
 }
 
@@ -1306,6 +1316,12 @@ lx_ptrace_traceme(void)
 		lwpd->br_ptrace_tracer = accord;
 		did_attach = B_TRUE;
 		error = 0;
+
+		/*
+		 * Speculatively bump l_ptrace now before dropping p_lock.
+		 * It will be reverted if the tracee attachment fails.
+		 */
+		ptolxproc(p)->l_ptrace++;
 	}
 	mutex_exit(&p->p_lock);
 
@@ -1318,8 +1334,6 @@ lx_ptrace_traceme(void)
 		mutex_enter(&accord->lxpa_tracees_lock);
 
 		if (!(accord->lxpa_flags & LX_ACC_TOMBSTONE)) {
-			lx_proc_data_t *procd = ttolxproc(curthread);
-
 			/*
 			 * Put ourselves in the tracee list for this accord.
 			 */
@@ -1328,11 +1342,6 @@ lx_ptrace_traceme(void)
 			mutex_exit(&accord->lxpa_tracees_lock);
 			lx_ptrace_accord_exit(accord);
 
-			/*
-			 * Bump traced-lwp count for the process.
-			 */
-			procd->l_ptrace++;
-
 			return (0);
 		}
 		mutex_exit(&accord->lxpa_tracees_lock);
@@ -1343,6 +1352,13 @@ lx_ptrace_traceme(void)
 		 */
 		error = ESRCH;
 		lx_ptrace_accord_exit(accord);
+
+		/*
+		 * Undo speculative increment of ptracer count.
+		 */
+		mutex_enter(&p->p_lock);
+		ptolxproc(p)->l_ptrace--;
+		mutex_exit(&p->p_lock);
 	}
 
 	/*
diff --git a/usr/src/uts/common/brand/lx/sys/lx_misc.h b/usr/src/uts/common/brand/lx/sys/lx_misc.h
index 2a8fc1ceca..7c1e50362c 100644
--- a/usr/src/uts/common/brand/lx/sys/lx_misc.h
+++ b/usr/src/uts/common/brand/lx/sys/lx_misc.h
@@ -34,6 +34,7 @@ extern void lx_freelwp(klwp_t *);
 extern void *lx_lwpdata_alloc(proc_t *);
 extern void lx_lwpdata_free(void *);
 extern void lx_initlwp(klwp_t *, void *);
+extern void lx_initlwp_post(klwp_t *);
 extern void lx_forklwp(klwp_t *, klwp_t *);
 
 extern void lx_set_gdt(int, user_desc_t *);
diff --git a/usr/src/uts/common/brand/sn1/sn1_brand.c b/usr/src/uts/common/brand/sn1/sn1_brand.c
index 446108fc3d..eb99142327 100644
--- a/usr/src/uts/common/brand/sn1/sn1_brand.c
+++ b/usr/src/uts/common/brand/sn1/sn1_brand.c
@@ -75,6 +75,7 @@ struct brand_ops sn1_brops = {
 	NULL,				/* b_lwpdata_alloc */
 	NULL,				/* b_lwpdata_free */
 	sn1_initlwp,			/* b_initlwp */
+	NULL,				/* b_initlwp_post */
 	sn1_forklwp,			/* b_forklwp */
 	sn1_freelwp,			/* b_freelwp */
 	sn1_lwpexit,			/* b_lwpexit */
diff --git a/usr/src/uts/common/brand/sngl/sngl_brand.c b/usr/src/uts/common/brand/sngl/sngl_brand.c
index e07f79705b..4a21cd1b6b 100644
--- a/usr/src/uts/common/brand/sngl/sngl_brand.c
+++ b/usr/src/uts/common/brand/sngl/sngl_brand.c
@@ -76,6 +76,7 @@ struct brand_ops sngl_brops = {
 	NULL,				/* b_lwpdata_alloc */
 	NULL,				/* b_lwpdata_free */
 	sngl_initlwp,			/* b_initlwp */
+	NULL,				/* b_initlwp_post */
 	sngl_forklwp,			/* b_forklwp */
 	sngl_freelwp,			/* b_freelwp */
 	sngl_lwpexit,			/* b_lwpexit */
diff --git a/usr/src/uts/common/brand/solaris10/s10_brand.c b/usr/src/uts/common/brand/solaris10/s10_brand.c
index 73313d3505..345d2151b3 100644
--- a/usr/src/uts/common/brand/solaris10/s10_brand.c
+++ b/usr/src/uts/common/brand/solaris10/s10_brand.c
@@ -80,6 +80,7 @@ struct brand_ops s10_brops = {
 	NULL,				/* b_lwpdata_alloc */
 	NULL,				/* b_lwpdata_free */
 	s10_initlwp,			/* b_initlwp */
+	NULL,				/* b_initlwp_post */
 	s10_forklwp,			/* b_forklwp */
 	s10_freelwp,			/* b_freelwp */
 	s10_lwpexit,			/* b_lwpexit */
diff --git a/usr/src/uts/common/os/lwp.c b/usr/src/uts/common/os/lwp.c
index 1775e22b0d..3aaf2c746c 100644
--- a/usr/src/uts/common/os/lwp.c
+++ b/usr/src/uts/common/os/lwp.c
@@ -697,7 +697,14 @@ grow:
 	t->t_pre_sys = 1;
 	t->t_post_sys = 1;
 
-	/* Complete branded lwp initialization */
+	/*
+	 * Perform lwp branding
+	 *
+	 * The b_initlwp hook is _not_ allowed to drop p->p_lock as it must be
+	 * continuously held between when the tidhash is sized and when the lwp
+	 * is inserted into it.  Operations requiring p->p_lock to be
+	 * temporarily dropped can be performed in b_initlwp_post.
+	 */
 	if (PROC_IS_BRANDED(p)) {
 		BROP(p)->b_initlwp(lwp, brand_data);
 		/*
@@ -732,6 +739,13 @@ grow:
 	lep->le_start = t->t_start;
 	lwp_hash_in(p, lep, p->p_tidhash, p->p_tidhash_sz, 1);
 
+	/*
+	 * Complete lwp branding
+	 */
+	if (PROC_IS_BRANDED(p) && BROP(p)->b_initlwp_post != NULL) {
+		BROP(p)->b_initlwp_post(lwp);
+	}
+
 	if (state == TS_RUN) {
 		/*
 		 * We set the new lwp running immediately.
diff --git a/usr/src/uts/common/os/pid.c b/usr/src/uts/common/os/pid.c
index 39db5cb27d..eba6147fab 100644
--- a/usr/src/uts/common/os/pid.c
+++ b/usr/src/uts/common/os/pid.c
@@ -21,6 +21,7 @@
 
 /*
  * Copyright (c) 1989, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2015 Joyent, Inc.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
@@ -534,6 +535,20 @@ sprunlock(proc_t *p)
 	THREAD_KPRI_RELEASE();
 }
 
+/*
+ * Undo effects of sprlock but without dropping p->p_lock
+ */
+void
+sprunprlock(proc_t *p)
+{
+	ASSERT(p->p_proc_flag & P_PR_LOCK);
+	ASSERT(MUTEX_HELD(&p->p_lock));
+
+	cv_signal(&pr_pid_cv[p->p_slot]);
+	p->p_proc_flag &= ~P_PR_LOCK;
+	THREAD_KPRI_RELEASE();
+}
+
 void
 pid_init(void)
 {
diff --git a/usr/src/uts/common/sys/brand.h b/usr/src/uts/common/sys/brand.h
index 4f2de94b12..f5a46182b9 100644
--- a/usr/src/uts/common/sys/brand.h
+++ b/usr/src/uts/common/sys/brand.h
@@ -117,7 +117,7 @@ struct execa;
  * b_proc_exit - Perform process brand exit processing
  * b_exec - Reset branded process state on exec
  * b_lwp_setrval - Set return code for forked child
- * b_initlwp - Initialize lwp brand data
+ * b_initlwp - Initialize lwp brand data (cannot drop p->p_lock)
  * b_forklwp - Copy lwp brand data during fork
  * b_freelwp - Free lwp brand data
  * b_lwpexit - Perform lwp-specific brand exit processing
@@ -131,6 +131,7 @@ struct execa;
  * b_lwpdata_alloc - Speculatively allocate data for use in b_initlwp
  * b_lwpdata_free - Free data from allocated by b_lwpdata_alloc if errors occur
  *                  during lwp creation before b_initlwp could be called.
+ * b_initlwp_post - Complete lwp branding (can temporarily drop p->p_lock)
  * b_exit_with_sig - Instead of sending SIGCLD, exit with custom behavior
  * b_psig_to_proc - Custom additional behavior during psig
  * b_wait_filter - Filter processes from being matched by waitid
@@ -163,6 +164,7 @@ struct brand_ops {
 	void	*(*b_lwpdata_alloc)(struct proc *);
 	void	(*b_lwpdata_free)(void *);
 	void	(*b_initlwp)(klwp_t *, void *);
+	void	(*b_initlwp_post)(klwp_t *);
 	void	(*b_forklwp)(klwp_t *, klwp_t *);
 	void	(*b_freelwp)(klwp_t *);
 	void	(*b_lwpexit)(klwp_t *);
diff --git a/usr/src/uts/common/sys/proc.h b/usr/src/uts/common/sys/proc.h
index 5d3e59dc71..ff4a1abce4 100644
--- a/usr/src/uts/common/sys/proc.h
+++ b/usr/src/uts/common/sys/proc.h
@@ -647,6 +647,7 @@ extern int sprtrylock_proc(proc_t *);
 extern void sprwaitlock_proc(proc_t *);
 extern void sprlock_proc(proc_t *);
 extern void sprunlock(proc_t *);
+extern void sprunprlock(proc_t *);
 extern void pid_init(void);
 extern proc_t *pid_entry(int);
 extern int pid_slot(proc_t *);