OS-5380 lxbrand cgroupfs deadlocks against pidlockrelease-20160428

Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>

2 files changed, 21 insertions, 10 deletions

diff --git a/usr/src/uts/common/brand/lx/cgroups/cgrps_vfsops.c b/usr/src/uts/common/brand/lx/cgroups/cgrps_vfsops.c
index 3bd9675996..a9bd783569 100644
--- a/usr/src/uts/common/brand/lx/cgroups/cgrps_vfsops.c
+++ b/usr/src/uts/common/brand/lx/cgroups/cgrps_vfsops.c
@@ -905,6 +905,8 @@ cgrp_run_rel_agent(void *a)
  * Launch the user-level release_agent manager. The event data is the
  * pathname (relative to the mount point of the file system) of the newly empty
  * cgroup.
+ *
+ * The cg_contents mutex is held on entry and dropped before returning.
  */
 void
 cgrp_rel_agent_event(cgrp_mnt_t *cgm, cgrp_node_t *cn)
@@ -922,13 +924,17 @@ cgrp_rel_agent_event(cgrp_mnt_t *cgm, cgrp_node_t *cn)
 	ASSERT(MUTEX_HELD(&cgm->cg_contents));
 
 	/* Nothing to do if the agent is not set */
-	if (cgm->cg_agent[0] == '\0')
+	if (cgm->cg_agent[0] == '\0') {
+		mutex_exit(&cgm->cg_contents);
 		return;
+	}
 
 	parent = cn->cgn_parent;
 	/* Cannot remove the top-level cgroup (only via unmount) */
-	if (parent == cn)
+	if (parent == cn) {
+		mutex_exit(&cgm->cg_contents);
 		return;
+	}
 
 	argstr = kmem_alloc(MAXPATHLEN, KM_SLEEP);
 	oldstr = kmem_alloc(MAXPATHLEN, KM_SLEEP);
@@ -977,6 +983,12 @@ cgrp_rel_agent_event(cgrp_mnt_t *cgm, cgrp_node_t *cn)
 	/* The release agent process cannot belong to our cgroup */
 	plwpd->br_cgroupid = 0;
 
+	/*
+	 * The cg_contents mutex cannot be held while taking the pool lock
+	 * or calling newproc.
+	 */
+	mutex_exit(&cgm->cg_contents);
+
 	if (z->zone_defaultcid > 0) {
 		cid = z->zone_defaultcid;
 	} else {
@@ -987,8 +999,6 @@ cgrp_rel_agent_event(cgrp_mnt_t *cgm, cgrp_node_t *cn)
 	if (cid == -1)
 		cid = defaultcid;
 
-	mutex_exit(&cgm->cg_contents);
-
 	if ((agent_err = newproc(cgrp_run_rel_agent, (void *)rarg, cid,
 	    minclsyspri - 1, NULL, -1)) != 0) {
 		/* There's nothing we can do if creating the proc fails. */
@@ -996,8 +1006,6 @@ cgrp_rel_agent_event(cgrp_mnt_t *cgm, cgrp_node_t *cn)
 		kmem_free(rarg->crraa_agent_path, sizeof (cgm->cg_agent));
 		kmem_free(rarg, sizeof (cgrp_rra_arg_t));
 	}
-
-	mutex_enter(&cgm->cg_contents);
 }
 
 /*ARGSUSED*/
@@ -1037,7 +1045,8 @@ cgrp_lwp_exit_helper(vfs_t *vfsp, uint_t cg_id, id_t tid, pid_t tpid)
 	if (cn->cgn_task_cnt == 0 && cn->cgn_dirents == N_DIRENTS(cgm) &&
 	    cn->cgn_notify == 1) {
 		cgrp_rel_agent_event(cgm, cn);
+		ASSERT(MUTEX_NOT_HELD(&cgm->cg_contents));
+	} else {
+		mutex_exit(&cgm->cg_contents);
 	}
-
-	mutex_exit(&cgm->cg_contents);
 }
diff --git a/usr/src/uts/common/brand/lx/cgroups/cgrps_vnops.c b/usr/src/uts/common/brand/lx/cgroups/cgrps_vnops.c
index d3020f60c7..bd571c8c18 100644
--- a/usr/src/uts/common/brand/lx/cgroups/cgrps_vnops.c
+++ b/usr/src/uts/common/brand/lx/cgroups/cgrps_vnops.c
@@ -183,8 +183,7 @@ cgrp_thr_move(cgrp_mnt_t *cgm, lx_lwp_data_t *plwpd, cgrp_node_t *ncn,
 		 */
 		mutex_exit(&p->p_lock);
 		cgrp_rel_agent_event(cgm, ocn);
-		mutex_exit(&cgm->cg_contents);
-
+		ASSERT(MUTEX_NOT_HELD(&cgm->cg_contents));
 		return (B_TRUE);
 	}
 
@@ -1364,10 +1363,13 @@ cgrp_rmdir(struct vnode *dvp, char *nm, struct vnode *cdir, struct cred *cred,
 	if (parent->cgn_task_cnt == 0 &&
 	    parent->cgn_dirents == N_DIRENTS(cgm) && parent->cgn_notify == 1) {
 		cgrp_rel_agent_event(cgm, parent);
+		ASSERT(MUTEX_NOT_HELD(&cgm->cg_contents));
+		goto dropped;
 	}
 
 done:
 	mutex_exit(&cgm->cg_contents);
+dropped:
 	vnevent_rmdir(CGNTOV(self), dvp, nm, ct);
 	cgnode_rele(self);