OS-4460 exec brands processes that still have multiple threads

Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Joshua M. Clulow <jmc@joyent.com>

6 files changed, 114 insertions, 27 deletions

diff --git a/usr/src/uts/common/brand/lx/os/lx_misc.c b/usr/src/uts/common/brand/lx/os/lx_misc.c
index ef94600bb2..67565379fe 100644
--- a/usr/src/uts/common/brand/lx/os/lx_misc.c
+++ b/usr/src/uts/common/brand/lx/os/lx_misc.c
@@ -262,9 +262,23 @@ lx_freelwp(klwp_t *lwp)
 	struct lx_lwp_data *lwpd = lwptolxlwp(lwp);
 	proc_t *p = lwptoproc(lwp);
 
-	VERIFY(lwpd != NULL);
 	VERIFY(MUTEX_NOT_HELD(&p->p_lock));
 
+	if (lwpd == NULL) {
+		/*
+		 * There is one case where an LX branded process will possess
+		 * LWPs which lack their own brand data.  During the course of
+		 * executing native binary, the process will be preemptively
+		 * branded to allow hooks such as b_native_exec to function.
+		 * If that process possesses multiple LWPS, they will _not_ be
+		 * branded since they will exit if the exec succeeds.  It's
+		 * during this LWP exit that lx_freelwp would be called on an
+		 * unbranded LWP.  When that is the case, it is acceptable to
+		 * bypass the hook.
+		 */
+		return;
+	}
+
 	/*
 	 * It is possible for the lx_freelwp hook to be called without a prior
 	 * call to lx_exitlwp being made.  This happens as part of lwp
diff --git a/usr/src/uts/common/os/brand.c b/usr/src/uts/common/os/brand.c
index 00f2ce0440..f00e1bf2c1 100644
--- a/usr/src/uts/common/os/brand.c
+++ b/usr/src/uts/common/os/brand.c
@@ -312,8 +312,8 @@ brand_unregister_zone(struct brand *bp)
 	mutex_exit(&brand_list_lock);
 }
 
-void
-brand_setbrand(proc_t *p)
+int
+brand_setbrand(proc_t *p, boolean_t lwps_ok)
 {
 	brand_t *bp = p->p_zone->zone_brand;
 	void *brand_data = NULL;
@@ -322,10 +322,48 @@ brand_setbrand(proc_t *p)
 	VERIFY(bp != NULL);
 
 	/*
-	 * We should only be called from exec() or getproc(), when we know the
-	 * process has 0 or 1 threads.
+	 * Process branding occurs during fork() and exec().  When it happens
+	 * during fork(), the LWP count will always be 0 since branding is
+	 * performed as part of getproc(), before LWPs have been associated.
+	 * The same is not true during exec(), where a multi-LWP process may
+	 * undergo branding just prior to gexec(). This is to ensure
+	 * exec-related brand hooks are available.  While it may seem
+	 * complicated to brand a multi-LWP process, the two possible outcomes
+	 * simplify things:
+	 *
+	 * 1. The exec() succeeds:  LWPs besides the caller will be killed and
+	 *    any further branding will occur in a single-LWP context.
+	 * 2. The exec() fails: The process will be promptly unbranded since
+	 *    the hooks are no longer needed.
+	 *
+	 * To prevent inconsistent brand state from being encountered during
+	 * the exec(), LWPs beyond the caller which are associated with this
+	 * process must be held temporarily.  They will be released either when
+	 * they are killed in the exec() success, or when the brand is cleared
+	 * after exec() failure.
 	 */
-	VERIFY((p->p_tlist == NULL) || (p->p_tlist == p->p_tlist->t_forw));
+	if (lwps_ok) {
+		/*
+		 * We've been called from a exec() context tolerating the
+		 * existence of multiple LWPs during branding is necessary.
+		 */
+		VERIFY(p == curproc);
+		VERIFY(p->p_tlist != NULL);
+
+		if (p->p_tlist != p->p_tlist->t_forw) {
+			/*
+			 * Multiple LWPs are present.  Hold all but the caller.
+			 */
+			if (!holdlwps(SHOLDFORK1)) {
+				return (-1);
+			}
+		}
+	} else {
+		/*
+		 * Processes branded during fork() should not have LWPs at all.
+		 */
+		VERIFY(p->p_tlist == NULL);
+	}
 
 	if (bp->b_data_size > 0) {
 		brand_data = kmem_zalloc(bp->b_data_size, KM_SLEEP);
@@ -338,10 +376,11 @@ brand_setbrand(proc_t *p)
 	ASSERT(PROC_IS_BRANDED(p));
 	BROP(p)->b_setbrand(p);
 	mutex_exit(&p->p_lock);
+	return (0);
 }
 
 void
-brand_clearbrand(proc_t *p)
+brand_clearbrand(proc_t *p, boolean_t lwps_ok)
 {
 	brand_t *bp = p->p_zone->zone_brand;
 	void *brand_data;
@@ -350,16 +389,30 @@ brand_clearbrand(proc_t *p)
 	VERIFY(bp != NULL);
 	VERIFY(PROC_IS_BRANDED(p));
 
-	/*
-	 * There cannot be more than one lwp associated with a process when
-	 * stripping the brand.
-	 */
-	VERIFY((p->p_tlist == NULL) || (p->p_tlist == p->p_tlist->t_forw));
-
 	mutex_enter(&p->p_lock);
 	p->p_brand = &native_brand;
 	brand_data = p->p_brand_data;
 	p->p_brand_data = NULL;
+
+	if (lwps_ok) {
+		VERIFY(p == curproc);
+		/*
+		 * A process with multiple LWPs is being de-branded after
+		 * failing an exec.  The other LWPs were held as part of the
+		 * procedure, so they must be resumed now.
+		 */
+		if (p->p_tlist != NULL && p->p_tlist != p->p_tlist->t_forw) {
+			continuelwps(p);
+		}
+	} else {
+		/*
+		 * While clearing the brand, it's ok for one LWP to be present.
+		 * This happens when a native binary is executed inside a
+		 * branded zone, since the brand will be removed during the
+		 * course of a successful exec.
+		 */
+		VERIFY(p->p_tlist == NULL || p->p_tlist == p->p_tlist->t_forw);
+	}
 	mutex_exit(&p->p_lock);
 
 	if (brand_data != NULL) {
diff --git a/usr/src/uts/common/os/exec.c b/usr/src/uts/common/os/exec.c
index 15e481da1a..ae82c5f4e8 100644
--- a/usr/src/uts/common/os/exec.c
+++ b/usr/src/uts/common/os/exec.c
@@ -303,25 +303,38 @@ exec_common(const char *fname, const char **argp, const char **envp,
 	if (brandme) {
 		void *brand_data = NULL;
 
-		brand_setbrand(p);
-		if (BROP(p)->b_lwpdata_alloc != NULL &&
-		    (brand_data = BROP(p)->b_lwpdata_alloc(p)) == NULL) {
+		/*
+		 * Process branding may fail if multiple LWPs are present and
+		 * holdlwps() cannot complete successfully.
+		 */
+		error = brand_setbrand(p, B_TRUE);
+
+		if (error == 0 && BROP(p)->b_lwpdata_alloc != NULL) {
+			brand_data = BROP(p)->b_lwpdata_alloc(p);
+			if (brand_data == NULL) {
+				error = 1;
+			}
+		}
+
+		if (error == 0) {
+			mutex_enter(&p->p_lock);
+			BROP(p)->b_initlwp(lwp, brand_data);
+			mutex_exit(&p->p_lock);
+		} else {
 			VN_RELE(vp);
-			if (dir != NULL)
+			if (dir != NULL) {
 				VN_RELE(dir);
+			}
 			pn_free(&resolvepn);
 			goto fail;
 		}
-		mutex_enter(&p->p_lock);
-		BROP(p)->b_initlwp(lwp, brand_data);
-		mutex_exit(&p->p_lock);
 	}
 
 	if ((error = gexec(&vp, &ua, &args, NULL, 0, &execsz,
 	    exec_file, p->p_cred, &brand_action)) != 0) {
 		if (brandme) {
 			BROP(p)->b_freelwp(lwp);
-			brand_clearbrand(p);
+			brand_clearbrand(p, B_TRUE);
 		}
 		VN_RELE(vp);
 		if (dir != NULL)
@@ -441,7 +454,7 @@ exec_common(const char *fname, const char **argp, const char **envp,
 	/* Unbrand ourself if necessary. */
 	if (PROC_IS_BRANDED(p) && (brand_action == EBA_NATIVE)) {
 		BROP(p)->b_freelwp(lwp);
-		brand_clearbrand(p);
+		brand_clearbrand(p, B_FALSE);
 	}
 
 	setregs(&args);
diff --git a/usr/src/uts/common/os/exit.c b/usr/src/uts/common/os/exit.c
index 6e3884946c..e0bd8a9a68 100644
--- a/usr/src/uts/common/os/exit.c
+++ b/usr/src/uts/common/os/exit.c
@@ -1356,7 +1356,7 @@ freeproc(proc_t *p)
 
 	/* Clear any remaining brand data */
 	if (PROC_IS_BRANDED(p)) {
-		brand_clearbrand(p);
+		brand_clearbrand(p, B_FALSE);
 	}
 
 
diff --git a/usr/src/uts/common/os/fork.c b/usr/src/uts/common/os/fork.c
index 522731f5e2..e8602fcd05 100644
--- a/usr/src/uts/common/os/fork.c
+++ b/usr/src/uts/common/os/fork.c
@@ -704,7 +704,7 @@ fork_fail(proc_t *cp)
 	if (PTOU(curproc)->u_cwd)
 		refstr_rele(PTOU(curproc)->u_cwd);
 	if (PROC_IS_BRANDED(cp)) {
-		brand_clearbrand(cp);
+		brand_clearbrand(cp, B_FALSE);
 	}
 }
 
@@ -1162,7 +1162,14 @@ getproc(proc_t **cpp, pid_t pid, uint_t flags)
 	mutex_exit(&pidlock);
 
 	if (PROC_IS_BRANDED(pp)) {
-		brand_setbrand(cp);
+		/*
+		 * The only reason why process branding should fail is when
+		 * the procedure is complicated by multiple LWPs on the scene.
+		 * With an LWP count of 0, this newly allocated process has no
+		 * reason to fail branding.
+		 */
+		VERIFY0(brand_setbrand(cp, B_FALSE));
+
 		BROP(pp)->b_copy_procdata(cp, pp);
 	}
 
diff --git a/usr/src/uts/common/sys/brand.h b/usr/src/uts/common/sys/brand.h
index f753902bab..4f2de94b12 100644
--- a/usr/src/uts/common/sys/brand.h
+++ b/usr/src/uts/common/sys/brand.h
@@ -235,8 +235,8 @@ extern brand_t	*brand_register_zone(struct brand_attr *);
 extern brand_t	*brand_find_name(char *);
 extern void	brand_unregister_zone(brand_t *);
 extern int	brand_zone_count(brand_t *);
-extern void	brand_setbrand(proc_t *);
-extern void	brand_clearbrand(proc_t *);
+extern int	brand_setbrand(proc_t *, boolean_t);
+extern void	brand_clearbrand(proc_t *, boolean_t);
 
 /*
  * The following functions can be shared among kernel brand modules which