OS-3438 lx brand: "start rsyslog" hangs

3 files changed, 35 insertions, 17 deletions

diff --git a/usr/src/uts/common/brand/lx/os/lx_brand.c b/usr/src/uts/common/brand/lx/os/lx_brand.c
index 1581d5c3b3..58ebdb3110 100644
--- a/usr/src/uts/common/brand/lx/os/lx_brand.c
+++ b/usr/src/uts/common/brand/lx/os/lx_brand.c
@@ -101,6 +101,7 @@ static int lx_elfexec(struct vnode *vp, struct execa *uap, struct uarg *args,
     caddr_t exec_file, struct cred *cred, int brand_action);
 
 static boolean_t lx_native_exec(uint8_t, const char **);
+static void lx_ptrace_exectrap(proc_t *);
 
 /* lx brand */
 struct brand_ops lx_brops = {
@@ -125,7 +126,8 @@ struct brand_ops lx_brops = {
 	NSIG,
 	lx_exit_with_sig,
 	lx_wait_filter,
-	lx_native_exec
+	lx_native_exec,
+	lx_ptrace_exectrap
 };
 
 struct brand_mach_ops lx_mops = {
@@ -461,21 +463,22 @@ lx_ptrace_stop_for_option(int option)
 	p->p_wcode = CLD_STOPPED;
 	sigcld(p, sqp);
 	mutex_exit(&pidlock);
+}
 
-	/*
-	 * If (p_proc_flag & P_PR_PTRACE) were set, then in stop() we would set:
-	 *	p->p_wcode = CLD_TRAPPED
-	 *	p->p_wdata = SIGTRAP
-	 * However, when using the extended ptrace options we disable
-	 * P_PR_PTRACE so that we don't stop twice on exec when
-	 * LX_PTRACE_O_TRACEEXEC is set. We could ensure P_PR_PTRACE is set
-	 * when using extended options but then we would stop on exec even when
-	 * LX_PTRACE_O_TRACEEXEC is not set, so that is clearly broken. Thus,
-	 * we have to set p_wcode and p_wdata ourselves so that waitid will
-	 * do the right thing for this process. We still rely on stop() to do
-	 * all of the other processing needed for our signal.
-	 */
-	p->p_wcode = CLD_TRAPPED;
+/*
+ * Brand entry to allow us to optionally generate the ptrace SIGTRAP on exec().
+ * This will only be called if ptrace is enabled -- and we only generate the
+ * SIGTRAP if LX_PTRACE_O_TRACEEXEC hasn't been set.
+ */
+void
+lx_ptrace_exectrap(proc_t *p)
+{
+	lx_proc_data_t *lpdp;
+
+	if ((lpdp = p->p_brand_data) == NULL ||
+	    !(lpdp->l_ptrace_opts & LX_PTRACE_O_TRACEEXEC)) {
+		psignal(p, SIGTRAP);
+	}
 }
 
 void
diff --git a/usr/src/uts/common/os/exec.c b/usr/src/uts/common/os/exec.c
index 1d16255cc5..e896bbd1e5 100644
--- a/usr/src/uts/common/os/exec.c
+++ b/usr/src/uts/common/os/exec.c
@@ -859,8 +859,22 @@ gexec(
 			if (pp->p_plist || (pp->p_proc_flag & P_PR_TRACE))
 				args->traceinval = 1;
 		}
-		if (pp->p_proc_flag & P_PR_PTRACE)
-			psignal(pp, SIGTRAP);
+
+		/*
+		 * If legacy ptrace is enabled, defer to the brand as to the
+		 * behavior as to the SIGTRAP generated during exec().  (If
+		 * we're not branded or the brand isn't interested in changing
+		 * the default behavior, we generate the SIGTRAP.)
+		 */
+		if (pp->p_proc_flag & P_PR_PTRACE) {
+			if (PROC_IS_BRANDED(pp) &&
+			    BROP(pp)->b_ptrace_exectrap != NULL) {
+				BROP(pp)->b_ptrace_exectrap(pp);
+			} else {
+				psignal(pp, SIGTRAP);
+			}
+		}
+
 		if (args->traceinval)
 			prinvalidate(&pp->p_user);
 	}
diff --git a/usr/src/uts/common/sys/brand.h b/usr/src/uts/common/sys/brand.h
index 04833705f3..a3add10399 100644
--- a/usr/src/uts/common/sys/brand.h
+++ b/usr/src/uts/common/sys/brand.h
@@ -130,6 +130,7 @@ struct brand_ops {
 	void	(*b_exit_with_sig)(proc_t *, sigqueue_t *, void *);
 	boolean_t (*b_wait_filter)(proc_t *, proc_t *);
 	boolean_t (*b_native_exec)(uint8_t, const char **);
+	void (*b_ptrace_exectrap)(proc_t *);
 };
 
 /*