OS-2894 lxbrand add PTRACE_O_TRACEFORK and PTRACE_O_TRACEEXEC support

8 files changed, 305 insertions, 54 deletions

diff --git a/usr/src/lib/brand/lx/lx_brand/common/clone.c b/usr/src/lib/brand/lx/lx_brand/common/clone.c
index 3686b61b54..8752d0b02f 100644
--- a/usr/src/lib/brand/lx/lx_brand/common/clone.c
+++ b/usr/src/lib/brand/lx/lx_brand/common/clone.c
@@ -145,6 +145,8 @@ lx_exit(uintptr_t p1)
 	lx_tsd->lxtsd_exit = LX_EXIT;
 	lx_tsd->lxtsd_exit_status = status;
 
+	lx_ptrace_stop_if_option(LX_PTRACE_O_TRACEEXIT);
+
 	/*
 	 * Block all signals in the exit context to avoid taking any signals
 	 * (to the degree possible) while exiting.
@@ -453,6 +455,7 @@ lx_clone(uintptr_t p1, uintptr_t p2, uintptr_t p3, uintptr_t p4,
 		if (cldstk)
 			lx_setup_clone(rp->lxr_gs, (void *)rp->lxr_eip, cldstk);
 
+		lx_ptrace_stop_if_option(LX_PTRACE_O_TRACECLONE);
 		return (0);
 	}
 
@@ -541,5 +544,8 @@ lx_clone(uintptr_t p1, uintptr_t p2, uintptr_t p3, uintptr_t p4,
 		rval = clone_res;
 	}
 
+	if (rval == 0)
+		lx_ptrace_stop_if_option(LX_PTRACE_O_TRACECLONE);
+
 	return (rval);
 }
diff --git a/usr/src/lib/brand/lx/lx_brand/common/fork.c b/usr/src/lib/brand/lx/lx_brand/common/fork.c
index 7e75efaa39..6087d4c8c3 100644
--- a/usr/src/lib/brand/lx/lx_brand/common/fork.c
+++ b/usr/src/lib/brand/lx/lx_brand/common/fork.c
@@ -22,10 +22,9 @@
 /*
  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2014 Joyent, Inc.  All rights reserved.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <errno.h>
 #include <unistd.h>
 #include <sys/lx_misc.h>
@@ -43,8 +42,11 @@ lx_fork(void)
 {
 	int ret = fork1();
 
-	if (ret == 0 && lx_is_rpm)
-		(void) sleep(lx_rpm_delay);
+	if (ret == 0) {
+		if (lx_is_rpm)
+			(void) sleep(lx_rpm_delay);
+		lx_ptrace_stop_if_option(LX_PTRACE_O_TRACEFORK);
+	}
 
 	return (ret == -1 ? -errno : ret);
 }
@@ -61,5 +63,9 @@ lx_vfork(void)
 {
 	int ret = fork1();
 
+	if (ret == 0) {
+		lx_ptrace_stop_if_option(LX_PTRACE_O_TRACEVFORK);
+	}
+
 	return (ret == -1 ? -errno : ret);
 }
diff --git a/usr/src/lib/brand/lx/lx_brand/common/misc.c b/usr/src/lib/brand/lx/lx_brand/common/misc.c
index 3e8397d659..466f7274e8 100644
--- a/usr/src/lib/brand/lx/lx_brand/common/misc.c
+++ b/usr/src/lib/brand/lx/lx_brand/common/misc.c
@@ -21,6 +21,7 @@
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2014 Joyent, Inc.  All rights reserved.
  */
 
 #include <assert.h>
@@ -485,6 +486,8 @@ lx_execve(uintptr_t p1, uintptr_t p2, uintptr_t p3)
 	if (argv == NULL)
 		argv = nullist;
 
+	lx_ptrace_stop_if_option(LX_PTRACE_O_TRACEEXEC);
+
 	/* This is a normal exec call. */
 	(void) execve(filename, argv, envp);
 
diff --git a/usr/src/lib/brand/lx/lx_brand/common/ptrace.c b/usr/src/lib/brand/lx/lx_brand/common/ptrace.c
index 45432c2c94..b3b6d1579b 100644
--- a/usr/src/lib/brand/lx/lx_brand/common/ptrace.c
+++ b/usr/src/lib/brand/lx/lx_brand/common/ptrace.c
@@ -123,16 +123,6 @@
 #define	LX_PTRACE_SYSCALL	24
 #define	LX_PTRACE_SETOPTIONS	0x4200
 
-/* Options for LX_PTRACE_SETOPTIONS */
-#define	LX_PTRACE_O_TRACESYSGOOD	0x0001
-#define	LX_PTRACE_O_TRACEFORK		0x0002
-#define	LX_PTRACE_O_TRACEVFORK		0x0004
-#define	LX_PTRACE_O_TRACECLONE		0x0008
-#define	LX_PTRACE_O_TRACEEXEC		0x0010
-#define	LX_PTRACE_O_TRACEVFORKDONE	0x0020
-#define	LX_PTRACE_O_TRACEEXIT		0x0040
-#define	LX_PTRACE_O_TRACESECCOMP	0x0080
-
 /*
  * This corresponds to the user_i387_struct Linux structure.
  */
@@ -827,19 +817,28 @@ setup_watchpoints(pid_t pid, uintptr_t *debugreg)
 
 /*
  * Returns TRUE if the process is traced, FALSE otherwise.  This is only true
- * if the process is currently stopped, and has been traced using PTRACE_TRACEME
- * or PTRACE_ATTACH.
+ * if the process is currently stopped, and has been traced using
+ * PTRACE_TRACEME, PTRACE_ATTACH or one of the Linux-specific trace options.
  */
 static int
 is_traced(pid_t pid)
 {
 	ptrace_monitor_map_t *p;
 	pstatus_t status;
+	uint_t curr_opts;
+
+	/*
+	 * First get the stop options since that is an indication that the
+	 * process is being traced.
+	 */
+	if (syscall(SYS_brand, B_PTRACE_EXT_OPTS, B_PTRACE_EXT_OPTS_GET, pid,
+	    &curr_opts) != 0)
+		return (0);
 
 	if (get_status(pid, &status) != 0)
 		return (0);
 
-	if ((status.pr_flags & PR_PTRACE) &&
+	if ((status.pr_flags & PR_PTRACE || curr_opts != 0) &&
 	    (status.pr_ppid == getpid()) &&
 	    (status.pr_lwp.pr_flags & PR_ISTOP))
 		return (1);
@@ -1687,37 +1686,62 @@ ptrace_syscall(pid_t lxpid, pid_t pid, lwpid_t lwpid, int sig)
 static int
 ptrace_setoptions(pid_t pid, int options)
 {
-	int fd;
-	int error;
-	pstatus_t status;
-
-	/* XXX currently error if unsupported option */
-	if (options & ~(LX_PTRACE_O_TRACEEXEC | LX_PTRACE_O_TRACEFORK))
-		return (-EINVAL);
+	int ret;
 
 	/*
-	 * TRACEEXEC and TRACEFORK are similar to ptrace_traceme except that it
-	 * stops the next time the child forks or execs
+	 * If we're setting the TRACEEXEC option for a process, its PR_PTRACE
+	 * option might also be set. We clear it so that the process doesn't
+	 * stop twice on exec.
 	 */
-	if ((error = get_status(pid, &status)) != 0)
-		return (error);
+	if (options & LX_PTRACE_O_TRACEEXEC) {
+		int fd;
+		int error;
+		long ctl[2];
+		pstatus_t status;
+
+		if ((ret = get_status(pid, &status)) != 0)
+			return (ret);
+
+		if ((fd = open_procfile(pid, O_WRONLY, "ctl")) < 0)
+			return (-errno);
+
+		ctl[0] = PCUNSET;
+		ctl[1] = PR_PTRACE;
+		error = 0;
+		if (write(fd, ctl, sizeof (ctl)) != sizeof (ctl) ||
+		    ptrace_trace_common(fd) != 0)
+			error = -errno;
 
-	if ((fd = open_procfile(pid, O_WRONLY, "ctl")) < 0)
-		return (-errno);
+		(void) close(fd);
 
-	error = 0;
+		if (error != 0)
+			return (error);
+	}
 
-	/* XXX Implement correct emulation for TRACEFORK and TREACEEXEC */
-	/*
-	 * if (options & LX_PTRACE_O_TRACEFORK) {
-	 * }
-	 *
-	 * if (options & LX_PTRACE_O_TRACEEXEC) {
-	 * }
-	 */
+	ret = syscall(SYS_brand, B_PTRACE_EXT_OPTS, B_PTRACE_EXT_OPTS_SET, pid,
+	    options);
 
-	(void) close(fd);
-	return (error);
+	return (-ret);
+}
+
+void
+lx_ptrace_stop_if_option(int option)
+{
+	pid_t pid;
+	uint_t curr_opts;
+
+	pid = getpid();
+	if (pid == 1)
+		pid = zoneinit_pid;
+
+	/* first we have to see if the stop option is set for this process */
+	if (syscall(SYS_brand, B_PTRACE_EXT_OPTS, B_PTRACE_EXT_OPTS_GET, pid,
+	    &curr_opts) != 0)
+		return;
+
+	/* if the option is set, this brand call will stop us */
+	if (curr_opts & option)
+		(void) syscall(SYS_brand, B_PTRACE_STOP_FOR_OPT, option);
 }
 
 int
@@ -1996,9 +2020,12 @@ set_dr6(pid_t pid, siginfo_t *infop)
 }
 
 /*
- * This is called from the emulation of the wait4 and waitpid system call to
- * take into account the monitor processes which we spawn to observe other
- * processes from ptrace_attach().
+ * This is called from the emulation of the wait4, waitpid and waitid system
+ * calls to take into account:
+ *  - the monitor processes which we spawn to observe other processes from
+ *    ptrace_attach().
+ *  - the extended si_status result we can get when extended ptrace options
+ *    are enabled.
  */
 int
 lx_ptrace_wait(siginfo_t *infop)
@@ -2027,13 +2054,28 @@ lx_ptrace_wait(siginfo_t *infop)
 	}
 	(void) mutex_unlock(&ptrace_map_mtx);
 
-	/*
-	 * If the traced process got a SIGWAITING, we must be in the middle
-	 * of a clone(2) with CLONE_PTRACE set.
-	 */
-	if (infop->si_code == CLD_TRAPPED && infop->si_status == SIGWAITING) {
-		ptrace_catch_fork(pid, 0);
-		return (-1);
+	if (infop->si_code == CLD_TRAPPED) {
+		/*
+		 * If the traced process got a SIGWAITING, we must be in the
+		 * middle of a clone(2) with CLONE_PTRACE set.
+		 */
+		if (infop->si_status == SIGWAITING) {
+			ptrace_catch_fork(pid, 0);
+			return (-1);
+		}
+
+		/*
+		 * If the traced process got a SIGTRAP then Linux ptrace
+		 * options might have been set, so setup the extended
+		 * si_status to contain the (possible) event.
+		 */
+		if (infop->si_status == SIGTRAP) {
+			uint_t event;
+
+			if (syscall(SYS_brand, B_PTRACE_EXT_OPTS,
+			    B_PTRACE_EXT_OPTS_EVT, pid, &event) == 0)
+				infop->si_status |= event;
+		}
 	}
 
 	if (get_status(pid, &status) == 0 &&
diff --git a/usr/src/lib/brand/lx/lx_brand/common/wait.c b/usr/src/lib/brand/lx/lx_brand/common/wait.c
index 18b2382385..7512838bf3 100644
--- a/usr/src/lib/brand/lx/lx_brand/common/wait.c
+++ b/usr/src/lib/brand/lx/lx_brand/common/wait.c
@@ -275,7 +275,7 @@ int
 lx_waitid(uintptr_t idtype, uintptr_t id, uintptr_t infop, uintptr_t opt)
 {
 	int rval, options;
-	siginfo_t s_infop = {0};
+	siginfo_t s_info = {0};
 	if ((options = ltos_options(opt)) == -1)
 		return (-1);
 	switch (idtype) {
@@ -291,8 +291,12 @@ lx_waitid(uintptr_t idtype, uintptr_t id, uintptr_t infop, uintptr_t opt)
 	default:
 		return (-EINVAL);
 	}
-	if ((rval = lx_waitid_helper(idtype, (id_t)id, &s_infop, options)) < 0)
+	if ((rval = lx_waitid_helper(idtype, (id_t)id, &s_info, options)) < 0)
 		return (rval);
 
-	return (stol_siginfo(&s_infop, (lx_siginfo_t *)infop));
+	/* If the WNOHANG flag was specified and no child was found return 0. */
+	if ((options & WNOHANG) && s_info.si_pid == 0)
+		return (0);
+
+	return (stol_siginfo(&s_info, (lx_siginfo_t *)infop));
 }
diff --git a/usr/src/lib/brand/lx/lx_brand/sys/lx_misc.h b/usr/src/lib/brand/lx/lx_brand/sys/lx_misc.h
index d1eb7beac6..9a48982baf 100644
--- a/usr/src/lib/brand/lx/lx_brand/sys/lx_misc.h
+++ b/usr/src/lib/brand/lx/lx_brand/sys/lx_misc.h
@@ -99,6 +99,33 @@ extern boolean_t lx_is_rpm;
 #define	LX_RUSAGE_CHILDREN	(-1)
 
 /*
+ * Based on code from brand_misc.h, but use of that is incompatible with the
+ * lx brand.
+ *
+ * These macros invoke a brandsys subcommand, B_TRUSS_POINT, which makes it
+ * easy to debug with DTrace.
+ */
+#define	B_TRUSS_POINT	6
+
+#define	B_TRACE_POINT_5(a0, a1, a2, a3, a4) \
+	(void) syscall(SYS_brand, B_TRUSS_POINT, (a0), (a1), (a2), (a3), (a4))
+
+#define	B_TRACE_POINT_4(a0, a1, a2, a3) \
+	B_TRACE_POINT_5((a0), (a1), (a2), (a3), 0)
+
+#define	B_TRACE_POINT_3(a0, a1, a2) \
+	B_TRACE_POINT_5((a0), (a1), (a2), 0, 0)
+
+#define	B_TRACE_POINT_2(a0, a1) \
+	B_TRACE_POINT_5((a0), (a1), 0, 0, 0)
+
+#define	B_TRACE_POINT_1(a0) \
+	B_TRACE_POINT_5((a0), 0, 0, 0, 0)
+
+#define	B_TRACE_POINT_0() \
+	B_TRACE_POINT_5(0, 0, 0, 0, 0)
+
+/*
  * normally we never want to write to stderr or stdout because it's unsafe
  * to make assumptions about the underlying file descriptors.  to protect
  * against writes to these file descriptors we go ahead and close them
@@ -125,6 +152,7 @@ extern int lx_lpid_to_spid(pid_t, pid_t *);
 
 extern int lx_ptrace_wait(siginfo_t *);
 extern void lx_ptrace_fork(void);
+extern void lx_ptrace_stop_if_option(int);
 
 extern int lx_check_alloca(size_t);
 #define	SAFE_ALLOCA(sz)	(lx_check_alloca(sz) ? alloca(sz) : NULL)
diff --git a/usr/src/uts/common/brand/lx/os/lx_brand.c b/usr/src/uts/common/brand/lx/os/lx_brand.c
index 4474869e70..7533101e7f 100644
--- a/usr/src/uts/common/brand/lx/os/lx_brand.c
+++ b/usr/src/uts/common/brand/lx/os/lx_brand.c
@@ -331,6 +331,118 @@ lx_ptrace_fire(void)
 	}
 }
 
+/*
+ * Supports Linux PTRACE_SETOPTIONS handling which is similar to PTRACE_TRACEME
+ * but return an event in the second byte of si_status.
+ */
+static int
+lx_ptrace_ext_opts(int cmd, pid_t pid, uintptr_t val, int64_t *rval)
+{
+	proc_t *p;
+	klwp_t *lwp;
+	lx_proc_data_t *lpdp;
+	uint_t ret;
+
+	if (cmd != B_PTRACE_EXT_OPTS_SET && cmd != B_PTRACE_EXT_OPTS_GET &&
+	    cmd != B_PTRACE_EXT_OPTS_EVT)
+		return (set_errno(EINVAL));
+
+	if ((p = sprlock(pid)) == NULL)
+		return (ESRCH);
+
+	if (priv_proc_cred_perm(curproc->p_cred, p, NULL, VWRITE) != 0) {
+		sprunlock(p);
+		return (EPERM);
+	}
+
+	if ((lpdp = p->p_brand_data) == NULL) {
+		sprunlock(p);
+		return (ESRCH);
+	}
+
+	if (cmd == B_PTRACE_EXT_OPTS_SET) {
+		lpdp->l_ptrace_opts = (uint_t)val;
+
+	} else if (cmd == B_PTRACE_EXT_OPTS_GET) {
+		ret = lpdp->l_ptrace_opts;
+
+	} else /* B_PTRACE_EXT_OPTS_EVT */ {
+		ret = lpdp->l_ptrace_event;
+		lpdp->l_ptrace_event = 0;
+	}
+
+	sprunlock(p);
+
+	if (cmd != B_PTRACE_EXT_OPTS_SET) {
+		if (copyout(&ret, (void *)val, sizeof (uint_t)) != 0)
+			return (EFAULT);
+	}
+
+	*rval = 0;
+	return (0);
+}
+
+/*
+ * Used to support Linux PTRACE_SETOPTIONS handling and similar to
+ * PTRACE_TRACEME. We signal ourselves to stop on return from this syscall and
+ * setup the event reason so the emulation can pull this out when someone
+ * 'waits' on this process.
+ */
+static void
+lx_ptrace_stop_for_option(int option)
+{
+	proc_t *p = ttoproc(curthread);
+	lx_proc_data_t *lpdp;
+
+	psignal(p, SIGTRAP);
+
+	if ((lpdp = p->p_brand_data) == NULL) {
+		/* this should never happen but just let the process stop */
+		return;
+	}
+
+	/* Track the event as the reason for stopping */
+	switch (option) {
+	case LX_PTRACE_O_TRACEFORK:
+		lpdp->l_ptrace_event = LX_PTRACE_EVENT_FORK;
+		break;
+	case LX_PTRACE_O_TRACEVFORK:
+		lpdp->l_ptrace_event = LX_PTRACE_EVENT_VFORK;
+		break;
+	case LX_PTRACE_O_TRACECLONE:
+		lpdp->l_ptrace_event = LX_PTRACE_EVENT_CLONE;
+		break;
+	case LX_PTRACE_O_TRACEEXEC:
+		lpdp->l_ptrace_event = LX_PTRACE_EVENT_EXEC;
+		break;
+	case LX_PTRACE_O_TRACEVFORKDONE:
+		lpdp->l_ptrace_event = LX_PTRACE_EVENT_VFORK_DONE;
+		break;
+	case LX_PTRACE_O_TRACEEXIT:
+		lpdp->l_ptrace_event = LX_PTRACE_EVENT_EXIT;
+		break;
+	case LX_PTRACE_O_TRACESECCOMP:
+		lpdp->l_ptrace_event = LX_PTRACE_EVENT_SECCOMP;
+		break;
+	}
+
+	/*
+	 * If (p_proc_flag & P_PR_PTRACE) were set, then in stop() we would set:
+	 *	p->p_wcode = CLD_TRAPPED
+	 *	p->p_wdata = SIGTRAP
+	 * However, when using the extended ptrace options we disable
+	 * P_PR_PTRACE so that we don't stop twice on exec when
+	 * LX_PTRACE_O_TRACEEXEC is set. We could ensure P_PR_PTRACE is set
+	 * when using extended options but then we would stop on exec even when
+	 * LX_PTRACE_O_TRACEEXEC is not set, so that is clearly broken. Thus,
+	 * we have to set p_wcode and p_wdata ourselves so that waitid will
+	 * do the right thing for this process. We still rely on stop() to do
+	 * all of the other processing needed for our signal.
+	 */
+	p->p_wdata = SIGTRAP;
+	p->p_wcode = CLD_TRAPPED;
+}
+
 void
 lx_brand_systrace_enable(void)
 {
@@ -500,6 +612,15 @@ lx_brandsys(int cmd, int64_t *rval, uintptr_t arg1, uintptr_t arg2,
 		}
 		return (*rval = 0);
 
+	/*
+	 * The B_TRUSS_POINT subcommand is used so that we can make a no-op
+	 * syscall for debugging purposes (dtracing) from within the user-level
+	 * emulation.
+	 */
+	case B_TRUSS_POINT:
+		*rval = 0;
+		return (0);
+
 	case B_LPID_TO_SPAIR:
 		/*
 		 * Given a Linux pid as arg1, return the Solaris pid in arg2 and
@@ -594,6 +715,17 @@ lx_brandsys(int cmd, int64_t *rval, uintptr_t arg1, uintptr_t arg2,
 		 */
 		return (lx_sched_affinity(cmd, arg1, arg2, arg3, rval));
 
+	case B_PTRACE_EXT_OPTS:
+		/*
+		 * Set or get the ptrace extended options or get the event
+		 * reason for the stop.
+		 */
+		return (lx_ptrace_ext_opts((int)arg1, (pid_t)arg2, arg3, rval));
+
+	case B_PTRACE_STOP_FOR_OPT:
+		lx_ptrace_stop_for_option((int)arg1);
+		return (0);
+
 	default:
 		linux_call = cmd - B_EMULATE_SYSCALL;
 		/*
diff --git a/usr/src/uts/common/brand/lx/sys/lx_brand.h b/usr/src/uts/common/brand/lx/sys/lx_brand.h
index c872d114a0..c71c3b8032 100644
--- a/usr/src/uts/common/brand/lx/sys/lx_brand.h
+++ b/usr/src/uts/common/brand/lx/sys/lx_brand.h
@@ -75,9 +75,37 @@ extern "C" {
 #define	B_PTRACE_SYSCALL	131
 #define	B_SET_AFFINITY_MASK	132
 #define	B_GET_AFFINITY_MASK	133
+#define	B_PTRACE_EXT_OPTS	134
+#define	B_PTRACE_STOP_FOR_OPT	135
 
 #define	B_EMULATE_SYSCALL	192
 
+/* B_PTRACE_EXT_OPTS subcommands */
+#define	 B_PTRACE_EXT_OPTS_SET	1
+#define	 B_PTRACE_EXT_OPTS_GET	2
+#define	 B_PTRACE_EXT_OPTS_EVT	3
+
+/*
+ * Support for Linux PTRACE_SETOPTIONS handling.
+ */
+#define	LX_PTRACE_O_TRACESYSGOOD	0x0001
+#define	LX_PTRACE_O_TRACEFORK		0x0002
+#define	LX_PTRACE_O_TRACEVFORK		0x0004
+#define	LX_PTRACE_O_TRACECLONE		0x0008
+#define	LX_PTRACE_O_TRACEEXEC		0x0010
+#define	LX_PTRACE_O_TRACEVFORKDONE	0x0020
+#define	LX_PTRACE_O_TRACEEXIT		0x0040
+#define	LX_PTRACE_O_TRACESECCOMP	0x0080
+
+/* siginfo si_status for traced events */
+#define	LX_PTRACE_EVENT_FORK		0x100
+#define	LX_PTRACE_EVENT_VFORK		0x200
+#define	LX_PTRACE_EVENT_CLONE		0x300
+#define	LX_PTRACE_EVENT_EXEC		0x400
+#define	LX_PTRACE_EVENT_VFORK_DONE	0x500
+#define	LX_PTRACE_EVENT_EXIT		0x600
+#define	LX_PTRACE_EVENT_SECCOMP		0x700
+
 #define	LX_VERSION_1		1
 #define	LX_VERSION		LX_VERSION_1
 
@@ -160,6 +188,8 @@ typedef struct lx_proc_data {
 	void (*l_sigrestorer[MAXSIG])(void); /* array of sigrestorer fns */
 	pid_t l_ppid;		/* pid of originating parent proc */
 	uint64_t l_ptrace;	/* process being observed with ptrace */
+	uint_t l_ptrace_opts;	/* process's extended ptrace options */
+	uint_t l_ptrace_event;	/* extended ptrace option trap event */
 	lx_elf_data_t l_elf_data; /* ELF data for linux executable */
 } lx_proc_data_t;