OS-4303 lxbrand ltp ptrace05 fails

Reviewed by: Joshua M. Clulow <jmc@joyent.com>

9 files changed, 44 insertions, 9 deletions

diff --git a/usr/src/lib/brand/lx/testing/ltp_skiplist b/usr/src/lib/brand/lx/testing/ltp_skiplist
index 6f3ca08907..0755aace85 100644
--- a/usr/src/lib/brand/lx/testing/ltp_skiplist
+++ b/usr/src/lib/brand/lx/testing/ltp_skiplist
@@ -191,9 +191,6 @@ process_vm_readv03
 process_vm_writev01
 process_vm_writev02
 prot_hsymlinks
-ptrace01		# OS-3915
-ptrace02
-ptrace05		# OS-4303
 read02
 readahead01
 readahead02
diff --git a/usr/src/uts/common/brand/lx/os/lx_brand.c b/usr/src/uts/common/brand/lx/os/lx_brand.c
index 620157a6ec..c22d782b68 100644
--- a/usr/src/uts/common/brand/lx/os/lx_brand.c
+++ b/usr/src/uts/common/brand/lx/os/lx_brand.c
@@ -263,6 +263,7 @@ struct brand_ops lx_brops = {
 	lx_waitid_helper,		/* b_waitid_helper */
 	lx_sigcld_repost,		/* b_sigcld_repost */
 	lx_ptrace_issig_stop,		/* b_issig_stop */
+	lx_ptrace_sig_ignorable,	/* b_sig_ignorable */
 	lx_savecontext,			/* b_savecontext */
 #if defined(_SYSCALL32_IMPL)
 	lx_savecontext32,		/* b_savecontext32 */
diff --git a/usr/src/uts/common/brand/lx/os/lx_ptrace.c b/usr/src/uts/common/brand/lx/os/lx_ptrace.c
index 95df7b2d53..1659da56ff 100644
--- a/usr/src/uts/common/brand/lx/os/lx_ptrace.c
+++ b/usr/src/uts/common/brand/lx/os/lx_ptrace.c
@@ -1677,6 +1677,22 @@ lx_ptrace_issig_stop(proc_t *p, klwp_t *lwp)
 	return (0);
 }
 
+boolean_t
+lx_ptrace_sig_ignorable(proc_t *p, int sig)
+{
+	lx_proc_data_t *lxpd = ptolxproc(p);
+
+	if (lxpd->l_ptrace != 0 && lx_stol_signo(sig, 0) != 0) {
+		/*
+		 * In order to preserve proper ptrace behavior when it comes to
+		 * signal handling, it is unacceptable to ignore any signals.
+		 * Doing so would bypass the logic in lx_ptrace_issig_stop.
+		 */
+		return (B_FALSE);
+	}
+	return (B_TRUE);
+}
+
 static void
 lx_ptrace_exit_tracer(proc_t *p, lx_lwp_data_t *lwpd,
     lx_ptrace_accord_t *accord)
diff --git a/usr/src/uts/common/brand/lx/sys/lx_misc.h b/usr/src/uts/common/brand/lx/sys/lx_misc.h
index c6161b2a98..ae6a891087 100644
--- a/usr/src/uts/common/brand/lx/sys/lx_misc.h
+++ b/usr/src/uts/common/brand/lx/sys/lx_misc.h
@@ -68,6 +68,7 @@ extern int lx_ptrace_stop_for_option(int, boolean_t, ulong_t, uintptr_t);
 extern int lx_ptrace_set_clone_inherit(int, boolean_t);
 extern int lx_sigcld_repost(proc_t *, sigqueue_t *);
 extern int lx_ptrace_issig_stop(proc_t *, klwp_t *);
+extern boolean_t lx_ptrace_sig_ignorable(proc_t *, int);
 
 extern int lx_helper_clone(int64_t *, int, void *, void *, void *);
 extern int lx_helper_setgroups(int, gid_t *);
diff --git a/usr/src/uts/common/brand/sn1/sn1_brand.c b/usr/src/uts/common/brand/sn1/sn1_brand.c
index 176044f68d..7f99dc5f33 100644
--- a/usr/src/uts/common/brand/sn1/sn1_brand.c
+++ b/usr/src/uts/common/brand/sn1/sn1_brand.c
@@ -92,6 +92,7 @@ struct brand_ops sn1_brops = {
 	NULL,				/* b_waitid_helper */
 	NULL,				/* b_sigcld_repost */
 	NULL,				/* b_issig_stop */
+	NULL,				/* b_sig_ignorable */
 	NULL,				/* b_savecontext */
 #if defined(_SYSCALL32_IMPL)
 	NULL,				/* b_savecontext32 */
diff --git a/usr/src/uts/common/brand/sngl/sngl_brand.c b/usr/src/uts/common/brand/sngl/sngl_brand.c
index 3d4da3c441..d5e1808d89 100644
--- a/usr/src/uts/common/brand/sngl/sngl_brand.c
+++ b/usr/src/uts/common/brand/sngl/sngl_brand.c
@@ -93,6 +93,7 @@ struct brand_ops sngl_brops = {
 	NULL,				/* b_waitid_helper */
 	NULL,				/* b_sigcld_repost */
 	NULL,				/* b_issig_stop */
+	NULL,				/* b_sig_ignorable */
 	NULL,				/* b_savecontext */
 #if defined(_SYSCALL32_IMPL)
 	NULL,				/* b_savecontext32 */
diff --git a/usr/src/uts/common/brand/solaris10/s10_brand.c b/usr/src/uts/common/brand/solaris10/s10_brand.c
index 1dc2671386..f70f54a788 100644
--- a/usr/src/uts/common/brand/solaris10/s10_brand.c
+++ b/usr/src/uts/common/brand/solaris10/s10_brand.c
@@ -97,6 +97,7 @@ struct brand_ops s10_brops = {
 	NULL,				/* b_waitid_helper */
 	NULL,				/* b_sigcld_repost */
 	NULL,				/* b_issig_stop */
+	NULL,				/* b_sig_ignorable */
 	NULL,				/* b_savecontext */
 #if defined(_SYSCALL32_IMPL)
 	NULL,				/* b_savecontext32 */
diff --git a/usr/src/uts/common/os/sig.c b/usr/src/uts/common/os/sig.c
index ae643c280e..bc48c6e6e8 100644
--- a/usr/src/uts/common/os/sig.c
+++ b/usr/src/uts/common/os/sig.c
@@ -142,6 +142,21 @@ signal_is_blocked(kthread_t *t, int sig)
 }
 
 /*
+ * Return true if the signal can safely be ignored.
+ * That is, if the signal is included in the p_ignore mask and doing so is not
+ * forbidden by any process branding.
+ */
+static int
+sig_ignorable(proc_t *p, int sig)
+{
+	return (sigismember(&p->p_ignore, sig) &&	/* sig in ignore mask */
+	    !(PROC_IS_BRANDED(p) &&			/* allowed by brand */
+	    BROP(p)->b_sig_ignorable != NULL &&
+	    BROP(p)->b_sig_ignorable(p, sig) == B_FALSE));
+
+}
+
+/*
  * Return true if the signal can safely be discarded on generation.
  * That is, if there is no need for the signal on the receiving end.
  * The answer is true if the process is a zombie or
@@ -158,7 +173,7 @@ sig_discardable(proc_t *p, int sig)
 	kthread_t *t = p->p_tlist;
 
 	return (t == NULL ||		/* if zombie or ... */
-	    (sigismember(&p->p_ignore, sig) &&	/* signal is ignored */
+	    (sig_ignorable(p, sig) &&		/* signal is ignored */
 	    t->t_forw == t &&			/* and single-threaded */
 	    !tracing(p, sig) &&			/* and no /proc tracing */
 	    !signal_is_blocked(t, sig) &&	/* and signal not blocked */
@@ -482,7 +497,7 @@ issig_justlooking(void)
 			if (sigismember(&set, sig) &&
 			    (tracing(p, sig) ||
 			    sigismember(&t->t_sigwait, sig) ||
-			    !sigismember(&p->p_ignore, sig))) {
+			    !sig_ignorable(p, sig))) {
 				/*
 				 * Don't promote a signal that will stop
 				 * the process when lwp_nostop is set.
@@ -656,7 +671,7 @@ issig_forreal(void)
 			lwp->lwp_cursig = 0;
 			lwp->lwp_extsig = 0;
 			if (sigismember(&t->t_sigwait, sig) ||
-			    (!sigismember(&p->p_ignore, sig) &&
+			    (!sig_ignorable(p, sig) &&
 			    !isjobstop(sig))) {
 				if (p->p_flag & (SEXITLWPS|SKILLED)) {
 					sig = SIGKILL;
@@ -708,7 +723,7 @@ issig_forreal(void)
 				toproc = 0;
 				if (tracing(p, sig) ||
 				    sigismember(&t->t_sigwait, sig) ||
-				    !sigismember(&p->p_ignore, sig)) {
+				    !sig_ignorable(p, sig)) {
 					if (sigismember(&t->t_extsig, sig))
 						ext = 1;
 					break;
@@ -722,7 +737,7 @@ issig_forreal(void)
 				toproc = 1;
 				if (tracing(p, sig) ||
 				    sigismember(&t->t_sigwait, sig) ||
-				    !sigismember(&p->p_ignore, sig)) {
+				    !sig_ignorable(p, sig)) {
 					if (sigismember(&p->p_extsig, sig))
 						ext = 1;
 					break;
@@ -1344,7 +1359,7 @@ psig(void)
 	 * this signal from pending to current (we dropped p->p_lock).
 	 * This can happen only in a multi-threaded process.
 	 */
-	if (sigismember(&p->p_ignore, sig) ||
+	if (sig_ignorable(p, sig) ||
 	    (func == SIG_DFL && sigismember(&stopdefault, sig))) {
 		lwp->lwp_cursig = 0;
 		lwp->lwp_extsig = 0;
diff --git a/usr/src/uts/common/sys/brand.h b/usr/src/uts/common/sys/brand.h
index 79e02dcf17..8183f643a4 100644
--- a/usr/src/uts/common/sys/brand.h
+++ b/usr/src/uts/common/sys/brand.h
@@ -141,6 +141,7 @@ struct execa;
  * b_waitid_helper - Generate synthetic results for waitid
  * b_sigcld_repost - Post synthetic SIGCLD signals
  * b_issig_stop - Alter/suppress signal delivery during issig
+ * b_sig_ignorable - Disallow discarding of signals
  * b_savecontext - Alter context during savecontext
  * b_restorecontext - Alter context during restorecontext
  * b_sendsig_stack - Override stack used for signal delivery
@@ -183,6 +184,7 @@ struct brand_ops {
 	    boolean_t *, int *);
 	int	(*b_sigcld_repost)(proc_t *, sigqueue_t *);
 	int	(*b_issig_stop)(proc_t *, klwp_t *);
+	boolean_t (*b_sig_ignorable)(proc_t *, int);
 	void	(*b_savecontext)(ucontext_t *);
 #if defined(_SYSCALL32_IMPL)
 	void	(*b_savecontext32)(ucontext32_t *);