OS-3742 lxbrand add support for signalfd

OS-4382 remove obsolete brand hooks added during lx development

17 files changed, 81 insertions, 37 deletions

diff --git a/usr/src/lib/brand/lx/lx_brand/common/lx_brand.c b/usr/src/lib/brand/lx/lx_brand/common/lx_brand.c
index 73cd0ce8af..12dd468a6c 100644
--- a/usr/src/lib/brand/lx/lx_brand/common/lx_brand.c
+++ b/usr/src/lib/brand/lx/lx_brand/common/lx_brand.c
@@ -1196,14 +1196,14 @@ static lx_syscall_handler_t lx_handlers[] = {
 	NULL,				/* 279: move_pages */
 	lx_utimensat,			/* 280: utimensat */
 	lx_epoll_pwait,			/* 281: epoll_pwait */
-	NULL,				/* 282: signalfd */
+	lx_signalfd,			/* 282: signalfd */
 	lx_timerfd_create,		/* 283: timerfd_create */
 	lx_eventfd,			/* 284: eventfd */
 	NULL,				/* 285: fallocate */
 	lx_timerfd_settime,		/* 286: timerfd_settime */
 	lx_timerfd_gettime,		/* 287: timerfd_gettime */
 	lx_accept4,			/* 288: accept4 */
-	NULL,				/* 289: signalfd4 */
+	lx_signalfd4,			/* 289: signalfd4 */
 	lx_eventfd2,			/* 290: eventfd2 */
 	lx_epoll_create1,		/* 291: epoll_create1 */
 	lx_dup3,			/* 292: dup3 */
@@ -1566,13 +1566,13 @@ static lx_syscall_handler_t lx_handlers[] = {
 	NULL,				/* 318: getcpu */
 	lx_epoll_pwait,			/* 319: epoll_pwait */
 	lx_utimensat,			/* 320: utimensat */
-	NULL,				/* 321: signalfd */
+	lx_signalfd,			/* 321: signalfd */
 	lx_timerfd_create,		/* 322: timerfd_create */
 	lx_eventfd,			/* 323: eventfd */
 	NULL,				/* 324: fallocate */
 	lx_timerfd_settime,		/* 325: timerfd_settime */
 	lx_timerfd_gettime,		/* 326: timerfd_gettime */
-	NULL,				/* 327: signalfd4 */
+	lx_signalfd4,			/* 327: signalfd4 */
 	lx_eventfd2,			/* 328: eventfd2 */
 	lx_epoll_create1,		/* 329: epoll_create1 */
 	lx_dup3,			/* 330: dup3 */
diff --git a/usr/src/lib/brand/lx/lx_brand/common/signal.c b/usr/src/lib/brand/lx/lx_brand/common/signal.c
index 7dc5a04004..92bd6cfeb4 100644
--- a/usr/src/lib/brand/lx/lx_brand/common/signal.c
+++ b/usr/src/lib/brand/lx/lx_brand/common/signal.c
@@ -57,6 +57,7 @@
 #include <stdio.h>
 #include <libintl.h>
 #include <ieeefp.h>
+#include <sys/signalfd.h>
 
 #if defined(_ILP32)
 extern int pselect_large_fdset(int nfds, fd_set *in0, fd_set *out0, fd_set *ex0,
@@ -2569,3 +2570,35 @@ lx_rt_tgsigqueueinfo(uintptr_t p1, uintptr_t p2, uintptr_t p3, uintptr_t p4)
 	return ((syscall(SYS_brand, B_HELPER_TGSIGQUEUE, tgid, tid, sig,
 	    &siginfo)) ? (-errno) : 0);
 }
+
+long
+lx_signalfd(int fd, uintptr_t mask, size_t msize)
+{
+	return (lx_signalfd4(fd, mask, msize, 0));
+}
+
+long
+lx_signalfd4(int fd, uintptr_t mask, size_t msize, int flags)
+{
+	sigset_t s_set;
+	int r;
+
+	if (msize != sizeof (int64_t))
+		return (-EINVAL);
+
+	if (ltos_sigset((lx_sigset_t *)mask, &s_set) != 0)
+		return (-errno);
+
+	r = signalfd(fd, &s_set, flags);
+
+	/*
+	 * signalfd(3C) may fail with ENOENT if /dev/signalfd is not available.
+	 * It is less jarring to Linux programs to tell them that internal
+	 * allocation failed than to report an error number they are not
+	 * expecting.
+	 */
+	if (r == -1 && errno == ENOENT)
+		return (-ENODEV);
+
+	return (r == -1 ? -errno : r);
+}
diff --git a/usr/src/lib/brand/lx/lx_brand/sys/lx_syscall.h b/usr/src/lib/brand/lx/lx_brand/sys/lx_syscall.h
index 14c7fe8371..46a775bb3e 100644
--- a/usr/src/lib/brand/lx/lx_brand/sys/lx_syscall.h
+++ b/usr/src/lib/brand/lx/lx_brand/sys/lx_syscall.h
@@ -313,6 +313,8 @@ extern long lx_setuid(uid_t);
 extern long lx_setregid(gid_t, gid_t);
 extern long lx_setreuid(uid_t, uid_t);
 extern long lx_shmdt(char *);
+extern long lx_signalfd(int, uintptr_t, size_t);
+extern long lx_signalfd4(int, uintptr_t, size_t, int);
 extern long lx_stime(const time_t *);
 extern long lx_symlink(const char *, const char *);
 extern long lx_syslog(int, char *, int);
diff --git a/usr/src/lib/brand/lx/testing/Readme_ltp b/usr/src/lib/brand/lx/testing/Readme_ltp
index d29be61e7a..42ca40d4ff 100644
--- a/usr/src/lib/brand/lx/testing/Readme_ltp
+++ b/usr/src/lib/brand/lx/testing/Readme_ltp
@@ -257,9 +257,6 @@ x settimeofday01
 # setxattr03
 # shmget05	OS-3326
 # sighold02	fails on sig 63 OS-3307
-# signalfd01
-# signalfd4_01
-# signalfd4_02
 # sigrelse01	fails on sig 63 OS-3307
 # splice01
 # splice02
diff --git a/usr/src/lib/brand/lx/testing/ltp_skiplist b/usr/src/lib/brand/lx/testing/ltp_skiplist
index 0755aace85..7d3a6d06eb 100644
--- a/usr/src/lib/brand/lx/testing/ltp_skiplist
+++ b/usr/src/lib/brand/lx/testing/ltp_skiplist
@@ -225,9 +225,6 @@ setxattr02
 setxattr03
 sgetmask01
 shmget05		# OS-3326
-signalfd01
-signalfd4_01
-signalfd4_02
 splice01
 splice02
 splice03
diff --git a/usr/src/lib/brand/lx/zone/platform.xml b/usr/src/lib/brand/lx/zone/platform.xml
index f841795530..ab5362909b 100644
--- a/usr/src/lib/brand/lx/zone/platform.xml
+++ b/usr/src/lib/brand/lx/zone/platform.xml
@@ -80,6 +80,7 @@
 	<device match="poll" />
 	<device match="pts/*" />
 	<device match="random" />
+	<device match="signalfd" />
 	<device match="tcp" />
 	<device match="tcp6" />
 	<device match="timerfd" />
diff --git a/usr/src/uts/common/brand/lx/os/lx_brand.c b/usr/src/uts/common/brand/lx/os/lx_brand.c
index 23990a0603..62cc2eca42 100644
--- a/usr/src/uts/common/brand/lx/os/lx_brand.c
+++ b/usr/src/uts/common/brand/lx/os/lx_brand.c
@@ -255,12 +255,11 @@ struct brand_ops lx_brops = {
 	lx_elfexec,			/* b_elfexec */
 	NULL,				/* b_sigset_native_to_brand */
 	NULL,				/* b_sigset_brand_to_native */
-	NULL,				/* b_psig_to_proc */
+	lx_sigfd_translate,		/* b_sigfd_translate */
 	NSIG,				/* b_nsig */
 	lx_exit_with_sig,		/* b_exit_with_sig */
 	lx_wait_filter,			/* b_wait_filter */
 	lx_native_exec,			/* b_native_exec */
-	NULL,				/* b_ptrace_exectrap */
 	lx_map32limit,			/* b_map32limit */
 	lx_stop_notify,			/* b_stop_notify */
 	lx_waitid_helper,		/* b_waitid_helper */
diff --git a/usr/src/uts/common/brand/lx/os/lx_misc.c b/usr/src/uts/common/brand/lx/os/lx_misc.c
index 1ce305ea41..a53282921b 100644
--- a/usr/src/uts/common/brand/lx/os/lx_misc.c
+++ b/usr/src/uts/common/brand/lx/os/lx_misc.c
@@ -38,6 +38,7 @@
 #include <sys/lx_brand.h>
 #include <sys/lx_misc.h>
 #include <sys/lx_futex.h>
+#include <lx_errno.h>
 #include <sys/cmn_err.h>
 #include <sys/siginfo.h>
 #include <sys/contract/process_impl.h>
@@ -782,3 +783,26 @@ lx_stol_hwaddr(const struct sockaddr_dl *src, struct sockaddr *dst, int *size)
 	bcopy(LLADDR(src), dst->sa_data, copy_size);
 	*size = copy_size;
 }
+
+/*
+ * Brand hook to convert native kernel siginfo signal number, errno, code, pid
+ * and si_status to Linux values. Similar to the stol_ksiginfo function but
+ * this one converts in-place, converts the pid, and does not copyout.
+ */
+void
+lx_sigfd_translate(k_siginfo_t *infop)
+{
+	infop->si_signo = lx_stol_signo(infop->si_signo, LX_SIGKILL);
+
+	infop->si_status = lx_stol_status(infop->si_status, LX_SIGKILL);
+
+	infop->si_code = lx_stol_sigcode(infop->si_code);
+
+	infop->si_errno = lx_errno(infop->si_errno, EINVAL);
+
+	if (infop->si_pid == curproc->p_zone->zone_proc_initpid) {
+		infop->si_pid = 1;
+	} else if (infop->si_pid == curproc->p_zone->zone_zsched->p_pid) {
+		infop->si_pid = 0;
+	}
+}
diff --git a/usr/src/uts/common/brand/lx/os/lx_syscall.c b/usr/src/uts/common/brand/lx/os/lx_syscall.c
index fff06a65a2..150f4a80ea 100644
--- a/usr/src/uts/common/brand/lx/os/lx_syscall.c
+++ b/usr/src/uts/common/brand/lx/os/lx_syscall.c
@@ -848,13 +848,13 @@ lx_sysent_t lx_sysent32[] = {
 	{"getcpu",	lx_getcpu,		0,		3}, /* 318 */
 	{"epoll_pwait",	NULL,			0,		5}, /* 319 */
 	{"utimensat",	NULL,			0,		4}, /* 320 */
-	{"signalfd",	NULL,			NOSYS_NULL,	0}, /* 321 */
+	{"signalfd",	NULL,			0,		3}, /* 321 */
 	{"timerfd_create", NULL,		0,		2}, /* 322 */
 	{"eventfd",	NULL,			0,		1}, /* 323 */
 	{"fallocate",	NULL,			NOSYS_NULL,	0}, /* 324 */
 	{"timerfd_settime", NULL,		0,		4}, /* 325 */
 	{"timerfd_gettime", NULL,		0,		2}, /* 326 */
-	{"signalfd4",	NULL,			NOSYS_NULL,	0}, /* 327 */
+	{"signalfd4",	NULL,			0,		4}, /* 327 */
 	{"eventfd2",	NULL,			0,		2}, /* 328 */
 	{"epoll_create1", NULL,			0,		1}, /* 329 */
 	{"dup3",	NULL,			0,		3}, /* 330 */
@@ -1176,14 +1176,14 @@ lx_sysent_t lx_sysent64[] = {
 	{"move_pages",	NULL,			NOSYS_NULL,	0}, /* 279 */
 	{"utimensat",	NULL,			0,		4}, /* 280 */
 	{"epoll_pwait",	NULL,			0,		5}, /* 281 */
-	{"signalfd",	NULL,			NOSYS_NULL,	0}, /* 282 */
+	{"signalfd",	NULL,			0,		3}, /* 282 */
 	{"timerfd_create", NULL,		0,		2}, /* 283 */
 	{"eventfd",	NULL,			0,		1}, /* 284 */
 	{"fallocate",	NULL,			NOSYS_NULL,	0}, /* 285 */
 	{"timerfd_settime", NULL,		0,		4}, /* 286 */
 	{"timerfd_gettime", NULL,		0,		2}, /* 287 */
 	{"accept4",	NULL,			0,		4}, /* 288 */
-	{"signalfd4",	NULL,			NOSYS_NULL,	0}, /* 289 */
+	{"signalfd4",	NULL,			0,		4}, /* 289 */
 	{"eventfd2",	NULL,			0,		2}, /* 290 */
 	{"epoll_create1", NULL,			0,		1}, /* 291 */
 	{"dup3",	NULL,			0,		3}, /* 292 */
diff --git a/usr/src/uts/common/brand/lx/sys/lx_misc.h b/usr/src/uts/common/brand/lx/sys/lx_misc.h
index a3df150eef..bd0b31fadc 100644
--- a/usr/src/uts/common/brand/lx/sys/lx_misc.h
+++ b/usr/src/uts/common/brand/lx/sys/lx_misc.h
@@ -20,6 +20,7 @@
 extern "C" {
 #endif
 
+#include <sys/siginfo.h>
 #include <sys/lx_brand.h>
 
 #ifdef _KERNEL
@@ -42,6 +43,7 @@ extern greg_t lx_fixsegreg(greg_t, model_t);
 extern uintptr_t lx_fsbase(klwp_t *, uintptr_t);
 extern void lx_exit_with_sig(proc_t *, sigqueue_t *);
 extern boolean_t lx_wait_filter(proc_t *, proc_t *);
+extern void lx_sigfd_translate(k_siginfo_t *);
 
 typedef enum lx_if_action {
 	LX_IF_FROMNATIVE,
diff --git a/usr/src/uts/common/brand/sn1/sn1_brand.c b/usr/src/uts/common/brand/sn1/sn1_brand.c
index 7f99dc5f33..446108fc3d 100644
--- a/usr/src/uts/common/brand/sn1/sn1_brand.c
+++ b/usr/src/uts/common/brand/sn1/sn1_brand.c
@@ -81,12 +81,11 @@ struct brand_ops sn1_brops = {
 	sn1_elfexec,			/* b_elfexec */
 	NULL,				/* b_sigset_native_to_brand */
 	NULL,				/* b_sigset_brand_to_native */
-	NULL,				/* b_psig_to_proc */
+	NULL,				/* b_sigfd_translate */
 	NSIG,				/* b_nsig */
 	NULL,				/* b_exit_with_sig */
 	NULL,				/* b_wait_filter */
 	NULL,				/* b_native_exec */
-	NULL,				/* b_ptrace_exectrap */
 	NULL,				/* b_map32limit */
 	NULL,				/* b_stop_notify */
 	NULL,				/* b_waitid_helper */
diff --git a/usr/src/uts/common/brand/sngl/sngl_brand.c b/usr/src/uts/common/brand/sngl/sngl_brand.c
index d5e1808d89..e07f79705b 100644
--- a/usr/src/uts/common/brand/sngl/sngl_brand.c
+++ b/usr/src/uts/common/brand/sngl/sngl_brand.c
@@ -82,12 +82,11 @@ struct brand_ops sngl_brops = {
 	sngl_elfexec,			/* b_elfexec */
 	NULL,				/* b_sigset_native_to_brand */
 	NULL,				/* b_sigset_brand_to_native */
-	NULL,				/* b_psig_to_proc */
+	NULL,				/* b_sigfd_translate */
 	NSIG,				/* b_nsig */
 	NULL,				/* b_exit_with_sig */
 	NULL,				/* b_wait_filter */
 	NULL,				/* b_native_exec */
-	NULL,				/* b_ptrace_exectrap */
 	NULL,				/* b_map32limit */
 	NULL,				/* b_stop_notify */
 	NULL,				/* b_waitid_helper */
diff --git a/usr/src/uts/common/brand/solaris10/s10_brand.c b/usr/src/uts/common/brand/solaris10/s10_brand.c
index f70f54a788..73313d3505 100644
--- a/usr/src/uts/common/brand/solaris10/s10_brand.c
+++ b/usr/src/uts/common/brand/solaris10/s10_brand.c
@@ -86,12 +86,11 @@ struct brand_ops s10_brops = {
 	s10_elfexec,			/* b_elfexec */
 	s10_sigset_native_to_s10,	/* b_sigset_native_to_brand */
 	s10_sigset_s10_to_native,	/* b_sigset_brand_to_native */
-	NULL,				/* b_psig_to_proc */
+	NULL,				/* b_sigfd_translate */
 	S10_NSIG,			/* b_nsig */
 	NULL,				/* b_exit_with_sig */
 	NULL,				/* b_wait_filter */
 	NULL,				/* b_native_exec */
-	NULL,				/* b_ptrace_exectrap */
 	NULL,				/* b_map32limit */
 	NULL,				/* b_stop_notify */
 	NULL,				/* b_waitid_helper */
diff --git a/usr/src/uts/common/io/signalfd.c b/usr/src/uts/common/io/signalfd.c
index 3b9b76e7f4..734cf3170a 100644
--- a/usr/src/uts/common/io/signalfd.c
+++ b/usr/src/uts/common/io/signalfd.c
@@ -97,6 +97,7 @@
 #include <sys/schedctl.h>
 #include <sys/id_space.h>
 #include <sys/sdt.h>
+#include <sys/brand.h>
 
 typedef struct signalfd_state signalfd_state_t;
 
@@ -390,6 +391,9 @@ consume_signal(k_sigset_t set, uio_t *uio, boolean_t block)
 	lwp->lwp_extsig = 0;
 	mutex_exit(&p->p_lock);
 
+	if (PROC_IS_BRANDED(p) && BROP(p)->b_sigfd_translate)
+		BROP(p)->b_sigfd_translate(infop);
+
 	/* Convert k_siginfo into external, datamodel independent, struct. */
 	bzero(ssp, sizeof (*ssp));
 	ssp->ssi_signo = infop->si_signo;
diff --git a/usr/src/uts/common/os/exec.c b/usr/src/uts/common/os/exec.c
index 7e32fb0506..15e481da1a 100644
--- a/usr/src/uts/common/os/exec.c
+++ b/usr/src/uts/common/os/exec.c
@@ -882,18 +882,10 @@ gexec(
 		}
 
 		/*
-		 * If legacy ptrace is enabled, defer to the brand as to the
-		 * behavior as to the SIGTRAP generated during exec().  (If
-		 * we're not branded or the brand isn't interested in changing
-		 * the default behavior, we generate the SIGTRAP.)
+		 * If legacy ptrace is enabled, generate the SIGTRAP.
 		 */
 		if (pp->p_proc_flag & P_PR_PTRACE) {
-			if (PROC_IS_BRANDED(pp) &&
-			    BROP(pp)->b_ptrace_exectrap != NULL) {
-				BROP(pp)->b_ptrace_exectrap(pp);
-			} else {
-				psignal(pp, SIGTRAP);
-			}
+			psignal(pp, SIGTRAP);
 		}
 
 		if (args->traceinval)
diff --git a/usr/src/uts/common/os/sig.c b/usr/src/uts/common/os/sig.c
index cc2e753526..a2cfb44ee7 100644
--- a/usr/src/uts/common/os/sig.c
+++ b/usr/src/uts/common/os/sig.c
@@ -1478,9 +1478,6 @@ psig(void)
 		DTRACE_PROC3(signal__handle, int, sig, k_siginfo_t *,
 		    sip, void (*)(void), func);
 
-		if (PROC_IS_BRANDED(p) && BROP(p)->b_psig_to_proc)
-			BROP(p)->b_psig_to_proc(p, t, sig);
-
 		lwp->lwp_cursig = 0;
 		lwp->lwp_extsig = 0;
 		if (lwp->lwp_curinfo) {
diff --git a/usr/src/uts/common/sys/brand.h b/usr/src/uts/common/sys/brand.h
index 8183f643a4..0af67a1908 100644
--- a/usr/src/uts/common/sys/brand.h
+++ b/usr/src/uts/common/sys/brand.h
@@ -172,12 +172,11 @@ struct brand_ops {
 	    struct cred *cred, int *brand_action);
 	void	(*b_sigset_native_to_brand)(sigset_t *);
 	void	(*b_sigset_brand_to_native)(sigset_t *);
-	void	(*b_psig_to_proc)(proc_t *, kthread_t *, int);
+	void	(*b_sigfd_translate)(k_siginfo_t *);
 	int	b_nsig;
 	void	(*b_exit_with_sig)(proc_t *, sigqueue_t *);
 	boolean_t (*b_wait_filter)(proc_t *, proc_t *);
 	boolean_t (*b_native_exec)(uint8_t, const char **);
-	void (*b_ptrace_exectrap)(proc_t *);
 	uint32_t (*b_map32limit)(proc_t *);
 	void	(*b_stop_notify)(proc_t *, klwp_t *, ushort_t, ushort_t);
 	int	(*b_waitid_helper)(idtype_t, id_t, k_siginfo_t *, int,