OS-3411 lxbrand 64bit signal assertion failure in lx_call_user_handler20141016release-20141016

4 files changed, 60 insertions, 34 deletions

diff --git a/usr/src/lib/brand/lx/lx_brand/amd64/lx_handler.s b/usr/src/lib/brand/lx/lx_brand/amd64/lx_handler.s
index f449f47be3..7cf509da13 100644
--- a/usr/src/lib/brand/lx/lx_brand/amd64/lx_handler.s
+++ b/usr/src/lib/brand/lx/lx_brand/amd64/lx_handler.s
@@ -285,6 +285,8 @@ lx_sigreturn_tolibc(uintptr_t sp)
 	 * |	| stuff we saved in our prologue		|
 	 * |	=================================================
 	 * |	| LX_SIGRT_MAGIC				|
+	 * |	=================================================
+	 * |	| {unused word to maintain ABI stack alignment} |
 	 * V	=================================================
 	 *	| Linux local data built by lx stk_builder()	|
 	 * 	=================================================
@@ -297,7 +299,7 @@ lx_sigreturn_tolibc(uintptr_t sp)
 	ENTRY_NP(lx_sigdeliver)
 	pushq   %rbp
 	movq    %rsp, %rbp
-	subq	$0x30, %rsp
+	subq	$0x40, %rsp		/* an extra word to maintain alignmnt */
 	movq	%rdi,  -8(%rbp)		/* sig */
 	movq	%rsi, -16(%rbp)		/* siginfo* */
 	movq	%rdx, -24(%rbp)		/* ucontext* */
@@ -305,16 +307,12 @@ lx_sigreturn_tolibc(uintptr_t sp)
 	movq	%r8,  -40(%rbp)		/* stack builder */
 	movq	%r9,  -48(%rbp)		/* Linux signal handler */
 
-					/*
-					 * create stack_size stack buffer as a
-					 * local varible
-					 */
-	subq    %rcx, %rsp
+	subq    %rcx, %rsp		/* create stack_size stack buffer */
 
-	movq	$LX_SIGRT_MAGIC, %rcx	/* load and "push" marker value onto */
-	movq	%rcx, -56(%rbp)		/*         stack for lx_rt_sigreturn */
+	movq	$LX_SIGRT_MAGIC, %rcx	/* load and place marker value onto */
+	movq	%rcx, -56(%rbp)		/*        stack for lx_rt_sigreturn */
 
-	movq	%rsp, %rcx		/* arg3 - %rcx stack pointer */
+	movq	%rsp, %rcx		/* arg3 - %rcx is stack pointer */
 					/* arg2 - %rdx is ucontext ptr */
 					/* arg1 - %rsi is siginfo ptr */
 					/* arg0 - %rdi is sig num */
@@ -371,10 +369,6 @@ lx_sigreturn_tolibc(uintptr_t sp)
 	 * Trampoline code is called by the return at the end of a Linux
 	 * signal handler to return control to the interrupted application
 	 * via the lx_rt_sigreturn() syscall.
-	 *
-	 * This routine must consist of the EXACT code sequence below
-	 * as gdb looks at the sequence of instructions a routine will return
-	 * to determine whether it is in a signal handler or not.
 	 */
 	ENTRY_NP(lx_rt_sigreturn_tramp)
 	movq	$LX_SYS_rt_sigreturn, %rax
diff --git a/usr/src/lib/brand/lx/lx_brand/common/lx_brand.c b/usr/src/lib/brand/lx/lx_brand/common/lx_brand.c
index 180e040a98..2f787a2368 100644
--- a/usr/src/lib/brand/lx/lx_brand/common/lx_brand.c
+++ b/usr/src/lib/brand/lx/lx_brand/common/lx_brand.c
@@ -448,6 +448,16 @@ static int
 lx_emulate_args(lx_regs_t *rp, struct lx_sysent *s, uintptr_t *args)
 {
 #if defined(_LP64)
+	/*
+	 * Note: Syscall argument passing is different from function call
+	 * argument passing on amd64.  For function calls, the fourth arg is
+	 * passed via %rcx, but for system calls the 4th arg is passed via %r10.
+	 * This is because in amd64, the syscall instruction puts the lower
+	 * 32 bits of %rflags in %r11 and puts the %rip value to %rcx.
+	 *
+	 * Appendix A of the amd64 ABI (Linux conventions) states that syscalls
+	 * are limited to 6 args and no arg is passed on the stack.
+	 */
 	args[0] = rp->lxr_rdi;
 	args[1] = rp->lxr_rsi;
 	args[2] = rp->lxr_rdx;
@@ -981,7 +991,16 @@ lx_syscall_regs(void)
 		assert(fr->fr_savpc != NULL);
 	}
 
+#if defined(_LP64)
+	/*
+	 * This is %rbp, update to be at the end of the frame for correct
+	 * struct offsets. lx_emulate only takes one parameter, a pointer to
+	 * lx_regs_t.
+	 */
+	return ((lx_regs_t *)(fr->fr_savfp - sizeof (lx_regs_t)));
+#else
 	return ((lx_regs_t *)((uintptr_t *)fr)[2]);
+#endif
 }
 
 int
diff --git a/usr/src/lib/brand/lx/lx_brand/common/signal.c b/usr/src/lib/brand/lx/lx_brand/common/signal.c
index 73689bd163..042b9e0227 100644
--- a/usr/src/lib/brand/lx/lx_brand/common/signal.c
+++ b/usr/src/lib/brand/lx/lx_brand/common/signal.c
@@ -302,7 +302,7 @@ struct lx_sigstack {
 	lx_siginfo_t si;	/* saved signal information */
 	lx_ucontext_t uc;	/* saved user context */
 	lx_fpstate_t fpstate;	/* saved FP state */
-	char trampoline[10];	/* code for trampoline to lx_rt_sigreturn() */
+	char pad[2];		/* stack alignment */
 };
 #else
 struct lx_sigstack {
@@ -779,6 +779,7 @@ lx_sigprocmask_common(uintptr_t how, uintptr_t l_setp, uintptr_t l_osetp,
 
 		s_setp = &set;
 
+		/* Only 32-bit code passes other than USE_SIGSET */
 		if (sigset_type == USE_SIGSET)
 			err = ltos_sigset((lx_sigset_t *)l_setp, s_setp);
 #if defined(_ILP32)
@@ -1067,7 +1068,12 @@ lx_rt_sigreturn(void)
 	 */
 	sp = (uintptr_t)rp->lxr_esp - 4;
 #else
-	sp = (uintptr_t)rp->lxr_rsp;
+	/*
+	 * We need to make an adjustment for 64-bit code as well. Since 64-bit
+	 * does not use the trampoline, it's probably for the same reason as
+	 * alluded to above.
+	 */
+	sp = (uintptr_t)rp->lxr_rsp - 8;
 #endif
 
 	/*
@@ -1090,6 +1096,9 @@ lx_rt_sigreturn(void)
 	 * Check for and remove LX_SIGRT_MAGIC from the stack.
 	 */
 #if defined(_LP64)
+	/* account for extra word used in lx_sigdeliver for stack alignment */
+	sp += 8;
+
 	if (*(uint64_t *)sp != LX_SIGRT_MAGIC)
 		lx_err_fatal("sp @ 0x%p, expected 0x%x, found 0x%x!",
 		    sp, LX_SIGRT_MAGIC, *(uint32_t *)sp);
@@ -1228,6 +1237,16 @@ lx_rt_sigreturn(void)
 static int
 lx_setcontext(const ucontext_t *ucp)
 {
+	extern int lx_traceflag;
+
+	/*
+	 * See we don't return via lx_emulate, issue a trace msg here if
+	 * necessary. We know this is only called in the 64-bit rt_sigreturn
+	 * code path to the syscall number is 15.
+	 */
+	if (lx_traceflag != 0) {
+		(void) syscall(SYS_brand, B_SYSRETURN, 15, 0);
+	}
 	return (syscall(SYS_brand, B_SIGNAL_RETURN, ucp));
 }
 #endif
@@ -1334,17 +1353,16 @@ lx_build_signal_frame(int lx_sig, siginfo_t *sip, void *p, void *sp)
 
 	if (lxsap && (lxsap->lxsa_flags & LX_SA_RESTORER) &&
 	    lxsap->lxsa_restorer) {
-#if defined(_LP64)
 		/*
-		 * lxsa_restorer is only set by sigaction in 32-bit code
-		 * XXX should lxsa_restorer ever be set for 64-bit code?
+		 * lxsa_restorer is explicitly set by sigaction in 32-bit code
+		 * but it can also be implicitly set for both 32 and 64 bit
+		 * code via lx_sigaction_common when we bcopy the user-supplied
+		 * lx_sigaction element into the proper slot in the sighandler
+		 * array.
 		 */
-		assert(0);
-#endif
 		lx_ssp->retaddr = lxsap->lxsa_restorer;
 		lx_debug("lxsa_restorer exists @ 0x%p", lx_ssp->retaddr);
 	} else {
-		/* We should always take this path in 64-bit code */
 		lx_ssp->retaddr = lx_rt_sigreturn_tramp;
 		lx_debug("lx_ssp->retaddr set to 0x%p", lx_rt_sigreturn_tramp);
 	}
@@ -1434,6 +1452,7 @@ lx_build_signal_frame(int lx_sig, siginfo_t *sip, void *p, void *sp)
 		lx_ucp->uc_sigcontext.sc_fpstate = NULL;
 	}
 
+#if defined(_ILP32)
 	/*
 	 * Believe it or not, gdb wants to SEE the sigreturn code on the
 	 * top of the stack to determine whether the stack frame belongs to
@@ -1443,6 +1462,7 @@ lx_build_signal_frame(int lx_sig, siginfo_t *sip, void *p, void *sp)
 	 */
 	bcopy((void *)lx_rt_sigreturn_tramp, lx_ssp->trampoline,
 	    sizeof (lx_ssp->trampoline));
+#endif
 
 #if defined(_LP64)
 	/*
@@ -1465,7 +1485,6 @@ lx_call_user_handler(int sig, siginfo_t *sip, void *p)
 {
 	void (*user_handler)();
 	void (*stk_builder)();
-
 #if defined(_ILP32)
 	lx_tsd_t *lx_tsd;
 	int err;
@@ -1503,6 +1522,10 @@ lx_call_user_handler(int sig, siginfo_t *sip, void *p)
 #if defined(_LP64)
 	/* %gs is ignored in the 64-bit lx_sigdeliver */
 	gs = 0;
+
+	stksize = sizeof (struct lx_sigstack);
+	stk_builder = lx_build_signal_frame;
+
 #else
 	if ((err = thr_getspecific(lx_tsd_key, (void **)&lx_tsd)) != 0)
 		lx_err_fatal("lx_call_user_handler: unable to read "
@@ -1518,21 +1541,15 @@ lx_call_user_handler(int sig, siginfo_t *sip, void *p)
 	 * bugs in the library. This is only applicable to 32-bit code.
 	 */
 	assert(gs != 0);
-#endif
 
 	if (lxsap->lxsa_flags & LX_SA_SIGINFO) {
 		stksize = sizeof (struct lx_sigstack);
 		stk_builder = lx_build_signal_frame;
 	} else  {
-#if defined(_LP64)
-		/* 64-bit code should never generate old-style signals */
-		stk_builder = NULL;	/* shutup the compiler */
-		assert(0);
-#else
 		stksize = sizeof (struct lx_oldsigstack);
 		stk_builder = lx_build_old_signal_frame;
-#endif
 	}
+#endif
 
 	user_handler = lxsap->lxsa_handler;
 
@@ -1787,11 +1804,6 @@ lx_rt_sigaction(uintptr_t lx_sig, uintptr_t actp, uintptr_t oactp,
 	if ((size_t)setsize != sizeof (lx_sigset_t))
 		return (-EINVAL);
 
-	/*
-	 * XXX - for 64 bit code this is the only sigaction. Why don't we have
-	 * the logic around lx_sigaction in this code path?
-	 */
-
 	return (lx_sigaction_common(lx_sig, (struct lx_sigaction *)actp,
 	    (struct lx_sigaction *)oactp));
 }
diff --git a/usr/src/lib/brand/lx/lx_brand/i386/lx_handler.s b/usr/src/lib/brand/lx/lx_brand/i386/lx_handler.s
index e8e944fd03..b6cdf3f4fe 100644
--- a/usr/src/lib/brand/lx/lx_brand/i386/lx_handler.s
+++ b/usr/src/lib/brand/lx/lx_brand/i386/lx_handler.s
@@ -360,6 +360,7 @@ lx_sigreturn_tolibc(uintptr_t sp)
 	 * These two routines must consist of the EXACT code sequences below
 	 * as gdb looks at the sequence of instructions a routine will return
 	 * to determine whether it is in a signal handler or not.
+	 * See the Linux code setup_signal_stack_sc() in arch/x86/um/signal.c.
 	 */
 	ENTRY_NP(lx_sigreturn_tramp)
 	popl	%eax