OS-3611 lx brand: 64-bit processes should not use VAs above VA hole

3 files changed, 29 insertions, 0 deletions

diff --git a/usr/src/uts/common/brand/lx/os/lx_brand.c b/usr/src/uts/common/brand/lx/os/lx_brand.c
index fcd60c7d01..141dd6e653 100644
--- a/usr/src/uts/common/brand/lx/os/lx_brand.c
+++ b/usr/src/uts/common/brand/lx/os/lx_brand.c
@@ -92,6 +92,15 @@ lx_systrace_f *lx_systrace_return_ptr;
 
 static int lx_systrace_enabled;
 
+/*
+ * While this is effectively mmu.hole_start - PAGESIZE, we don't particularly
+ * want an MMU dependency here (and should there be a microprocessor without
+ * a hole, we don't want to start allocating from the top of the VA range).
+ */
+#define	LX_MAXSTACK64	0x7ffffff00000
+
+uint64_t lx_maxstack64 = LX_MAXSTACK64;
+
 static int lx_elfexec(struct vnode *vp, struct execa *uap, struct uarg *args,
     struct intpdata *idata, int level, long *execsz, int setid,
     caddr_t exec_file, struct cred *cred, int brand_action);
@@ -1265,6 +1274,22 @@ lx_elfexec(struct vnode *vp, struct execa *uap, struct uarg *args,
 	args->brandname = LX_BRANDNAME;
 	args->emulator = lib_path;
 
+#if defined(_LP64)
+	/*
+	 * To conform with the way Linux lays out the address space, we clamp
+	 * the stack to be the top of the lower region of the x86-64 canonical
+	 * form address space -- which has the side-effect of laying out the
+	 * entire address space in that lower region.  Note that this only
+	 * matters on 64-bit processes (this value will always be greater than
+	 * the size of a 32-bit address space) and doesn't actually affect
+	 * USERLIMIT:  if a Linux-branded processes wishes to map something
+	 * into the top half of the address space, it can do so -- but with
+	 * the user stack starting at the top of the bottom region, those high
+	 * virtual addresses won't be used unless explicitly directed.
+	 */
+	args->maxstack = lx_maxstack64;
+#endif
+
 	/*
 	 * We will first exec the brand library, then map in the linux
 	 * executable and the linux linker.
diff --git a/usr/src/uts/common/os/exec.c b/usr/src/uts/common/os/exec.c
index d044b2a08d..cfb5c27302 100644
--- a/usr/src/uts/common/os/exec.c
+++ b/usr/src/uts/common/os/exec.c
@@ -1949,6 +1949,9 @@ exec_args(execa_t *uap, uarg_t *args, intpdata_t *intp, void **auxvpp)
 		usrstack = (char *)USRSTACK32;
 	}
 
+	if (args->maxstack != 0 && (uintptr_t)usrstack > args->maxstack)
+		usrstack = (char *)args->maxstack;
+
 	ASSERT(P2PHASE((uintptr_t)usrstack, args->stk_align) == 0);
 
 #if defined(__sparc)
diff --git a/usr/src/uts/common/sys/exec.h b/usr/src/uts/common/sys/exec.h
index 73c5cccee7..50786aca30 100644
--- a/usr/src/uts/common/sys/exec.h
+++ b/usr/src/uts/common/sys/exec.h
@@ -111,6 +111,7 @@ typedef struct uarg {
 	char	*auxp_brand; /* address of first brand auxv on user stack */
 	cred_t	*pfcred;
 	boolean_t scrubenv;
+	uintptr_t maxstack;
 } uarg_t;
 
 /*