OS-6088 fix mlock errno values and remove tests from the skiplist

Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Approved by: Patrick Mooney <patrick.mooney@joyent.com>

3 files changed, 39 insertions, 16 deletions

diff --git a/usr/src/lib/brand/lx/testing/ltp_skiplist b/usr/src/lib/brand/lx/testing/ltp_skiplist
index 87d8b30187..f5f0a3b8d9 100644
--- a/usr/src/lib/brand/lx/testing/ltp_skiplist
+++ b/usr/src/lib/brand/lx/testing/ltp_skiplist
@@ -39,9 +39,6 @@ mem01
 migrate_pages01
 migrate_pages02
 mknod01
-mlockall02
-mlockall03
-mlock02
 mmap12
 mount03
 mount05
diff --git a/usr/src/uts/common/brand/lx/syscall/lx_mem.c b/usr/src/uts/common/brand/lx/syscall/lx_mem.c
index d1919307af..84bc79eb60 100644
--- a/usr/src/uts/common/brand/lx/syscall/lx_mem.c
+++ b/usr/src/uts/common/brand/lx/syscall/lx_mem.c
@@ -64,6 +64,9 @@ extern int memcntl(caddr_t, size_t, int, caddr_t, int, int);
 /* For convenience */
 #define	LX_PROT_GROWMASK	(LX_PROT_GROWSUP|LX_PROT_GROWSDOWN)
 
+/* From lx_rlimit.c */
+extern void lx_get_rctl(char *, struct rlimit64 *);
+
 static int
 lx_mlock_common(int op, uintptr_t addr, size_t len)
 {
@@ -80,18 +83,31 @@ lx_mlock_common(int op, uintptr_t addr, size_t len)
 		return (set_errno(EINVAL));
 	}
 
-	if (lx_kern_release_cmp(curzone, "2.6.9") < 0) {
-		if ((err = secpolicy_lock_memory(CRED())) != 0)
-			return (set_errno(err));
-	}
-
 	err = as_ctl(as, (caddr_t)align_addr, align_len, op, 0, 0, NULL, 0);
+	if (err == EAGAIN)
+		err = ENOMEM;
 	return (err == 0 ? 0 : set_errno(err));
 }
 
 int
 lx_mlock(uintptr_t addr, size_t len)
 {
+	int err;
+
+	/*
+	 * If the the caller is not privileged and either the limit is 0, or
+	 * the kernel version is earlier than 2.6.9, then fail with EPERM. See
+	 * LTP mlock2.c.
+	 */
+	if ((err = secpolicy_lock_memory(CRED())) != 0) {
+		struct rlimit64 rlim64;
+
+		lx_get_rctl("process.max-locked-memory", &rlim64);
+		if (rlim64.rlim_cur == 0 ||
+		    lx_kern_release_cmp(curzone, "2.6.9") < 0)
+			return (set_errno(err));
+	}
+
 	return (lx_mlock_common(MC_LOCK, addr, len));
 }
 
@@ -107,8 +123,17 @@ lx_mlockall(int flags)
 	int err;
 	struct as *as = curproc->p_as;
 
-	if (lx_kern_release_cmp(curzone, "2.6.9") < 0) {
-		if ((err = secpolicy_lock_memory(CRED())) != 0)
+	/*
+	 * If the the caller is not privileged and either the limit is 0, or
+	 * the kernel version is earlier than 2.6.9, then fail with EPERM. See
+	 * LTP mlockall2.c.
+	 */
+	if ((err = secpolicy_lock_memory(CRED())) != 0) {
+		struct rlimit64 rlim64;
+
+		lx_get_rctl("process.max-locked-memory", &rlim64);
+		if (rlim64.rlim_cur == 0 ||
+		    lx_kern_release_cmp(curzone, "2.6.9") < 0)
 			return (set_errno(err));
 	}
 
@@ -116,6 +141,8 @@ lx_mlockall(int flags)
 		return (set_errno(EINVAL));
 
 	err = as_ctl(as, 0, 0, MC_LOCKAS, 0, (uintptr_t)flags, NULL, 0);
+	if (err == EAGAIN)
+		err = ENOMEM;
 	return (err == 0 ? 0 : set_errno(err));
 }
 
diff --git a/usr/src/uts/common/brand/lx/syscall/lx_rlimit.c b/usr/src/uts/common/brand/lx/syscall/lx_rlimit.c
index 7c8f7ba733..8fadf8d391 100644
--- a/usr/src/uts/common/brand/lx/syscall/lx_rlimit.c
+++ b/usr/src/uts/common/brand/lx/syscall/lx_rlimit.c
@@ -85,7 +85,7 @@ static int l_to_r[LX_RLIMIT_NLIMITS] = {
  */
 #define	LX_RLIM_INFINITY_N	ULONG_MAX
 
-static void
+void
 lx_get_rctl(char *nm, struct rlimit64 *rlp64)
 {
 	rctl_hndl_t hndl;
@@ -418,11 +418,10 @@ lx_setrlimit_common(int lx_resource, uint64_t rlim_cur, uint64_t rlim_max)
 		break;
 
 	case LX_RLIMIT_MEMLOCK:
-		/* Do not exceed zone.max-locked-memory */
-		if (rlim_max > curzone->zone_locked_mem_ctl ||
-		    rlim_cur > curzone->zone_locked_mem_ctl)
-			return (set_errno(EINVAL));
-
+		/*
+		 * We allow setting to unlimited (LX_RLIM_INFINITY_N). The zone
+		 * limit will always apply.
+		 */
 		rl64.rlim_cur = rlim_cur;
 		rl64.rlim_max = rlim_max;
 		err = lx_set_rctl("process.max-locked-memory", &rl64);