OS-5701 lxbrand doesn't support per-process RLIMIT_MEMLOCK

Reviewed by: Ryan Zezeski <ryan.zeseski@joyent.com>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Approved by: Ryan Zezeski <ryan.zeseski@joyent.com>

4 files changed, 72 insertions, 22 deletions

diff --git a/usr/src/man/man5/resource_controls.5 b/usr/src/man/man5/resource_controls.5
index 47e1521f41..1315880e70 100644
--- a/usr/src/man/man5/resource_controls.5
+++ b/usr/src/man/man5/resource_controls.5
@@ -1,10 +1,10 @@
 '\" te
 .\" Copyright (c) 2007, Sun Microsystems, Inc. All Rights Reserved.
-.\" Copyright (c) 2012, Joyent, Inc. All Rights Reserved.
+.\" Copyright 2017, Joyent, Inc.
 .\" The contents of this file are subject to the terms of the Common Development and Distribution License (the "License").  You may not use this file except in compliance with the License.
 .\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE or http://www.opensolaris.org/os/licensing.  See the License for the specific language governing permissions and limitations under the License.
 .\" When distributing Covered Code, include this CDDL HEADER in each file and include the License file at usr/src/OPENSOLARIS.LICENSE.  If applicable, add the following below this CDDL HEADER, with the fields enclosed by brackets "[]" replaced with your own identifying information: Portions Copyright [yyyy] [name of copyright owner]
-.TH RESOURCE_CONTROLS 5 "April 9, 2016"
+.TH RESOURCE_CONTROLS 5 "April 18, 2017"
 .SH NAME
 resource_controls \- resource controls available through projects and zones
 .SH DESCRIPTION
@@ -123,6 +123,17 @@ number of bytes.
 .sp
 .ne 2
 .na
+\fB\fBprocess.max-locked-memory\fR\fR
+.ad
+.sp .6
+.RS 4n
+Total amount of physical memory that can be locked by this process, expressed
+as a number of bytes.
+.RE
+
+.sp
+.ne 2
+.na
 \fB\fBprocess.max-msg-messages\fR\fR
 .ad
 .sp .6
diff --git a/usr/src/uts/common/brand/lx/syscall/lx_rlimit.c b/usr/src/uts/common/brand/lx/syscall/lx_rlimit.c
index 6581ead25b..7c8f7ba733 100644
--- a/usr/src/uts/common/brand/lx/syscall/lx_rlimit.c
+++ b/usr/src/uts/common/brand/lx/syscall/lx_rlimit.c
@@ -10,7 +10,7 @@
  */
 
 /*
- * Copyright 2015 Joyent, Inc.
+ * Copyright 2017 Joyent, Inc.
  */
 
 #include <sys/systm.h>
@@ -55,9 +55,9 @@ typedef struct {
 } lx_rlimit32_t;
 
 /*
- * Linux supports many of the same resources that we do, but on Illumos these
+ * Linux supports many of the same resources that we do, but on illumos these
  * are rctls. Instead of using rlimit, we use rctls for all of the limits.
- * This table is used to translate Linux rlimit keys into the Illumos legacy
+ * This table is used to translate Linux rlimit keys into the illumos legacy
  * rlimit. We then primarily use the rctl/rlimit compatability code to
  * manage these.
  */
@@ -167,9 +167,13 @@ lx_getrlimit_common(int lx_resource, uint64_t *rlim_curp, uint64_t *rlim_maxp)
 		break;
 
 	case LX_RLIMIT_MEMLOCK:
-		/* zone.max-locked-memory */
-		rlim64.rlim_cur = rlim64.rlim_max =
-		    curzone->zone_locked_mem_ctl;
+		lx_get_rctl("process.max-locked-memory", &rlim64);
+
+		/* If unlimited, use zone.max-locked-memory */
+		if (rlim64.rlim_max == RLIM64_INFINITY)
+			rlim64.rlim_max = curzone->zone_locked_mem_ctl;
+		if (rlim64.rlim_cur == RLIM64_INFINITY)
+			rlim64.rlim_cur = curzone->zone_locked_mem_ctl;
 		break;
 
 	case LX_RLIMIT_SIGPENDING:
@@ -414,18 +418,23 @@ lx_setrlimit_common(int lx_resource, uint64_t rlim_cur, uint64_t rlim_max)
 		break;
 
 	case LX_RLIMIT_MEMLOCK:
-		/*
-		 * zone.max-locked-memory
-		 * Since we're emulating the value via a zone rctl, we can't
-		 * set that from within the zone. Lie and say we set the value.
-		 */
+		/* Do not exceed zone.max-locked-memory */
+		if (rlim_max > curzone->zone_locked_mem_ctl ||
+		    rlim_cur > curzone->zone_locked_mem_ctl)
+			return (set_errno(EINVAL));
+
+		rl64.rlim_cur = rlim_cur;
+		rl64.rlim_max = rlim_max;
+		err = lx_set_rctl("process.max-locked-memory", &rl64);
+		if (err != 0)
+			return (set_errno(err));
 		break;
 
 	case LX_RLIMIT_SIGPENDING:
 		/*
 		 * On Ubuntu at least, the login and sshd processes expect to
 		 * set this limit to 16k and login will fail if this fails. On
-		 * Illumos we have a system limit of 8k and normally the
+		 * illumos we have a system limit of 8k and normally the
 		 * privileged limit is 512. We simply pretend this works to
 		 * allow login to work.
 		 */
diff --git a/usr/src/uts/common/os/rctl.c b/usr/src/uts/common/os/rctl.c
index 09b80323d5..fb5b07a939 100644
--- a/usr/src/uts/common/os/rctl.c
+++ b/usr/src/uts/common/os/rctl.c
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2017, Joyent, Inc.
  */
 
 #include <sys/atomic.h>
@@ -194,6 +195,8 @@ id_space_t *rctl_ids;
 kmem_cache_t *rctl_cache;	/* kmem cache for rctl structures */
 kmem_cache_t *rctl_val_cache;	/* kmem cache for rctl values */
 
+extern rctl_hndl_t rc_process_maxlockedmem;
+
 kmutex_t rctl_lists_lock;
 rctl_dict_entry_t *rctl_lists[RC_MAX_ENTITY + 1];
 
@@ -2872,12 +2875,12 @@ rctl_init(void)
  * rctl_incr_locked_mem(proc_t *p, kproject_t *proj, rctl_qty_t inc,
  *     int chargeproc)
  *
- * Increments the amount of locked memory on a project, and
- * zone. If proj is non-NULL the project must be held by the
- * caller; if it is NULL the proj and zone of proc_t p are used.
- * If chargeproc is non-zero, then the charged amount is cached
- * on p->p_locked_mem so that the charge can be migrated when a
- * process changes projects.
+ * Increments the amount of locked memory on a process, project, and
+ * zone. If 'proj' is non-NULL, the project must be held by the
+ * caller; if it is NULL, the project and zone of process 'p' are used.
+ * If 'chargeproc' is non-zero, then the charged amount is added
+ * to p->p_locked_mem. This is also used so that the charge can be
+ * migrated when a process changes projects.
  *
  * Return values
  *    0 - success
@@ -2895,6 +2898,7 @@ rctl_incr_locked_mem(proc_t *p, kproject_t *proj, rctl_qty_t inc,
 
 	ASSERT(p != NULL);
 	ASSERT(MUTEX_HELD(&p->p_lock));
+
 	if (proj != NULL) {
 		projp = proj;
 		zonep = proj->kpj_zone;
@@ -2938,11 +2942,30 @@ rctl_incr_locked_mem(proc_t *p, kproject_t *proj, rctl_qty_t inc,
 		}
 	}
 
-	zonep->zone_locked_mem += inc;
-	projp->kpj_data.kpd_locked_mem += inc;
 	if (chargeproc != 0) {
+		rlim64_t p_max;
+
+		p_max = rctl_enforced_value(rc_process_maxlockedmem, p->p_rctls,
+		    p);
+
+		/* Check for overflow */
+		if ((p->p_locked_mem + inc) < p->p_locked_mem) {
+			ret = EAGAIN;
+			goto out;
+		}
+		if ((p->p_locked_mem + inc) > p_max) {
+			if (rctl_test_entity(rc_process_maxlockedmem,
+			    p->p_rctls, p, &e, inc, 0) & RCT_DENY) {
+				ret = EAGAIN;
+				goto out;
+			}
+		}
+
 		p->p_locked_mem += inc;
 	}
+
+	zonep->zone_locked_mem += inc;
+	projp->kpj_data.kpd_locked_mem += inc;
 out:
 	mutex_exit(&zonep->zone_mem_lock);
 	return (ret);
diff --git a/usr/src/uts/common/os/rctl_proc.c b/usr/src/uts/common/os/rctl_proc.c
index 9b7324fe7b..6daba9cf29 100644
--- a/usr/src/uts/common/os/rctl_proc.c
+++ b/usr/src/uts/common/os/rctl_proc.c
@@ -21,6 +21,7 @@
 /*
  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2017 Joyent, Inc.
  */
 
 #include <sys/types.h>
@@ -66,6 +67,7 @@ rctl_hndl_t rc_process_semmsl;
 rctl_hndl_t rc_process_semopm;
 rctl_hndl_t rc_process_portev;
 rctl_hndl_t rc_process_sigqueue;
+rctl_hndl_t rc_process_maxlockedmem;
 
 /*
  * process.max-cpu-time / RLIMIT_CPU
@@ -383,6 +385,11 @@ rctlproc_init(void)
 	rctl_add_default_limit("process.max-sigqueue-size",
 	    _SIGQUEUE_SIZE_PRIVILEGED, RCPRIV_PRIVILEGED, RCTL_LOCAL_DENY);
 
+	rc_process_maxlockedmem = rctl_register("process.max-locked-memory",
+	    RCENTITY_PROCESS, RCTL_GLOBAL_LOWERABLE | RCTL_GLOBAL_DENY_ALWAYS |
+	    RCTL_GLOBAL_SIGNAL_NEVER | RCTL_GLOBAL_BYTES,
+	    ULONG_MAX, UINT32_MAX, &rctl_default_ops);
+
 	/*
 	 * Place minimal set of controls on "sched" process for inheritance by
 	 * processes created via newproc().