6810086 panic in rctl_incr_swap() due to freed up proc structure

6557272 assertion failed: zone->zone_max_swap >= swap, panic during dism stress test

6 files changed, 33 insertions, 24 deletions

diff --git a/usr/src/uts/common/os/fork.c b/usr/src/uts/common/os/fork.c
index 05344477bd..fd4972f6d7 100644
--- a/usr/src/uts/common/os/fork.c
+++ b/usr/src/uts/common/os/fork.c
@@ -20,15 +20,13 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
 /*	  All Rights Reserved  	*/
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/sysmacros.h>
@@ -270,7 +268,7 @@ cfork(int isvfork, int isfork1, int flags)
 		p->p_flag |= SFORKING;
 		mutex_exit(&p->p_lock);
 
-		error = as_dup(p->p_as, &cp->p_as);
+		error = as_dup(p->p_as, cp);
 		if (error != 0) {
 			mutex_enter(&p->p_lock);
 			sprunlock(p);
diff --git a/usr/src/uts/common/os/rctl.c b/usr/src/uts/common/os/rctl.c
index 41da98ee02..dee5330bca 100644
--- a/usr/src/uts/common/os/rctl.c
+++ b/usr/src/uts/common/os/rctl.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -2908,6 +2908,13 @@ rctl_incr_locked_mem(proc_t *p, kproject_t *proj, rctl_qty_t inc,
 
 	e.rcep_p.proj = projp;
 	e.rcep_t = RCENTITY_PROJECT;
+
+	/* check for overflow */
+	if ((projp->kpj_data.kpd_locked_mem + inc) <
+	    projp->kpj_data.kpd_locked_mem) {
+		ret = EAGAIN;
+		goto out;
+	}
 	if (projp->kpj_data.kpd_locked_mem + inc >
 	    projp->kpj_data.kpd_locked_mem_ctl) {
 		if (rctl_test_entity(rc_project_locked_mem, projp->kpj_rctls,
@@ -2918,6 +2925,12 @@ rctl_incr_locked_mem(proc_t *p, kproject_t *proj, rctl_qty_t inc,
 	}
 	e.rcep_p.zone = zonep;
 	e.rcep_t = RCENTITY_ZONE;
+
+	/* Check for overflow */
+	if ((zonep->zone_locked_mem + inc) < zonep->zone_locked_mem) {
+		ret = EAGAIN;
+		goto out;
+	}
 	if (zonep->zone_locked_mem + inc > zonep->zone_locked_mem_ctl) {
 		if (rctl_test_entity(rc_zone_locked_mem, zonep->zone_rctls,
 		    p, &e, inc, 0) & RCT_DENY) {
@@ -3003,6 +3016,11 @@ rctl_incr_swap(proc_t *proc, zone_t *zone, size_t swap)
 
 	mutex_enter(&zone->zone_mem_lock);
 
+	/* Check for overflow */
+	if ((zone->zone_max_swap + swap) < zone->zone_max_swap) {
+		mutex_exit(&zone->zone_mem_lock);
+		return (EAGAIN);
+	}
 	if ((zone->zone_max_swap + swap) >
 	    zone->zone_max_swap_ctl) {
 
diff --git a/usr/src/uts/common/os/task.c b/usr/src/uts/common/os/task.c
index 628fcde30b..aa407dd97b 100644
--- a/usr/src/uts/common/os/task.c
+++ b/usr/src/uts/common/os/task.c
@@ -19,12 +19,10 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/atomic.h>
 #include <sys/cmn_err.h>
 #include <sys/exacct.h>
@@ -467,7 +465,7 @@ task_create(projid_t projid, zone_t *zone)
 	 */
 	(void) mod_hash_reserve(task_hash, &hndl);
 	mutex_enter(&task_hash_lock);
-	ASSERT(task_find(tkid, getzoneid()) == NULL);
+	ASSERT(task_find(tkid, zone->zone_id) == NULL);
 	if (mod_hash_insert_reserve(task_hash, (mod_hash_key_t)(uintptr_t)tkid,
 	    (mod_hash_val_t *)tk, hndl) != 0) {
 		mod_hash_cancel(task_hash, &hndl);
@@ -638,9 +636,9 @@ task_change(task_t *newtk, proc_t *p)
 	ASSERT(oldtk != NULL);
 	ASSERT(oldtk->tk_memb_list != NULL);
 
-	mutex_enter(&p->p_zone->zone_nlwps_lock);
+	mutex_enter(&oldtk->tk_zone->zone_nlwps_lock);
 	oldtk->tk_nlwps -= p->p_lwpcnt;
-	mutex_exit(&p->p_zone->zone_nlwps_lock);
+	mutex_exit(&oldtk->tk_zone->zone_nlwps_lock);
 
 	mutex_enter(&newtk->tk_zone->zone_nlwps_lock);
 	newtk->tk_nlwps += p->p_lwpcnt;
diff --git a/usr/src/uts/common/os/zone.c b/usr/src/uts/common/os/zone.c
index 6b4ba04b7b..6135b02e52 100644
--- a/usr/src/uts/common/os/zone.c
+++ b/usr/src/uts/common/os/zone.c
@@ -5149,6 +5149,8 @@ zone_enter(zoneid_t zoneid)
 	pp->p_task->tk_proj->kpj_data.kpd_crypto_mem -= pp->p_crypto_mem;
 	mutex_exit(&(pp->p_task->tk_proj->kpj_data.kpd_crypto_lock));
 
+	pp->p_flag |= SZONETOP;
+	pp->p_zone = zone;
 	mutex_exit(&pp->p_lock);
 	AS_LOCK_EXIT(pp->p_as, &pp->p_as->a_lock);
 
@@ -5189,9 +5191,6 @@ zone_enter(zoneid_t zoneid)
 	oldtk = task_join(tk, 0);
 	mutex_exit(&cpu_lock);
 
-	pp->p_flag |= SZONETOP;
-	pp->p_zone = zone;
-
 	/*
 	 * call RCTLOP_SET functions on this proc
 	 */
diff --git a/usr/src/uts/common/vm/as.h b/usr/src/uts/common/vm/as.h
index 381bdbaedc..8ee8cb5276 100644
--- a/usr/src/uts/common/vm/as.h
+++ b/usr/src/uts/common/vm/as.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -39,8 +39,6 @@
 #ifndef	_VM_AS_H
 #define	_VM_AS_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/watchpoint.h>
 #include <vm/seg.h>
 #include <vm/faultcode.h>
@@ -253,7 +251,7 @@ void	as_rangelock(struct as *as);
 void	as_rangeunlock(struct as *as);
 struct	as *as_alloc();
 void	as_free(struct as *as);
-int	as_dup(struct as *as, struct as **outas);
+int	as_dup(struct as *as, struct proc *forkedproc);
 struct	seg *as_findseg(struct as *as, caddr_t addr, int tail);
 int	as_addseg(struct as *as, struct seg *newseg);
 struct	seg *as_removeseg(struct as *as, struct seg *seg);
diff --git a/usr/src/uts/common/vm/vm_as.c b/usr/src/uts/common/vm/vm_as.c
index 567cc3c16b..3e8df474df 100644
--- a/usr/src/uts/common/vm/vm_as.c
+++ b/usr/src/uts/common/vm/vm_as.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -36,8 +36,6 @@
  * contributors.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 /*
  * VM - address spaces.
  */
@@ -779,7 +777,7 @@ retry:
 }
 
 int
-as_dup(struct as *as, struct as **outas)
+as_dup(struct as *as, struct proc *forkedproc)
 {
 	struct as *newas;
 	struct seg *seg, *newseg;
@@ -789,7 +787,7 @@ as_dup(struct as *as, struct as **outas)
 	as_clearwatch(as);
 	newas = as_alloc();
 	newas->a_userlimit = as->a_userlimit;
-	newas->a_proc = as->a_proc->p_child;
+	newas->a_proc = forkedproc;
 
 	AS_LOCK_ENTER(newas, &newas->a_lock, RW_WRITER);
 
@@ -856,7 +854,7 @@ as_dup(struct as *as, struct as **outas)
 		as_free(newas);
 		return (error);
 	}
-	*outas = newas;
+	forkedproc->p_as = newas;
 	return (0);
 }