6875154 6801244 fix misses corner case

Contributed by Chad Mynhier <cmynhier@gmail.com>

3 files changed, 29 insertions, 41 deletions

diff --git a/usr/src/uts/common/vm/seg_dev.c b/usr/src/uts/common/vm/seg_dev.c
index e3ea694c05..9d214024a5 100644
--- a/usr/src/uts/common/vm/seg_dev.c
+++ b/usr/src/uts/common/vm/seg_dev.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -37,8 +37,6 @@
  * contributors.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 /*
  * VM - segment of a mapped device.
  *
@@ -428,6 +426,15 @@ segdev_create(struct seg *seg, void *argsp)
 		sdp->devmap_data = NULL;
 		hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size,
 		    HAT_UNLOAD_UNMAP);
+	} else {
+		/*
+		 * Mappings of /dev/null don't count towards the VSZ of a
+		 * process.  Mappings of /dev/null have no mapping type.
+		 */
+		if ((SEGOP_GETTYPE(seg, (seg)->s_base) & (MAP_SHARED |
+		    MAP_PRIVATE)) == 0) {
+			seg->s_as->a_resvsize -= seg->s_size;
+		}
 	}
 
 	return (error);
diff --git a/usr/src/uts/common/vm/seg_vn.c b/usr/src/uts/common/vm/seg_vn.c
index 3649a8da21..666b98f389 100644
--- a/usr/src/uts/common/vm/seg_vn.c
+++ b/usr/src/uts/common/vm/seg_vn.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -651,6 +651,15 @@ segvn_create(struct seg *seg, void *argsp)
 	}
 
 	/*
+	 * MAP_NORESERVE mappings don't count towards the VSZ of a process
+	 * until we fault the pages in.
+	 */
+	if ((a->vp == NULL || a->vp->v_type != VREG) &&
+	    a->flags & MAP_NORESERVE) {
+		seg->s_as->a_resvsize -= seg->s_size;
+	}
+
+	/*
 	 * If more than one segment in the address space, and they're adjacent
 	 * virtually, try to concatenate them.  Don't concatenate if an
 	 * explicit anon_map structure was supplied (e.g., SystemV shared
diff --git a/usr/src/uts/common/vm/vm_as.c b/usr/src/uts/common/vm/vm_as.c
index e8ccade6e8..01ad32e0b1 100644
--- a/usr/src/uts/common/vm/vm_as.c
+++ b/usr/src/uts/common/vm/vm_as.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -1425,6 +1425,7 @@ top:
 		 * We'll handle MAP_NORESERVE cases in segvn_unmap(). (Again,
 		 * we have to do this check here while we have seg.)
 		 */
+		rsize = 0;
 		if (!SEG_IS_DEVNULL_MAPPING(seg) &&
 		    !SEG_IS_PARTIAL_RESV(seg))
 			rsize = ssize;
@@ -1504,7 +1505,8 @@ retry:
 		}
 
 		as->a_size -= ssize;
-		as->a_resvsize -= rsize;
+		if (rsize)
+			as->a_resvsize -= rsize;
 		raddr += ssize;
 	}
 	AS_LOCK_EXIT(as, &as->a_lock);
@@ -1545,12 +1547,7 @@ as_map_segvn_segs(struct as *as, caddr_t addr, size_t size, uint_t szcvec,
 			seg_free(seg);
 		} else {
 			as->a_size += size;
-			/*
-			 * We'll count MAP_NORESERVE mappings as we fault
-			 * pages in.
-			 */
-			if (!SEG_IS_PARTIAL_RESV(seg))
-				as->a_resvsize += size;
+			as->a_resvsize += size;
 		}
 		return (error);
 	}
@@ -1583,14 +1580,7 @@ as_map_segvn_segs(struct as *as, caddr_t addr, size_t size, uint_t szcvec,
 				return (error);
 			}
 			as->a_size += segsize;
-			/*
-			 * We'll count MAP_NORESERVE mappings as we fault
-			 * pages in.  We don't count /dev/null mappings at all.
-			 */
-			if (!SEG_IS_DEVNULL_MAPPING(seg) &&
-			    !SEG_IS_PARTIAL_RESV(seg))
-				as->a_resvsize += segsize;
-
+			as->a_resvsize += segsize;
 			*segcreated = 1;
 			if (do_off) {
 				vn_a->offset += segsize;
@@ -1619,14 +1609,7 @@ as_map_segvn_segs(struct as *as, caddr_t addr, size_t size, uint_t szcvec,
 				return (error);
 			}
 			as->a_size += segsize;
-			/*
-			 * We'll count MAP_NORESERVE mappings as we fault
-			 * pages in.  We don't count /dev/null mappings at all.
-			 */
-			if (!SEG_IS_DEVNULL_MAPPING(seg) &&
-			    !SEG_IS_PARTIAL_RESV(seg))
-				as->a_resvsize += segsize;
-
+			as->a_resvsize += segsize;
 			*segcreated = 1;
 			if (do_off) {
 				vn_a->offset += segsize;
@@ -1677,12 +1660,7 @@ again:
 			seg_free(seg);
 		} else {
 			as->a_size += size;
-			/*
-			 * We'll count MAP_NORESERVE mappings as we fault
-			 * pages in.
-			 */
-			if (!SEG_IS_PARTIAL_RESV(seg))
-				as->a_resvsize += size;
+			as->a_resvsize += size;
 		}
 		return (error);
 	}
@@ -1842,13 +1820,7 @@ as_map_locked(struct as *as, caddr_t addr, size_t size, int (*crfp)(),
 		 * Add size now so as_unmap will work if as_ctl fails.
 		 */
 		as->a_size += rsize;
-		/*
-		 * We'll count MAP_NORESERVE mappings as we fault
-		 * pages in.  We don't count /dev/null mappings at all.
-		 */
-		if (!SEG_IS_DEVNULL_MAPPING(seg) &&
-		    !SEG_IS_PARTIAL_RESV(seg))
-			as->a_resvsize += rsize;
+		as->a_resvsize += rsize;
 	}
 
 	as_setwatch(as);