6322069 Unscalability of AH_LOCK_SIZE causes anonhash_lock contention on larger systems

10 files changed, 125 insertions, 86 deletions

diff --git a/usr/src/uts/common/conf/param.c b/usr/src/uts/common/conf/param.c
index 09e529b934..bb0cb4de1e 100644
--- a/usr/src/uts/common/conf/param.c
+++ b/usr/src/uts/common/conf/param.c
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1983, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #include <sys/types.h>
@@ -114,6 +113,7 @@ const unsigned long	_pgthresh	= (unsigned long)PGTHRESH;
 const unsigned int	_maxslp		= (unsigned int)MAXSLP;
 const unsigned long	_maxhandspreadpages = (unsigned long)MAXHANDSPREADPAGES;
 const int		_ncpu 		= (int)NCPU;
+const int		_ncpu_log2	= (int)NCPU_LOG2;
 const unsigned long	_defaultstksz	= (unsigned long)DEFAULTSTKSZ;
 const unsigned int	_nbpg		= (unsigned int)MMU_PAGESIZE;
 
diff --git a/usr/src/uts/common/fs/swapfs/swap_vnops.c b/usr/src/uts/common/fs/swapfs/swap_vnops.c
index d38aef7dae..74c86d4cb9 100644
--- a/usr/src/uts/common/fs/swapfs/swap_vnops.c
+++ b/usr/src/uts/common/fs/swapfs/swap_vnops.c
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #include <sys/types.h>
@@ -241,7 +240,7 @@ again:
 				    PAGESIZE, flags, cr, NULL);
 
 				if (!err) {
-					ahm = &anonhash_lock[AH_LOCK(vp, off)];
+					ahm = AH_MUTEX(vp, off);
 					mutex_enter(ahm);
 
 					ap = swap_anon(vp, off);
@@ -422,7 +421,7 @@ swap_getconpage(
 				struct anon *ap;
 				kmutex_t *ahm;
 
-				ahm = &anonhash_lock[AH_LOCK(vp, off)];
+				ahm = AH_MUTEX(vp, off);
 				mutex_enter(ahm);
 				ap = swap_anon(vp, off);
 				if (ap == NULL)
diff --git a/usr/src/uts/common/os/vm_subr.c b/usr/src/uts/common/os/vm_subr.c
index b25c04fc81..6f4abb6de5 100644
--- a/usr/src/uts/common/os/vm_subr.c
+++ b/usr/src/uts/common/os/vm_subr.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
@@ -37,8 +35,6 @@
  * contributors.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/types.h>
 #include <sys/t_lock.h>
 #include <sys/param.h>
@@ -114,9 +110,8 @@ void
 physio_bufs_init(void)
 {
 	physio_buf_cache = kmem_cache_create("physio_buf_cache",
-		sizeof (struct buf), 0,
-		physio_buf_constructor, physio_buf_destructor,
-		NULL, NULL, NULL, 0);
+	    sizeof (struct buf), 0, physio_buf_constructor,
+	    physio_buf_destructor, NULL, NULL, NULL, 0);
 }
 
 
@@ -149,10 +144,10 @@ default_physio(int (*strat)(struct buf *), struct buf *bp, dev_t dev,
 
 	/* Kernel probe */
 	TNF_PROBE_4(physio_start, "io rawio", /* CSTYLED */,
-		tnf_device,	device,		dev,
-		tnf_offset,	offset,		uio->uio_loffset,
-		tnf_size,	size,		uio->uio_resid,
-		tnf_bioflags,	rw,		rw);
+	    tnf_device,		device,		dev,
+	    tnf_offset,		offset,		uio->uio_loffset,
+	    tnf_size,		size,		uio->uio_resid,
+	    tnf_bioflags,	rw,		rw);
 
 	if (rw == B_READ) {
 		CPU_STATS_ADD_K(sys, phread, 1);
@@ -161,7 +156,7 @@ default_physio(int (*strat)(struct buf *), struct buf *bp, dev_t dev,
 	}
 
 	TRACE_1(TR_FAC_PHYSIO, TR_PHYSIO_GETBUF_START,
-		"getbuf_start: bp %p", bp);
+	    "getbuf_start: bp %p", bp);
 
 	if (bp == NULL) {
 		bp = kmem_cache_alloc(physio_buf_cache, KM_SLEEP);
@@ -248,8 +243,7 @@ default_physio(int (*strat)(struct buf *), struct buf *bp, dev_t dev,
 			if (error != 0) {
 				bp->b_flags |= B_ERROR;
 				bp->b_error = error;
-				bp->b_flags &=
-					~(B_BUSY|B_WANTED|B_PHYS);
+				bp->b_flags &= ~(B_BUSY|B_WANTED|B_PHYS);
 				break;
 			}
 			bp->b_shadow = pplist;
@@ -267,13 +261,13 @@ default_physio(int (*strat)(struct buf *), struct buf *bp, dev_t dev,
 			 * unlock the pages
 			 */
 			TRACE_1(TR_FAC_PHYSIO, TR_PHYSIO_UNLOCK_START,
-				"as_pageunlock_start: bp %p", bp);
+			    "as_pageunlock_start: bp %p", bp);
 
 			as_pageunlock(asp, pplist, a, c,
-				rw == B_READ? S_WRITE : S_READ);
+			    rw == B_READ? S_WRITE : S_READ);
 
 			TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_UNLOCK_END,
-				"as_pageunlock_end:");
+			    "as_pageunlock_end:");
 
 			c -= bp->b_resid;
 			iov->iov_base += c;
@@ -373,7 +367,7 @@ cow_mapin(struct as *as, caddr_t uaddr, caddr_t kaddr, struct page **cached_ppp,
 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
 		seg = as_findseg(as, uaddr, 0);
 		if ((seg == NULL) || ((base = seg->s_base) > uaddr) ||
-			(uaddr + total) > base + seg->s_size) {
+		    (uaddr + total) > base + seg->s_size) {
 			AS_LOCK_EXIT(as, &as->a_lock);
 			return (EINVAL);
 		}
@@ -415,8 +409,7 @@ tryagain:
 			 * segment. The disadvantage is that it locks the
 			 * page from being used by anybody else.
 			 */
-			ahm = &anonhash_lock[
-			    AH_LOCK(pp->p_vnode, pp->p_offset)];
+			ahm = AH_MUTEX(pp->p_vnode, pp->p_offset);
 			mutex_enter(ahm);
 			*app = swap_anon(pp->p_vnode, pp->p_offset);
 			/*
diff --git a/usr/src/uts/common/sys/param.h b/usr/src/uts/common/sys/param.h
index 0d541568ac..40af8ce04d 100644
--- a/usr/src/uts/common/sys/param.h
+++ b/usr/src/uts/common/sys/param.h
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 /*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T	*/
@@ -380,6 +379,7 @@ extern const uintptr_t _argsbase;
 extern const unsigned long _defaultstksz;
 extern const unsigned int _nbpg;
 extern const int _ncpu;
+extern const int _ncpu_log2;
 extern const int _clsize;
 #endif	/* defined(_KERNEL) && !defined(_ASM) */
 
@@ -398,6 +398,7 @@ extern const int _clsize;
 #define	ARGSBASE	_argsbase
 #define	DEFAULTSTKSZ	_defaultstksz
 #define	NCPU		_ncpu
+#define	NCPU_LOG2	_ncpu_log2
 
 #endif	/* defined(_MACHDEP) */
 
diff --git a/usr/src/uts/common/vm/anon.h b/usr/src/uts/common/vm/anon.h
index 13672d5c0b..652fcc0951 100644
--- a/usr/src/uts/common/vm/anon.h
+++ b/usr/src/uts/common/vm/anon.h
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
@@ -39,8 +38,6 @@
 #ifndef	_VM_ANON_H
 #define	_VM_ANON_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/cred.h>
 #include <sys/zone.h>
 #include <vm/seg.h>
@@ -116,7 +113,7 @@ struct anon {
  */
 extern kmutex_t anoninfo_lock;
 extern kmutex_t swapinfo_lock;
-extern kmutex_t anonhash_lock[];
+extern pad_mutex_t *anonhash_lock;
 extern pad_mutex_t anon_array_lock[];
 extern kcondvar_t anon_array_cv[];
 
@@ -130,8 +127,11 @@ extern struct anon **anon_hash;
 #define	ANON_HASH(VP, OFF)	\
 ((((uintptr_t)(VP) >> 7)  ^ ((OFF) >> PAGESHIFT)) & (ANON_HASH_SIZE - 1))
 
-#define	AH_LOCK_SIZE	64
-#define	AH_LOCK(vp, off) (ANON_HASH((vp), (off)) & (AH_LOCK_SIZE -1))
+#define	AH_LOCK_SIZE	(2 << NCPU_LOG2)
+
+#define	AH_MUTEX(vp, off)				\
+	(&anonhash_lock[(ANON_HASH((vp), (off)) &	\
+	    (AH_LOCK_SIZE - 1))].pad_mutex)
 
 #endif	/* _KERNEL */
 
diff --git a/usr/src/uts/common/vm/vm_anon.c b/usr/src/uts/common/vm/vm_anon.c
index 5018c41c2e..6ded5d7192 100644
--- a/usr/src/uts/common/vm/vm_anon.c
+++ b/usr/src/uts/common/vm/vm_anon.c
@@ -143,6 +143,19 @@ struct anon **anon_hash;
 static struct kmem_cache *anon_cache;
 static struct kmem_cache *anonmap_cache;
 
+pad_mutex_t	*anonhash_lock;
+
+/*
+ * Used to make the increment of all refcnts of all anon slots of a large
+ * page appear to be atomic.  The lock is grabbed for the first anon slot of
+ * a large page.
+ */
+pad_mutex_t	*anonpages_hash_lock;
+
+#define	APH_MUTEX(vp, off)				\
+	(&anonpages_hash_lock[(ANON_HASH((vp), (off)) &	\
+	    (AH_LOCK_SIZE - 1))].pad_mutex)
+
 #ifdef VM_STATS
 static struct anonvmstats_str {
 	ulong_t getpages[30];
@@ -179,19 +192,32 @@ anonmap_cache_destructor(void *buf, void *cdrarg)
 	mutex_destroy(&amp->a_purgemtx);
 }
 
-kmutex_t	anonhash_lock[AH_LOCK_SIZE];
-kmutex_t	anonpages_hash_lock[AH_LOCK_SIZE];
-
 void
 anon_init(void)
 {
 	int i;
+	pad_mutex_t *tmp;
 
-	anon_hash_size = 1L << highbit(physmem / ANON_HASHAVELEN);
+	/* These both need to be powers of 2 so round up to the next power */
+	anon_hash_size = 1L << highbit((physmem / ANON_HASHAVELEN) - 1);
+
+	/*
+	 * We need to align the anonhash_lock and anonpages_hash_lock arrays
+	 * to a 64B boundary to avoid false sharing.  We add 63B to our
+	 * allocation so that we can get a 64B aligned address to use.
+	 * We allocate both of these together to avoid wasting an additional
+	 * 63B.
+	 */
+	tmp = kmem_zalloc((2 * AH_LOCK_SIZE * sizeof (pad_mutex_t)) + 63,
+	    KM_SLEEP);
+	anonhash_lock = (pad_mutex_t *)P2ROUNDUP((uintptr_t)tmp, 64);
+	anonpages_hash_lock = anonhash_lock + AH_LOCK_SIZE;
 
 	for (i = 0; i < AH_LOCK_SIZE; i++) {
-		mutex_init(&anonhash_lock[i], NULL, MUTEX_DEFAULT, NULL);
-		mutex_init(&anonpages_hash_lock[i], NULL, MUTEX_DEFAULT, NULL);
+		mutex_init(&anonhash_lock[i].pad_mutex, NULL, MUTEX_DEFAULT,
+		    NULL);
+		mutex_init(&anonpages_hash_lock[i].pad_mutex, NULL,
+		    MUTEX_DEFAULT, NULL);
 	}
 
 	for (i = 0; i < ANON_LOCKSIZE; i++) {
@@ -225,7 +251,7 @@ anon_addhash(struct anon *ap)
 {
 	int index;
 
-	ASSERT(MUTEX_HELD(&anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]));
+	ASSERT(MUTEX_HELD(AH_MUTEX(ap->an_vp, ap->an_off)));
 	index = ANON_HASH(ap->an_vp, ap->an_off);
 	ap->an_hash = anon_hash[index];
 	anon_hash[index] = ap;
@@ -236,7 +262,7 @@ anon_rmhash(struct anon *ap)
 {
 	struct anon **app;
 
-	ASSERT(MUTEX_HELD(&anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]));
+	ASSERT(MUTEX_HELD(AH_MUTEX(ap->an_vp, ap->an_off)));
 
 	for (app = &anon_hash[ANON_HASH(ap->an_vp, ap->an_off)];
 	    *app; app = &((*app)->an_hash)) {
@@ -942,7 +968,7 @@ anon_alloc(struct vnode *vp, anoff_t off)
 	ap->an_refcnt = 1;
 	ap->an_pvp = NULL;
 	ap->an_poff = 0;
-	ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)];
+	ahm = AH_MUTEX(ap->an_vp, ap->an_off);
 	mutex_enter(ahm);
 	anon_addhash(ap);
 	mutex_exit(ahm);
@@ -975,7 +1001,7 @@ anon_swap_free(struct anon *ap, page_t *pp)
 		return;
 
 	page_io_lock(pp);
-	ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)];
+	ahm = AH_MUTEX(ap->an_vp, ap->an_off);
 	mutex_enter(ahm);
 
 	ASSERT(ap->an_refcnt != 0);
@@ -1007,7 +1033,7 @@ anon_decref(struct anon *ap)
 	anoff_t off;
 	kmutex_t *ahm;
 
-	ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)];
+	ahm = AH_MUTEX(ap->an_vp, ap->an_off);
 	mutex_enter(ahm);
 	ASSERT(ap->an_refcnt != 0);
 	if (ap->an_refcnt == 0)
@@ -1063,7 +1089,7 @@ anon_szcshare(struct anon_hdr *ahp, ulong_t anon_index)
 	if (ap == NULL)
 		return (0);
 
-	ahmpages = &anonpages_hash_lock[AH_LOCK(ap->an_vp, ap->an_off)];
+	ahmpages = APH_MUTEX(ap->an_vp, ap->an_off);
 	mutex_enter(ahmpages);
 	ASSERT(ap->an_refcnt >= 1);
 	if (ap->an_refcnt == 1) {
@@ -1128,7 +1154,7 @@ anon_decref_pages(
 	VM_STAT_ADD(anonvmstats.decrefpages[0]);
 
 	if (ap != NULL) {
-		ahmpages = &anonpages_hash_lock[AH_LOCK(ap->an_vp, ap->an_off)];
+		ahmpages = APH_MUTEX(ap->an_vp, ap->an_off);
 		mutex_enter(ahmpages);
 		ASSERT((refcnt = ap->an_refcnt) != 0);
 		VM_STAT_ADD(anonvmstats.decrefpages[1]);
@@ -1156,8 +1182,7 @@ anon_decref_pages(
 			pp = page_lookup(vp, (u_offset_t)off, SE_EXCL);
 			if (pp == NULL || pp->p_szc == 0) {
 				VM_STAT_ADD(anonvmstats.decrefpages[3]);
-				ahm = &anonhash_lock[AH_LOCK(ap->an_vp,
-				    ap->an_off)];
+				ahm = AH_MUTEX(ap->an_vp, ap->an_off);
 				(void) anon_set_ptr(ahp, an_idx + i, NULL,
 				    ANON_SLEEP);
 				mutex_enter(ahm);
@@ -1224,8 +1249,7 @@ anon_decref_pages(
 					ap = anon_get_ptr(ahp, an_idx + j);
 					ASSERT(ap != NULL &&
 					    ap->an_refcnt == 1);
-					ahm = &anonhash_lock[AH_LOCK(ap->an_vp,
-					    ap->an_off)];
+					ahm = AH_MUTEX(ap->an_vp, ap->an_off);
 					(void) anon_set_ptr(ahp, an_idx + j,
 					    NULL, ANON_SLEEP);
 					mutex_enter(ahm);
@@ -1262,7 +1286,7 @@ anon_decref_pages(
 		} else {
 			VM_STAT_ADD(anonvmstats.decrefpages[8]);
 			(void) anon_set_ptr(ahp, an_idx + i, NULL, ANON_SLEEP);
-			ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)];
+			ahm = AH_MUTEX(ap->an_vp, ap->an_off);
 			mutex_enter(ahm);
 			ap->an_refcnt--;
 			mutex_exit(ahm);
@@ -1305,7 +1329,7 @@ anon_dup(struct anon_hdr *old, ulong_t old_idx, struct anon_hdr *new,
 			break;
 
 		(void) anon_set_ptr(new, new_idx + off, ap, ANON_SLEEP);
-		ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)];
+		ahm = AH_MUTEX(ap->an_vp, ap->an_off);
 
 		mutex_enter(ahm);
 		ap->an_refcnt++;
@@ -1397,11 +1421,9 @@ anon_dup_fill_holes(
 				 * getting an anonpages_hash_lock for the
 				 * first anon slot of a large page.
 				 */
-				int hash = AH_LOCK(ap->an_vp, ap->an_off);
-
 				VM_STAT_ADD(anonvmstats.dupfillholes[2]);
 
-				ahmpages = &anonpages_hash_lock[hash];
+				ahmpages = APH_MUTEX(ap->an_vp, ap->an_off);
 				mutex_enter(ahmpages);
 				/*LINTED*/
 				ASSERT(refcnt = ap->an_refcnt);
@@ -1411,7 +1433,7 @@ anon_dup_fill_holes(
 			}
 			(void) anon_set_ptr(new, new_idx + off + i, ap,
 			    ANON_SLEEP);
-			ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)];
+			ahm = AH_MUTEX(ap->an_vp, ap->an_off);
 			mutex_enter(ahm);
 			ASSERT(ahmpages != NULL || ap->an_refcnt == 1);
 			ASSERT(i == 0 || ahmpages == NULL ||
@@ -1680,7 +1702,7 @@ anon_disclaim(struct anon_map *amp, ulong_t index, size_t size)
 			continue;
 		}
 
-		ahm = &anonhash_lock[AH_LOCK(vp, off)];
+		ahm = AH_MUTEX(vp, off);
 		mutex_enter(ahm);
 		ASSERT(ap->an_refcnt != 0);
 		/*
@@ -1767,7 +1789,7 @@ anon_disclaim(struct anon_map *amp, ulong_t index, size_t size)
 				if (ap == NULL)
 					break;
 				swap_xlate(ap, &vp, &off);
-				ahm = &anonhash_lock[AH_LOCK(vp, off)];
+				ahm = AH_MUTEX(vp, off);
 				mutex_enter(ahm);
 				ASSERT(ap->an_refcnt != 0);
 
@@ -1830,7 +1852,7 @@ anon_getpage(
 	 * routine does.
 	 */
 	if (pl != NULL && (pp = page_lookup(vp, (u_offset_t)off, SE_SHARED))) {
-		ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)];
+		ahm = AH_MUTEX(ap->an_vp, ap->an_off);
 		mutex_enter(ahm);
 		if (ap->an_refcnt == 1)
 			*protp = PROT_ALL;
@@ -1854,7 +1876,7 @@ anon_getpage(
 	    seg, addr, rw, cred, NULL);
 
 	if (err == 0 && pl != NULL) {
-		ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)];
+		ahm = AH_MUTEX(ap->an_vp, ap->an_off);
 		mutex_enter(ahm);
 		if (ap->an_refcnt != 1)
 			*protp &= ~PROT_WRITE;	/* make read-only */
@@ -2511,8 +2533,7 @@ anon_map_privatepages(
 	 * first anon slot of a large page.
 	 */
 	if (ap != NULL) {
-		ahmpages = &anonpages_hash_lock[AH_LOCK(ap->an_vp,
-		    ap->an_off)];
+		ahmpages = APH_MUTEX(ap->an_vp, ap->an_off);
 		mutex_enter(ahmpages);
 		if (ap->an_refcnt == 1) {
 			VM_STAT_ADD(anonvmstats.privatepages[4]);
@@ -2820,8 +2841,7 @@ anon_map_createpages(
 			 */
 			if (ap->an_pvp != NULL) {
 				page_io_lock(pp);
-				ahm = &anonhash_lock[AH_LOCK(ap->an_vp,
-				    ap->an_off)];
+				ahm = AH_MUTEX(ap->an_vp, ap->an_off);
 				mutex_enter(ahm);
 				if (ap->an_pvp != NULL) {
 					swap_phys_free(ap->an_pvp,
@@ -3007,7 +3027,7 @@ anon_try_demote_pages(
 	ap = anon_get_ptr(ahp, sidx);
 	if (ap != NULL && private) {
 		VM_STAT_ADD(anonvmstats.demotepages[1]);
-		ahmpages = &anonpages_hash_lock[AH_LOCK(ap->an_vp, ap->an_off)];
+		ahmpages = APH_MUTEX(ap->an_vp, ap->an_off);
 		mutex_enter(ahmpages);
 	}
 
diff --git a/usr/src/uts/common/vm/vm_swap.c b/usr/src/uts/common/vm/vm_swap.c
index 61210201aa..ed325d18d4 100644
--- a/usr/src/uts/common/vm/vm_swap.c
+++ b/usr/src/uts/common/vm/vm_swap.c
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1987, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
@@ -357,7 +356,7 @@ swap_anon(struct vnode *vp, u_offset_t off)
 {
 	struct anon *ap;
 
-	ASSERT(MUTEX_HELD(&anonhash_lock[AH_LOCK(vp, off)]));
+	ASSERT(MUTEX_HELD(AH_MUTEX(vp, off)));
 
 	for (ap = anon_hash[ANON_HASH(vp, off)]; ap != NULL; ap = ap->an_hash) {
 		if (ap->an_vp == vp && ap->an_off == off)
@@ -1452,7 +1451,8 @@ swapdel(
 	 * may change under us.
 	 */
 	for (app = anon_hash; app < &anon_hash[ANON_HASH_SIZE]; app++) {
-		ahm = &anonhash_lock[(app-anon_hash) & (AH_LOCK_SIZE - 1)];
+		ahm = &anonhash_lock[(app - anon_hash) &
+		    (AH_LOCK_SIZE - 1)].pad_mutex;
 		mutex_enter(ahm);
 top:
 		for (ap = *app; ap != NULL; ap = ap->an_hash) {
@@ -1612,7 +1612,7 @@ again:
 	 */
 	if (!alloc_pg)
 		page_io_lock(pp);
-	ahm = &anonhash_lock[AH_LOCK(vp, off)];
+	ahm = AH_MUTEX(vp, off);
 	mutex_enter(ahm);
 	ap = swap_anon(vp, off);
 	if ((ap == NULL || ap->an_pvp == NULL) && alloc_pg) {
@@ -1681,7 +1681,7 @@ swap_newphysname(
 		 * No swap available so return error unless requested
 		 * offset is already backed in which case return that.
 		 */
-		ahm = &anonhash_lock[AH_LOCK(vp, offset)];
+		ahm = AH_MUTEX(vp, offset);
 		mutex_enter(ahm);
 		if ((ap = swap_anon(vp, offset)) == NULL) {
 			error = SE_NOANON;
@@ -1709,7 +1709,7 @@ swap_newphysname(
 
 	for (off = start, poff = pstart; poff < pstart + plen;
 	    off += PAGESIZE, poff += PAGESIZE) {
-		ahm = &anonhash_lock[AH_LOCK(vp, off)];
+		ahm = AH_MUTEX(vp, off);
 		mutex_enter(ahm);
 		if ((ap = swap_anon(vp, off)) != NULL) {
 			/* Free old slot if any, and assign new one */
@@ -1779,7 +1779,7 @@ swap_getphysname(
 	int error = 0;
 	kmutex_t *ahm;
 
-	ahm = &anonhash_lock[AH_LOCK(vp, off)];
+	ahm = AH_MUTEX(vp, off);
 	mutex_enter(ahm);
 
 	/* Get anon slot for vp, off */
diff --git a/usr/src/uts/i86pc/sys/machparam.h b/usr/src/uts/i86pc/sys/machparam.h
index ee66fd1d94..ed3c5d5fcc 100644
--- a/usr/src/uts/i86pc/sys/machparam.h
+++ b/usr/src/uts/i86pc/sys/machparam.h
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 /*	Copyright (c) 1988 AT&T	*/
@@ -55,8 +54,10 @@ extern "C" {
 
 #if defined(__amd64)
 #define	NCPU	256
+#define	NCPU_LOG2	8
 #elif defined(__i386)
 #define	NCPU	32
+#define	NCPU_LOG2	5
 #endif
 
 /*
diff --git a/usr/src/uts/sun4u/sys/machparam.h b/usr/src/uts/sun4u/sys/machparam.h
index df9b11b794..e60d02a2cc 100644
--- a/usr/src/uts/sun4u/sys/machparam.h
+++ b/usr/src/uts/sun4u/sys/machparam.h
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 /*	Copyright (c) 1988 AT&T	*/
@@ -29,8 +28,6 @@
 #ifndef _SYS_MACHPARAM_H
 #define	_SYS_MACHPARAM_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #ifdef	__cplusplus
 extern "C" {
 #endif
@@ -80,6 +77,32 @@ extern "C" {
 #endif
 #endif	/* _STARFIRE && !lint */
 
+#if	(NCPU <= 1)
+#define	NCPU_LOG2	0
+#elif	(NCPU <= 2)
+#define	NCPU_LOG2	1
+#elif	(NCPU <= 4)
+#define	NCPU_LOG2	2
+#elif	(NCPU <= 8)
+#define	NCPU_LOG2	3
+#elif	(NCPU <= 16)
+#define	NCPU_LOG2	4
+#elif	(NCPU <= 32)
+#define	NCPU_LOG2	5
+#elif	(NCPU <= 64)
+#define	NCPU_LOG2	6
+#elif	(NCPU <= 128)
+#define	NCPU_LOG2	7
+#elif	(NCPU <= 256)
+#define	NCPU_LOG2	8
+#elif	(NCPU <= 512)
+#define	NCPU_LOG2	9
+#elif	(NCPU <= 1024)
+#define	NCPU_LOG2	10
+#else
+#error	"add test for larger NCPU"
+#endif
+
 /*
  * Maximum number of processors that we support.  With CMP processors, the
  * portid may not be equal to cpuid.  MAX_CPU_CHIPID can be defined in a
diff --git a/usr/src/uts/sun4v/sys/machparam.h b/usr/src/uts/sun4v/sys/machparam.h
index eff5dd33e7..4fe46b985b 100644
--- a/usr/src/uts/sun4v/sys/machparam.h
+++ b/usr/src/uts/sun4v/sys/machparam.h
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #ifndef _SYS_MACHPARAM_H
@@ -59,6 +58,9 @@ extern "C" {
  */
 #ifndef NCPU
 #define	NCPU	512
+#define	NCPU_LOG2	9
+#elif (!defined(NCPU_LOG2))
+#error "Must define NCPU_LOG2 together with NCPU"
 #endif
 
 /*