6638112 eliminate HMELOCK_ENTER() to avoid xcall timeout panic and improve GET_TTE() performance

6620705 sfmmu_hblk_hash_add/rm should not access hme buckets via va
6638127 TSB_UPDATE_TL()/TSB_UPDATE_TL_PN() shouldn't spin forever in TSB_LOCK_ENTRY()

4 files changed, 608 insertions, 640 deletions

diff --git a/usr/src/uts/sfmmu/ml/sfmmu_asm.s b/usr/src/uts/sfmmu/ml/sfmmu_asm.s
index 784e5e4a33..dbd9d98983 100644
--- a/usr/src/uts/sfmmu/ml/sfmmu_asm.s
+++ b/usr/src/uts/sfmmu/ml/sfmmu_asm.s
@@ -156,7 +156,7 @@
  *
  * tsbep = TSBE va (ro)
  * tmp1, tmp2 = scratch registers (clobbered)
- * label = label to use for branches (text)
+ * label = label to jump to if we fail to lock the tsb entry
  * %asi = ASI to use for TSB access
  *
  * NOTE that we flush the TSB using fast VIS instructions that
@@ -169,15 +169,14 @@
 
 #define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
 	lda	[tsbep]ASI_MEM, tmp1					;\
-label:									;\
 	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
 	cmp	tmp1, tmp2 						;\
-	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
-	  lda	[tsbep]ASI_MEM, tmp1					;\
+	be,a,pn	%icc, label		/* if locked ignore */		;\
+	  nop								;\
 	casa	[tsbep]ASI_MEM, tmp1, tmp2				;\
 	cmp	tmp1, tmp2 						;\
-	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
-	  lda	[tsbep]ASI_MEM, tmp1					;\
+	bne,a,pn %icc, label		/* didn't lock so ignore */	;\
+	  nop								;\
 	/* tsbe lock acquired */					;\
 	membar #StoreStore
 
@@ -185,15 +184,14 @@ label:									;\
 
 #define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
 	lda	[tsbep]%asi, tmp1					;\
-label:									;\
 	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
 	cmp	tmp1, tmp2 						;\
-	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
-	  lda	[tsbep]%asi, tmp1					;\
+	be,a,pn	%icc, label		/* if locked ignore */		;\
+	  nop								;\
 	casa	[tsbep]%asi, tmp1, tmp2					;\
 	cmp	tmp1, tmp2 						;\
-	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
-	  lda	[tsbep]%asi, tmp1					;\
+	bne,a,pn %icc, label		/* didn't lock so ignore */	;\
+	  nop								;\
 	/* tsbe lock acquired */					;\
 	membar #StoreStore
 
@@ -235,7 +233,7 @@ label:									;\
  *   we need to mask off the context and leave only the va (clobbered)
  * ttepa = pointer to the TTE to retrieve/load as pa (ro)
  * tmp1, tmp2 = scratch registers
- * label = label to use for branches (text)
+ * label = label to jump to if we fail to lock the tsb entry
  * %asi = ASI to use for TSB access
  */
 
@@ -254,7 +252,7 @@ label:									;\
 	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
 	sethi	%hi(TSBTAG_INVALID), tmp2				;\
 	add	tsbep, TSBE_TAG, tmp1					;\
-	brgez,a,pn tte, label/**/f					;\
+	brgez,a,pn tte, label						;\
 	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
 	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
 label:
@@ -272,7 +270,7 @@ label:
 	ldxa	[ttepa]ASI_MEM, tte					;\
 	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
 	sethi	%hi(TSBTAG_INVALID), tmp2				;\
-	brgez,a,pn tte, label/**/f					;\
+	brgez,a,pn tte, label						;\
 	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
 	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
 label:
@@ -308,7 +306,7 @@ label:
 	ldxa	[ttepa]ASI_MEM, tte					;\
 	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
 	sethi	%hi(TSBTAG_INVALID), tmp2				;\
-	brgez,a,pn tte, label/**/f					;\
+	brgez,a,pn tte, label						;\
 	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
 	or	tte, tmp1, tte						;\
 	andn	tte, TTE_EXECPRM_INT, tte				;\
@@ -377,7 +375,7 @@ label:									;\
 	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
 	sethi	%hi(TSBTAG_INVALID), tmp2				;\
 	add	tsbep, TSBE_TAG, tmp1					;\
-	brgez,a,pn tteva, label/**/f					;\
+	brgez,a,pn tteva, label						;\
 	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
 	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
 label:
@@ -389,7 +387,7 @@ label:
 	ldx	[tteva], tteva			/* load tte */		;\
 	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
 	sethi	%hi(TSBTAG_INVALID), tmp2				;\
-	brgez,a,pn tteva, label/**/f					;\
+	brgez,a,pn tteva, label						;\
 	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
 	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
 label:
@@ -1425,7 +1423,7 @@ sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift)
 	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
 
 	SETUP_TSB_ASI(%o3, %g3)
-	TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1)
+	TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, locked_tsb_l8)
 
 	wrpr	%g0, %o5, %pstate		/* enable interrupts */
 	
@@ -1479,7 +1477,7 @@ sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift)
 
 	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
 	/* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */
-	TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1)
+	TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, locked_tsb_l9)
 
 	wrpr	%g0, %o5, %pstate		! enable interrupts
 	retl
@@ -1541,296 +1539,6 @@ sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr)
 
 #endif /* !lint */
 
-
-#if defined (lint)
-/*
- * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the
- * the hash list.
- */
-/* ARGSUSED */
-void
-sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
-	uint64_t hblkpa)
-{
-}
-
-/*
- * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the
- * hash list.
- */
-/* ARGSUSED */
-void
-sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
-	uint64_t hblkpa, struct hme_blk *prev_hblkp)
-{
-}
-#else /* lint */
-
-/*
- * Functions to grab/release hme bucket list lock.  I only use a byte
- * instead of the whole int because eventually we might want to
- * put some counters on the other bytes (of course, these routines would
- * have to change).  The code that grab this lock should execute
- * with interrupts disabled and hold the lock for the least amount of time
- * possible.
- */
-
-/*
- * Even though hmeh_listlock is updated using pa there's no need to flush
- * dcache since hmeh_listlock will be restored to the original value (0)
- * before interrupts are reenabled.
- */
-
-/*
- * For sparcv9 hme hash buckets may not be in the nucleus.  hme hash update
- * routines still use virtual addresses to update the bucket fields. But they
- * must not cause a TLB miss after grabbing the low level bucket lock. To
- * achieve this we must make sure the bucket structure is completely within an
- * 8K page.
- */
-
-#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1))
-#error - the size of hmehash_bucket structure is not power of 2
-#endif
-
-/*
- * Enable backoff to significantly reduce locking overhead and reduce a chance
- * of xcall timeout. This is only enabled for sun4v as a Makefile compile-
- * time option.
- * The rd %ccr is better for performance compared to a non pipeline releasing
- * tight spin on N2/VF.
- * Backoff based fix is a temporary solution and doesn't allow scaling above
- * lock saturation point. The final fix is to eliminate HMELOCK_ENTER()
- * to avoid xcall timeouts and improve GET_TTE() performance.
- */
-
-#ifdef HMELOCK_BACKOFF_ENABLE
-
-#define HMELOCK_BACKOFF(reg, val)                               \
-	set     val, reg                                        ;\
-	rd	%ccr, %g0                                       ;\
-	brnz	reg, .-4                                        ;\
-	dec	reg
-
-#define CAS_HME(tmp1, tmp2, exitlabel, asi)                     \
-	mov     0xff, tmp2                                      ;\
-	casa    [tmp1]asi, %g0, tmp2                            ;\
-	brz,a,pt tmp2, exitlabel                                ;\
-	membar  #LoadLoad
-
-#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label, asi)            \
-	mov     0xff, tmp2                                      ;\
-	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
-	casa    [tmp1]asi, %g0, tmp2                            ;\
-	brz,a,pt tmp2, label/**/2                               ;\
-	membar  #LoadLoad                                       ;\
-	HMELOCK_BACKOFF(tmp2,0x8)                               ;\
-	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
-	HMELOCK_BACKOFF(tmp2,0x10)                              ;\
-	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
-	HMELOCK_BACKOFF(tmp2,0x20)                              ;\
-	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
-	HMELOCK_BACKOFF(tmp2,0x40)                              ;\
-	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
-	HMELOCK_BACKOFF(tmp2,0x80)                              ;\
-	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
-label/**/1:                                                     ;\
-	HMELOCK_BACKOFF(tmp2,0x100)                             ;\
-	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
-	HMELOCK_BACKOFF(tmp2,0x200)                             ;\
-	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
-	HMELOCK_BACKOFF(tmp2,0x400)                             ;\
-	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
-	HMELOCK_BACKOFF(tmp2,0x800)                             ;\
-	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
-	HMELOCK_BACKOFF(tmp2,0x1000)                            ;\
-	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
-	HMELOCK_BACKOFF(tmp2,0x2000)                            ;\
-	mov     0xff, tmp2                                      ;\
-	casa    [tmp1]asi, %g0, tmp2                            ;\
-	brnz,pn tmp2, label/**/1     /* reset backoff */        ;\
-	membar  #LoadLoad                                       ;\
-label/**/2:
-
-#else /* HMELOCK_BACKOFF_ENABLE */
-
-#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi)           \
-	mov     0xff, tmp2                                      ;\
-	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
-label1:                                                         ;\
-	casa    [tmp1]asi, %g0, tmp2                            ;\
-	brnz,pn tmp2, label1                                    ;\
-	mov     0xff, tmp2                                      ;\
-	membar  #LoadLoad
-
-#endif /* HMELOCK_BACKOFF_ENABLE */
-
-#define HMELOCK_EXIT(hmebp, tmp1, asi)                          \
-	membar  #LoadStore|#StoreStore                          ;\
-	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
-	sta     %g0, [tmp1]asi
-
-	.seg	".data"
-hblk_add_panic1:
-	.ascii	"sfmmu_hblk_hash_add: interrupts disabled"
-	.byte	0
-hblk_add_panic2:
-	.ascii	"sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not"
-	.byte	0
-	.align	4
-	.seg	".text"
-
-	ENTRY_NP(sfmmu_hblk_hash_add)
-	/*
-	 * %o0 = hmebp
-	 * %o1 = hmeblkp
-	 * %o2 = hblkpa
-	 */
-	rdpr	%pstate, %o5
-#ifdef DEBUG
-	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
-	bnz,pt %icc, 3f				/* disabled, panic	 */
-	  nop
-	save	%sp, -SA(MINFRAME), %sp
-	sethi	%hi(hblk_add_panic1), %o0
-	call	panic
-	 or	%o0, %lo(hblk_add_panic1), %o0
-	ret
-	restore
-
-3:
-#endif /* DEBUG */
-	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
-	mov	%o2, %g1
-
-	/*
-	 * g1 = hblkpa
-	 */
-	ldn	[%o0 + HMEBUCK_HBLK], %o4	/* next hmeblk */
-	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = next hblkpa */
-#ifdef	DEBUG
-	cmp	%o4, %g0
-	bne,pt %xcc, 1f
-	 nop
-	brz,pt %g2, 1f
-	 nop
-	wrpr	%g0, %o5, %pstate		/* enable interrupts */
-	save	%sp, -SA(MINFRAME), %sp
-	sethi	%hi(hblk_add_panic2), %o0
-	call	panic
-	  or	%o0, %lo(hblk_add_panic2), %o0
-	ret
-	restore
-1:
-#endif /* DEBUG */
-	/*
-	 * We update hmeblks entries before grabbing lock because the stores
-	 * could take a tlb miss and require the hash lock.  The buckets
-	 * are part of the nucleus so we are cool with those stores.
-	 *
-	 * if buckets are not part of the nucleus our game is to
-	 * not touch any other page via va until we drop the lock.
-	 * This guarantees we won't get a tlb miss before the lock release
-	 * since interrupts are disabled.
-	 */
-	stn	%o4, [%o1 + HMEBLK_NEXT]	/* update hmeblk's next */
-	stx	%g2, [%o1 + HMEBLK_NEXTPA]	/* update hmeblk's next pa */
-	HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N)
-	stn	%o1, [%o0 + HMEBUCK_HBLK]	/* update bucket hblk next */
-	stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* add hmeblk to list */
-	HMELOCK_EXIT(%o0, %g2, ASI_N)
-	retl
-	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
-	SET_SIZE(sfmmu_hblk_hash_add)
-
-	ENTRY_NP(sfmmu_hblk_hash_rm)
-	/*
-	 * This function removes an hmeblk from the hash chain. 
-	 * It is written to guarantee we don't take a tlb miss
-	 * by using physical addresses to update the list.
-	 * 
-	 * %o0 = hmebp
-	 * %o1 = hmeblkp
-	 * %o2 = hmeblkp previous pa
-	 * %o3 = hmeblkp previous
-	 */
-
-	mov	%o3, %o4			/* o4 = hmeblkp previous */
-
-	rdpr	%pstate, %o5
-#ifdef DEBUG
-	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l4, %g1)
-#endif /* DEBUG */
-	/*
-	 * disable interrupts, clear Address Mask to access 64 bit physaddr
-	 */
-	andn    %o5, PSTATE_IE, %g1
-	wrpr    %g1, 0, %pstate
-
-#ifndef sun4v
-	sethi   %hi(dcache_line_mask), %g4
-	ld      [%g4 + %lo(dcache_line_mask)], %g4
-#endif /* sun4v */
-
-	/*
-	 * if buckets are not part of the nucleus our game is to
-	 * not touch any other page via va until we drop the lock.
-	 * This guarantees we won't get a tlb miss before the lock release
-	 * since interrupts are disabled.
-	 */
-	HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N)
-	ldn	[%o0 + HMEBUCK_HBLK], %g2	/* first hmeblk in list */
-	cmp	%g2, %o1
-	bne,pt	%ncc,1f
-	 mov	ASI_MEM, %asi
-	/*
-	 * hmeblk is first on list
-	 */
-	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = hmeblk pa */
-	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
-	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
-	stn	%o3, [%o0 + HMEBUCK_HBLK]	/* write va */
-	ba,pt	%xcc, 2f
-	  stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* write pa */
-1:
-	/* hmeblk is not first on list */
-
-	mov	%o2, %g3
-#ifndef sun4v
-	GET_CPU_IMPL(%g2)
-	cmp 	%g2, CHEETAH_IMPL
-	bge,a,pt %icc, hblk_hash_rm_1
-	  and	%o4, %g4, %g2
-	cmp	%g2, SPITFIRE_IMPL
-	blt	%icc, hblk_hash_rm_2		/* no flushing needed for OPL */
-	  and	%o4, %g4, %g2
-	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev pa from dcache */
-	add	%o4, HMEBLK_NEXT, %o4
-	and	%o4, %g4, %g2
-	ba	hblk_hash_rm_2
-	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev va from dcache */
-hblk_hash_rm_1:
-
-	stxa	%g0, [%g3]ASI_DC_INVAL		/* flush prev pa from dcache */
-	membar	#Sync
-	add     %g3, HMEBLK_NEXT, %g2
-	stxa	%g0, [%g2]ASI_DC_INVAL		/* flush prev va from dcache */
-hblk_hash_rm_2:
-	membar	#Sync
-#endif /* sun4v */
-	ldxa	[%g3 + HMEBLK_NEXTPA] %asi, %g2	/* g2 = hmeblk pa */ 
-	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
-	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
-	stna	%o3, [%g3 + HMEBLK_NEXT] %asi	/* write va */
-	stxa	%g1, [%g3 + HMEBLK_NEXTPA] %asi	/* write pa */
-2:
-	HMELOCK_EXIT(%o0, %g2, ASI_N)
-	retl
-	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
-	SET_SIZE(sfmmu_hblk_hash_rm)
-
-#endif /* lint */
-
 /*
  * These macros are used to update global sfmmu hme hash statistics
  * in perf critical paths. It is only enabled in debug kernels or
@@ -2336,28 +2044,25 @@ label/**/2:								;\
 /*
  * Function to traverse hmeblk hash link list and find corresponding match.
  * The search is done using physical pointers. It returns the physical address
- * and virtual address pointers to the hmeblk that matches with the tag
- * provided.
+ * pointer to the hmeblk that matches with the tag provided.
  * Parameters:
  * hmebp	= register that points to hme hash bucket, also used as
  *		  tmp reg (clobbered)
  * hmeblktag	= register with hmeblk tag match
  * hatid	= register with hatid
  * hmeblkpa	= register where physical ptr will be stored
- * hmeblkva	= register where virtual ptr will be stored
  * tmp1		= tmp reg
  * label: temporary label
  */
 
-#define	HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva,	\
-	tsbarea, tmp1, label)					 	\
+#define	HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, tsbarea, 	\
+	tmp1, label)							\
 	add     hmebp, HMEBUCK_NEXTPA, hmeblkpa				;\
 	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
-	add     hmebp, HMEBUCK_HBLK, hmeblkva				;\
-	ldxa    [hmeblkva]ASI_MEM, hmeblkva				;\
 	HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
 label/**/1:								;\
-	brz,pn	hmeblkva, label/**/2					;\
+	cmp	hmeblkpa, HMEBLK_ENDPA					;\
+	be,pn   %xcc, label/**/2					;\
 	HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
 	add	hmeblkpa, HMEBLK_TAG, hmebp				;\
 	ldxa	[hmebp]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
@@ -2367,9 +2072,7 @@ label/**/1:								;\
 	xor	hmebp, hatid, hmebp					;\
 	or	hmebp, tmp1, hmebp					;\
 	brz,pn	hmebp, label/**/2	/* branch on hit */		;\
-	  add	hmeblkpa, HMEBLK_NEXT, hmebp				;\
-	ldna	[hmebp]ASI_MEM, hmeblkva	/* hmeblk ptr va */	;\
-	add	hmeblkpa, HMEBLK_NEXTPA, hmebp				;\
+	  add	hmeblkpa, HMEBLK_NEXTPA, hmebp				;\
 	ba,pt	%xcc, label/**/1					;\
 	  ldxa	[hmebp]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */	;\
 label/**/2:
@@ -2377,22 +2080,22 @@ label/**/2:
 /*
  * Function to traverse hmeblk hash link list and find corresponding match.
  * The search is done using physical pointers. It returns the physical address
- * and virtual address pointers to the hmeblk that matches with the tag
+ * pointer to the hmeblk that matches with the tag
  * provided.
  * Parameters:
  * hmeblktag	= register with hmeblk tag match (rid field is 0)
  * hatid	= register with hatid (pointer to SRD)
  * hmeblkpa	= register where physical ptr will be stored
- * hmeblkva	= register where virtual ptr will be stored
  * tmp1		= tmp reg
  * tmp2		= tmp reg
  * label: temporary label
  */
 
-#define	HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, hmeblkva,	\
-	tsbarea, tmp1, tmp2, label)			 		\
+#define	HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, tsbarea,	\
+	tmp1, tmp2, label)			 			\
 label/**/1:								;\
-	brz,pn	hmeblkva, label/**/4					;\
+	cmp	hmeblkpa, HMEBLK_ENDPA					;\
+	be,pn   %xcc, label/**/4					;\
 	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			;\
 	add	hmeblkpa, HMEBLK_TAG, tmp2				;\
 	ldxa	[tmp2]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
@@ -2401,16 +2104,14 @@ label/**/1:								;\
 	xor	tmp1, hmeblktag, tmp1					;\
 	xor	tmp2, hatid, tmp2					;\
 	brz,pn	tmp2, label/**/3	/* branch on hit */		;\
-	  add	hmeblkpa, HMEBLK_NEXT, tmp2				;\
+	  add	hmeblkpa, HMEBLK_NEXTPA, tmp2				;\
 label/**/2:								;\
-	ldna	[tmp2]ASI_MEM, hmeblkva	/* hmeblk ptr va */		;\
-	add	hmeblkpa, HMEBLK_NEXTPA, tmp2				;\
 	ba,pt	%xcc, label/**/1					;\
 	  ldxa	[tmp2]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */		;\
 label/**/3:								;\
 	cmp	tmp1, SFMMU_MAX_HME_REGIONS				;\
 	bgeu,pt	%xcc, label/**/2					;\
-	  add	hmeblkpa, HMEBLK_NEXT, tmp2				;\
+	  add	hmeblkpa, HMEBLK_NEXTPA, tmp2				;\
 	and	tmp1, BT_ULMASK, tmp2					;\
 	srlx	tmp1, BT_ULSHIFT, tmp1					;\
 	sllx	tmp1, CLONGSHIFT, tmp1					;\
@@ -2419,7 +2120,7 @@ label/**/3:								;\
 	srlx	tmp1, tmp2, tmp1					;\
 	btst	0x1, tmp1						;\
 	bz,pn	%xcc, label/**/2					;\
-	  add	hmeblkpa, HMEBLK_NEXT, tmp2				;\
+	  add	hmeblkpa, HMEBLK_NEXTPA, tmp2				;\
 label/**/4:
 
 #if ((1 << SFHME_SHIFT) != SFHME_SIZE)
@@ -2458,25 +2159,25 @@ label1:
  * hatid	= sfmmu pointer for TSB miss (in)
  * tte		= tte for TLB miss if found, otherwise clobbered (out)
  * hmeblkpa	= PA of hment if found, otherwise clobbered (out)
- * hmeblkva	= VA of hment if found, otherwise clobbered (out)
  * tsbarea	= pointer to the tsbmiss area for this cpu. (in)
  * hmemisc	= hblk_misc if TTE is found (out), otherwise clobbered
  * hmeshift	= constant/register to shift VA to obtain the virtual pfn
  *		  for this page size.
  * hashno	= constant/register hash number
+ * tmp		= temp value - clobbered
  * label	= temporary label for branching within macro.
  * foundlabel	= label to jump to when tte is found.
  * suspendlabel= label to jump to when tte is suspended.	
  * exitlabel	= label to jump to when tte is not found.
  *
  */
-#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmemisc, \
-		hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \
+#define GET_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, hmeshift, \
+		 hashno, tmp, label, foundlabel, suspendlabel, exitlabel) \
 									;\
 	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
 	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
 	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
-		hmeblkpa, label/**/5, hmemisc, hmeblkva)		;\
+		hmeblkpa, label/**/5, hmemisc, tmp)			;\
 									;\
 	/*								;\
 	 * tagacc = tagacc						;\
@@ -2484,7 +2185,7 @@ label1:
 	 * tsbarea = tsbarea						;\
 	 * tte   = hmebp (hme bucket pointer)				;\
 	 * hmeblkpa  = vapg  (virtual page)				;\
-	 * hmemisc, hmeblkva = scratch					;\
+	 * hmemisc, tmp = scratch					;\
 	 */								;\
 	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc)	;\
 	or	hmemisc, SFMMU_INVALID_SHMERID, hmemisc			;\
@@ -2495,22 +2196,20 @@ label1:
 	 * tte   = hmebp						;\
 	 * hmeblkpa  = CLOBBERED					;\
 	 * hmemisc  = htag_bspage+hashno+invalid_rid			;\
-	 * hmeblkva  = scratch						;\
+	 * tmp  = scratch						;\
 	 */								;\
 	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
-	HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM)	;\
-	HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, hmeblkva, 	\
+	HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, 	 		\
 		tsbarea, tagacc, label/**/1)				;\
 	/*								;\
 	 * tagacc = CLOBBERED						;\
 	 * tte = CLOBBERED						;\
 	 * hmeblkpa = hmeblkpa						;\
-	 * hmeblkva = hmeblkva						;\
+	 * tmp = scratch						;\
 	 */								;\
-	brnz,pt	hmeblkva, label/**/4	/* branch if hmeblk found */	;\
+	cmp	hmeblkpa, HMEBLK_ENDPA					;\
+	bne,pn   %xcc, label/**/4       /* branch if hmeblk found */    ;\
 	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
-	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva	;\
-	HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM)  /* drop lock */	;\
 	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
 	  nop								;\
 label/**/4:								;\
@@ -2523,7 +2222,7 @@ label/**/4:								;\
 	 * tte   = clobbered						;\
 	 * hmeblkpa  = hmeblkpa						;\
 	 * hmemisc  = hblktag						;\
-	 * hmeblkva  = hmeblkva 					;\
+	 * tmp = scratch						;\
 	 */								;\
 	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte,		\
 		label/**/2)						;\
@@ -2534,15 +2233,13 @@ label/**/4:								;\
 	 * tte   = clobbered						;\
 	 * hmeblkpa  = hmeblkpa						;\
 	 * hmemisc  = hblk_misc						;\
-	 * hmeblkva  = hmeblkva 					;\
+	 * tmp = scratch						;\
 	 */								;\
 									;\
 	add	hatid, SFHME_TTE, hatid					;\
 	add	hmeblkpa, hatid, hmeblkpa				;\
 	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
-	add	hmeblkva, hatid, hmeblkva				;\
 	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
-	HMELOCK_EXIT(hatid, hatid, ASI_MEM)	/* drop lock */		;\
 	set	TTE_SUSPEND, hatid					;\
 	TTE_SUSPEND_INT_SHIFT(hatid)					;\
 	btst	tte, hatid						;\
@@ -2561,14 +2258,14 @@ label/**/4:								;\
  * If valid tte is found, hmemisc = shctx flag, i.e., shme is
  * either 0 (not part of scd) or 1 (part of scd).
  */
-#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea,	\
-		hmemisc, hmeshift, hashno, label, foundlabel,		\
+#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, 	\
+		hmeshift, hashno, tmp, label, foundlabel,		\
 		suspendlabel, exitlabel)				\
 									;\
 	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
 	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
 	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
-		hmeblkpa, label/**/5, hmemisc, hmeblkva)		;\
+		hmeblkpa, label/**/5, hmemisc, tmp)			;\
 									;\
 	/*								;\
 	 * tagacc = tagacc						;\
@@ -2576,7 +2273,7 @@ label/**/4:								;\
 	 * tsbarea = tsbarea						;\
 	 * tte   = hmebp (hme bucket pointer)				;\
 	 * hmeblkpa  = vapg  (virtual page)				;\
-	 * hmemisc, hmeblkva = scratch					;\
+	 * hmemisc, tmp = scratch					;\
 	 */								;\
 	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc)	;\
 									;\
@@ -2587,30 +2284,26 @@ label/**/4:								;\
 	 * tte   = hmebp						;\
 	 * hmemisc  = htag_bspage + hashno + 0 (for rid)		;\
 	 * hmeblkpa  = CLOBBERED					;\
-	 * hmeblkva  = scratch						;\
+	 * tmp = scratch						;\
 	 */								;\
 	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
-	HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM)	;\
 									;\
 	add     tte, HMEBUCK_NEXTPA, hmeblkpa				;\
 	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
-	add     tte, HMEBUCK_HBLK, hmeblkva				;\
-	ldxa    [hmeblkva]ASI_MEM, hmeblkva				;\
 	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte)			;\
 									;\
 label/**/8:								;\
-	HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, hmeblkva, 	\
+	HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa,			\
 		tsbarea, tagacc, tte, label/**/1)			;\
 	/*								;\
 	 * tagacc = CLOBBERED						;\
 	 * tte = CLOBBERED						;\
 	 * hmeblkpa = hmeblkpa						;\
-	 * hmeblkva = hmeblkva						;\
+	 * tmp = scratch						;\
 	 */								;\
-	brnz,pt	hmeblkva, label/**/4	/* branch if hmeblk found */	;\
+	cmp	hmeblkpa, HMEBLK_ENDPA					;\
+	bne,pn   %xcc, label/**/4       /* branch if hmeblk found */    ;\
 	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
-	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva	;\
-	HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM)  /* drop lock */	;\
 	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
 	  nop								;\
 label/**/4:								;\
@@ -2623,8 +2316,8 @@ label/**/4:								;\
 	 * tte   = clobbered						;\
 	 * hmeblkpa  = hmeblkpa						;\
 	 * hmemisc  = hblktag						;\
-	 * hmeblkva  = hmeblkva 					;\
 	 * tsbarea = tsbmiss area					;\
+	 * tmp = scratch						;\
 	 */								;\
 	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte,		\
 		label/**/2)						;\
@@ -2635,15 +2328,15 @@ label/**/4:								;\
 	 * tte = clobbered						;\
 	 * hmeblkpa  = hmeblkpa						;\
 	 * hmemisc  = hblk_misc						;\
-	 * hmeblkva  = hmeblkva						;\
 	 * tsbarea = tsbmiss area					;\
+	 * tmp = scratch						;\
 	 */								;\
 									;\
 	add	hatid, SFHME_TTE, hatid					;\
 	add	hmeblkpa, hatid, hmeblkpa				;\
 	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
 	brlz,pt tte, label/**/6						;\
-	  add	hmeblkva, hatid, hmeblkva				;\
+	  nop								;\
 	btst	HBLK_SZMASK, hmemisc					;\
 	bnz,a,pt %icc, label/**/7					;\
 	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
@@ -2657,8 +2350,6 @@ label/**/4:								;\
 	sub	hmeblkpa, hatid, hmeblkpa				;\
 	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
 	srlx	tagacc, hmeshift, tte					;\
-	add	hmeblkpa, HMEBLK_NEXT, hmeblkva				;\
-	ldxa	[hmeblkva]ASI_MEM, hmeblkva				;\
 	add	hmeblkpa, HMEBLK_NEXTPA, hmeblkpa			;\
 	ldxa	[hmeblkpa]ASI_MEM, hmeblkpa				;\
 	MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc)		;\
@@ -2667,7 +2358,6 @@ label/**/6:								;\
 	GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc)		;\
 	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
 label/**/7:								;\
-	HMELOCK_EXIT(hatid, hatid, ASI_MEM)	/* drop lock */		;\
 	set	TTE_SUSPEND, hatid					;\
 	TTE_SUSPEND_INT_SHIFT(hatid)					;\
 	btst	tte, hatid						;\
@@ -3343,8 +3033,8 @@ udtlb_miss_probesecond:
 	 */
 2:
 
-	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
-		MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte,
+	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
+		MMU_PAGESHIFT64K, TTE64K, %g5, tsb_l8K, tsb_checktte,
 		sfmmu_suspend_tl, tsb_512K)
 	/* NOT REACHED */
 
@@ -3372,8 +3062,8 @@ tsb_512K:
 	 * 512K hash
 	 */
 	
-	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
-		MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte,
+	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
+		MMU_PAGESHIFT512K, TTE512K, %g5, tsb_l512K, tsb_checktte,
 		sfmmu_suspend_tl, tsb_4M)
 	/* NOT REACHED */
 
@@ -3389,8 +3079,8 @@ tsb_4M:
 	 * 4M hash
 	 */
 
-	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
-		MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte,
+	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
+		MMU_PAGESHIFT4M, TTE4M, %g5, tsb_l4M, tsb_checktte,
 		sfmmu_suspend_tl, tsb_32M)
 	/* NOT REACHED */
 
@@ -3410,8 +3100,8 @@ tsb_32M:
 	 * 32M hash
 	 */
 
-	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
-		MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte,
+	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
+		MMU_PAGESHIFT32M, TTE32M, %g5, tsb_l32M, tsb_checktte,
 		sfmmu_suspend_tl, tsb_256M)
 	/* NOT REACHED */
 	
@@ -3428,8 +3118,8 @@ tsb_256M:
 	 * 256M hash
 	 */
 		
-	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
-	    MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte,
+	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
+	    MMU_PAGESHIFT256M, TTE256M, %g5, tsb_l256M, tsb_checktte,
 	    sfmmu_suspend_tl, tsb_shme)
 	/* NOT REACHED */
 
@@ -3439,7 +3129,6 @@ tsb_checktte:
 	 * g2 = tagacc
 	 * g3 = tte
 	 * g4 = tte pa
-	 * g5 = tte va
 	 * g6 = tsbmiss area
 	 * g7 = hatid
 	 */
@@ -3464,8 +3153,8 @@ tsb_shme:
 	brz,pn	%g7, tsb_pagefault
 	  nop
 
-	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
-		MMU_PAGESHIFT64K, TTE64K, tsb_shme_l8K, tsb_shme_checktte,
+	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
+		MMU_PAGESHIFT64K, TTE64K, %g5, tsb_shme_l8K, tsb_shme_checktte,
 		sfmmu_suspend_tl, tsb_shme_512K)
 	/* NOT REACHED */
 
@@ -3479,8 +3168,8 @@ tsb_shme_512K:
 	 * 512K hash
 	 */
 	
-	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
-		MMU_PAGESHIFT512K, TTE512K, tsb_shme_l512K, tsb_shme_checktte,
+	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
+		MMU_PAGESHIFT512K, TTE512K, %g5, tsb_shme_l512K, tsb_shme_checktte,
 		sfmmu_suspend_tl, tsb_shme_4M)
 	/* NOT REACHED */
 
@@ -3493,8 +3182,8 @@ tsb_shme_4M:
 	/*
 	 * 4M hash
 	 */
-	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
-		MMU_PAGESHIFT4M, TTE4M, tsb_shme_l4M, tsb_shme_checktte,
+	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
+		MMU_PAGESHIFT4M, TTE4M, %g5, tsb_shme_l4M, tsb_shme_checktte,
 		sfmmu_suspend_tl, tsb_shme_32M)
 	/* NOT REACHED */
 
@@ -3508,8 +3197,8 @@ tsb_shme_32M:
 	 * 32M hash
 	 */
 
-	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
-		MMU_PAGESHIFT32M, TTE32M, tsb_shme_l32M, tsb_shme_checktte,
+	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
+		MMU_PAGESHIFT32M, TTE32M, %g5, tsb_shme_l32M, tsb_shme_checktte,
 		sfmmu_suspend_tl, tsb_shme_256M)
 	/* NOT REACHED */
 
@@ -3523,8 +3212,8 @@ tsb_shme_256M:
 	 * 256M hash
 	 */
 		
-	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
-	    MMU_PAGESHIFT256M, TTE256M, tsb_shme_l256M, tsb_shme_checktte,
+	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
+	    MMU_PAGESHIFT256M, TTE256M, %g5, tsb_shme_l256M, tsb_shme_checktte,
 	    sfmmu_suspend_tl, tsb_pagefault)
 	/* NOT REACHED */
 
@@ -3536,7 +3225,6 @@ tsb_shme_checktte:
 	 * g1 = ctx1 flag
 	 * g3 = tte
 	 * g4 = tte pa
-	 * g5 = tte va
 	 * g6 = tsbmiss area
 	 * g7 = tt
 	 */
@@ -3554,7 +3242,6 @@ tsb_validtte:
 	/*
 	 * g3 = tte
 	 * g4 = tte pa
-	 * g5 = tte va
 	 * g6 = tsbmiss area
 	 * g7 = tt
 	 */
@@ -3566,7 +3253,7 @@ tsb_validtte:
 	bne,pt	%icc, 4f
 	  nop
 
-	TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod,
+	TTE_SET_REFMOD_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_refmod,
 	    tsb_protfault) 
 
 	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
@@ -3602,7 +3289,7 @@ tsb_validtte:
 	/*
 	 * Set reference bit if not already set
 	 */
-	TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref) 
+	TTE_SET_REF_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_ref) 
 
 	/*
 	 * Now, load into TSB/TLB.  At this point:
@@ -3688,7 +3375,7 @@ tsb_user8k:
 	mov	%g7, %asi
 #endif /* !UTSB_PHYS */
 
-	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5)
+	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l3)
 
 	rdpr    %tt, %g5
 #ifdef sun4v
@@ -3731,7 +3418,7 @@ tsb_user4m:
 	mov	%g7, %asi
 #endif /* UTSB_PHYS */
 
-        TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6)
+        TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l4)
 
 5:
 	rdpr    %tt, %g5
@@ -3786,7 +3473,7 @@ tsb_user_pn_synth:
 
 	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
 	mov	%g7, %asi
-	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */
+	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l5) /* update TSB */
 5:
         DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
         retry
@@ -3802,7 +3489,7 @@ tsb_user_itlb_synth:
 
 	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
 	mov	%g7, %asi
-	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */
+	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l6) /* update TSB */
 7:
 	SET_TTE4M_PN(%g5, %g7)			/* add TTE4M pagesize to TTE */
         ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
@@ -3832,7 +3519,7 @@ tsb_kernel_patch_asi:
 	or	%g0, RUNTIME_PATCH, %g6
 	mov	%g6, %asi	! XXX avoid writing to %asi !!
 #endif
-	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7)
+	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l7)
 3:
 #ifdef sun4v
 	cmp	%g5, T_INSTR_MMU_MISS
@@ -3912,8 +3599,8 @@ tsb_ism_32M:
 	 * 32M hash.
 	 */
 		
-	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M,
-	    TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl,
+	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT32M,
+	    TTE32M, %g5, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl,
 	    tsb_ism_4M)
 	/* NOT REACHED */
 	
@@ -3931,8 +3618,8 @@ tsb_ism_256M:
 	/*
 	 * 256M hash.
 	 */
-	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M,
-	    TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl,
+	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT256M,
+	    TTE256M, %g5, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl,
 	    tsb_ism_4M)
 
 tsb_ism_256M_found:
@@ -3943,8 +3630,8 @@ tsb_ism_4M:
 	/*
 	 * 4M hash.
 	 */
-	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M,
-	    TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl,
+	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT4M,
+	    TTE4M, %g5, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl,
 	    tsb_ism_8K)
 	/* NOT REACHED */
 
@@ -3957,8 +3644,8 @@ tsb_ism_8K:
 	 * 8K and 64K hash.
 	 */
 	
-	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K,
-	    TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl,
+	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT64K,
+	    TTE64K, %g5, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl,
 	    tsb_pagefault)
 	/* NOT REACHED */
 
@@ -4216,7 +3903,7 @@ sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno)
 	 */
 	set     TAGACC_CTX_MASK, %g1
 	andn    %o0, %g1, %o0
-	GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5,
+	GET_TTE(%o0, %o4, %g1, %g2, %o5, %g4, %g6, %g5, %g3, 
 		vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk)
 
 kvtop_hblk_found:
@@ -4226,7 +3913,7 @@ kvtop_hblk_found:
 	 * o2 = ttep
 	 * g1 = tte
 	 * g2 = tte pa
-	 * g3 = tte va
+	 * g3 = scratch
 	 * o2 = tsbmiss area
 	 * o1 = hat id
 	 */
@@ -4357,7 +4044,7 @@ vatopfn_nokernel:
 	 */
 	srlx	%o0, MMU_PAGESHIFT, %o0
 	sllx	%o0, MMU_PAGESHIFT, %o0
-	GET_TTE(%o0, %o4, %g3, %g4, %g5, %g1, %o5, %g6, %o1,
+	GET_TTE(%o0, %o4, %g3, %g4, %g1, %o5, %g6, %o1, %g5, 
 		kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk,
 		kvaszc2pfn_nohblk)
 
@@ -4643,12 +4330,16 @@ label/**/_ok:
 	mov	%g1, %asi
 #endif
 
-	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
-	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
+	/*
+	 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set)
+	 * If we fail to lock the TSB entry then just load the tte into the
+	 * TLB.
+	 */
+	TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l1)
 
 	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
 	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
-
+locked_tsb_l1:
 	DTLB_STUFF(%g5, %g1, %g2, %g4, %g6)
 
 	/* KPMLOCK_EXIT(kpmlckp, asi) */
@@ -4883,12 +4574,16 @@ label/**/_ok:
 	mov	%g1, %asi
 #endif
 
-	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
-	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
+	/*
+	 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set)
+	 * If we fail to lock the TSB entry then just load the tte into the
+	 * TLB.
+	 */
+	TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l2)
 
 	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
 	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
-
+locked_tsb_l2:
 	DTLB_STUFF(%g5, %g2, %g4, %g5, %g6)
 
 	/* KPMLOCK_EXIT(kpmlckp, asi) */
diff --git a/usr/src/uts/sfmmu/ml/sfmmu_kdi.s b/usr/src/uts/sfmmu/ml/sfmmu_kdi.s
index ded048bd9d..346a08aa5f 100644
--- a/usr/src/uts/sfmmu/ml/sfmmu_kdi.s
+++ b/usr/src/uts/sfmmu/ml/sfmmu_kdi.s
@@ -162,7 +162,8 @@ bspage:	/* TTE_PAGE_SHIFT in %g5 */				\
 	add	%g4, HMEBUCK_NEXTPA, %g4; /* %g4 is hmebucket PA */	\
 search_loop:								\
 	ldxa	[%g4]ASI_MEM, %g4;					\
-	brz	%g4, search_done;					\
+	cmp	%g4, HMEBLK_ENDPA;					\
+	be,pn	%xcc, search_done;					\
 	nop;								\
 									\
 	add	%g4, HMEBLK_TAG, %g4;	/* %g4 is now hmeblk PA */	\
diff --git a/usr/src/uts/sfmmu/vm/hat_sfmmu.c b/usr/src/uts/sfmmu/vm/hat_sfmmu.c
index b480d48a05..206d22d960 100644
--- a/usr/src/uts/sfmmu/vm/hat_sfmmu.c
+++ b/usr/src/uts/sfmmu/vm/hat_sfmmu.c
@@ -167,6 +167,13 @@ static nucleus_hblk8_info_t	nucleus_hblk8;
 static nucleus_hblk1_info_t	nucleus_hblk1;
 
 /*
+ * Data to manage per-cpu hmeblk pending queues, hmeblks are queued here
+ * after the initial phase of removing an hmeblk from the hash chain, see
+ * the detailed comment in sfmmu_hblk_hash_rm() for further details.
+ */
+static cpu_hme_pend_t		*cpu_hme_pend;
+static uint_t			cpu_hme_pend_thresh;
+/*
  * SFMMU specific hat functions
  */
 void	hat_pagecachectl(struct page *, int);
@@ -405,15 +412,13 @@ static caddr_t	sfmmu_hblk_unload(struct hat *, struct hme_blk *, caddr_t,
 			caddr_t, demap_range_t *, uint_t);
 static caddr_t	sfmmu_hblk_sync(struct hat *, struct hme_blk *, caddr_t,
 			caddr_t, int);
-static void	sfmmu_hblk_free(struct hmehash_bucket *, struct hme_blk *,
-			uint64_t, struct hme_blk **);
-static void	sfmmu_hblks_list_purge(struct hme_blk **);
+static void	sfmmu_hblk_free(struct hme_blk **);
+static void	sfmmu_hblks_list_purge(struct hme_blk **, int);
 static uint_t	sfmmu_get_free_hblk(struct hme_blk **, uint_t);
 static uint_t	sfmmu_put_free_hblk(struct hme_blk *, uint_t);
 static struct hme_blk *sfmmu_hblk_steal(int);
 static int	sfmmu_steal_this_hblk(struct hmehash_bucket *,
-			struct hme_blk *, uint64_t, uint64_t,
-			struct hme_blk *);
+			struct hme_blk *, uint64_t, struct hme_blk *);
 static caddr_t	sfmmu_hblk_unlock(struct hme_blk *, caddr_t, caddr_t);
 
 static void	hat_do_memload_array(struct hat *, caddr_t, size_t,
@@ -509,6 +514,11 @@ static void	sfmmu_hblkcache_destructor(void *, void *);
 static void	sfmmu_hblkcache_reclaim(void *);
 static void	sfmmu_shadow_hcleanup(sfmmu_t *, struct hme_blk *,
 			struct hmehash_bucket *);
+static void	sfmmu_hblk_hash_rm(struct hmehash_bucket *, struct hme_blk *,
+			struct hme_blk *, struct hme_blk **, int);
+static void	sfmmu_hblk_hash_add(struct hmehash_bucket *, struct hme_blk *,
+			uint64_t);
+static struct hme_blk *sfmmu_check_pending_hblks(int);
 static void	sfmmu_free_hblks(sfmmu_t *, caddr_t, caddr_t, int);
 static void	sfmmu_cleanup_rhblk(sf_srd_t *, caddr_t, uint_t, int);
 static void	sfmmu_unload_hmeregion_va(sf_srd_t *, uint_t, caddr_t, caddr_t,
@@ -1090,10 +1100,12 @@ hat_init(void)
 	for (i = 0; i < khmehash_num; i++) {
 		mutex_init(&khme_hash[i].hmehash_mutex, NULL,
 		    MUTEX_DEFAULT, NULL);
+		khme_hash[i].hmeh_nextpa = HMEBLK_ENDPA;
 	}
 	for (i = 0; i < uhmehash_num; i++) {
 		mutex_init(&uhme_hash[i].hmehash_mutex, NULL,
 		    MUTEX_DEFAULT, NULL);
+		uhme_hash[i].hmeh_nextpa = HMEBLK_ENDPA;
 	}
 	khmehash_num--;		/* make sure counter starts from 0 */
 	uhmehash_num--;		/* make sure counter starts from 0 */
@@ -1420,6 +1432,21 @@ hat_init(void)
 	 */
 	hrm_hashtab = kmem_zalloc(HRM_HASHSIZE * sizeof (struct hrmstat *),
 	    KM_SLEEP);
+
+	/* Allocate per-cpu pending freelist of hmeblks */
+	cpu_hme_pend = kmem_zalloc((NCPU * sizeof (cpu_hme_pend_t)) + 64,
+	    KM_SLEEP);
+	cpu_hme_pend = (cpu_hme_pend_t *)P2ROUNDUP(
+	    (uintptr_t)cpu_hme_pend, 64);
+
+	for (i = 0; i < NCPU; i++) {
+		mutex_init(&cpu_hme_pend[i].chp_mutex, NULL, MUTEX_DEFAULT,
+		    NULL);
+	}
+
+	if (cpu_hme_pend_thresh == 0) {
+		cpu_hme_pend_thresh = CPU_HME_PEND_THRESH;
+	}
 }
 
 /*
@@ -1853,7 +1880,6 @@ hat_swapout(struct hat *sfmmup)
 	struct hme_blk *pr_hblk = NULL;
 	struct hme_blk *nx_hblk;
 	int i;
-	uint64_t hblkpa, prevpa, nx_pa;
 	struct hme_blk *list = NULL;
 	hatlock_t *hatlockp;
 	struct tsb_info *tsbinfop;
@@ -1884,8 +1910,6 @@ hat_swapout(struct hat *sfmmup)
 		hmebp = &uhme_hash[i];
 		SFMMU_HASH_LOCK(hmebp);
 		hmeblkp = hmebp->hmeblkp;
-		hblkpa = hmebp->hmeh_nextpa;
-		prevpa = 0;
 		pr_hblk = NULL;
 		while (hmeblkp) {
 
@@ -1900,23 +1924,19 @@ hat_swapout(struct hat *sfmmup)
 				    NULL, HAT_UNLOAD);
 			}
 			nx_hblk = hmeblkp->hblk_next;
-			nx_pa = hmeblkp->hblk_nextpa;
 			if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) {
 				ASSERT(!hmeblkp->hblk_lckcnt);
-				sfmmu_hblk_hash_rm(hmebp, hmeblkp,
-				    prevpa, pr_hblk);
-				sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list);
+				sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk,
+				    &list, 0);
 			} else {
 				pr_hblk = hmeblkp;
-				prevpa = hblkpa;
 			}
 			hmeblkp = nx_hblk;
-			hblkpa = nx_pa;
 		}
 		SFMMU_HASH_UNLOCK(hmebp);
 	}
 
-	sfmmu_hblks_list_purge(&list);
+	sfmmu_hblks_list_purge(&list, 0);
 
 	/*
 	 * Now free up the ctx so that others can reuse it.
@@ -2853,9 +2873,6 @@ sfmmu_tteload_find_hmeblk(sfmmu_t *sfmmup, struct hmehash_bucket *hmebp,
 	hmeblk_tag hblktag;
 	int hmeshift;
 	struct hme_blk *hmeblkp, *pr_hblk, *list = NULL;
-	uint64_t hblkpa, prevpa;
-	struct kmem_cache *sfmmu_cache;
-	uint_t forcefree;
 
 	SFMMU_VALIDATE_HMERID(sfmmup, rid, vaddr, TTEBYTES(size));
 
@@ -2868,8 +2885,7 @@ sfmmu_tteload_find_hmeblk(sfmmu_t *sfmmup, struct hmehash_bucket *hmebp,
 
 ttearray_realloc:
 
-	HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa,
-	    pr_hblk, prevpa, &list);
+	HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list);
 
 	/*
 	 * We block until hblk_reserve_lock is released; it's held by
@@ -2901,8 +2917,8 @@ ttearray_realloc:
 		if (get_hblk_ttesz(hmeblkp) != size) {
 			ASSERT(!hmeblkp->hblk_vcnt);
 			ASSERT(!hmeblkp->hblk_hmecnt);
-			sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk);
-			sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list);
+			sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk,
+			    &list, 0);
 			goto ttearray_realloc;
 		}
 		if (hmeblkp->hblk_shw_bit) {
@@ -2923,22 +2939,12 @@ ttearray_realloc:
 	}
 
 	/*
-	 * hat_memload() should never call kmem_cache_free(); see block
-	 * comment showing the stacktrace in sfmmu_hblk_alloc();
-	 * enqueue each hblk in the list to reserve list if it's created
-	 * from sfmmu8_cache *and* sfmmup == KHATID.
+	 * hat_memload() should never call kmem_cache_free() for kernel hmeblks;
+	 * see block comment showing the stacktrace in sfmmu_hblk_alloc();
+	 * set the flag parameter to 1 so that sfmmu_hblks_list_purge() will
+	 * just add these hmeblks to the per-cpu pending queue.
 	 */
-	forcefree = (sfmmup == KHATID) ? 1 : 0;
-	while ((pr_hblk = list) != NULL) {
-		list = pr_hblk->hblk_next;
-		sfmmu_cache = get_hblk_cache(pr_hblk);
-		if ((sfmmu_cache == sfmmu8_cache) &&
-		    sfmmu_put_free_hblk(pr_hblk, forcefree))
-			continue;
-
-		ASSERT(sfmmup != KHATID);
-		kmem_cache_free(sfmmu_cache, pr_hblk);
-	}
+	sfmmu_hblks_list_purge(&list, 1);
 
 	ASSERT(get_hblk_ttesz(hmeblkp) == size);
 	ASSERT(!hmeblkp->hblk_shw_bit);
@@ -3653,7 +3659,6 @@ sfmmu_free_hblks(sfmmu_t *sfmmup, caddr_t addr, caddr_t endaddr,
 	struct hmehash_bucket *hmebp;
 	struct hme_blk *hmeblkp;
 	struct hme_blk *nx_hblk, *pr_hblk, *list = NULL;
-	uint64_t hblkpa, prevpa, nx_pa;
 
 	ASSERT(hashno > 0);
 	hblktag.htag_id = sfmmup;
@@ -3668,11 +3673,8 @@ sfmmu_free_hblks(sfmmu_t *sfmmup, caddr_t addr, caddr_t endaddr,
 		SFMMU_HASH_LOCK(hmebp);
 		/* inline HME_HASH_SEARCH */
 		hmeblkp = hmebp->hmeblkp;
-		hblkpa = hmebp->hmeh_nextpa;
-		prevpa = 0;
 		pr_hblk = NULL;
 		while (hmeblkp) {
-			ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp));
 			if (HTAGS_EQ(hmeblkp->hblk_tag, hblktag)) {
 				/* found hme_blk */
 				ASSERT(!hmeblkp->hblk_shared);
@@ -3697,17 +3699,13 @@ sfmmu_free_hblks(sfmmu_t *sfmmup, caddr_t addr, caddr_t endaddr,
 			}
 
 			nx_hblk = hmeblkp->hblk_next;
-			nx_pa = hmeblkp->hblk_nextpa;
 			if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) {
-				sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa,
-				    pr_hblk);
-				sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list);
+				sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk,
+				    &list, 0);
 			} else {
 				pr_hblk = hmeblkp;
-				prevpa = hblkpa;
 			}
 			hmeblkp = nx_hblk;
-			hblkpa = nx_pa;
 		}
 
 		SFMMU_HASH_UNLOCK(hmebp);
@@ -3725,7 +3723,7 @@ sfmmu_free_hblks(sfmmu_t *sfmmup, caddr_t addr, caddr_t endaddr,
 			    (1 << hmeshift));
 		}
 	}
-	sfmmu_hblks_list_purge(&list);
+	sfmmu_hblks_list_purge(&list, 0);
 }
 
 /*
@@ -3741,7 +3739,6 @@ sfmmu_cleanup_rhblk(sf_srd_t *srdp, caddr_t addr, uint_t rid, int ttesz)
 	struct hme_blk *hmeblkp;
 	struct hme_blk *pr_hblk;
 	struct hme_blk *list = NULL;
-	uint64_t hblkpa, prevpa;
 
 	ASSERT(SFMMU_IS_SHMERID_VALID(rid));
 	ASSERT(rid < SFMMU_MAX_HME_REGIONS);
@@ -3754,8 +3751,7 @@ sfmmu_cleanup_rhblk(sf_srd_t *srdp, caddr_t addr, uint_t rid, int ttesz)
 	hmebp = HME_HASH_FUNCTION(srdp, addr, hmeshift);
 
 	SFMMU_HASH_LOCK(hmebp);
-	HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, pr_hblk,
-	    prevpa, &list);
+	HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list);
 	if (hmeblkp != NULL) {
 		ASSERT(hmeblkp->hblk_shared);
 		ASSERT(!hmeblkp->hblk_shw_bit);
@@ -3763,11 +3759,11 @@ sfmmu_cleanup_rhblk(sf_srd_t *srdp, caddr_t addr, uint_t rid, int ttesz)
 			panic("sfmmu_cleanup_rhblk: valid hmeblk");
 		}
 		ASSERT(!hmeblkp->hblk_lckcnt);
-		sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk);
-		sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list);
+		sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk,
+		    &list, 0);
 	}
 	SFMMU_HASH_UNLOCK(hmebp);
-	sfmmu_hblks_list_purge(&list);
+	sfmmu_hblks_list_purge(&list, 0);
 }
 
 /* ARGSUSED */
@@ -3791,7 +3787,6 @@ sfmmu_unload_hmeregion_va(sf_srd_t *srdp, uint_t rid, caddr_t addr,
 	struct hme_blk *hmeblkp;
 	struct hme_blk *pr_hblk;
 	struct hme_blk *list = NULL;
-	uint64_t hblkpa, prevpa;
 
 	ASSERT(SFMMU_IS_SHMERID_VALID(rid));
 	ASSERT(rid < SFMMU_MAX_HME_REGIONS);
@@ -3805,8 +3800,7 @@ sfmmu_unload_hmeregion_va(sf_srd_t *srdp, uint_t rid, caddr_t addr,
 	hmebp = HME_HASH_FUNCTION(srdp, addr, hmeshift);
 
 	SFMMU_HASH_LOCK(hmebp);
-	HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, pr_hblk,
-	    prevpa, &list);
+	HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list);
 	if (hmeblkp != NULL) {
 		ASSERT(hmeblkp->hblk_shared);
 		ASSERT(!hmeblkp->hblk_lckcnt);
@@ -3816,11 +3810,11 @@ sfmmu_unload_hmeregion_va(sf_srd_t *srdp, uint_t rid, caddr_t addr,
 			ASSERT(*eaddrp > addr);
 		}
 		ASSERT(!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt);
-		sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk);
-		sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list);
+		sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk,
+		    &list, 0);
 	}
 	SFMMU_HASH_UNLOCK(hmebp);
-	sfmmu_hblks_list_purge(&list);
+	sfmmu_hblks_list_purge(&list, 0);
 }
 
 static void
@@ -3962,7 +3956,7 @@ hat_unlock(struct hat *sfmmup, caddr_t addr, size_t len)
 		}
 	}
 
-	sfmmu_hblks_list_purge(&list);
+	sfmmu_hblks_list_purge(&list, 0);
 }
 
 void
@@ -3981,7 +3975,6 @@ hat_unlock_region(struct hat *sfmmup, caddr_t addr, size_t len,
 	struct hme_blk *hmeblkp;
 	struct hme_blk *pr_hblk;
 	struct hme_blk *list;
-	uint64_t hblkpa, prevpa;
 
 	if (rcookie == HAT_INVALID_REGION_COOKIE) {
 		hat_unlock(sfmmup, addr, len);
@@ -4025,8 +4018,8 @@ hat_unlock_region(struct hat *sfmmup, caddr_t addr, size_t len,
 			hblktag.htag_id = srdp;
 			hmebp = HME_HASH_FUNCTION(srdp, va, hmeshift);
 			SFMMU_HASH_LOCK(hmebp);
-			HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa,
-			    pr_hblk, prevpa, &list);
+			HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk,
+			    &list);
 			if (hmeblkp == NULL) {
 				SFMMU_HASH_UNLOCK(hmebp);
 				ttesz--;
@@ -4044,7 +4037,7 @@ hat_unlock_region(struct hat *sfmmup, caddr_t addr, size_t len,
 			    "addr %p hat %p", (void *)va, (void *)sfmmup);
 		}
 	}
-	sfmmu_hblks_list_purge(&list);
+	sfmmu_hblks_list_purge(&list, 0);
 }
 
 /*
@@ -4921,7 +4914,7 @@ sfmmu_chgattr(struct hat *sfmmup, caddr_t addr, size_t len, uint_t attr,
 		}
 	}
 
-	sfmmu_hblks_list_purge(&list);
+	sfmmu_hblks_list_purge(&list, 0);
 	DEMAP_RANGE_FLUSH(&dmr);
 	cpuset = sfmmup->sfmmu_cpusran;
 	xt_sync(cpuset);
@@ -5274,7 +5267,7 @@ hat_chgprot(struct hat *sfmmup, caddr_t addr, size_t len, uint_t vprot)
 		}
 	}
 
-	sfmmu_hblks_list_purge(&list);
+	sfmmu_hblks_list_purge(&list, 0);
 	DEMAP_RANGE_FLUSH(&dmr);
 	cpuset = sfmmup->sfmmu_cpusran;
 	xt_sync(cpuset);
@@ -5499,7 +5492,6 @@ hat_unload_large_virtual(
 	struct hme_blk *nx_hblk;
 	struct hme_blk *list = NULL;
 	int i;
-	uint64_t hblkpa, prevpa, nx_pa;
 	demap_range_t dmr, *dmrp;
 	cpuset_t cpuset;
 	caddr_t	endaddr = startaddr + len;
@@ -5524,12 +5516,9 @@ hat_unload_large_virtual(
 		hmebp = &uhme_hash[i];
 		SFMMU_HASH_LOCK(hmebp);
 		hmeblkp = hmebp->hmeblkp;
-		hblkpa = hmebp->hmeh_nextpa;
-		prevpa = 0;
 		pr_hblk = NULL;
 		while (hmeblkp) {
 			nx_hblk = hmeblkp->hblk_next;
-			nx_pa = hmeblkp->hblk_nextpa;
 
 			/*
 			 * skip if not this context, if a shadow block or
@@ -5540,7 +5529,6 @@ hat_unload_large_virtual(
 			    (sa = (caddr_t)get_hblk_base(hmeblkp)) >= endaddr ||
 			    (ea = get_hblk_endaddr(hmeblkp)) <= startaddr) {
 				pr_hblk = hmeblkp;
-				prevpa = hblkpa;
 				goto next_block;
 			}
 
@@ -5561,12 +5549,10 @@ hat_unload_large_virtual(
 			    !hmeblkp->hblk_vcnt &&
 			    !hmeblkp->hblk_hmecnt) {
 				ASSERT(!hmeblkp->hblk_lckcnt);
-				sfmmu_hblk_hash_rm(hmebp, hmeblkp,
-				    prevpa, pr_hblk);
-				sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list);
+				sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk,
+				    &list, 0);
 			} else {
 				pr_hblk = hmeblkp;
-				prevpa = hblkpa;
 			}
 
 			if (callback == NULL)
@@ -5601,12 +5587,11 @@ hat_unload_large_virtual(
 
 next_block:
 			hmeblkp = nx_hblk;
-			hblkpa = nx_pa;
 		}
 		SFMMU_HASH_UNLOCK(hmebp);
 	}
 
-	sfmmu_hblks_list_purge(&list);
+	sfmmu_hblks_list_purge(&list, 0);
 	if (dmrp != NULL) {
 		DEMAP_RANGE_FLUSH(dmrp);
 		cpuset = sfmmup->sfmmu_cpusran;
@@ -5650,7 +5635,6 @@ hat_unload_callback(
 	struct hme_blk *hmeblkp, *pr_hblk, *list = NULL;
 	caddr_t endaddr;
 	cpuset_t cpuset;
-	uint64_t hblkpa, prevpa;
 	int addr_count = 0;
 	int a;
 	caddr_t cb_start_addr[MAX_CB_ADDR];
@@ -5745,8 +5729,7 @@ hat_unload_callback(
 
 		SFMMU_HASH_LOCK(hmebp);
 
-		HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, pr_hblk,
-		    prevpa, &list);
+		HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list);
 		if (hmeblkp == NULL) {
 			/*
 			 * didn't find an hmeblk. skip the appropiate
@@ -5805,9 +5788,8 @@ hat_unload_callback(
 			    get_hblk_span(hmeblkp));
 			if ((flags & HAT_UNLOAD_UNMAP) ||
 			    (iskernel && !issegkmap)) {
-				sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa,
-				    pr_hblk);
-				sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list);
+				sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk,
+				    &list, 0);
 			}
 			SFMMU_HASH_UNLOCK(hmebp);
 
@@ -5871,9 +5853,7 @@ hat_unload_callback(
 
 		if (((flags & HAT_UNLOAD_UNMAP) || (iskernel && !issegkmap)) &&
 		    !hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) {
-			sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa,
-			    pr_hblk);
-			sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list);
+			sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, &list, 0);
 		}
 		SFMMU_HASH_UNLOCK(hmebp);
 
@@ -5923,7 +5903,7 @@ hat_unload_callback(
 		}
 	}
 
-	sfmmu_hblks_list_purge(&list);
+	sfmmu_hblks_list_purge(&list, 0);
 	DEMAP_RANGE_FLUSH(dmrp);
 	if (dmrp != NULL) {
 		cpuset = sfmmup->sfmmu_cpusran;
@@ -6344,7 +6324,7 @@ hat_sync(struct hat *sfmmup, caddr_t addr, size_t len, uint_t clearflag)
 			hashno++;
 		}
 	}
-	sfmmu_hblks_list_purge(&list);
+	sfmmu_hblks_list_purge(&list, 0);
 	cpuset = sfmmup->sfmmu_cpusran;
 	xt_sync(cpuset);
 }
@@ -9094,12 +9074,45 @@ static void
 sfmmu_hblkcache_reclaim(void *cdrarg)
 {
 	int i;
-	uint64_t hblkpa, prevpa, nx_pa;
 	struct hmehash_bucket *hmebp;
 	struct hme_blk *hmeblkp, *nx_hblk, *pr_hblk = NULL;
 	static struct hmehash_bucket *uhmehash_reclaim_hand;
 	static struct hmehash_bucket *khmehash_reclaim_hand;
-	struct hme_blk *list = NULL;
+	struct hme_blk *list = NULL, *last_hmeblkp;
+	cpuset_t cpuset = cpu_ready_set;
+	cpu_hme_pend_t *cpuhp;
+
+	/* Free up hmeblks on the cpu pending lists */
+	for (i = 0; i < NCPU; i++) {
+		cpuhp = &cpu_hme_pend[i];
+		if (cpuhp->chp_listp != NULL)  {
+			mutex_enter(&cpuhp->chp_mutex);
+			if (cpuhp->chp_listp == NULL) {
+				mutex_exit(&cpuhp->chp_mutex);
+				continue;
+			}
+			for (last_hmeblkp = cpuhp->chp_listp;
+			    last_hmeblkp->hblk_next != NULL;
+			    last_hmeblkp = last_hmeblkp->hblk_next)
+				;
+			last_hmeblkp->hblk_next = list;
+			list = cpuhp->chp_listp;
+			cpuhp->chp_listp = NULL;
+			cpuhp->chp_count = 0;
+			mutex_exit(&cpuhp->chp_mutex);
+		}
+
+	}
+
+	if (list != NULL) {
+		kpreempt_disable();
+		CPUSET_DEL(cpuset, CPU->cpu_id);
+		xt_sync(cpuset);
+		xt_sync(cpuset);
+		kpreempt_enable();
+		sfmmu_hblk_free(&list);
+		list = NULL;
+	}
 
 	hmebp = uhmehash_reclaim_hand;
 	if (hmebp == NULL || hmebp > &uhme_hash[UHMEHASH_SZ])
@@ -9109,24 +9122,17 @@ sfmmu_hblkcache_reclaim(void *cdrarg)
 	for (i = UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) {
 		if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) {
 			hmeblkp = hmebp->hmeblkp;
-			hblkpa = hmebp->hmeh_nextpa;
-			prevpa = 0;
 			pr_hblk = NULL;
 			while (hmeblkp) {
 				nx_hblk = hmeblkp->hblk_next;
-				nx_pa = hmeblkp->hblk_nextpa;
 				if (!hmeblkp->hblk_vcnt &&
 				    !hmeblkp->hblk_hmecnt) {
 					sfmmu_hblk_hash_rm(hmebp, hmeblkp,
-					    prevpa, pr_hblk);
-					sfmmu_hblk_free(hmebp, hmeblkp,
-					    hblkpa, &list);
+					    pr_hblk, &list, 0);
 				} else {
 					pr_hblk = hmeblkp;
-					prevpa = hblkpa;
 				}
 				hmeblkp = nx_hblk;
-				hblkpa = nx_pa;
 			}
 			SFMMU_HASH_UNLOCK(hmebp);
 		}
@@ -9142,31 +9148,24 @@ sfmmu_hblkcache_reclaim(void *cdrarg)
 	for (i = KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) {
 		if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) {
 			hmeblkp = hmebp->hmeblkp;
-			hblkpa = hmebp->hmeh_nextpa;
-			prevpa = 0;
 			pr_hblk = NULL;
 			while (hmeblkp) {
 				nx_hblk = hmeblkp->hblk_next;
-				nx_pa = hmeblkp->hblk_nextpa;
 				if (!hmeblkp->hblk_vcnt &&
 				    !hmeblkp->hblk_hmecnt) {
 					sfmmu_hblk_hash_rm(hmebp, hmeblkp,
-					    prevpa, pr_hblk);
-					sfmmu_hblk_free(hmebp, hmeblkp,
-					    hblkpa, &list);
+					    pr_hblk, &list, 0);
 				} else {
 					pr_hblk = hmeblkp;
-					prevpa = hblkpa;
 				}
 				hmeblkp = nx_hblk;
-				hblkpa = nx_pa;
 			}
 			SFMMU_HASH_UNLOCK(hmebp);
 		}
 		if (hmebp++ == &khme_hash[KHMEHASH_SZ])
 			hmebp = khme_hash;
 	}
-	sfmmu_hblks_list_purge(&list);
+	sfmmu_hblks_list_purge(&list, 0);
 }
 
 /*
@@ -10677,6 +10676,7 @@ sfmmu_get_free_hblk(struct hme_blk **hmeblkpp, uint_t critical)
 {
 	struct  hme_blk *hblkp;
 
+
 	if (freehblkp != NULL) {
 		mutex_enter(&freehblkp_lock);
 		if (freehblkp != NULL) {
@@ -10697,10 +10697,28 @@ sfmmu_get_free_hblk(struct hme_blk **hmeblkpp, uint_t critical)
 			mutex_exit(&freehblkp_lock);
 			hblkp->hblk_next = NULL;
 			SFMMU_STAT(sf_get_free_success);
+
+			ASSERT(hblkp->hblk_hmecnt == 0);
+			ASSERT(hblkp->hblk_vcnt == 0);
+			ASSERT(hblkp->hblk_nextpa == va_to_pa((caddr_t)hblkp));
+
 			return (1);
 		}
 		mutex_exit(&freehblkp_lock);
 	}
+
+	/* Check cpu hblk pending queues */
+	if ((*hmeblkpp = sfmmu_check_pending_hblks(TTE8K)) != NULL) {
+		hblkp = *hmeblkpp;
+		hblkp->hblk_next = NULL;
+		hblkp->hblk_nextpa = va_to_pa((caddr_t)hblkp);
+
+		ASSERT(hblkp->hblk_hmecnt == 0);
+		ASSERT(hblkp->hblk_vcnt == 0);
+
+		return (1);
+	}
+
 	SFMMU_STAT(sf_get_free_fail);
 	return (0);
 }
@@ -10710,6 +10728,10 @@ sfmmu_put_free_hblk(struct hme_blk *hmeblkp, uint_t critical)
 {
 	struct  hme_blk *hblkp;
 
+	ASSERT(hmeblkp->hblk_hmecnt == 0);
+	ASSERT(hmeblkp->hblk_vcnt == 0);
+	ASSERT(hmeblkp->hblk_nextpa == va_to_pa((caddr_t)hmeblkp));
+
 	/*
 	 * If the current thread is mapping into kernel space,
 	 * let it succede even if freehblkcnt is max
@@ -10761,13 +10783,14 @@ static void
 sfmmu_hblk_swap(struct hme_blk *new)
 {
 	struct hme_blk *old, *hblkp, *prev;
-	uint64_t hblkpa, prevpa, newpa;
+	uint64_t newpa;
 	caddr_t	base, vaddr, endaddr;
 	struct hmehash_bucket *hmebp;
 	struct sf_hment *osfhme, *nsfhme;
 	page_t *pp;
 	kmutex_t *pml;
 	tte_t tte;
+	struct hme_blk *list = NULL;
 
 #ifdef	DEBUG
 	hmeblk_tag		hblktag;
@@ -10802,15 +10825,13 @@ sfmmu_hblk_swap(struct hme_blk *new)
 
 	/*
 	 * search hash chain for hblk_reserve; this needs to be performed
-	 * after adding new, otherwise prevpa and prev won't correspond
-	 * to the hblk which is prior to old in hash chain when we call
-	 * sfmmu_hblk_hash_rm to remove old later.
-	 */
-	for (prevpa = 0, prev = NULL,
-	    hblkpa = hmebp->hmeh_nextpa, hblkp = hmebp->hmeblkp;
-	    hblkp != NULL && hblkp != old;
-	    prevpa = hblkpa, prev = hblkp,
-	    hblkpa = hblkp->hblk_nextpa, hblkp = hblkp->hblk_next)
+	 * after adding new, otherwise prev won't correspond to the hblk which
+	 * is prior to old in hash chain when we call sfmmu_hblk_hash_rm to
+	 * remove old later.
+	 */
+	for (prev = NULL,
+	    hblkp = hmebp->hmeblkp; hblkp != NULL && hblkp != old;
+	    prev = hblkp, hblkp = hblkp->hblk_next)
 		;
 
 	if (hblkp != old)
@@ -10857,7 +10878,7 @@ sfmmu_hblk_swap(struct hme_blk *new)
 	/*
 	 * remove old from hash chain
 	 */
-	sfmmu_hblk_hash_rm(hmebp, old, prevpa, prev);
+	sfmmu_hblk_hash_rm(hmebp, old, prev, &list, 1);
 
 #ifdef	DEBUG
 
@@ -11225,9 +11246,10 @@ fill_hblk:
 			 * let's donate this hblk to our reserve list if
 			 * we are not mapping kernel range
 			 */
-			if (size == TTE8K && sfmmup != KHATID)
+			if (size == TTE8K && sfmmup != KHATID) {
 				if (sfmmu_put_free_hblk(hmeblkp, 0))
 					goto fill_hblk;
+			}
 		}
 	} else {
 		/*
@@ -11365,7 +11387,7 @@ hblk_init:
 	hmeblkp->hblk_tag = hblktag;
 	hmeblkp->hblk_shadow = shw_hblkp;
 	hblkpa = hmeblkp->hblk_nextpa;
-	hmeblkp->hblk_nextpa = 0;
+	hmeblkp->hblk_nextpa = HMEBLK_ENDPA;
 
 	ASSERT(get_hblk_ttesz(hmeblkp) == size);
 	ASSERT(get_hblk_span(hmeblkp) == HMEBLK_SPAN(size));
@@ -11378,95 +11400,46 @@ hblk_init:
 }
 
 /*
- * This function performs any cleanup required on the hme_blk
- * and returns it to the free list.
+ * This function cleans up the hme_blk and returns it to the free list.
  */
 /* ARGSUSED */
 static void
-sfmmu_hblk_free(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
-	uint64_t hblkpa, struct hme_blk **listp)
+sfmmu_hblk_free(struct hme_blk **listp)
 {
-	int shw_size, vshift;
-	struct hme_blk *shw_hblkp;
-	uint_t		shw_mask, newshw_mask;
-	caddr_t		vaddr;
+	struct hme_blk *hmeblkp, *next_hmeblkp;
 	int		size;
 	uint_t		critical;
+	uint64_t	hblkpa;
 
-	ASSERT(hmeblkp);
-	ASSERT(!hmeblkp->hblk_hmecnt);
-	ASSERT(!hmeblkp->hblk_vcnt);
-	ASSERT(!hmeblkp->hblk_lckcnt);
-	ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp));
-	ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve);
-
-	critical = (hblktosfmmu(hmeblkp) == KHATID) ? 1 : 0;
-
-	size = get_hblk_ttesz(hmeblkp);
-	shw_hblkp = hmeblkp->hblk_shadow;
-	if (shw_hblkp) {
-		ASSERT(hblktosfmmu(hmeblkp) != KHATID);
-		ASSERT(!hmeblkp->hblk_shared);
-		if (mmu_page_sizes == max_mmu_page_sizes) {
-			ASSERT(size < TTE256M);
-		} else {
-			ASSERT(size < TTE4M);
-		}
+	ASSERT(*listp != NULL);
 
-		shw_size = get_hblk_ttesz(shw_hblkp);
-		vaddr = (caddr_t)get_hblk_base(hmeblkp);
-		vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size);
-		ASSERT(vshift < 8);
-		/*
-		 * Atomically clear shadow mask bit
-		 */
-		do {
-			shw_mask = shw_hblkp->hblk_shw_mask;
-			ASSERT(shw_mask & (1 << vshift));
-			newshw_mask = shw_mask & ~(1 << vshift);
-			newshw_mask = cas32(&shw_hblkp->hblk_shw_mask,
-			    shw_mask, newshw_mask);
-		} while (newshw_mask != shw_mask);
-		hmeblkp->hblk_shadow = NULL;
-	}
-	hmeblkp->hblk_next = NULL;
-	hmeblkp->hblk_nextpa = hblkpa;
-	hmeblkp->hblk_shw_bit = 0;
-
-	if (hmeblkp->hblk_shared) {
-		sf_srd_t	*srdp;
-		sf_region_t	*rgnp;
-		uint_t		rid;
-
-		srdp = hblktosrd(hmeblkp);
-		ASSERT(srdp != NULL && srdp->srd_refcnt != 0);
-		rid = hmeblkp->hblk_tag.htag_rid;
-		ASSERT(SFMMU_IS_SHMERID_VALID(rid));
-		ASSERT(rid < SFMMU_MAX_HME_REGIONS);
-		rgnp = srdp->srd_hmergnp[rid];
-		ASSERT(rgnp != NULL);
-		SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid);
-		hmeblkp->hblk_shared = 0;
-	}
-
-	if (hmeblkp->hblk_nuc_bit == 0) {
+	hmeblkp = *listp;
+	while (hmeblkp != NULL) {
+		next_hmeblkp = hmeblkp->hblk_next;
+		ASSERT(!hmeblkp->hblk_hmecnt);
+		ASSERT(!hmeblkp->hblk_vcnt);
+		ASSERT(!hmeblkp->hblk_lckcnt);
+		ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve);
+		ASSERT(hmeblkp->hblk_shared == 0);
+		ASSERT(hmeblkp->hblk_shw_bit == 0);
+		ASSERT(hmeblkp->hblk_shadow == NULL);
 
-		if (size == TTE8K && sfmmu_put_free_hblk(hmeblkp, critical))
-			return;
+		hblkpa = va_to_pa((caddr_t)hmeblkp);
+		ASSERT(hblkpa != (uint64_t)-1);
+		critical = (hblktosfmmu(hmeblkp) == KHATID) ? 1 : 0;
 
-		hmeblkp->hblk_next = *listp;
-		*listp = hmeblkp;
-	}
-}
+		size = get_hblk_ttesz(hmeblkp);
+		hmeblkp->hblk_next = NULL;
+		hmeblkp->hblk_nextpa = hblkpa;
 
-static void
-sfmmu_hblks_list_purge(struct hme_blk **listp)
-{
-	struct hme_blk	*hmeblkp;
+		if (hmeblkp->hblk_nuc_bit == 0) {
 
-	while ((hmeblkp = *listp) != NULL) {
-		*listp = hmeblkp->hblk_next;
-		kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp);
+			if (size != TTE8K ||
+			    !sfmmu_put_free_hblk(hmeblkp, critical))
+				kmem_cache_free(get_hblk_cache(hmeblkp),
+				    hmeblkp);
+		}
+		hmeblkp = next_hmeblkp;
 	}
 }
 
@@ -11489,11 +11462,19 @@ sfmmu_hblk_steal(int size)
 	static struct hmehash_bucket *uhmehash_steal_hand = NULL;
 	struct hmehash_bucket *hmebp;
 	struct hme_blk *hmeblkp = NULL, *pr_hblk;
-	uint64_t hblkpa, prevpa;
+	uint64_t hblkpa;
 	int i;
 	uint_t loop_cnt = 0, critical;
 
 	for (;;) {
+		/* Check cpu hblk pending queues */
+		if ((hmeblkp = sfmmu_check_pending_hblks(size)) != NULL) {
+			hmeblkp->hblk_nextpa = va_to_pa((caddr_t)hmeblkp);
+			ASSERT(hmeblkp->hblk_hmecnt == 0);
+			ASSERT(hmeblkp->hblk_vcnt == 0);
+			return (hmeblkp);
+		}
+
 		if (size == TTE8K) {
 			critical =
 			    (++loop_cnt > SFMMU_HBLK_STEAL_THRESHOLD) ? 1 : 0;
@@ -11510,7 +11491,6 @@ sfmmu_hblk_steal(int size)
 			SFMMU_HASH_LOCK(hmebp);
 			hmeblkp = hmebp->hmeblkp;
 			hblkpa = hmebp->hmeh_nextpa;
-			prevpa = 0;
 			pr_hblk = NULL;
 			while (hmeblkp) {
 				/*
@@ -11532,8 +11512,7 @@ sfmmu_hblk_steal(int size)
 					    hmeblkp->hblk_hmecnt == 0) || (i >=
 					    BUCKETS_TO_SEARCH_BEFORE_UNLOAD)) {
 						if (sfmmu_steal_this_hblk(hmebp,
-						    hmeblkp, hblkpa, prevpa,
-						    pr_hblk)) {
+						    hmeblkp, hblkpa, pr_hblk)) {
 							/*
 							 * Hblk is unloaded
 							 * successfully
@@ -11543,7 +11522,6 @@ sfmmu_hblk_steal(int size)
 					}
 				}
 				pr_hblk = hmeblkp;
-				prevpa = hblkpa;
 				hblkpa = hmeblkp->hblk_nextpa;
 				hmeblkp = hmeblkp->hblk_next;
 			}
@@ -11565,7 +11543,6 @@ sfmmu_hblk_steal(int size)
 			SFMMU_HASH_LOCK(hmebp);
 			hmeblkp = hmebp->hmeblkp;
 			hblkpa = hmebp->hmeh_nextpa;
-			prevpa = 0;
 			pr_hblk = NULL;
 			while (hmeblkp) {
 				/*
@@ -11576,7 +11553,7 @@ sfmmu_hblk_steal(int size)
 				    (hmeblkp->hblk_vcnt == 0) &&
 				    (hmeblkp->hblk_hmecnt == 0)) {
 					if (sfmmu_steal_this_hblk(hmebp,
-					    hmeblkp, hblkpa, prevpa, pr_hblk)) {
+					    hmeblkp, hblkpa, pr_hblk)) {
 						break;
 					} else {
 						/*
@@ -11588,7 +11565,6 @@ sfmmu_hblk_steal(int size)
 				}
 
 				pr_hblk = hmeblkp;
-				prevpa = hblkpa;
 				hblkpa = hmeblkp->hblk_nextpa;
 				hmeblkp = hmeblkp->hblk_next;
 			}
@@ -11614,12 +11590,13 @@ sfmmu_hblk_steal(int size)
  */
 static int
 sfmmu_steal_this_hblk(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
-	uint64_t hblkpa, uint64_t prevpa, struct hme_blk *pr_hblk)
+	uint64_t hblkpa, struct hme_blk *pr_hblk)
 {
 	int shw_size, vshift;
 	struct hme_blk *shw_hblkp;
 	caddr_t vaddr;
 	uint_t shw_mask, newshw_mask;
+	struct hme_blk *list = NULL;
 
 	ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));
 
@@ -11652,7 +11629,7 @@ sfmmu_steal_this_hblk(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
 	ASSERT(hmeblkp->hblk_lckcnt == 0);
 	ASSERT(hmeblkp->hblk_vcnt == 0 && hmeblkp->hblk_hmecnt == 0);
 
-	sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk);
+	sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, &list, 1);
 	hmeblkp->hblk_nextpa = hblkpa;
 
 	shw_hblkp = hmeblkp->hblk_shadow;
@@ -15558,3 +15535,291 @@ sfmmu_scdcache_destructor(void *buf, void *cdrarg)
 
 	mutex_destroy(&scdp->scd_mutex);
 }
+
+/*
+ * The listp parameter is a pointer to a list of hmeblks which are partially
+ * freed as result of calling sfmmu_hblk_hash_rm(), the last phase of the
+ * freeing process is to cross-call all cpus to ensure that there are no
+ * remaining cached references.
+ *
+ * If the local generation number is less than the global then we can free
+ * hmeblks which are already on the pending queue as another cpu has completed
+ * the cross-call.
+ *
+ * We cross-call to make sure that there are no threads on other cpus accessing
+ * these hmblks and then complete the process of freeing them under the
+ * following conditions:
+ * 	The total number of pending hmeblks is greater than the threshold
+ *	The reserve list has fewer than HBLK_RESERVE_CNT hmeblks
+ *	It is at least 1 second since the last time we cross-called
+ *
+ * Otherwise, we add the hmeblks to the per-cpu pending queue.
+ */
+static void
+sfmmu_hblks_list_purge(struct hme_blk **listp, int dontfree)
+{
+	struct hme_blk *hblkp, *pr_hblkp = NULL;
+	int		count = 0;
+	cpuset_t	cpuset = cpu_ready_set;
+	cpu_hme_pend_t	*cpuhp;
+	timestruc_t	now;
+	int		one_second_expired = 0;
+
+	gethrestime_lasttick(&now);
+
+	for (hblkp = *listp; hblkp != NULL; hblkp = hblkp->hblk_next) {
+		ASSERT(hblkp->hblk_shw_bit == 0);
+		ASSERT(hblkp->hblk_shared == 0);
+		count++;
+		pr_hblkp = hblkp;
+	}
+
+	cpuhp = &cpu_hme_pend[CPU->cpu_seqid];
+	mutex_enter(&cpuhp->chp_mutex);
+
+	if ((cpuhp->chp_count + count) == 0) {
+		mutex_exit(&cpuhp->chp_mutex);
+		return;
+	}
+
+	if ((now.tv_sec - cpuhp->chp_timestamp) > 1) {
+		one_second_expired  = 1;
+	}
+
+	if (!dontfree && (freehblkcnt < HBLK_RESERVE_CNT ||
+	    (cpuhp->chp_count + count) > cpu_hme_pend_thresh ||
+	    one_second_expired)) {
+		/* Append global list to local */
+		if (pr_hblkp == NULL) {
+			*listp = cpuhp->chp_listp;
+		} else {
+			pr_hblkp->hblk_next = cpuhp->chp_listp;
+		}
+		cpuhp->chp_listp = NULL;
+		cpuhp->chp_count = 0;
+		cpuhp->chp_timestamp = now.tv_sec;
+		mutex_exit(&cpuhp->chp_mutex);
+
+		kpreempt_disable();
+		CPUSET_DEL(cpuset, CPU->cpu_id);
+		xt_sync(cpuset);
+		xt_sync(cpuset);
+		kpreempt_enable();
+
+		/*
+		 * At this stage we know that no trap handlers on other
+		 * cpus can have references to hmeblks on the list.
+		 */
+		sfmmu_hblk_free(listp);
+	} else if (*listp != NULL) {
+		pr_hblkp->hblk_next = cpuhp->chp_listp;
+		cpuhp->chp_listp = *listp;
+		cpuhp->chp_count += count;
+		*listp = NULL;
+		mutex_exit(&cpuhp->chp_mutex);
+	} else {
+		mutex_exit(&cpuhp->chp_mutex);
+	}
+}
+
+/*
+ * Add an hmeblk to the the hash list.
+ */
+void
+sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
+	uint64_t hblkpa)
+{
+	ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));
+#ifdef	DEBUG
+	if (hmebp->hmeblkp == NULL) {
+		ASSERT(hmebp->hmeh_nextpa == HMEBLK_ENDPA);
+	}
+#endif /* DEBUG */
+
+	hmeblkp->hblk_nextpa = hmebp->hmeh_nextpa;
+	/*
+	 * Since the TSB miss handler now does not lock the hash chain before
+	 * walking it, make sure that the hmeblks nextpa is globally visible
+	 * before we make the hmeblk globally visible by updating the chain root
+	 * pointer in the hash bucket.
+	 */
+	membar_producer();
+	hmebp->hmeh_nextpa = hblkpa;
+	hmeblkp->hblk_next = hmebp->hmeblkp;
+	hmebp->hmeblkp = hmeblkp;
+
+}
+
+/*
+ * This function is the first part of a 2 part process to remove an hmeblk
+ * from the hash chain. In this phase we unlink the hmeblk from the hash chain
+ * but leave the next physical pointer unchanged. The hmeblk is then linked onto
+ * a per-cpu pending list using the virtual address pointer.
+ *
+ * TSB miss trap handlers that start after this phase will no longer see
+ * this hmeblk. TSB miss handlers that still cache this hmeblk in a register
+ * can still use it for further chain traversal because we haven't yet modifed
+ * the next physical pointer or freed it.
+ *
+ * In the second phase of hmeblk removal we'll issue a barrier xcall before
+ * we reuse or free this hmeblk. This will make sure all lingering references to
+ * the hmeblk after first phase disappear before we finally reclaim it.
+ * This scheme eliminates the need for TSB miss handlers to lock hmeblk chains
+ * during their traversal.
+ *
+ * The hmehash_mutex must be held when calling this function.
+ *
+ * Input:
+ *	 hmebp - hme hash bucket pointer
+ *	 hmeblkp - address of hmeblk to be removed
+ *	 pr_hblk - virtual address of previous hmeblkp
+ *	 listp - pointer to list of hmeblks linked by virtual address
+ *	 free_now flag - indicates that a complete removal from the hash chains
+ *			 is necessary.
+ *
+ * It is inefficient to use the free_now flag as a cross-call is required to
+ * remove a single hmeblk from the hash chain but is necessary when hmeblks are
+ * in short supply.
+ */
+void
+sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
+    struct hme_blk *pr_hblk, struct hme_blk **listp,
+    int free_now)
+{
+	int shw_size, vshift;
+	struct hme_blk *shw_hblkp;
+	uint_t		shw_mask, newshw_mask;
+	caddr_t		vaddr;
+	int		size;
+	cpuset_t cpuset = cpu_ready_set;
+
+	ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));
+
+	if (hmebp->hmeblkp == hmeblkp) {
+		hmebp->hmeh_nextpa = hmeblkp->hblk_nextpa;
+		hmebp->hmeblkp = hmeblkp->hblk_next;
+	} else {
+		pr_hblk->hblk_nextpa = hmeblkp->hblk_nextpa;
+		pr_hblk->hblk_next = hmeblkp->hblk_next;
+	}
+
+	size = get_hblk_ttesz(hmeblkp);
+	shw_hblkp = hmeblkp->hblk_shadow;
+	if (shw_hblkp) {
+		ASSERT(hblktosfmmu(hmeblkp) != KHATID);
+		ASSERT(!hmeblkp->hblk_shared);
+#ifdef	DEBUG
+		if (mmu_page_sizes == max_mmu_page_sizes) {
+			ASSERT(size < TTE256M);
+		} else {
+			ASSERT(size < TTE4M);
+		}
+#endif /* DEBUG */
+
+		shw_size = get_hblk_ttesz(shw_hblkp);
+		vaddr = (caddr_t)get_hblk_base(hmeblkp);
+		vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size);
+		ASSERT(vshift < 8);
+		/*
+		 * Atomically clear shadow mask bit
+		 */
+		do {
+			shw_mask = shw_hblkp->hblk_shw_mask;
+			ASSERT(shw_mask & (1 << vshift));
+			newshw_mask = shw_mask & ~(1 << vshift);
+			newshw_mask = cas32(&shw_hblkp->hblk_shw_mask,
+			    shw_mask, newshw_mask);
+		} while (newshw_mask != shw_mask);
+		hmeblkp->hblk_shadow = NULL;
+	}
+	hmeblkp->hblk_shw_bit = 0;
+
+	if (hmeblkp->hblk_shared) {
+#ifdef	DEBUG
+		sf_srd_t	*srdp;
+		sf_region_t	*rgnp;
+		uint_t		rid;
+
+		srdp = hblktosrd(hmeblkp);
+		ASSERT(srdp != NULL && srdp->srd_refcnt != 0);
+		rid = hmeblkp->hblk_tag.htag_rid;
+		ASSERT(SFMMU_IS_SHMERID_VALID(rid));
+		ASSERT(rid < SFMMU_MAX_HME_REGIONS);
+		rgnp = srdp->srd_hmergnp[rid];
+		ASSERT(rgnp != NULL);
+		SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid);
+#endif /* DEBUG */
+		hmeblkp->hblk_shared = 0;
+	}
+	if (free_now) {
+		kpreempt_disable();
+		CPUSET_DEL(cpuset, CPU->cpu_id);
+		xt_sync(cpuset);
+		xt_sync(cpuset);
+		kpreempt_enable();
+
+		hmeblkp->hblk_nextpa = HMEBLK_ENDPA;
+		hmeblkp->hblk_next = NULL;
+	} else {
+		/* Append hmeblkp to listp for processing later. */
+		hmeblkp->hblk_next = *listp;
+		*listp = hmeblkp;
+	}
+}
+
+/*
+ * This routine is called when memory is in short supply and returns a free
+ * hmeblk of the requested size from the cpu pending lists.
+ */
+static struct hme_blk *
+sfmmu_check_pending_hblks(int size)
+{
+	int i;
+	struct hme_blk *hmeblkp = NULL, *last_hmeblkp;
+	int found_hmeblk;
+	cpuset_t cpuset = cpu_ready_set;
+	cpu_hme_pend_t *cpuhp;
+
+	/* Flush cpu hblk pending queues */
+	for (i = 0; i < NCPU; i++) {
+		cpuhp = &cpu_hme_pend[i];
+		if (cpuhp->chp_listp != NULL)  {
+			mutex_enter(&cpuhp->chp_mutex);
+			if (cpuhp->chp_listp == NULL)  {
+				mutex_exit(&cpuhp->chp_mutex);
+				continue;
+			}
+			found_hmeblk = 0;
+			last_hmeblkp = NULL;
+			for (hmeblkp = cpuhp->chp_listp; hmeblkp != NULL;
+			    hmeblkp = hmeblkp->hblk_next) {
+				if (get_hblk_ttesz(hmeblkp) == size) {
+					if (last_hmeblkp == NULL) {
+						cpuhp->chp_listp =
+						    hmeblkp->hblk_next;
+					} else {
+						last_hmeblkp->hblk_next =
+						    hmeblkp->hblk_next;
+					}
+					ASSERT(cpuhp->chp_count > 0);
+					cpuhp->chp_count--;
+					found_hmeblk = 1;
+					break;
+				} else {
+					last_hmeblkp = hmeblkp;
+				}
+			}
+			mutex_exit(&cpuhp->chp_mutex);
+
+			if (found_hmeblk) {
+				kpreempt_disable();
+				CPUSET_DEL(cpuset, CPU->cpu_id);
+				xt_sync(cpuset);
+				xt_sync(cpuset);
+				kpreempt_enable();
+				return (hmeblkp);
+			}
+		}
+	}
+	return (NULL);
+}
diff --git a/usr/src/uts/sfmmu/vm/hat_sfmmu.h b/usr/src/uts/sfmmu/vm/hat_sfmmu.h
index dc63b5e5df..e6ccb33185 100644
--- a/usr/src/uts/sfmmu/vm/hat_sfmmu.h
+++ b/usr/src/uts/sfmmu/vm/hat_sfmmu.h
@@ -301,6 +301,8 @@ struct tsb_info {
 #define	SFMMU_PRIVATE	0
 #define	SFMMU_SHARED	1
 
+#define	HMEBLK_ENDPA	1
+
 #ifndef _ASM
 
 #define	SFMMU_MAX_ISM_REGIONS		(64)
@@ -515,6 +517,24 @@ typedef struct sf_srd_bucket {
 }
 
 /*
+ *  Per cpu pending freelist of hmeblks.
+ */
+typedef struct cpu_hme_pend {
+	struct   hme_blk *chp_listp;
+	kmutex_t chp_mutex;
+	time_t	 chp_timestamp;
+	uint_t   chp_count;
+	uint8_t	 chp_pad[36];		/* pad to 64 bytes */
+} cpu_hme_pend_t;
+
+/*
+ * The default value of the threshold for the per cpu pending queues of hmeblks.
+ * The queues are flushed if either the number of hmeblks on the queue is above
+ * the threshold, or one second has elapsed since the last flush.
+ */
+#define	CPU_HME_PEND_THRESH 1000
+
+/*
  * Per-MMU context domain kstats.
  *
  * TSB Miss Exceptions
@@ -1247,7 +1267,7 @@ struct hme_blk_misc {
 };
 
 struct hme_blk {
-	uint64_t	hblk_nextpa;	/* physical address for hash list */
+	volatile uint64_t hblk_nextpa;	/* physical address for hash list */
 
 	hmeblk_tag	hblk_tag;	/* tag used to obtain an hmeblk match */
 
@@ -1367,7 +1387,7 @@ struct hme_blk {
  */
 struct hmehash_bucket {
 	kmutex_t	hmehash_mutex;
-	uint64_t	hmeh_nextpa;	/* physical address for hash list */
+	volatile uint64_t hmeh_nextpa;	/* physical address for hash list */
 	struct hme_blk *hmeblkp;
 	uint_t		hmeh_listlock;
 };
@@ -1527,16 +1547,12 @@ struct hmehash_bucket {
  * will be set to NULL, otherwise it will point to the correct hme_blk.
  * This macro also cleans empty hblks.
  */
-#define	HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, hblkpa,		\
-	pr_hblk, prevpa, listp)						\
+#define	HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, pr_hblk, listp)	\
 {									\
 	struct hme_blk *nx_hblk;					\
-	uint64_t 	nx_pa;						\
 									\
 	ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));				\
 	hblkp = hmebp->hmeblkp;						\
-	hblkpa = hmebp->hmeh_nextpa;					\
-	prevpa = 0;							\
 	pr_hblk = NULL;							\
 	while (hblkp) {							\
 		if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) {		\
@@ -1544,26 +1560,21 @@ struct hmehash_bucket {
 			break;						\
 		}							\
 		nx_hblk = hblkp->hblk_next;				\
-		nx_pa = hblkp->hblk_nextpa;				\
 		if (!hblkp->hblk_vcnt && !hblkp->hblk_hmecnt) {		\
-			sfmmu_hblk_hash_rm(hmebp, hblkp, prevpa, pr_hblk); \
-			sfmmu_hblk_free(hmebp, hblkp, hblkpa, listp);   \
+			sfmmu_hblk_hash_rm(hmebp, hblkp, pr_hblk,	\
+			    listp, 0);					\
 		} else {						\
 			pr_hblk = hblkp;				\
-			prevpa = hblkpa;				\
 		}							\
 		hblkp = nx_hblk;					\
-		hblkpa = nx_pa;						\
 	}								\
 }
 
 #define	HME_HASH_SEARCH(hmebp, hblktag, hblkp, listp)			\
 {									\
 	struct hme_blk *pr_hblk;					\
-	uint64_t hblkpa, prevpa;					\
 									\
-	HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, hblkpa, pr_hblk,	\
-		prevpa, listp);						\
+	HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp,  pr_hblk, listp);	\
 }
 
 /*
@@ -2274,10 +2285,6 @@ extern void	sfmmu_copytte(tte_t *, tte_t *);
 extern int	sfmmu_modifytte(tte_t *, tte_t *, tte_t *);
 extern int	sfmmu_modifytte_try(tte_t *, tte_t *, tte_t *);
 extern pfn_t	sfmmu_ttetopfn(tte_t *, caddr_t);
-extern void	sfmmu_hblk_hash_rm(struct hmehash_bucket *,
-			struct hme_blk *, uint64_t, struct hme_blk *);
-extern void	sfmmu_hblk_hash_add(struct hmehash_bucket *, struct hme_blk *,
-			uint64_t);
 extern uint_t	sfmmu_disable_intrs(void);
 extern void	sfmmu_enable_intrs(uint_t);
 /*