6550300 TLB invalidation on Intel processors requires additional code

3 files changed, 32 insertions, 20 deletions

diff --git a/usr/src/uts/i86pc/vm/hat_i86.c b/usr/src/uts/i86pc/vm/hat_i86.c
index 2221f84316..a585f73ced 100644
--- a/usr/src/uts/i86pc/vm/hat_i86.c
+++ b/usr/src/uts/i86pc/vm/hat_i86.c
@@ -458,6 +458,16 @@ mmu_init(void)
 		mmu.pt_nx = 0;
 
 	/*
+	 * Intel CPUs allow speculative caching (in TLB-like h/w) of
+	 * entries in upper page tables even though there may not be
+	 * any valid entries in lower tables. This implies we have to
+	 * re-INVLPG at every upper page table entry invalidation.
+	 */
+	if (cpuid_getvendor(CPU) == X86_VENDOR_Intel)
+		mmu.inval_nonleaf = 1;
+	else
+		mmu.inval_nonleaf = 0;
+	/*
 	 * Use CPU info to set various MMU parameters
 	 */
 	cpuid_get_addrsize(CPU, &pa_bits, &va_bits);
diff --git a/usr/src/uts/i86pc/vm/hat_pte.h b/usr/src/uts/i86pc/vm/hat_pte.h
index 9bd8c31c48..ea94cfd5ed 100644
--- a/usr/src/uts/i86pc/vm/hat_pte.h
+++ b/usr/src/uts/i86pc/vm/hat_pte.h
@@ -142,6 +142,7 @@ struct hat_mmu_info {
 	uint_t max_page_level;	/* maximum level at which we can map a page */
 	uint_t ptes_per_table;	/* # of entries in lower level page tables */
 	uint_t top_level_count;	/* # of entries in top most level page table */
+	uint_t inval_nonleaf;	/* invlpg after invalidates in inner tables */
 
 	uint_t	hash_cnt;	/* cnt of entries in htable_hash_cache */
 	uint_t	vlp_hash_cnt;	/* cnt of entries in vlp htable_hash_cache */
diff --git a/usr/src/uts/i86pc/vm/htable.c b/usr/src/uts/i86pc/vm/htable.c
index 71a8f3d349..00c197348a 100644
--- a/usr/src/uts/i86pc/vm/htable.c
+++ b/usr/src/uts/i86pc/vm/htable.c
@@ -854,6 +854,7 @@ unlink_ptp(htable_t *higher, htable_t *old, uintptr_t vaddr)
 	uint_t		entry = htable_va2entry(vaddr, higher);
 	x86pte_t	expect = MAKEPTP(old->ht_pfn, old->ht_level);
 	x86pte_t	found;
+	hat_t		*hat = old->ht_hat;
 
 	ASSERT(higher->ht_busy > 0);
 	ASSERT(higher->ht_valid_cnt > 0);
@@ -862,6 +863,19 @@ unlink_ptp(htable_t *higher, htable_t *old, uintptr_t vaddr)
 	if (found != expect)
 		panic("Bad PTP found=" FMT_PTE ", expected=" FMT_PTE,
 		    found, expect);
+
+	/*
+	 * When any top level VLP page table entry changes, we must issue
+	 * a reload of cr3 on all processors. Also some CPU types require
+	 * invalidating when inner table entries are invalidated.
+	 */
+	if (!(hat->hat_flags & HAT_FREEING)) {
+		if (higher->ht_flags & HTABLE_VLP)
+			hat_tlb_inval(hat, DEMAP_ALL_ADDR);
+		else if (mmu.inval_nonleaf)
+			hat_tlb_inval(hat, old->ht_vaddr);
+	}
+
 	HTABLE_DEC(higher->ht_valid_cnt);
 }
 
@@ -885,6 +899,13 @@ link_ptp(htable_t *higher, htable_t *new, uintptr_t vaddr)
 	found = x86pte_cas(higher, entry, 0, newptp);
 	if ((found & ~PT_REF) != 0)
 		panic("HAT: ptp not 0, found=" FMT_PTE, found);
+
+	/*
+	 * When any top level VLP page table entry changes, we must issue
+	 * a reload of cr3 on all processors using it.
+	 */
+	if (higher->ht_flags & HTABLE_VLP)
+		hat_tlb_inval(higher->ht_hat, DEMAP_ALL_ADDR);
 }
 
 /*
@@ -968,14 +989,6 @@ htable_release(htable_t *ht)
 			unlink_ptp(higher, ht, va);
 
 			/*
-			 * When any top level VLP page table entry changes, we
-			 * must issue a reload of cr3 on all processors.
-			 */
-			if ((hat->hat_flags & HAT_VLP) &&
-			    level == VLP_LEVEL - 1)
-				hat_tlb_inval(hat, DEMAP_ALL_ADDR);
-
-			/*
 			 * remove this htable from its hash list
 			 */
 			if (ht->ht_next)
@@ -1162,18 +1175,6 @@ try_again:
 			if (higher != NULL) {
 				link_ptp(higher, ht, base);
 				ht->ht_parent = higher;
-
-				/*
-				 * When any top level VLP page table changes,
-				 * we must reload cr3 on all processors.
-				 */
-#ifdef __i386
-				if (mmu.pae_hat &&
-#else /* !__i386 */
-				if ((hat->hat_flags & HAT_VLP) &&
-#endif /* __i386 */
-				    l == VLP_LEVEL - 1)
-					hat_tlb_inval(hat, DEMAP_ALL_ADDR);
 			}
 			ht->ht_next = hat->hat_ht_hash[h];
 			ASSERT(ht->ht_prev == NULL);