HVM-174 Remove use of p_private

4 files changed, 71 insertions, 30 deletions

diff --git a/kvm.c b/kvm.c
index 597139f..9c40a9c 100644
--- a/kvm.c
+++ b/kvm.c
@@ -27,6 +27,7 @@
 #include <sys/vm.h>
 #include <sys/proc.h>
 #include <vm/seg_kpm.h>
+#include <sys/avl.h>
 
 #include "vmx.h"
 #include "msr-index.h"
@@ -218,7 +219,7 @@ extern void update_divide_count(struct kvm_lapic *);
 extern void cli(void);
 extern void sti(void);
 static void kvm_destroy_vm(struct kvm *);
-
+static int kvm_avlmmucmp(const void *, const void *);
 
 int get_ept_level(void);
 static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg);
@@ -1618,12 +1619,12 @@ kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, uint64_t *parent_pte)
 	    sizeof (*sp));
 	sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGESIZE);
 	sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGESIZE);
-#ifndef XXX
-	set_page_private(virt_to_page((caddr_t)sp->spt), (void *)sp);
-#else
-	XXX_KVM_PROBE;
-	sp->hpa = (hat_getpfnum(kas.a_hat, (caddr_t)sp->spt)<< PAGESHIFT);
-#endif
+	sp->kmp_avlspt = (uintptr_t)virt_to_page((caddr_t)sp->spt);
+
+	mutex_enter(&vcpu->kvm->kvm_avllock);
+	avl_add(&vcpu->kvm->kvm_avlmp, sp);
+	mutex_exit(&vcpu->kvm->kvm_avllock);
+
 	list_insert_head(&vcpu->kvm->arch.active_mmu_pages, sp);
 #ifdef XXX
 	/* XXX don't see this used anywhere */
@@ -1643,15 +1644,21 @@ typedef int (*mmu_parent_walk_fn) (struct kvm_vcpu *, struct kvm_mmu_page *);
 extern uint64_t kvm_va2pa(caddr_t va);
 
 struct kvm_mmu_page *
-page_private(page_t *page)
-{
-	return ((struct kvm_mmu_page *)page->p_private);
+page_private(kvm_t *kvmp, page_t *page)
+{
+	kvm_mmu_page_t mp, *res;
+	mp.kmp_avlspt = (uintptr_t)page;
+	mutex_enter(&kvmp->kvm_avllock);
+	res = avl_find(&kvmp->kvm_avlmp, &mp, NULL);
+	mutex_exit(&kvmp->kvm_avllock);
+	ASSERT(res != NULL);
+	return (res);
 }
 
 inline struct kvm_mmu_page *
-page_header(hpa_t shadow_page)
+page_header(kvm_t *kvmp, hpa_t shadow_page)
 {
-	return (page_private(pfn_to_page(shadow_page >> PAGESHIFT)));
+	return (page_private(kvmp, pfn_to_page(shadow_page >> PAGESHIFT)));
 }
 
 static void
@@ -1664,7 +1671,8 @@ mmu_parent_walk(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
 	int i;
 
 	if (!sp->multimapped && sp->parent_pte) {
-		parent_sp = page_header(kvm_va2pa((caddr_t)sp->parent_pte));
+		parent_sp = page_header(vcpu->kvm,
+		    kvm_va2pa((caddr_t)sp->parent_pte));
 
 		fn(vcpu, parent_sp);
 		mmu_parent_walk(vcpu, parent_sp, fn);
@@ -1677,7 +1685,7 @@ mmu_parent_walk(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
 			if (!pte_chain->parent_ptes[i])
 				break;
 
-			parent_sp = page_header(kvm_va2pa(
+			parent_sp = page_header(vcpu->kvm, kvm_va2pa(
 			    (caddr_t)pte_chain->parent_ptes[i]));
 			fn(vcpu, parent_sp);
 			mmu_parent_walk(vcpu, parent_sp, fn);
@@ -1689,7 +1697,7 @@ static void
 kvm_mmu_update_unsync_bitmap(uint64_t *spte, struct kvm *kvm)
 {
 	unsigned int index;
-	struct kvm_mmu_page *sp = page_header(kvm_va2pa((caddr_t)spte));
+	struct kvm_mmu_page *sp = page_header(kvm, kvm_va2pa((caddr_t)spte));
 
 	index = spte - sp->spt;
 	if (!__test_and_set_bit(index, sp->unsync_child_bitmap))
@@ -1848,6 +1856,9 @@ kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp)
 	XXX_KVM_PROBE;
 #endif
 
+	mutex_enter(&kvm->kvm_avllock);
+	avl_remove(&kvm->kvm_avlmp, sp);
+	mutex_exit(&kvm->kvm_avllock);
 	list_remove(&kvm->arch.active_mmu_pages, sp);
 	if (sp)
 		kmem_cache_free(mmu_page_header_cache, sp);
@@ -1892,7 +1903,7 @@ __mmu_unsync_walk(struct kvm_mmu_page *sp, struct kvm_mmu_pages *pvec,
 
 		if (is_shadow_present_pte(ent) && !is_large_pte(ent)) {
 			struct kvm_mmu_page *child;
-			child = page_header(ent & PT64_BASE_ADDR_MASK);
+			child = page_header(kvm, ent & PT64_BASE_ADDR_MASK);
 
 			if (child->unsync_children) {
 				if (mmu_pages_add(pvec, child, i))
@@ -2111,8 +2122,8 @@ kvm_mmu_page_unlink_children(struct kvm *kvm, struct kvm_mmu_page *sp)
 		if (is_shadow_present_pte(ent)) {
 			if (!is_last_spte(ent, sp->role.level)) {
 				ent &= PT64_BASE_ADDR_MASK;
-				mmu_page_remove_parent_pte(page_header(ent),
-				    &pt[i]);
+				mmu_page_remove_parent_pte(page_header(kvm,
+				    ent), &pt[i]);
 			} else {
 				rmap_remove(kvm, &pt[i]);
 			}
@@ -2577,7 +2588,7 @@ mmu_sync_roots(struct kvm_vcpu *vcpu)
 
 	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
 		hpa_t root = vcpu->arch.mmu.root_hpa;
-		sp = page_header(root);
+		sp = page_header(vcpu->kvm, root);
 		mmu_sync_children(vcpu, sp);
 		return;
 	}
@@ -2587,7 +2598,7 @@ mmu_sync_roots(struct kvm_vcpu *vcpu)
 
 		if (root && VALID_PAGE(root)) {
 			root &= PT64_BASE_ADDR_MASK;
-			sp = page_header(root);
+			sp = page_header(vcpu->kvm, root);
 			mmu_sync_children(vcpu, sp);
 		}
 	}
@@ -3747,6 +3758,9 @@ kvm_create_vm(void)
 	mutex_init(&kvmp->lock, NULL, MUTEX_DRIVER, NULL);
 	mutex_init(&kvmp->irq_lock, NULL, MUTEX_DRIVER, NULL);
 	mutex_init(&kvmp->slots_lock, NULL, MUTEX_DRIVER, NULL);
+	mutex_init(&kvmp->kvm_avllock, NULL, MUTEX_DRIVER, NULL);
+	avl_create(&kvmp->kvm_avlmp, kvm_avlmmucmp, sizeof (kvm_mmu_page_t),
+	    offsetof(kvm_mmu_page_t, kmp_avlnode));
 	kvmp->kvmid = kvmid++;
 	mutex_enter(&kvm_lock);
 	kvmp->users_count = 1;
@@ -3763,6 +3777,7 @@ static void
 kvm_destroy_vm(struct kvm *kvmp)
 {
 	int ii;
+	void *cookie;
 
 	if (kvmp == NULL)
 		return;
@@ -3774,6 +3789,13 @@ kvm_destroy_vm(struct kvm *kvmp)
 #endif
 
 	list_remove(&vm_list, kvmp);
+	/*
+	 * XXX: The fact that we're cleaning these up here means that we aren't
+	 * properly cleaning them up somewhere else.
+	 */
+	while (avl_destroy_nodes(&kvmp->kvm_avlmp, &cookie) != NULL);
+	avl_destroy(&kvmp->kvm_avlmp);
+	mutex_destroy(&kvmp->kvm_avllock);
 	mutex_destroy(&kvmp->slots_lock);
 	mutex_destroy(&kvmp->irq_lock);
 	mutex_destroy(&kvmp->lock);
@@ -8312,7 +8334,8 @@ mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
 		if (is_last_spte(pte, sp->role.level)) {
 			rmap_remove(vcpu->kvm, spte);
 		} else {
-			child = page_header(pte & PT64_BASE_ADDR_MASK);
+			child = page_header(vcpu->kvm,
+			    pte & PT64_BASE_ADDR_MASK);
 			mmu_page_remove_parent_pte(child, spte);
 		}
 	}
@@ -15893,4 +15916,17 @@ kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
 	kvm_arch_vcpu_uninit(vcpu);
 	ddi_umem_free(vcpu->cookie);
 }
+
+static int
+kvm_avlmmucmp(const void *arg1, const void *arg2)
+{
+	const kvm_mmu_page_t *mp1 = arg1;
+	const kvm_mmu_page_t *mp2 = arg2;
+	if (mp1->kmp_avlspt > mp2->kmp_avlspt)
+		return (1);
+	if (mp1->kmp_avlspt < mp2->kmp_avlspt)
+		return (-1);
+	ASSERT(mp1->kmp_avlspt == mp2->kmp_avlspt);
+	return (0);
+}
 /* END CSTYLED */
diff --git a/kvm.h b/kvm.h
index b6f2d66..b48e2c8 100644
--- a/kvm.h
+++ b/kvm.h
@@ -7,6 +7,7 @@
 #include <sys/ddi.h>
 #include <sys/sunddi.h>
 #include <sys/sdt.h>
+#include <sys/avl.h>
 #undef _ASM  /* cyclic.h expects this not defined */
 #include <sys/cyclic.h>
 #define _ASM
@@ -1215,6 +1216,8 @@ typedef struct kvm {
 #endif
 	int kvmid;  /* unique identifier for this kvm */
 	int kvm_clones;
+	kmutex_t kvm_avllock;
+	avl_tree_t kvm_avlmp;		/* avl tree for mmu to page_t mapping */
 } kvm_t;
 #endif /*_KERNEL*/
 
@@ -1968,7 +1971,6 @@ typedef struct kvm_pit {
 #define RW_STATE_WORD1 4
 
 #define page_to_pfn(page) (page->p_pagenum)
-#define set_page_private(page, v)	((page)->p_private = (v))
 
 #ifdef XXX
 #define __ex(x) __kvm_handle_fault_on_reboot(x)
diff --git a/kvm_x86.c b/kvm_x86.c
index a964ac8..fb5d7b2 100644
--- a/kvm_x86.c
+++ b/kvm_x86.c
@@ -2698,7 +2698,6 @@ mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache, int min)
 		if (!page)
 			return (-ENOMEM);
 
-		set_page_private(page, 0);
 		cache->objects[cache->nobjs++] = page_address(page);
 	}
 
@@ -2928,7 +2927,7 @@ is_nx(struct kvm_vcpu *vcpu)
 	return (vcpu->arch.efer & EFER_NX);
 }
 
-extern struct kvm_mmu_page *page_header(hpa_t shadow_page);
+extern struct kvm_mmu_page *page_header(kvm_t *, hpa_t);
 
 pfn_t
 spte_to_pfn(uint64_t pte)
@@ -3051,7 +3050,7 @@ rmap_remove(struct kvm *kvm, uint64_t *spte)
 
 	if (!is_rmap_spte(*spte))
 		return;
-	sp = page_header(kvm_va2pa((caddr_t)spte));
+	sp = page_header(kvm, kvm_va2pa((caddr_t)spte));
 	pfn = spte_to_pfn(*spte);
 	if (*spte & shadow_accessed_mask)
 		kvm_set_pfn_accessed(kvm, pfn);
@@ -3122,7 +3121,7 @@ rmap_add(struct kvm_vcpu *vcpu, uint64_t *spte, gfn_t gfn)
 		return (count);
 
 	gfn = unalias_gfn(vcpu->kvm, gfn);
-	sp = page_header(kvm_va2pa((caddr_t)spte));
+	sp = page_header(vcpu->kvm, kvm_va2pa((caddr_t)spte));
 	sp->gfns[spte - sp->spt] = gfn;
 	rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
 	if (!*rmapp) {
@@ -3176,7 +3175,7 @@ static void
 page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
 {
 	int slot = memslot_id(kvm, gfn);
-	struct kvm_mmu_page *sp = page_header(kvm_va2pa(pte));
+	struct kvm_mmu_page *sp = page_header(kvm, kvm_va2pa(pte));
 
 	__set_bit(slot, sp->slot_bitmap);
 }
@@ -3398,7 +3397,8 @@ mmu_set_spte(struct kvm_vcpu *vcpu, uint64_t *sptep, unsigned pt_access,
 			struct kvm_mmu_page *child;
 			uint64_t pte = *sptep;
 
-			child = page_header(pte & PT64_BASE_ADDR_MASK);
+			child = page_header(vcpu->kvm,
+			    pte & PT64_BASE_ADDR_MASK);
 			mmu_page_remove_parent_pte(child, sptep);
 		} else if (pfn != spte_to_pfn(*sptep)) {
 			rmap_remove(vcpu->kvm, sptep);
@@ -3634,7 +3634,7 @@ mmu_free_roots(struct kvm_vcpu *vcpu)
 	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
 		hpa_t root = vcpu->arch.mmu.root_hpa;
 
-		sp = page_header(root);
+		sp = page_header(vcpu->kvm, root);
 		--sp->root_count;
 		if (!sp->root_count && sp->role.invalid)
 			kvm_mmu_zap_page(vcpu->kvm, sp);
@@ -3648,7 +3648,7 @@ mmu_free_roots(struct kvm_vcpu *vcpu)
 
 		if (root) {
 			root &= PT64_BASE_ADDR_MASK;
-			sp = page_header(root);
+			sp = page_header(vcpu->kvm, root);
 			--sp->root_count;
 			if (!sp->root_count && sp->role.invalid)
 				kvm_mmu_zap_page(vcpu->kvm, sp);
diff --git a/kvm_x86host.h b/kvm_x86host.h
index 4fdab47..6640c30 100644
--- a/kvm_x86host.h
+++ b/kvm_x86host.h
@@ -28,6 +28,7 @@
 
 #endif
 
+#include <sys/avl.h>
 #include <sys/ksynch.h>
 #include <sys/list.h>
 #include <sys/bitmap.h>
@@ -197,6 +198,7 @@ union kvm_mmu_page_role {
 };
 
 typedef struct kvm_mmu_page {
+	avl_node_t kmp_avlnode;
 	struct list_node link;
 	struct list_node hash_link;
 
@@ -211,6 +213,7 @@ typedef struct kvm_mmu_page {
 	union kvm_mmu_page_role role;
 
 	uint64_t *spt;
+	uintptr_t kmp_avlspt;
 	/* hold the gfn of each spte inside spt */
 	gfn_t *gfns;
 	/*