6789139 px_mmu_detach destroys a vmem arena before its address-space allocations are free'd

6 files changed, 111 insertions, 23 deletions

diff --git a/usr/src/uts/sun4/io/px/px_lib.h b/usr/src/uts/sun4/io/px/px_lib.h
index f4d12395b5..4b773256c4 100644
--- a/usr/src/uts/sun4/io/px/px_lib.h
+++ b/usr/src/uts/sun4/io/px/px_lib.h
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #ifndef	_SYS_PX_LIB_H
@@ -91,6 +90,7 @@ extern int px_lib_dma_bypass_rngchk(dev_info_t *dip, ddi_dma_attr_t *attr_p,
     uint64_t *lo_p, uint64_t *hi_p);
 extern int px_lib_iommu_getbypass(dev_info_t *dip, r_addr_t ra,
     io_attributes_t attr, io_addr_t *io_addr_p);
+extern int px_lib_iommu_detach(px_t *px_p);
 extern uint64_t px_lib_ro_bypass(dev_info_t *dip, io_attributes_t attr,
     uint64_t io_addr);
 extern int px_lib_dma_sync(dev_info_t *dip, dev_info_t *rdip,
diff --git a/usr/src/uts/sun4/io/px/px_mmu.c b/usr/src/uts/sun4/io/px/px_mmu.c
index 8cd02870ab..0c68271053 100644
--- a/usr/src/uts/sun4/io/px/px_mmu.c
+++ b/usr/src/uts/sun4/io/px/px_mmu.c
@@ -19,12 +19,9 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 /*
  * PX mmu initialization and configuration
  */
@@ -153,6 +150,8 @@ px_mmu_detach(px_t *px_p)
 {
 	px_mmu_t *mmu_p = px_p->px_mmu_p;
 
+	(void) px_lib_iommu_detach(px_p);
+
 	/*
 	 * Free the dvma resource map.
 	 */
diff --git a/usr/src/uts/sun4u/io/px/px_hlib.c b/usr/src/uts/sun4u/io/px/px_hlib.c
index 732c7a5a5a..11c529dfa7 100644
--- a/usr/src/uts/sun4u/io/px/px_hlib.c
+++ b/usr/src/uts/sun4u/io/px/px_hlib.c
@@ -18,9 +18,9 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #include <sys/types.h>
@@ -1697,7 +1697,7 @@ mmu_tsb_entries(caddr_t csr_base, pxu_t *pxu_p)
 void
 hvio_mmu_init(caddr_t csr_base, pxu_t *pxu_p)
 {
-	uint64_t	val, i, obp_tsb_pa, *base_tte_addr;
+	uint64_t	val, i, obp_tsb_pa;
 	uint_t obp_tsb_entries;
 
 	bzero(pxu_p->tsb_vaddr, pxu_p->tsb_size);
@@ -1709,17 +1709,12 @@ hvio_mmu_init(caddr_t csr_base, pxu_t *pxu_p)
 
 	obp_tsb_entries = mmu_tsb_entries(csr_base, pxu_p);
 
-	base_tte_addr = pxu_p->tsb_vaddr +
-	    ((pxu_p->tsb_size >> 3) - obp_tsb_entries);
+	/* save "shape" of OBP's TSB for use during Detach */
+	pxu_p->obp_tsb_paddr = obp_tsb_pa;
+	pxu_p->obp_tsb_entries = obp_tsb_entries;
 
-	for (i = 0; i < obp_tsb_entries; i++) {
-		uint64_t tte = lddphys(obp_tsb_pa + i * 8);
-
-		if (!MMU_TTE_VALID(tte))
-			continue;
-
-		base_tte_addr[i] = tte;
-	}
+	/* For each Valid TTE in OBP's TSB, save its value in px's IOTSB */
+	hvio_obptsb_attach(pxu_p);
 
 	/*
 	 * Invalidate the TLB through the diagnostic register.
@@ -1906,6 +1901,80 @@ hvio_iommu_getmap(devhandle_t dev_hdl, pxu_t *pxu_p, tsbid_t tsbid,
 	return (ret);
 }
 
+/*
+ * Copy each Valid OBP TTE from OBP's IOTSB to px's IOTSB.
+ */
+void
+hvio_obptsb_attach(pxu_t *pxu_p)
+{
+	uint64_t	obp_tsb_pa;
+	uint64_t	*base_tte_addr;
+	uint64_t	i;
+	uint_t		obp_tsb_entries;
+
+	obp_tsb_pa = pxu_p->obp_tsb_paddr;
+	obp_tsb_entries = pxu_p->obp_tsb_entries;
+
+	/*
+	 * Compute the starting addr of the area reserved for
+	 * OBP's TTEs; OBP's TTEs are stored at the highest addrs
+	 * of px's IOTSB.
+	 */
+	base_tte_addr = pxu_p->tsb_vaddr +
+	    ((pxu_p->tsb_size >> 3) - obp_tsb_entries);
+
+	for (i = 0; i < obp_tsb_entries; i++) {
+		uint64_t tte = lddphys(obp_tsb_pa + i * 8);
+
+		if (!MMU_TTE_VALID(tte))
+			continue;
+
+		base_tte_addr[i] = tte;
+	}
+}
+
+/*
+ * For each Valid OBP TTE, deallocate space from the vmem Arena used
+ * to manage the TTE's associated DVMA addr space.  (Allocation from
+ * the DVMA Arena was done in px_mmu_attach).
+ */
+void
+hvio_obptsb_detach(px_t *px_p)
+{
+	uint64_t	obp_tsb_pa;
+	uint64_t	i;
+	uint_t		obp_tsb_entries;
+	uint_t		obp_tsb_bias;
+	px_mmu_t	*mmu_p = px_p->px_mmu_p;
+	vmem_t		*dvma_map;
+	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
+
+	dvma_map = mmu_p->mmu_dvma_map;
+
+	obp_tsb_pa = pxu_p->obp_tsb_paddr;
+	obp_tsb_entries = pxu_p->obp_tsb_entries;
+	/*
+	 * OBP's TTEs are located at the high end of px's IOTSB.
+	 * Equivalently, OBP's DVMA space is allocated at the high end
+	 * of px's DVMA space.  Compute the bias that references
+	 * OBP's first possible page of DVMA space.
+	 */
+	obp_tsb_bias = (pxu_p->tsb_size >> 3) - obp_tsb_entries;
+
+	for (i = 0; i < obp_tsb_entries; i++) {
+		caddr_t va;
+		uint64_t tte = lddphys(obp_tsb_pa + i * 8);
+
+		if (!MMU_TTE_VALID(tte))
+			continue;
+
+		/* deallocate the TTE's associated page of DVMA space */
+		va = (caddr_t)(MMU_PTOB(mmu_p->dvma_base_pg + obp_tsb_bias +
+		    i));
+		vmem_xfree(dvma_map, va, MMU_PAGE_SIZE);
+	}
+}
+
 /* ARGSUSED */
 uint64_t
 hvio_get_bypass_base(pxu_t *pxu_p)
diff --git a/usr/src/uts/sun4u/io/px/px_lib4u.c b/usr/src/uts/sun4u/io/px/px_lib4u.c
index c59ecf43cb..57cbeac22f 100644
--- a/usr/src/uts/sun4u/io/px/px_lib4u.c
+++ b/usr/src/uts/sun4u/io/px/px_lib4u.c
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #include <sys/types.h>
@@ -574,6 +573,17 @@ px_lib_iommu_getmap(dev_info_t *dip, tsbid_t tsbid, io_attributes_t *attr_p,
 	return (DDI_SUCCESS);
 }
 
+int
+px_lib_iommu_detach(px_t *px_p)
+{
+	/*
+	 * Deallocate DVMA addr space that was reserved for OBP TTE's
+	 * during Attach.
+	 */
+	hvio_obptsb_detach(px_p);
+
+	return (DDI_SUCCESS);
+}
 
 /*
  * Checks dma attributes against system bypass ranges
diff --git a/usr/src/uts/sun4u/io/px/px_lib4u.h b/usr/src/uts/sun4u/io/px/px_lib4u.h
index 6828929cad..030d62bc33 100644
--- a/usr/src/uts/sun4u/io/px/px_lib4u.h
+++ b/usr/src/uts/sun4u/io/px/px_lib4u.h
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #ifndef _SYS_PX_LIB4U_H
@@ -111,6 +110,8 @@ typedef struct pxu {
 	/* sun4u specific vars */
 	caddr_t			px_address[4];
 	ddi_acc_handle_t	px_ac[4];
+	uint64_t		obp_tsb_paddr;
+	uint_t			obp_tsb_entries;
 
 	/* PCItool */
 	caddr_t		pcitool_addr;
@@ -328,6 +329,8 @@ extern uint64_t hvio_iommu_getbypass(devhandle_t dev_hdl, pxu_t *pxu_p,
 extern uint64_t hvio_get_bypass_base(pxu_t *pxu_p);
 extern uint64_t hvio_get_bypass_end(pxu_t *pxu_p);
 extern uint64_t px_get_range_prop(px_t *px_p, pci_ranges_t *rp, int bank);
+extern void hvio_obptsb_attach(pxu_t *pxu_p);
+extern void hvio_obptsb_detach(px_t *px_p);
 
 
 /*
diff --git a/usr/src/uts/sun4v/io/px/px_lib4v.c b/usr/src/uts/sun4v/io/px/px_lib4v.c
index a14bbfbf5b..54bcd03bef 100644
--- a/usr/src/uts/sun4v/io/px/px_lib4v.c
+++ b/usr/src/uts/sun4v/io/px/px_lib4v.c
@@ -555,6 +555,13 @@ px_lib_iommu_getmap(dev_info_t *dip, tsbid_t tsbid, io_attributes_t *attr_p,
 }
 
 /*ARGSUSED*/
+int
+px_lib_iommu_detach(px_t *px_p)
+{
+	return (DDI_SUCCESS);
+}
+
+/*ARGSUSED*/
 uint64_t
 px_get_rng_parent_hi_mask(px_t *px_p)
 {