6841893 Need to add quiesce method to amd_iommu in order for fast reboot to work.

4 files changed, 113 insertions, 34 deletions

diff --git a/usr/src/uts/i86pc/io/amd_iommu/amd_iommu.c b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu.c
index 1f638061c0..950b5e1e5b 100644
--- a/usr/src/uts/i86pc/io/amd_iommu/amd_iommu.c
+++ b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu.c
@@ -54,6 +54,7 @@ static int amd_iommu_open(dev_t *devp, int flag, int otyp, cred_t *credp);
 static int amd_iommu_close(dev_t dev, int flag, int otyp, cred_t *credp);
 static int amd_iommu_ioctl(dev_t dev, int cmd, intptr_t arg, int mode,
     cred_t *credp, int *rvalp);
+static int amd_iommu_quiesce(dev_info_t *dip);
 
 static struct cb_ops amd_iommu_cb_ops = {
 	amd_iommu_open,		/* cb_open */
@@ -87,7 +88,8 @@ static struct dev_ops amd_iommu_dev_ops = {
 	nodev,			/* devo_reset */
 	&amd_iommu_cb_ops,	/* devo_cb_ops */
 	NULL,			/* devo_bus_ops */
-	nulldev			/* devo_power */
+	nulldev,		/* devo_power */
+	amd_iommu_quiesce,	/* devo_quiesce */
 };
 
 static struct modldrv modldrv = {
@@ -442,3 +444,26 @@ amd_iommu_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
 
 	return (ENOTTY);
 }
+
+static int
+amd_iommu_quiesce(dev_info_t *dip)
+{
+	int instance = ddi_get_instance(dip);
+	struct amd_iommu_state *statep;
+	const char *f = "amd_iommu_quiesce";
+
+	statep = ddi_get_soft_state(amd_iommu_statep, instance);
+	if (statep == NULL) {
+		cmn_err(CE_WARN, "%s: cannot get soft state: instance %d",
+		    f, instance);
+		return (DDI_FAILURE);
+	}
+
+	if (amd_iommu_teardown(dip, statep, AMD_IOMMU_QUIESCE) != DDI_SUCCESS) {
+		cmn_err(CE_WARN, "%s: Unable to quiesce AMD IOMMU "
+		    "%s%d", f, ddi_driver_name(dip), instance);
+		return (DDI_FAILURE);
+	}
+
+	return (DDI_SUCCESS);
+}
diff --git a/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_impl.c b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_impl.c
index 8deb468c2f..d2f625b219 100644
--- a/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_impl.c
+++ b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_impl.c
@@ -35,7 +35,7 @@
 #include "amd_iommu_acpi.h"
 #include "amd_iommu_page_tables.h"
 
-static int amd_iommu_fini(amd_iommu_t *iommu);
+static int amd_iommu_fini(amd_iommu_t *iommu, int type);
 static void amd_iommu_teardown_interrupts(amd_iommu_t *iommu);
 static void amd_iommu_stop(amd_iommu_t *iommu);
 
@@ -481,7 +481,7 @@ amd_iommu_setup_tables_and_buffers(amd_iommu_t *iommu)
 }
 
 static void
-amd_iommu_teardown_tables_and_buffers(amd_iommu_t *iommu)
+amd_iommu_teardown_tables_and_buffers(amd_iommu_t *iommu, int type)
 {
 	dev_info_t *dip = iommu->aiomt_dip;
 	int instance = ddi_get_instance(dip);
@@ -493,12 +493,22 @@ amd_iommu_teardown_tables_and_buffers(amd_iommu_t *iommu)
 	    AMD_IOMMU_EVENTBASE, 0);
 	AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
 	    AMD_IOMMU_EVENTLEN, 0);
+	AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
+	    AMD_IOMMU_EVENTHEADPTR, 0);
+	AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
+	    AMD_IOMMU_EVENTTAILPTR, 0);
+
 
 	iommu->aiomt_cmdbuf = NULL;
 	AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
 	    AMD_IOMMU_COMBASE, 0);
 	AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
 	    AMD_IOMMU_COMLEN, 0);
+	AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
+	    AMD_IOMMU_CMDHEADPTR, 0);
+	AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
+	    AMD_IOMMU_CMDTAILPTR, 0);
+
 
 	iommu->aiomt_devtbl = NULL;
 	AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
@@ -506,7 +516,7 @@ amd_iommu_teardown_tables_and_buffers(amd_iommu_t *iommu)
 	AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
 	    AMD_IOMMU_DEVTABSIZE, 0);
 
-	if (iommu->aiomt_dmahdl == NULL)
+	if (iommu->aiomt_dmahdl == NULL || type == AMD_IOMMU_QUIESCE)
 		return;
 
 	/* Unbind the handle */
@@ -1050,7 +1060,7 @@ amd_iommu_init(dev_info_t *dip, ddi_acc_handle_t handle, int idx,
 		    "control regs. Skipping IOMMU idx=%d", f, driver,
 		    instance, idx);
 		mutex_exit(&iommu->aiomt_mutex);
-		(void) amd_iommu_fini(iommu);
+		(void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
 		return (NULL);
 	}
 
@@ -1094,13 +1104,13 @@ amd_iommu_init(dev_info_t *dip, ddi_acc_handle_t handle, int idx,
 	 */
 	if (amd_iommu_setup_tables_and_buffers(iommu) != DDI_SUCCESS) {
 		mutex_exit(&iommu->aiomt_mutex);
-		(void) amd_iommu_fini(iommu);
+		(void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
 		return (NULL);
 	}
 
 	if (amd_iommu_setup_exclusion(iommu) != DDI_SUCCESS) {
 		mutex_exit(&iommu->aiomt_mutex);
-		(void) amd_iommu_fini(iommu);
+		(void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
 		return (NULL);
 	}
 
@@ -1108,7 +1118,7 @@ amd_iommu_init(dev_info_t *dip, ddi_acc_handle_t handle, int idx,
 
 	if (amd_iommu_setup_interrupts(iommu) != DDI_SUCCESS) {
 		mutex_exit(&iommu->aiomt_mutex);
-		(void) amd_iommu_fini(iommu);
+		(void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
 		return (NULL);
 	}
 
@@ -1125,20 +1135,20 @@ amd_iommu_init(dev_info_t *dip, ddi_acc_handle_t handle, int idx,
 	 */
 	if (amd_iommu_setup_passthru(iommu) != DDI_SUCCESS) {
 		mutex_exit(&iommu->aiomt_mutex);
-		(void) amd_iommu_fini(iommu);
+		(void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
 		return (NULL);
 	}
 
 	if (amd_iommu_start(iommu) != DDI_SUCCESS) {
 		mutex_exit(&iommu->aiomt_mutex);
-		(void) amd_iommu_fini(iommu);
+		(void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
 		return (NULL);
 	}
 
 	/* xxx register/start race  */
 	if (amd_iommu_register(iommu) != DDI_SUCCESS) {
 		mutex_exit(&iommu->aiomt_mutex);
-		(void) amd_iommu_fini(iommu);
+		(void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
 		return (NULL);
 	}
 
@@ -1151,7 +1161,7 @@ amd_iommu_init(dev_info_t *dip, ddi_acc_handle_t handle, int idx,
 }
 
 static int
-amd_iommu_fini(amd_iommu_t *iommu)
+amd_iommu_fini(amd_iommu_t *iommu, int type)
 {
 	int idx = iommu->aiomt_idx;
 	dev_info_t *dip = iommu->aiomt_dip;
@@ -1159,17 +1169,28 @@ amd_iommu_fini(amd_iommu_t *iommu)
 	const char *driver = ddi_driver_name(dip);
 	const char *f = "amd_iommu_fini";
 
-	mutex_enter(&iommu->aiomt_mutex);
-	if (amd_iommu_unregister(iommu) != DDI_SUCCESS) {
-		cmn_err(CE_NOTE, "%s: %s%d: Fini of IOMMU unit failed. "
-		    "idx = %d", f, driver, instance, idx);
-		return (DDI_FAILURE);
+	if (type == AMD_IOMMU_TEARDOWN) {
+		mutex_enter(&iommu->aiomt_mutex);
+		if (amd_iommu_unregister(iommu) != DDI_SUCCESS) {
+			cmn_err(CE_NOTE, "%s: %s%d: Fini of IOMMU unit failed. "
+			    "idx = %d", f, driver, instance, idx);
+			return (DDI_FAILURE);
+		}
 	}
+
 	amd_iommu_stop(iommu);
-	amd_iommu_fini_page_tables(iommu);
-	amd_iommu_teardown_interrupts(iommu);
-	amd_iommu_teardown_exclusion(iommu);
-	amd_iommu_teardown_tables_and_buffers(iommu);
+
+	if (type == AMD_IOMMU_TEARDOWN) {
+		amd_iommu_fini_page_tables(iommu);
+		amd_iommu_teardown_interrupts(iommu);
+		amd_iommu_teardown_exclusion(iommu);
+	}
+
+	amd_iommu_teardown_tables_and_buffers(iommu, type);
+
+	if (type == AMD_IOMMU_QUIESCE)
+		return (DDI_SUCCESS);
+
 	if (iommu->aiomt_va != NULL) {
 		hat_unload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va,
 		    iommu->aiomt_reg_size, HAT_UNLOAD_UNLOCK);
@@ -1246,7 +1267,7 @@ amd_iommu_setup(dev_info_t *dip, amd_iommu_state_t *statep)
 		/* check if cap ID is secure device cap id */
 		if (id != PCI_CAP_ID_SECURE_DEV) {
 			if (amd_iommu_debug) {
-				cmn_err(CE_WARN,
+				cmn_err(CE_NOTE,
 				    "%s: %s%d: skipping IOMMU: idx(0x%x) "
 				    "cap ID (0x%x) != secure dev capid (0x%x)",
 				    f, driver, instance, idx, id,
@@ -1299,20 +1320,21 @@ amd_iommu_setup(dev_info_t *dip, amd_iommu_state_t *statep)
 }
 
 int
-amd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep)
+amd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep, int type)
 {
 	int instance = ddi_get_instance(dip);
 	const char *driver = ddi_driver_name(dip);
-	amd_iommu_t *iommu;
+	amd_iommu_t *iommu, *next_iommu;
 	int teardown;
 	int error = DDI_SUCCESS;
 	const char *f = "amd_iommu_teardown";
 
 	teardown = 0;
 	for (iommu = statep->aioms_iommu_start; iommu;
-	    iommu = iommu->aiomt_next) {
+	    iommu = next_iommu) {
 		ASSERT(statep->aioms_nunits > 0);
-		if (amd_iommu_fini(iommu) != DDI_SUCCESS) {
+		next_iommu = iommu->aiomt_next;
+		if (amd_iommu_fini(iommu, type) != DDI_SUCCESS) {
 			error = DDI_FAILURE;
 			continue;
 		}
@@ -1394,7 +1416,7 @@ map_current_window(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp,
 	mutex_enter(&amd_iommu_pgtable_lock);
 
 	if (amd_iommu_debug == AMD_IOMMU_DEBUG_PAGE_TABLES) {
-		cmn_err(CE_WARN, "%s: %s%d: idx=%d Attempting to get cookies "
+		cmn_err(CE_NOTE, "%s: %s%d: idx=%d Attempting to get cookies "
 		    "from handle for device %s",
 		    f, driver, instance, idx, path);
 	}
diff --git a/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_page_tables.c b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_page_tables.c
index e59a3dde38..5fbee01479 100644
--- a/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_page_tables.c
+++ b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_page_tables.c
@@ -510,14 +510,14 @@ amd_iommu_set_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,
 {
 	uint64_t *devtbl_entry;
 	amd_iommu_cmdargs_t cmdargs = {0};
-	int error;
+	int error, flags;
 	dev_info_t *idip = iommu->aiomt_dip;
 	const char *driver = ddi_driver_name(idip);
 	int instance = ddi_get_instance(idip);
 	const char *f = "amd_iommu_set_devtbl_entry";
 
 	if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
-		cmn_err(CE_WARN, "%s: attempting to set devtbl entry for %s",
+		cmn_err(CE_NOTE, "%s: attempting to set devtbl entry for %s",
 		    f, path);
 	}
 
@@ -536,10 +536,39 @@ amd_iommu_set_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,
 	    [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];
 
 	if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
-		cmn_err(CE_WARN, "%s: deviceid=%u devtbl entry (%p) for %s",
+		cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s",
 		    f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path);
 	}
 
+	/*
+	 * Flush internal caches, need to do this if we came up from
+	 * fast boot
+	 */
+	cmdargs.ca_deviceid = deviceid;
+	error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
+	    &cmdargs, 0, 0);
+	if (error != DDI_SUCCESS) {
+		cmn_err(CE_WARN, "%s: idx=%d: deviceid=%d"
+		    "Failed to invalidate domain in IOMMU HW cache",
+		    f, iommu->aiomt_idx, deviceid);
+		return (error);
+	}
+
+	cmdargs.ca_domainid = (uint16_t)domainid;
+	cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
+	flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
+	    AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
+
+	error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
+	    &cmdargs, flags, 0);
+	if (error != DDI_SUCCESS) {
+		cmn_err(CE_WARN, "%s: idx=%d: domainid=%d"
+		    "Failed to invalidate translations in IOMMU HW cache",
+		    f, iommu->aiomt_idx, cmdargs.ca_domainid);
+		return (error);
+	}
+
+	/* Initialize device table entry */
 	if (init_devtbl(iommu, devtbl_entry, domainid, dp)) {
 		cmdargs.ca_deviceid = deviceid;
 		error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
@@ -582,7 +611,7 @@ amd_iommu_clear_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,
 	    [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];
 
 	if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
-		cmn_err(CE_WARN, "%s: deviceid=%u devtbl entry (%p) for %s",
+		cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s",
 		    f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path);
 	}
 
@@ -1548,7 +1577,7 @@ amd_iommu_map_pa2va(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp,
 	for (pfn = pfn_start; pfn <= pfn_end; pfn++, pg++) {
 
 		if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
-			cmn_err(CE_WARN, "%s: attempting to create page tables "
+			cmn_err(CE_NOTE, "%s: attempting to create page tables "
 			    "for pfn = %p, va = %p, path = %s",
 			    f, (void *)(uintptr_t)(pfn << MMU_PAGESHIFT),
 			    (void *)(uintptr_t)(pg << MMU_PAGESHIFT), path);
@@ -1568,7 +1597,7 @@ amd_iommu_map_pa2va(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp,
 		}
 
 		if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
-			cmn_err(CE_WARN, "%s: successfuly created page tables "
+			cmn_err(CE_NOTE, "%s: successfuly created page tables "
 			    "for pfn = %p, vapg = %p, path = %s",
 			    f, (void *)(uintptr_t)pfn,
 			    (void *)(uintptr_t)pg, path);
diff --git a/usr/src/uts/i86pc/sys/amd_iommu.h b/usr/src/uts/i86pc/sys/amd_iommu.h
index d2e969f355..1977d063aa 100644
--- a/usr/src/uts/i86pc/sys/amd_iommu.h
+++ b/usr/src/uts/i86pc/sys/amd_iommu.h
@@ -43,8 +43,11 @@ typedef struct amd_iommu_state {
 	int aioms_nunits;			/* # of IOMMUs in function */
 } amd_iommu_state_t;
 
+#define	AMD_IOMMU_QUIESCE	(0)
+#define	AMD_IOMMU_TEARDOWN	(1)
+
 int amd_iommu_setup(dev_info_t *dip, amd_iommu_state_t *statep);
-int amd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep);
+int amd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep, int type);
 int amd_iommu_lookup_src_bdf(uint16_t bdf, uint16_t *src_bdfp);
 
 #endif	/* _KERNEL */