11369 PCIe errors on passthru devices shouldn't cause a panic

Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Reviewed by: Toomas Soome <tsoome@me.com>
Reviewed by: Andy Fiddaman <andy@omniosce.org>
Approved by: Richard Lowe <richlowe@richlowe.net>

4 files changed, 40 insertions, 7 deletions

diff --git a/usr/src/uts/common/io/pciex/pcie.c b/usr/src/uts/common/io/pciex/pcie.c
index 5e3d92d5a4..136d2fe336 100644
--- a/usr/src/uts/common/io/pciex/pcie.c
+++ b/usr/src/uts/common/io/pciex/pcie.c
@@ -684,6 +684,7 @@ pcie_init_pfd(dev_info_t *dip)
 
 	pfd_p->pe_bus_p = bus_p;
 	pfd_p->pe_severity_flags = 0;
+	pfd_p->pe_severity_mask = 0;
 	pfd_p->pe_orig_severity_flags = 0;
 	pfd_p->pe_lock = B_FALSE;
 	pfd_p->pe_valid = B_FALSE;
@@ -840,6 +841,7 @@ pcie_rc_init_pfd(dev_info_t *dip, pf_data_t *pfd_p)
 {
 	pfd_p->pe_bus_p = PCIE_DIP2DOWNBUS(dip);
 	pfd_p->pe_severity_flags = 0;
+	pfd_p->pe_severity_mask = 0;
 	pfd_p->pe_orig_severity_flags = 0;
 	pfd_p->pe_lock = B_FALSE;
 	pfd_p->pe_valid = B_FALSE;
@@ -921,7 +923,7 @@ pcie_rc_init_bus(dev_info_t *dip)
 	bus_p->bus_aer_off = (uint16_t)-1;
 
 	/* Needed only for handle lookup */
-	bus_p->bus_fm_flags |= PF_FM_READY;
+	atomic_or_uint(&bus_p->bus_fm_flags, PF_FM_READY);
 
 	ndi_set_bus_private(dip, B_FALSE, DEVI_PORT_TYPE_PCI, bus_p);
 
@@ -1340,7 +1342,7 @@ caps_done:
 	}
 
 	bus_p->bus_soft_state = PCI_SOFT_STATE_CLOSED;
-	bus_p->bus_fm_flags = 0;
+	(void) atomic_swap_uint(&bus_p->bus_fm_flags, 0);
 	bus_p->bus_mps = 0;
 
 	ndi_set_bus_private(dip, B_TRUE, DEVI_PORT_TYPE_PCI, (void *)bus_p);
diff --git a/usr/src/uts/common/io/pciex/pcie_fault.c b/usr/src/uts/common/io/pciex/pcie_fault.c
index c23d3557e2..6a335db3e2 100644
--- a/usr/src/uts/common/io/pciex/pcie_fault.c
+++ b/usr/src/uts/common/io/pciex/pcie_fault.c
@@ -920,6 +920,13 @@ pf_default_hdl(dev_info_t *dip, pf_impl_t *impl)
 	}
 
 	/*
+	 * If this is a device used for PCI passthrough into a virtual machine,
+	 * don't let any error it caused panic the system.
+	 */
+	if (bus_p->bus_fm_flags & PF_FM_IS_PASSTHRU)
+		pfd_p->pe_severity_mask |= PF_ERR_PANIC;
+
+	/*
 	 * Read vendor/device ID and check with cached data; if it doesn't
 	 * match, it could very well mean that the device is no longer
 	 * responding.  In this case, we return PF_SCAN_BAD_RESPONSE; should
@@ -952,6 +959,7 @@ pf_default_hdl(dev_info_t *dip, pf_impl_t *impl)
 
 	pf_pci_regs_gather(pfd_p, bus_p);
 	pf_pci_regs_clear(pfd_p, bus_p);
+
 	if (PCIE_IS_RP(bus_p))
 		pf_pci_find_rp_fault(pfd_p, bus_p);
 
@@ -986,6 +994,22 @@ done:
 }
 
 /*
+ * Set the passthru flag on a device bus_p. Called by passthru drivers to
+ * indicate when a device is or is no longer under passthru control.
+ */
+void
+pf_set_passthru(dev_info_t *dip, boolean_t is_passthru)
+{
+	pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
+
+	if (is_passthru) {
+		atomic_or_uint(&bus_p->bus_fm_flags, PF_FM_IS_PASSTHRU);
+	} else {
+		atomic_and_uint(&bus_p->bus_fm_flags, ~PF_FM_IS_PASSTHRU);
+	}
+}
+
+/*
  * Called during postattach to initialize a device's error handling
  * capabilities.  If the devices has already been hardened, then there isn't
  * much needed.  Otherwise initialize the device's default FMA capabilities.
@@ -1028,7 +1052,7 @@ pf_init(dev_info_t *dip, ddi_iblock_cookie_t ibc, ddi_attach_cmd_t cmd)
 		    DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE);
 		cap &= (DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE);
 
-		bus_p->bus_fm_flags |= PF_FM_IS_NH;
+		atomic_or_uint(&bus_p->bus_fm_flags, PF_FM_IS_NH);
 
 		if (cmd == DDI_ATTACH) {
 			ddi_fm_init(dip, &cap, &ibc);
@@ -1043,7 +1067,7 @@ pf_init(dev_info_t *dip, ddi_iblock_cookie_t ibc, ddi_attach_cmd_t cmd)
 
 	/* If ddi_fm_init fails for any reason RETURN */
 	if (!fmhdl) {
-		bus_p->bus_fm_flags = 0;
+		(void) atomic_swap_uint(&bus_p->bus_fm_flags, 0);
 		return;
 	}
 
@@ -1053,7 +1077,7 @@ pf_init(dev_info_t *dip, ddi_iblock_cookie_t ibc, ddi_attach_cmd_t cmd)
 			ddi_fm_handler_register(dip, pf_dummy_cb, NULL);
 	}
 
-	bus_p->bus_fm_flags |= PF_FM_READY;
+	atomic_or_uint(&bus_p->bus_fm_flags, PF_FM_READY);
 }
 
 /* undo FMA lock, called at predetach */
@@ -1070,7 +1094,7 @@ pf_fini(dev_info_t *dip, ddi_detach_cmd_t cmd)
 		return;
 
 	/* no other code should set the flag to false */
-	bus_p->bus_fm_flags &= ~PF_FM_READY;
+	atomic_and_uint(&bus_p->bus_fm_flags, ~PF_FM_READY);
 
 	/*
 	 * Grab the mutex to make sure device isn't in the middle of
@@ -1084,7 +1108,7 @@ pf_fini(dev_info_t *dip, ddi_detach_cmd_t cmd)
 	/* undo non-hardened drivers */
 	if (bus_p->bus_fm_flags & PF_FM_IS_NH) {
 		if (cmd == DDI_DETACH) {
-			bus_p->bus_fm_flags &= ~PF_FM_IS_NH;
+			atomic_and_uint(&bus_p->bus_fm_flags, ~PF_FM_IS_NH);
 			pci_ereport_teardown(dip);
 			/*
 			 * ddi_fini itself calls ddi_handler_unregister,
@@ -1459,6 +1483,8 @@ done:
 		/* Have pciev_eh adjust the severity */
 		pfd_p->pe_severity_flags = pciev_eh(pfd_p, impl);
 
+		pfd_p->pe_severity_flags &= ~pfd_p->pe_severity_mask;
+
 		error_flags |= pfd_p->pe_severity_flags;
 	}
 
@@ -3062,6 +3088,7 @@ pf_reset_pfd(pf_data_t *pfd_p)
 	pcie_bus_t	*bus_p = PCIE_PFD2BUS(pfd_p);
 
 	pfd_p->pe_severity_flags = 0;
+	pfd_p->pe_severity_mask = 0;
 	pfd_p->pe_orig_severity_flags = 0;
 	/* pe_lock and pe_valid were reset in pf_send_ereport */
 
diff --git a/usr/src/uts/common/sys/pcie_impl.h b/usr/src/uts/common/sys/pcie_impl.h
index 9e9e96e8fd..5e2ab33dba 100644
--- a/usr/src/uts/common/sys/pcie_impl.h
+++ b/usr/src/uts/common/sys/pcie_impl.h
@@ -397,6 +397,7 @@ struct pf_data {
 	boolean_t		pe_lock;
 	boolean_t		pe_valid;
 	uint32_t		pe_severity_flags;	/* Severity of error */
+	uint32_t		pe_severity_mask;
 	uint32_t		pe_orig_severity_flags; /* Original severity */
 	pf_affected_dev_t	*pe_affected_dev;
 	pcie_bus_t		*pe_bus_p;
@@ -425,6 +426,7 @@ typedef struct pf_impl {
 /* bus_fm_flags field */
 #define	PF_FM_READY		(1 << 0)	/* bus_fm_lock initialized */
 #define	PF_FM_IS_NH		(1 << 1)	/* known as non-hardened */
+#define	PF_FM_IS_PASSTHRU	(1 << 2)	/* device is controlled by VM */
 
 /*
  * PCIe fabric handle lookup address flags.  Used to define what type of
@@ -619,6 +621,7 @@ extern void pf_eh_enter(pcie_bus_t *bus_p);
 extern void pf_eh_exit(pcie_bus_t *bus_p);
 extern int pf_scan_fabric(dev_info_t *rpdip, ddi_fm_error_t *derr,
     pf_data_t *root_pfd_p);
+extern void pf_set_passthru(dev_info_t *, boolean_t);
 extern void pf_init(dev_info_t *, ddi_iblock_cookie_t, ddi_attach_cmd_t);
 extern void pf_fini(dev_info_t *, ddi_detach_cmd_t);
 extern int pf_hdl_lookup(dev_info_t *, uint64_t, uint32_t, uint64_t,
diff --git a/usr/src/uts/sun4/io/px/px_fm.c b/usr/src/uts/sun4/io/px/px_fm.c
index fe958d56bd..9db9e7e50a 100644
--- a/usr/src/uts/sun4/io/px/px_fm.c
+++ b/usr/src/uts/sun4/io/px/px_fm.c
@@ -756,6 +756,7 @@ px_get_pfd(px_t *px_p) {
 	}
 
 	pfd_p->pe_severity_flags = 0;
+	pfd_p->pe_severity_mask = 0;
 	pfd_p->pe_orig_severity_flags = 0;
 	pfd_p->pe_valid = B_TRUE;