8579 ixgbe infinite loops in rar and vmdq clearing

Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Michael Speer <michael.speer@pluribusnetworks.com>
Approved by: Richard Lowe <richlowe@richlowe.net>

7 files changed, 57 insertions, 11 deletions

diff --git a/usr/src/uts/common/io/ixgbe/core/README.illumos b/usr/src/uts/common/io/ixgbe/core/README.illumos
new file mode 100644
index 0000000000..a09765894d
--- /dev/null
+++ b/usr/src/uts/common/io/ixgbe/core/README.illumos
@@ -0,0 +1,14 @@
+This directory contains the 'common code' to the ixgbe driver. This code
+comes from FreeBSD.
+
+The following are a subset of the current modifications that are being
+made to this common code to facilitate illumos specific features:
+
+o IXGBE_REMOVED
+
+The IXGBE_REMOVED macro normally taks a pointer to the hardware address.
+However, that does not allow us to go through and actually do a useful
+check. Instead we modify all users of this macro to pass the struct
+ixgbe_hw of the ixgbe_t. Importantly, we also remove the current
+definition of IXGBE_REMOVED in ixbe_common.h and instead use the
+definition in ixgbe_osdep.h.
diff --git a/usr/src/uts/common/io/ixgbe/core/ixgbe_common.c b/usr/src/uts/common/io/ixgbe/core/ixgbe_common.c
index 79d460f2a4..eb98709d14 100644
--- a/usr/src/uts/common/io/ixgbe/core/ixgbe_common.c
+++ b/usr/src/uts/common/io/ixgbe/core/ixgbe_common.c
@@ -3130,7 +3130,7 @@ s32 ixgbe_disable_pcie_master(struct ixgbe_hw *hw)
 
 	/* Exit if master requests are blocked */
 	if (!(IXGBE_READ_REG(hw, IXGBE_STATUS) & IXGBE_STATUS_GIO) ||
-	    IXGBE_REMOVED(hw->hw_addr))
+	    IXGBE_REMOVED(hw))
 		goto out;
 
 	/* Poll for master request bit to clear */
@@ -3162,7 +3162,7 @@ s32 ixgbe_disable_pcie_master(struct ixgbe_hw *hw)
 	for (i = 0; i < poll; i++) {
 		usec_delay(100);
 		value = IXGBE_READ_PCIE_WORD(hw, IXGBE_PCI_DEVICE_STATUS);
-		if (IXGBE_REMOVED(hw->hw_addr))
+		if (IXGBE_REMOVED(hw))
 			goto out;
 		if (!(value & IXGBE_PCI_DEVICE_STATUS_TRANSACTION_PENDING))
 			goto out;
@@ -3590,7 +3590,7 @@ u16 ixgbe_get_pcie_msix_count_generic(struct ixgbe_hw *hw)
 
 	DEBUGFUNC("ixgbe_get_pcie_msix_count_generic");
 	msix_count = IXGBE_READ_PCIE_WORD(hw, pcie_offset);
-	if (IXGBE_REMOVED(hw->hw_addr))
+	if (IXGBE_REMOVED(hw))
 		msix_count = 0;
 	msix_count &= IXGBE_PCIE_MSIX_TBL_SZ_MASK;
 
@@ -3695,7 +3695,7 @@ s32 ixgbe_clear_vmdq_generic(struct ixgbe_hw *hw, u32 rar, u32 vmdq)
 	mpsar_lo = IXGBE_READ_REG(hw, IXGBE_MPSAR_LO(rar));
 	mpsar_hi = IXGBE_READ_REG(hw, IXGBE_MPSAR_HI(rar));
 
-	if (IXGBE_REMOVED(hw->hw_addr))
+	if (IXGBE_REMOVED(hw))
 		goto done;
 
 	if (!mpsar_lo && !mpsar_hi)
@@ -4631,7 +4631,7 @@ void ixgbe_clear_tx_pending(struct ixgbe_hw *hw)
 	for (i = 0; i < poll; i++) {
 		usec_delay(100);
 		value = IXGBE_READ_PCIE_WORD(hw, IXGBE_PCI_DEVICE_STATUS);
-		if (IXGBE_REMOVED(hw->hw_addr))
+		if (IXGBE_REMOVED(hw))
 			goto out;
 		if (!(value & IXGBE_PCI_DEVICE_STATUS_TRANSACTION_PENDING))
 			goto out;
diff --git a/usr/src/uts/common/io/ixgbe/core/ixgbe_common.h b/usr/src/uts/common/io/ixgbe/core/ixgbe_common.h
index e685f5b32a..069fc88c96 100644
--- a/usr/src/uts/common/io/ixgbe/core/ixgbe_common.h
+++ b/usr/src/uts/common/io/ixgbe/core/ixgbe_common.h
@@ -41,7 +41,6 @@
 		IXGBE_WRITE_REG(hw, reg, (u32) value); \
 		IXGBE_WRITE_REG(hw, reg + 4, (u32) (value >> 32)); \
 	} while (0)
-#define IXGBE_REMOVED(a) (0)
 #if !defined(NO_READ_PBA_RAW) || !defined(NO_WRITE_PBA_RAW)
 struct ixgbe_pba {
 	u16 word[2];
diff --git a/usr/src/uts/common/io/ixgbe/ixgbe_osdep.c b/usr/src/uts/common/io/ixgbe/ixgbe_osdep.c
index ae820a58cf..fcb695abfd 100644
--- a/usr/src/uts/common/io/ixgbe/ixgbe_osdep.c
+++ b/usr/src/uts/common/io/ixgbe/ixgbe_osdep.c
@@ -24,10 +24,10 @@
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright (c) 2017, Joyent, Inc.
  */
 
-#include "ixgbe_osdep.h"
-#include "ixgbe_api.h"
+#include "ixgbe_sw.h"
 
 uint16_t
 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg)
@@ -40,3 +40,30 @@ ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint32_t val)
 {
 	pci_config_put16(OS_DEP(hw)->cfg_handle, reg, val);
 }
+
+/*
+ * This is our last line of defense against a hardware device that has decided
+ * to somehow disappear without our knowledge of it. To try and deal with this,
+ * we'll read the status register and see if it returns all 1s, indicating an
+ * invalid read. Note the status register is defined to have bits in all current
+ * revisions that are hardwired to zero.
+ */
+boolean_t
+ixgbe_removed(struct ixgbe_hw *hw)
+{
+	uint32_t val;
+
+	val = IXGBE_READ_REG(hw, IXGBE_STATUS);
+	if (val == PCI_EINVAL32) {
+		ixgbe_t *ixgbe = OS_DEP(hw)->ixgbe;
+
+		ixgbe_error(ixgbe, "failed to read status register: device "
+		    "may be gone");
+		if (ixgbe->ixgbe_ks != NULL) {
+			ixgbe_stat_t *s = ixgbe->ixgbe_ks->ks_data;
+			s->dev_gone.value.ui64++;
+		}
+		return (B_TRUE);
+	}
+	return (B_FALSE);
+}
diff --git a/usr/src/uts/common/io/ixgbe/ixgbe_osdep.h b/usr/src/uts/common/io/ixgbe/ixgbe_osdep.h
index 9526364b2f..0dc911bfd0 100644
--- a/usr/src/uts/common/io/ixgbe/ixgbe_osdep.h
+++ b/usr/src/uts/common/io/ixgbe/ixgbe_osdep.h
@@ -27,7 +27,7 @@
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 /*
- * Copyright (c) 2012, Joyent, Inc. All rights reserved.
+ * Copyright (c) 2017, Joyent, Inc.
  * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved.
  */
 
@@ -65,6 +65,7 @@ extern "C" {
 struct ixgbe_hw;
 uint16_t ixgbe_read_pci_cfg(struct ixgbe_hw *, uint32_t);
 void ixgbe_write_pci_cfg(struct ixgbe_hw *, uint32_t, uint32_t);
+boolean_t ixgbe_removed(struct ixgbe_hw *);
 
 #define	usec_delay(x)		drv_usecwait(x)
 #define	msec_delay(x)		drv_usecwait(x * 1000)
@@ -128,6 +129,7 @@ void ixgbe_write_pci_cfg(struct ixgbe_hw *, uint32_t, uint32_t);
 #define	UNREFERENCED_4PARAMETER(_p, _q, _r, _s)	_NOTE(ARGUNUSED(_p, _q,_r, _s))
 
 
+#define	IXGBE_REMOVED(hw) ixgbe_removed(hw)
 
 typedef	int8_t		s8;
 typedef	int16_t		s16;
diff --git a/usr/src/uts/common/io/ixgbe/ixgbe_stat.c b/usr/src/uts/common/io/ixgbe/ixgbe_stat.c
index 96b5aeb646..b5892b3c45 100644
--- a/usr/src/uts/common/io/ixgbe/ixgbe_stat.c
+++ b/usr/src/uts/common/io/ixgbe/ixgbe_stat.c
@@ -27,7 +27,7 @@
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
  * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved.
- * Copyright 2017 Joyent, Inc.
+ * Copyright (c) 2017, Joyent, Inc.
  */
 
 #include "ixgbe_sw.h"
@@ -466,6 +466,9 @@ ixgbe_init_stats(ixgbe_t *ixgbe)
 	    KSTAT_DATA_UINT64);
 	kstat_named_init(&ixgbe_ks->lroc, "lro_pkt_count",
 	    KSTAT_DATA_UINT64);
+
+	kstat_named_init(&ixgbe_ks->dev_gone, "device_gone",
+	    KSTAT_DATA_UINT64);
 	/*
 	 * Function to provide kernel stat update on demand
 	 */
diff --git a/usr/src/uts/common/io/ixgbe/ixgbe_sw.h b/usr/src/uts/common/io/ixgbe/ixgbe_sw.h
index 986061bfad..6cd03d571d 100644
--- a/usr/src/uts/common/io/ixgbe/ixgbe_sw.h
+++ b/usr/src/uts/common/io/ixgbe/ixgbe_sw.h
@@ -27,7 +27,7 @@
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 Saso Kiselkov. All rights reserved.
  * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved.
- * Copyright 2017 Joyent, Inc.
+ * Copyright (c) 2017, Joyent, Inc.
  */
 
 #ifndef	_IXGBE_SW_H
@@ -819,6 +819,7 @@ typedef struct ixgbe_stat {
 	kstat_named_t mptc;	/* Multicast Packets Xmited Count */
 	kstat_named_t bptc;	/* Broadcast Packets Xmited Count */
 	kstat_named_t lroc;	/* LRO Packets Received Count */
+	kstat_named_t dev_gone;	/* Number of device gone events encountered */
 } ixgbe_stat_t;
 
 /*