6878578 while processing a fatal error recursive mutex panic encountered in nxge

6877282 while plumbing a hybrid vnet in the service domain panic occurs in nxge
6873178 assertion failed: nmp, file: ../../common/io/nxge/nxge_txdma.c, line: 688
6879284 Foxxy FEM hot plug issue with C10 NEM
6873102 guest domain panics when running network load tests with hybrid I/O
6856725 nxge may unmap memory while dblks are still referencing it

8 files changed, 85 insertions, 42 deletions

diff --git a/usr/src/uts/common/io/nxge/nxge_espc.c b/usr/src/uts/common/io/nxge/nxge_espc.c
index 2d19888892..9b467e503c 100644
--- a/usr/src/uts/common/io/nxge/nxge_espc.c
+++ b/usr/src/uts/common/io/nxge/nxge_espc.c
@@ -254,6 +254,8 @@ nxge_vpd_info_get(p_nxge_t nxgep)
 	} else if (strncmp(nxgep->vpd_info.bd_model,
 	    NXGE_RFEM_BM_STR, strlen(NXGE_RFEM_BM_STR)) == 0) {
 		nxgep->hot_swappable_phy = B_TRUE;
+		nxgep->platform_type = P_NEPTUNE_GENERIC;
+		nxgep->niu_type = NEPTUNE_2_10GF;
 	}
 
 	/* If Alonso platform, replace "mif" for the last 2 ports phy-type */
diff --git a/usr/src/uts/common/io/nxge/nxge_hio.c b/usr/src/uts/common/io/nxge/nxge_hio.c
index b58acde5e8..a90843863c 100644
--- a/usr/src/uts/common/io/nxge/nxge_hio.c
+++ b/usr/src/uts/common/io/nxge/nxge_hio.c
@@ -132,9 +132,10 @@ nxge_hio_init(nxge_t *nxge)
 	int i;
 
 	nhd = (nxge_hio_data_t *)nxge->nxge_hw_p->hio;
-	if (nhd == 0) {
+	if (nhd == NULL) {
 		nhd = KMEM_ZALLOC(sizeof (*nhd), KM_SLEEP);
 		MUTEX_INIT(&nhd->lock, NULL, MUTEX_DRIVER, NULL);
+		nhd->type = NXGE_HIO_TYPE_SERVICE;
 		nxge->nxge_hw_p->hio = (uintptr_t)nhd;
 	}
 
@@ -966,8 +967,7 @@ static void nxge_hio_dc_unshare(nxge_t *, nxge_hio_vr_t *,
  *	Any domain
  */
 int
-nxge_hio_init(
-	nxge_t *nxge)
+nxge_hio_init(nxge_t *nxge)
 {
 	nxge_hio_data_t *nhd;
 	int i, region;
@@ -976,6 +976,10 @@ nxge_hio_init(
 	if (nhd == 0) {
 		nhd = KMEM_ZALLOC(sizeof (*nhd), KM_SLEEP);
 		MUTEX_INIT(&nhd->lock, NULL, MUTEX_DRIVER, NULL);
+		if (isLDOMguest(nxge))
+			nhd->type = NXGE_HIO_TYPE_GUEST;
+		else
+			nhd->type = NXGE_HIO_TYPE_SERVICE;
 		nxge->nxge_hw_p->hio = (uintptr_t)nhd;
 	}
 
@@ -1917,15 +1921,12 @@ nxge_hio_unshare(
 }
 
 int
-nxge_hio_addres(
-	nxge_hio_vr_t *vr,
-	mac_ring_type_t type,
-	uint64_t *map)
+nxge_hio_addres(nxge_hio_vr_t *vr, mac_ring_type_t type, uint64_t *map)
 {
 	nxge_t		*nxge = (nxge_t *)vr->nxge;
 	nxge_grp_t	*group;
 	int		groupid;
-	int		i;
+	int		i, rv = 0;
 	int		max_dcs;
 
 	NXGE_DEBUG_MSG((nxge, HIO_CTL, "==> nxge_hio_addres"));
@@ -1958,8 +1959,6 @@ nxge_hio_addres(
 
 	for (i = 0; i < max_dcs; i++) {
 		if (group->map & (1 << i)) {
-			int rv;
-
 			if ((rv = nxge_hio_dc_share(nxge, vr, type, i)) < 0) {
 				if (*map == 0) /* Couldn't get even one DC. */
 					return (-rv);
@@ -1970,8 +1969,13 @@ nxge_hio_addres(
 		}
 	}
 
-	NXGE_DEBUG_MSG((nxge, HIO_CTL, "<== nxge_hio_addres"));
+	if ((*map == 0) || (rv != 0)) {
+		NXGE_DEBUG_MSG((nxge, HIO_CTL,
+		    "<== nxge_hio_addres: rv(%x)", rv));
+		return (EIO);
+	}
 
+	NXGE_DEBUG_MSG((nxge, HIO_CTL, "<== nxge_hio_addres"));
 	return (0);
 }
 
diff --git a/usr/src/uts/common/io/nxge/nxge_hio_guest.c b/usr/src/uts/common/io/nxge/nxge_hio_guest.c
index 78c1bb53a6..3c552f2058 100644
--- a/usr/src/uts/common/io/nxge/nxge_hio_guest.c
+++ b/usr/src/uts/common/io/nxge/nxge_hio_guest.c
@@ -192,13 +192,21 @@ nxge_hio_vr_add(nxge_t *nxge)
 
 	NXGE_DEBUG_MSG((nxge, HIO_CTL, "==> nxge_hio_vr_add"));
 
+	if (nhd->type == NXGE_HIO_TYPE_SERVICE) {
+		/*
+		 * Can't add VR to the service domain from which we came.
+		 */
+		ASSERT(nhd->type == NXGE_HIO_TYPE_GUEST);
+		return (DDI_FAILURE);
+	}
+
 	/*
 	 * Get our HV cookie.
 	 */
 	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, nxge->dip,
 	    0, "reg", &reg_val, &reg_len) != DDI_PROP_SUCCESS) {
 		NXGE_DEBUG_MSG((nxge, VPD_CTL, "`reg' property not found"));
-		return (NXGE_ERROR);
+		return (DDI_FAILURE);
 	}
 
 	cookie = (uint32_t)(reg_val[0]);
@@ -209,7 +217,7 @@ nxge_hio_vr_add(nxge_t *nxge)
 	if (hv_rv != 0) {
 		NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL,
 		    "vr->getinfo() failed"));
-		return (NXGE_ERROR);
+		return (DDI_FAILURE);
 	}
 
 	/*
@@ -239,7 +247,7 @@ nxge_hio_vr_add(nxge_t *nxge)
 		NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL,
 		    "nxge_hio_vr_add(%d): cookie(0x%x)\n",
 		    nxge->instance, cookie));
-		return (NXGE_ERROR);
+		return (DDI_FAILURE);
 	}
 
 	vr = &nhd->vr[vr_index];
@@ -265,7 +273,7 @@ nxge_hio_vr_add(nxge_t *nxge)
 	if (nxge_hio_intr_init(nxge) != NXGE_OK) {
 		NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL,
 		    "nxge_hio_intr_init() failed"));
-		return (NXGE_ERROR);
+		return (DDI_FAILURE);
 	}
 
 	/*
@@ -282,7 +290,7 @@ nxge_hio_vr_add(nxge_t *nxge)
 		if (hv_rv != 0) {
 			NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL,
 			    "tx->get_map() failed"));
-			return (NXGE_ERROR);
+			return (DDI_FAILURE);
 		}
 		res_map_parse(nxge, NXGE_TRANSMIT_GROUP, tx_map);
 
@@ -297,7 +305,7 @@ nxge_hio_vr_add(nxge_t *nxge)
 				if (dc == 0) {
 					NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL,
 					    "DC add failed"));
-					return (NXGE_ERROR);
+					return (DDI_FAILURE);
 				}
 				dc->channel = (nxge_channel_t)i;
 			}
@@ -315,7 +323,7 @@ nxge_hio_vr_add(nxge_t *nxge)
 		if (hv_rv != 0) {
 			NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL,
 			    "rx->get_map() failed"));
-			return (NXGE_ERROR);
+			return (DDI_FAILURE);
 		}
 		res_map_parse(nxge, NXGE_RECEIVE_GROUP, rx_map);
 
@@ -330,7 +338,7 @@ nxge_hio_vr_add(nxge_t *nxge)
 				if (dc == 0) {
 					NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL,
 					    "DC add failed"));
-					return (NXGE_ERROR);
+					return (DDI_FAILURE);
 				}
 				dc->channel = (nxge_channel_t)i;
 			}
@@ -341,14 +349,14 @@ nxge_hio_vr_add(nxge_t *nxge)
 	if (status != NXGE_OK) {
 		cmn_err(CE_WARN, "nxge(%d): nxge_mac_register failed\n",
 		    nxge->instance);
-		return (status);
+		return (DDI_FAILURE);
 	}
 
 	nxge->hio_vr = vr;	/* For faster lookups. */
 
 	NXGE_DEBUG_MSG((nxge, HIO_CTL, "<== nxge_hio_vr_add"));
 
-	return (NXGE_OK);
+	return (DDI_SUCCESS);
 }
 
 /*
diff --git a/usr/src/uts/common/io/nxge/nxge_mac.c b/usr/src/uts/common/io/nxge/nxge_mac.c
index 9d8ef7bc5c..8a8c8652ca 100644
--- a/usr/src/uts/common/io/nxge/nxge_mac.c
+++ b/usr/src/uts/common/io/nxge/nxge_mac.c
@@ -312,6 +312,10 @@ nxge_get_xcvr_type(p_nxge_t nxgep)
 	 */
 	if (nxgep->mac.portmode == PORT_HSP_MODE) {
 		nxgep->hot_swappable_phy = B_TRUE;
+		if (portn > 1) {
+			return (NXGE_ERROR);
+		}
+
 		/*
 		 * If this is the 2nd NIU port, then check 2 addresses
 		 * to take care of the Goa NEM card. Port 1 can have addr 17
diff --git a/usr/src/uts/common/io/nxge/nxge_main.c b/usr/src/uts/common/io/nxge/nxge_main.c
index bedec1c472..28c6f1272d 100644
--- a/usr/src/uts/common/io/nxge/nxge_main.c
+++ b/usr/src/uts/common/io/nxge/nxge_main.c
@@ -128,10 +128,12 @@ nxge_rxbuf_type_t nxge_rx_buf_size_type = RCR_PKTBUFSZ_0;
 nxge_rxbuf_threshold_t nxge_rx_threshold_lo = NXGE_RX_COPY_3;
 
 /* Use kmem_alloc() to allocate data buffers. */
-#if defined(_BIG_ENDIAN)
+#if defined(__sparc)
 uint32_t	nxge_use_kmem_alloc = 1;
-#else
+#elif defined(__i386)
 uint32_t	nxge_use_kmem_alloc = 0;
+#else
+uint32_t	nxge_use_kmem_alloc = 1;
 #endif
 
 rtrace_t npi_rtracebuf;
@@ -822,11 +824,12 @@ nxge_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 	if (isLDOMguest(nxgep)) {
 		/* Find our VR & channel sets. */
 		status = nxge_hio_vr_add(nxgep);
-		if (status != NXGE_OK) {
-			NXGE_DEBUG_MSG((nxgep, DDI_CTL,
+		if (status != DDI_SUCCESS) {
+			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
 			    "nxge_hio_vr_add failed"));
 			(void) hsvc_unregister(&nxgep->niu_hsvc);
 			nxgep->niu_hsvc_available = B_FALSE;
+			goto nxge_attach_fail;
 		}
 		goto nxge_attach_exit;
 	}
@@ -6530,6 +6533,8 @@ nxge_init_common_dev(p_nxge_t nxgep)
 	p_nxge_hw_list_t	hw_p;
 	dev_info_t 		*p_dip;
 
+	ASSERT(nxgep != NULL);
+
 	NXGE_DEBUG_MSG((nxgep, MOD_CTL, "==> nxge_init_common_device"));
 
 	p_dip = nxgep->p_dip;
@@ -6644,6 +6649,8 @@ nxge_uninit_common_dev(p_nxge_t nxgep)
 	p_nxge_hw_pt_cfg_t	p_cfgp;
 	dev_info_t 		*p_dip;
 
+	ASSERT(nxgep != NULL);
+
 	NXGE_DEBUG_MSG((nxgep, MOD_CTL, "==> nxge_uninit_common_device"));
 	if (nxgep->nxge_hw_p == NULL) {
 		NXGE_DEBUG_MSG((nxgep, MOD_CTL,
diff --git a/usr/src/uts/common/io/nxge/nxge_rxdma.c b/usr/src/uts/common/io/nxge/nxge_rxdma.c
index 4b427d1a8d..c7459b16ec 100644
--- a/usr/src/uts/common/io/nxge/nxge_rxdma.c
+++ b/usr/src/uts/common/io/nxge/nxge_rxdma.c
@@ -18,6 +18,7 @@
  *
  * CDDL HEADER END
  */
+
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
@@ -4268,12 +4269,13 @@ nxge_rxdma_start_channel(p_nxge_t nxgep, uint16_t channel,
 
 	if (isLDOMguest(nxgep)) {
 		/* Add interrupt handler for this channel. */
-		if (nxge_hio_intr_add(nxgep, VP_BOUND_RX, channel)
-		    != NXGE_OK) {
+		status = nxge_hio_intr_add(nxgep, VP_BOUND_RX, channel);
+		if (status != NXGE_OK) {
 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
 			    " nxge_rxdma_start_channel: "
 			    " nxge_hio_intr_add failed (0x%08x channel %d)",
-		    status, channel));
+			    status, channel));
+			return (status);
 		}
 	}
 
@@ -4556,7 +4558,6 @@ nxge_rxdma_fatal_err_recover(p_nxge_t nxgep, uint16_t channel)
 	rbrp = (p_rx_rbr_ring_t)nxgep->rx_rbr_rings->rbr_rings[channel];
 	rcrp = (p_rx_rcr_ring_t)nxgep->rx_rcr_rings->rcr_rings[channel];
 
-	MUTEX_ENTER(&rcrp->lock);
 	MUTEX_ENTER(&rbrp->lock);
 	MUTEX_ENTER(&rbrp->post_lock);
 
@@ -4651,20 +4652,17 @@ nxge_rxdma_fatal_err_recover(p_nxge_t nxgep, uint16_t channel)
 
 	MUTEX_EXIT(&rbrp->post_lock);
 	MUTEX_EXIT(&rbrp->lock);
-	MUTEX_EXIT(&rcrp->lock);
 
 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
 	    "Recovery Successful, RxDMAChannel#%d Restored",
 	    channel));
 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fatal_err_recover"));
-
 	return (NXGE_OK);
+
 fail:
 	MUTEX_EXIT(&rbrp->post_lock);
 	MUTEX_EXIT(&rbrp->lock);
-	MUTEX_EXIT(&rcrp->lock);
 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed"));
-
 	return (NXGE_ERROR | rs);
 }
 
@@ -4673,6 +4671,7 @@ nxge_rx_port_fatal_err_recover(p_nxge_t nxgep)
 {
 	nxge_grp_set_t *set = &nxgep->rx_set;
 	nxge_status_t status = NXGE_OK;
+	p_rx_rcr_ring_t rcrp;
 	int rdc;
 
 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_port_fatal_err_recover"));
@@ -4689,10 +4688,16 @@ nxge_rx_port_fatal_err_recover(p_nxge_t nxgep)
 
 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
 		if ((1 << rdc) & set->owned.map) {
-			if (nxge_rxdma_fatal_err_recover(nxgep, rdc)
-			    != NXGE_OK) {
-				NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
-				    "Could not recover channel %d", rdc));
+			rcrp = nxgep->rx_rcr_rings->rcr_rings[rdc];
+			if (rcrp != NULL) {
+				MUTEX_ENTER(&rcrp->lock);
+				if (nxge_rxdma_fatal_err_recover(nxgep,
+				    rdc) != NXGE_OK) {
+					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
+					    "Could not recover "
+					    "channel %d", rdc));
+				}
+				MUTEX_EXIT(&rcrp->lock);
 			}
 		}
 	}
diff --git a/usr/src/uts/common/io/nxge/nxge_txdma.c b/usr/src/uts/common/io/nxge/nxge_txdma.c
index b995aeb641..68b823b01c 100644
--- a/usr/src/uts/common/io/nxge/nxge_txdma.c
+++ b/usr/src/uts/common/io/nxge/nxge_txdma.c
@@ -209,11 +209,13 @@ nxge_uninit_txdma_channel(p_nxge_t nxgep, int channel)
 		nxgep->statsp->tdc_ksp[channel] = 0;
 	}
 
-	(void) nxge_txdma_stop_channel(nxgep, channel);
+	if (nxge_txdma_stop_channel(nxgep, channel) != NXGE_OK)
+		goto nxge_uninit_txdma_channel_exit;
+
 	nxge_unmap_txdma_channel(nxgep, channel);
 
-	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
-	    "<== nxge_uninit_txdma_channel"));
+nxge_uninit_txdma_channel_exit:
+	NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_uninit_txdma_channel"));
 }
 
 void
@@ -684,6 +686,8 @@ nxge_fill_tx_hdr(p_mblk_t mp, boolean_t fill_len,
 				    (mblk_len < stuff_len)) {
 					stuff_len -= mblk_len;
 					nmp = nmp->b_cont;
+					if (nmp)
+						mblk_len = MBLKL(nmp);
 				}
 				ASSERT(nmp);
 				up = (uint16_t *)(nmp->b_rptr + stuff_len);
@@ -2946,7 +2950,16 @@ nxge_txdma_stop_channel(p_nxge_t nxgep, uint16_t channel)
 	 */
 	(void) nxge_txdma_stop_inj_err(nxgep, channel);
 
+	if (nxgep->tx_rings == NULL) {
+		status = NXGE_ERROR;
+		goto nxge_txdma_stop_channel_exit;
+	}
+
 	tx_ring_p = nxgep->tx_rings->rings[channel];
+	if (tx_ring_p == NULL) {
+		status = NXGE_ERROR;
+		goto nxge_txdma_stop_channel_exit;
+	}
 
 	/*
 	 * Reset TXDMA channel
diff --git a/usr/src/uts/common/sys/nxge/nxge_hio.h b/usr/src/uts/common/sys/nxge/nxge_hio.h
index b18f32e346..82e0e8fa6d 100644
--- a/usr/src/uts/common/sys/nxge/nxge_hio.h
+++ b/usr/src/uts/common/sys/nxge/nxge_hio.h
@@ -99,8 +99,8 @@ typedef struct {
 #define	NXGE_VR_SR_MAX		8 /* There are 8 subregions (SR). */
 
 typedef enum {
-	NXGE_HIO_TYPE_SERVICE,	/* We are a service domain driver. */
-	NXGE_HIO_TYPE_GUEST	/* We are a guest domain driver. */
+	NXGE_HIO_TYPE_SERVICE = 0x80,	/* We are a service domain driver. */
+	NXGE_HIO_TYPE_GUEST		/* We are a guest domain driver. */
 } nxge_hio_type_t;
 
 typedef enum {