6603354 VTS xnetlb loopback test fails on XAUI optical card on maramba.

6661084 nxge driver does not process phy-type option in nxge.conf correctly
6662776 Spurious values show up for some kstat counters on Neptune ports 2 and 3
6665961 NEM10G instance numbers should be preserved on nxge driver
6665964 Add hot swappable PHY support for NEM10G with Neptune RFEM

10 files changed, 263 insertions, 83 deletions

diff --git a/usr/src/uts/common/io/nxge/nxge_espc.c b/usr/src/uts/common/io/nxge/nxge_espc.c
index 0333de9517..0fb9cc2fb7 100644
--- a/usr/src/uts/common/io/nxge/nxge_espc.c
+++ b/usr/src/uts/common/io/nxge/nxge_espc.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -253,6 +253,9 @@ nxge_vpd_info_get(p_nxge_t nxgep)
 	} else if (strncmp(nxgep->vpd_info.bd_model,
 	    NXGE_ALONSO_BM_STR, strlen(NXGE_ALONSO_BM_STR)) == 0) {
 		nxgep->platform_type = P_NEPTUNE_ALONSO;
+	} else if (strncmp(nxgep->vpd_info.bd_model,
+	    NXGE_RFEM_BM_STR, strlen(NXGE_RFEM_BM_STR)) == 0) {
+		nxgep->hot_swappable_phy = B_TRUE;
 	}
 
 	/* If Alonso platform, replace "mif" for the last 2 ports phy-type */
diff --git a/usr/src/uts/common/io/nxge/nxge_hw.c b/usr/src/uts/common/io/nxge/nxge_hw.c
index 484c8d184f..877a174efe 100644
--- a/usr/src/uts/common/io/nxge/nxge_hw.c
+++ b/usr/src/uts/common/io/nxge/nxge_hw.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -60,12 +60,15 @@ static void
 nxge_rtrace_ioctl(p_nxge_t, queue_t *, mblk_t *, struct iocblk *);
 
 /* ARGSUSED */
-void
+nxge_status_t
 nxge_global_reset(p_nxge_t nxgep)
 {
+	nxge_status_t	status = NXGE_OK;
+
 	NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_global_reset"));
 
-	(void) nxge_link_monitor(nxgep, LINK_MONITOR_STOP);
+	if ((status = nxge_link_monitor(nxgep, LINK_MONITOR_STOP)) != NXGE_OK)
+		return (status);
 	(void) nxge_intr_hw_disable(nxgep);
 
 	if ((nxgep->suspended) ||
@@ -77,13 +80,18 @@ nxge_global_reset(p_nxge_t nxgep)
 			nxge_lb_serdes1000) ||
 			(nxgep->statsp->port_stats.lb_mode ==
 			nxge_lb_serdes10g))) {
-		(void) nxge_link_init(nxgep);
+		if ((status = nxge_link_init(nxgep)) != NXGE_OK)
+			return (status);
 	}
-	(void) nxge_link_monitor(nxgep, LINK_MONITOR_START);
-	(void) nxge_mac_init(nxgep);
+
+	if ((status = nxge_link_monitor(nxgep, LINK_MONITOR_START)) != NXGE_OK)
+		return (status);
+	if ((status = nxge_mac_init(nxgep)) != NXGE_OK)
+		return (status);
 	(void) nxge_intr_hw_enable(nxgep);
 
 	NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_global_reset"));
+	return (status);
 }
 
 /* ARGSUSED */
@@ -881,7 +889,12 @@ nxge_set_lb(p_nxge_t nxgep, queue_t *wq, p_mblk_t mp)
 				"!nxge%d: Returning to normal operation",
 				nxgep->instance);
 		}
-		nxge_set_lb_normal(nxgep);
+		if (nxge_set_lb_normal(nxgep) != NXGE_OK) {
+			status = B_FALSE;
+			cmn_err(CE_NOTE,
+			    "!nxge%d: Failed to return to normal operation",
+			    nxgep->instance);
+		}
 		goto nxge_set_lb_exit;
 	}
 	nxgep->statsp->port_stats.lb_mode = lb_mode;
@@ -923,10 +936,14 @@ nxge_set_lb(p_nxge_t nxgep, queue_t *wq, p_mblk_t mp)
 		(nxgep->statsp->port_stats.lb_mode == nxge_lb_phy1000) ||
 		(nxgep->statsp->port_stats.lb_mode == nxge_lb_phy)) {
 
-		(void) nxge_link_monitor(nxgep, LINK_MONITOR_STOP);
-		(void) nxge_xcvr_find(nxgep);
-		(void) nxge_link_init(nxgep);
-		(void) nxge_link_monitor(nxgep, LINK_MONITOR_START);
+		if (nxge_link_monitor(nxgep, LINK_MONITOR_STOP) != NXGE_OK)
+			goto nxge_set_lb_err;
+		if (nxge_xcvr_find(nxgep) != NXGE_OK)
+			goto nxge_set_lb_err;
+		if (nxge_link_init(nxgep) != NXGE_OK)
+			goto nxge_set_lb_err;
+		if (nxge_link_monitor(nxgep, LINK_MONITOR_START) != NXGE_OK)
+			goto nxge_set_lb_err;
 	}
 	if (lb_info->lb_type == internal) {
 		if ((nxgep->statsp->port_stats.lb_mode == nxge_lb_mac10g) ||
@@ -948,19 +965,29 @@ nxge_set_lb(p_nxge_t nxgep, queue_t *wq, p_mblk_t mp)
 		nxgep->statsp->mac_stats.link_duplex = 2;
 		nxgep->statsp->mac_stats.link_up = 1;
 	}
-	nxge_global_reset(nxgep);
+	if (nxge_global_reset(nxgep) != NXGE_OK)
+		goto nxge_set_lb_err;
 
 nxge_set_lb_exit:
 	NXGE_DEBUG_MSG((nxgep, DDI_CTL,
 		"<== nxge_set_lb status = 0x%08x", status));
 	return (status);
+nxge_set_lb_err:
+	status = B_FALSE;
+	cmn_err(CE_NOTE,
+	    "!nxge%d: Failed to put adapter in %s loopback mode",
+	    nxgep->instance, lb_info->key);
+	return (status);
 }
 
 /* ARGSUSED */
-void
+nxge_status_t
 nxge_set_lb_normal(p_nxge_t nxgep)
 {
+	nxge_status_t	status = NXGE_OK;
+
 	NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_set_lb_normal"));
+
 	nxgep->statsp->port_stats.lb_mode = nxge_lb_normal;
 	nxgep->param_arr[param_autoneg].value =
 		nxgep->param_arr[param_autoneg].old_value;
@@ -979,14 +1006,20 @@ nxge_set_lb_normal(p_nxge_t nxgep)
 	nxgep->param_arr[param_master_cfg_value].value =
 		nxgep->param_arr[param_master_cfg_value].old_value;
 
-	nxge_global_reset(nxgep);
+	if ((status = nxge_global_reset(nxgep)) != NXGE_OK)
+		return (status);
 
-	(void) nxge_link_monitor(nxgep, LINK_MONITOR_STOP);
-	(void) nxge_xcvr_find(nxgep);
-	(void) nxge_link_init(nxgep);
-	(void) nxge_link_monitor(nxgep, LINK_MONITOR_START);
+	if ((status = nxge_link_monitor(nxgep, LINK_MONITOR_STOP)) != NXGE_OK)
+		return (status);
+	if ((status = nxge_xcvr_find(nxgep)) != NXGE_OK)
+		return (status);
+	if ((status = nxge_link_init(nxgep)) != NXGE_OK)
+		return (status);
+	status = nxge_link_monitor(nxgep, LINK_MONITOR_START);
 
 	NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_set_lb_normal"));
+
+	return (status);
 }
 
 /* ARGSUSED */
diff --git a/usr/src/uts/common/io/nxge/nxge_kstats.c b/usr/src/uts/common/io/nxge/nxge_kstats.c
index e89d0d6300..6ed1b7027d 100644
--- a/usr/src/uts/common/io/nxge/nxge_kstats.c
+++ b/usr/src/uts/common/io/nxge/nxge_kstats.c
@@ -839,12 +839,14 @@ nxge_xmac_stat_update(kstat_t *ksp, int rw)
 		statsp->rx_viol_err_cnt =
 			xmac_kstatsp->rx_viol_err_cnt.value.ul;
 		statsp->rx_byte_cnt = xmac_kstatsp->rx_byte_cnt.value.ul;
+		statsp->rx_frame_cnt = xmac_kstatsp->rx_frame_cnt.value.ul;
 		statsp->rx_hist1_cnt = xmac_kstatsp->rx_hist1_cnt.value.ul;
 		statsp->rx_hist2_cnt = xmac_kstatsp->rx_hist2_cnt.value.ul;
 		statsp->rx_hist3_cnt = xmac_kstatsp->rx_hist3_cnt.value.ul;
 		statsp->rx_hist4_cnt = xmac_kstatsp->rx_hist4_cnt.value.ul;
 		statsp->rx_hist5_cnt = xmac_kstatsp->rx_hist5_cnt.value.ul;
 		statsp->rx_hist6_cnt = xmac_kstatsp->rx_hist6_cnt.value.ul;
+		statsp->rx_hist7_cnt = xmac_kstatsp->rx_hist7_cnt.value.ul;
 		statsp->rx_mult_cnt = xmac_kstatsp->rx_mult_cnt.value.ul;
 		statsp->rx_frag_cnt = xmac_kstatsp->rx_frag_cnt.value.ul;
 		statsp->rx_frame_align_err_cnt =
@@ -887,12 +889,14 @@ nxge_xmac_stat_update(kstat_t *ksp, int rw)
 		xmac_kstatsp->rx_viol_err_cnt.value.ul =
 			statsp->rx_viol_err_cnt;
 		xmac_kstatsp->rx_byte_cnt.value.ul = statsp->rx_byte_cnt;
+		xmac_kstatsp->rx_frame_cnt.value.ul = statsp->rx_frame_cnt;
 		xmac_kstatsp->rx_hist1_cnt.value.ul = statsp->rx_hist1_cnt;
 		xmac_kstatsp->rx_hist2_cnt.value.ul = statsp->rx_hist2_cnt;
 		xmac_kstatsp->rx_hist3_cnt.value.ul = statsp->rx_hist3_cnt;
 		xmac_kstatsp->rx_hist4_cnt.value.ul = statsp->rx_hist4_cnt;
 		xmac_kstatsp->rx_hist5_cnt.value.ul = statsp->rx_hist5_cnt;
 		xmac_kstatsp->rx_hist6_cnt.value.ul = statsp->rx_hist6_cnt;
+		xmac_kstatsp->rx_hist7_cnt.value.ul = statsp->rx_hist7_cnt;
 		xmac_kstatsp->rx_mult_cnt.value.ul = statsp->rx_mult_cnt;
 		xmac_kstatsp->rx_frag_cnt.value.ul = statsp->rx_frag_cnt;
 		xmac_kstatsp->rx_frame_align_err_cnt.value.ul =
@@ -1416,7 +1420,7 @@ nxge_setup_kstats(p_nxge_t nxgep)
 	kstat_named_init(&nxgekp->mdt_pld_bind_fail, "mdt_pld_bind_fail",
 		KSTAT_DATA_ULONG);
 #endif
-#ifdef ACNXGEPT_JUMBO
+#ifdef ACCEPT_JUMBO
 	kstat_named_init(&nxgekp->tx_jumbo_pkts, "tx_jumbo_pkts",
 		KSTAT_DATA_ULONG);
 #endif
@@ -1424,7 +1428,7 @@ nxge_setup_kstats(p_nxge_t nxgep)
 	/*
 	 * Rx Statistics.
 	 */
-#ifdef ACNXGEPT_JUMBO
+#ifdef ACCEPT_JUMBO
 	kstat_named_init(&nxgekp->rx_jumbo_pkts, "rx_jumbo_pkts",
 		KSTAT_DATA_ULONG);
 #endif
@@ -1472,6 +1476,8 @@ nxge_xmac_init_kstats(struct kstat *ksp)
 		KSTAT_DATA_ULONG);
 
 	/* Receive MAC statistics */
+	kstat_named_init(&nxgekp->rx_frame_cnt, "rxmac_frame_cnt",
+	    KSTAT_DATA_ULONG);
 	kstat_named_init(&nxgekp->rx_overflow_err, "rxmac_overflow_err",
 		KSTAT_DATA_ULONG);
 	kstat_named_init(&nxgekp->rx_underflow_err, "rxmac_underflow_err",
@@ -1499,6 +1505,8 @@ nxge_xmac_init_kstats(struct kstat *ksp)
 		KSTAT_DATA_ULONG);
 	kstat_named_init(&nxgekp->rx_hist6_cnt, "rxmac_1024_1522_cnt",
 		KSTAT_DATA_ULONG);
+	kstat_named_init(&nxgekp->rx_hist7_cnt, "rxmac_jumbo_cnt",
+	    KSTAT_DATA_ULONG);
 	kstat_named_init(&nxgekp->rx_broadcast_cnt, "rxmac_broadcast_cnt",
 		KSTAT_DATA_ULONG);
 	kstat_named_init(&nxgekp->rx_mult_cnt, "rxmac_multicast_cnt",
@@ -1612,6 +1620,8 @@ nxge_mac_init_kstats(p_nxge_t nxgep, struct kstat *ksp)
 	kstat_named_init(&nxgekp->rx_frame_align_err_cnt,
 		"rxmac_alignment_err",
 		KSTAT_DATA_ULONG);
+	kstat_named_init(&nxgekp->rx_frame_cnt, "rxmac_frame_cnt",
+	    KSTAT_DATA_ULONG);
 	if (nxgep->mac.porttype == PORT_TYPE_XMAC) {
 		kstat_named_init(&nxgekp->rx_hist1_cnt, "rxmac_64_cnt",
 			KSTAT_DATA_ULONG);
@@ -1625,6 +1635,8 @@ nxge_mac_init_kstats(p_nxge_t nxgep, struct kstat *ksp)
 			KSTAT_DATA_ULONG);
 		kstat_named_init(&nxgekp->rx_hist6_cnt, "rxmac_1024_1522_cnt",
 			KSTAT_DATA_ULONG);
+		kstat_named_init(&nxgekp->rx_hist7_cnt, "rxmac_jumbo_cnt",
+		    KSTAT_DATA_ULONG);
 		kstat_named_init(&nxgekp->rx_broadcast_cnt,
 			"rxmac_broadcast_cnt",
 			KSTAT_DATA_ULONG);
@@ -1641,9 +1653,6 @@ nxge_mac_init_kstats(p_nxge_t nxgep, struct kstat *ksp)
 		kstat_named_init(&nxgekp->rx_local_fault_err_cnt,
 			"rxmac_local_faults",
 			KSTAT_DATA_ULONG);
-	} else if (nxgep->mac.porttype == PORT_TYPE_BMAC) {
-		kstat_named_init(&nxgekp->rx_frame_cnt, "rxmac_frame_cnt",
-			KSTAT_DATA_ULONG);
 	}
 }
 
@@ -1742,11 +1751,15 @@ nxge_port_kstat_update(kstat_t *ksp, int rw)
 		/*
 		 * Rx Statistics.
 		 */
-#ifdef ACNXGEPT_JUMBO
+#ifdef ACCEPT_JUMBO
 		statsp->port_stats.rx_jumbo_pkts =
 			nxgekp->rx_jumbo_pkts.value.ul;
 #endif
-		(void) nxge_xmac_stat_update(ksp, KSTAT_WRITE);
+		if (nxgep->mac.porttype == PORT_TYPE_XMAC) {
+			(void) nxge_xmac_stat_update(ksp, KSTAT_WRITE);
+		} else {
+			(void) nxge_bmac_stat_update(ksp, KSTAT_WRITE);
+		}
 		return (0);
 	} else {
 		if (nxgep->filter.all_phys_cnt)
@@ -1883,7 +1896,11 @@ nxge_port_kstat_update(kstat_t *ksp, int rw)
 		nxgekp->rx_jumbo_pkts.value.ul =
 			statsp->port_stats.rx_jumbo_pkts;
 #endif
-		(void) nxge_xmac_stat_update(ksp, KSTAT_READ);
+		if (nxgep->mac.porttype == PORT_TYPE_XMAC) {
+			(void) nxge_xmac_stat_update(ksp, KSTAT_READ);
+		} else {
+			(void) nxge_bmac_stat_update(ksp, KSTAT_READ);
+		}
 	}
 
 	NXGE_DEBUG_MSG((nxgep, KST_CTL, "<== nxge_port_kstat_update"));
@@ -1928,50 +1945,84 @@ nxge_save_cntrs(p_nxge_t nxgep)
 		XMAC_REG_RD(handle, portn, XRXMAC_CRC_ER_CNT_REG, &val);
 		statsp->xmac_stats.rx_crc_err_cnt +=
 			(val & XRXMAC_CRC_ER_CNT_MASK);
+
 		XMAC_REG_RD(handle, portn, XRXMAC_MPSZER_CNT_REG, &val);
 		statsp->xmac_stats.rx_len_err_cnt +=
 			(val & XRXMAC_MPSZER_CNT_MASK);
+
 		XMAC_REG_RD(handle, portn, XRXMAC_CD_VIO_CNT_REG, &val);
 		statsp->xmac_stats.rx_viol_err_cnt +=
 			(val & XRXMAC_CD_VIO_CNT_MASK);
+
 		XMAC_REG_RD(handle, portn, XRXMAC_BT_CNT_REG, &val);
 		statsp->xmac_stats.rx_byte_cnt += (val & XRXMAC_BT_CNT_MASK);
+
 		XMAC_REG_RD(handle, portn, XRXMAC_HIST_CNT1_REG, &val);
 		statsp->xmac_stats.rx_hist1_cnt +=
 			(val & XRXMAC_HIST_CNT1_MASK);
+		statsp->xmac_stats.rx_frame_cnt +=
+		    (val & XRXMAC_HIST_CNT1_MASK);
+
 		XMAC_REG_RD(handle, portn, XRXMAC_HIST_CNT2_REG, &val);
 		statsp->xmac_stats.rx_hist2_cnt +=
 			(val & XRXMAC_HIST_CNT2_MASK);
+		statsp->xmac_stats.rx_frame_cnt +=
+		    (val & XRXMAC_HIST_CNT2_MASK);
+
 		XMAC_REG_RD(handle, portn, XRXMAC_HIST_CNT3_REG, &val);
 		statsp->xmac_stats.rx_hist3_cnt +=
 			(val & XRXMAC_HIST_CNT3_MASK);
+		statsp->xmac_stats.rx_frame_cnt +=
+		    (val & XRXMAC_HIST_CNT3_MASK);
+
 		XMAC_REG_RD(handle, portn, XRXMAC_HIST_CNT4_REG, &val);
 		statsp->xmac_stats.rx_hist4_cnt +=
 			(val & XRXMAC_HIST_CNT4_MASK);
+		statsp->xmac_stats.rx_frame_cnt +=
+		    (val & XRXMAC_HIST_CNT4_MASK);
+
 		XMAC_REG_RD(handle, portn, XRXMAC_HIST_CNT5_REG, &val);
 		statsp->xmac_stats.rx_hist5_cnt +=
 			(val & XRXMAC_HIST_CNT5_MASK);
+		statsp->xmac_stats.rx_frame_cnt +=
+		    (val & XRXMAC_HIST_CNT5_MASK);
+
 		XMAC_REG_RD(handle, portn, XRXMAC_HIST_CNT6_REG, &val);
 		statsp->xmac_stats.rx_hist6_cnt +=
 			(val & XRXMAC_HIST_CNT6_MASK);
+		statsp->xmac_stats.rx_frame_cnt +=
+		    (val & XRXMAC_HIST_CNT6_MASK);
+
+		XMAC_REG_RD(handle, portn, XRXMAC_HIST_CNT7_REG, &val);
+		statsp->xmac_stats.rx_hist7_cnt +=
+		    (val & XRXMAC_HIST_CNT7_MASK);
+		statsp->xmac_stats.rx_frame_cnt +=
+		    (val & XRXMAC_HIST_CNT7_MASK);
+
 		XMAC_REG_RD(handle, portn, XRXMAC_BC_FRM_CNT_REG, &val);
 		statsp->xmac_stats.rx_broadcast_cnt +=
 			(val & XRXMAC_BC_FRM_CNT_MASK);
+
 		XMAC_REG_RD(handle, portn, XRXMAC_MC_FRM_CNT_REG, &val);
 		statsp->xmac_stats.rx_mult_cnt +=
 			(val & XRXMAC_MC_FRM_CNT_MASK);
+
 		XMAC_REG_RD(handle, portn, XRXMAC_FRAG_CNT_REG, &val);
 		statsp->xmac_stats.rx_frag_cnt += (val & XRXMAC_FRAG_CNT_MASK);
+
 		XMAC_REG_RD(handle, portn, XRXMAC_AL_ER_CNT_REG, &val);
 		statsp->xmac_stats.rx_frame_align_err_cnt +=
 			(val & XRXMAC_AL_ER_CNT_MASK);
+
 		XMAC_REG_RD(handle, portn, XMAC_LINK_FLT_CNT_REG, &val);
 		statsp->xmac_stats.rx_linkfault_err_cnt +=
 			(val & XMAC_LINK_FLT_CNT_MASK);
+
 		(void) npi_xmac_xpcs_read(handle, portn,
 			XPCS_REG_DESCWERR_COUNTER, &cnt32);
 		statsp->xmac_stats.xpcs_deskew_err_cnt +=
 			(val & XMAC_XPCS_DESKEW_ERR_CNT_MASK);
+
 #ifdef	NXGE_DEBUG_SYMBOL_ERR
 		(void) npi_xmac_xpcs_read(handle, portn,
 			XPCS_REG_SYMBOL_ERR_L0_1_COUNTER, &cnt32);
@@ -1994,28 +2045,50 @@ nxge_save_cntrs(p_nxge_t nxgep)
 		 */
 		BMAC_REG_RD(handle, portn, BTXMAC_FRM_CNT_REG, &val);
 		statsp->bmac_stats.tx_frame_cnt += (val & BTXMAC_FRM_CNT_MASK);
-		XMAC_REG_RD(handle, portn, BTXMAC_BYTE_CNT_REG, &val);
+		/* Clear register as it is not auto clear on read */
+		BMAC_REG_WR(handle, portn, BTXMAC_FRM_CNT_REG, 0);
+
+		BMAC_REG_RD(handle, portn, BTXMAC_BYTE_CNT_REG, &val);
 		statsp->bmac_stats.tx_byte_cnt += (val & BTXMAC_BYTE_CNT_MASK);
+		/* Clear register as it is not auto clear on read */
+		BMAC_REG_WR(handle, portn, BTXMAC_BYTE_CNT_REG, 0);
 
 		/*
 		 * Receive MAC statistics.
 		 */
-		XMAC_REG_RD(handle, portn, RXMAC_FRM_CNT_REG, &val);
+		BMAC_REG_RD(handle, portn, RXMAC_FRM_CNT_REG, &val);
 		statsp->bmac_stats.rx_frame_cnt += (val & RXMAC_FRM_CNT_MASK);
-		XMAC_REG_RD(handle, portn, BRXMAC_BYTE_CNT_REG, &val);
+		/* Clear register as it is not auto clear on read */
+		BMAC_REG_WR(handle, portn, RXMAC_FRM_CNT_REG, 0);
+
+		BMAC_REG_RD(handle, portn, BRXMAC_BYTE_CNT_REG, &val);
 		statsp->bmac_stats.rx_byte_cnt += (val & BRXMAC_BYTE_CNT_MASK);
-		XMAC_REG_RD(handle, portn, BMAC_AL_ER_CNT_REG, &val);
+		/* Clear register as it is not auto clear on read */
+		BMAC_REG_WR(handle, portn, BRXMAC_BYTE_CNT_REG, 0);
+
+		BMAC_REG_RD(handle, portn, BMAC_AL_ER_CNT_REG, &val);
 		statsp->bmac_stats.rx_align_err_cnt +=
 			(val & BMAC_AL_ER_CNT_MASK);
-		XMAC_REG_RD(handle, portn, MAC_LEN_ER_CNT_REG, &val);
+		/* Clear register as it is not auto clear on read */
+		BMAC_REG_WR(handle, portn, BMAC_AL_ER_CNT_REG, 0);
+
+		BMAC_REG_RD(handle, portn, MAC_LEN_ER_CNT_REG, &val);
 		statsp->bmac_stats.rx_len_err_cnt +=
 			(val & MAC_LEN_ER_CNT_MASK);
-		XMAC_REG_RD(handle, portn, BMAC_CRC_ER_CNT_REG, &val);
+		/* Clear register as it is not auto clear on read */
+		BMAC_REG_WR(handle, portn, MAC_LEN_ER_CNT_REG, 0);
+
+		BMAC_REG_RD(handle, portn, BMAC_CRC_ER_CNT_REG, &val);
 		statsp->bmac_stats.rx_crc_err_cnt +=
 			(val & BMAC_CRC_ER_CNT_MASK);
-		XMAC_REG_RD(handle, portn, BMAC_CD_VIO_CNT_REG, &val);
+		/* Clear register as it is not auto clear on read */
+		BMAC_REG_WR(handle, portn, BMAC_CRC_ER_CNT_REG, 0);
+
+		BMAC_REG_RD(handle, portn, BMAC_CD_VIO_CNT_REG, &val);
 		statsp->bmac_stats.rx_viol_err_cnt +=
 			(val & BMAC_CD_VIO_CNT_MASK);
+		/* Clear register as it is not auto clear on read */
+		BMAC_REG_WR(handle, portn, BMAC_CD_VIO_CNT_REG, 0);
 	}
 	/* Update IPP counters */
 	(void) npi_ipp_get_ecc_err_count(handle, portn, &cnt8);
diff --git a/usr/src/uts/common/io/nxge/nxge_mac.c b/usr/src/uts/common/io/nxge/nxge_mac.c
index 06ad7392fc..9c15cb2137 100644
--- a/usr/src/uts/common/io/nxge/nxge_mac.c
+++ b/usr/src/uts/common/io/nxge/nxge_mac.c
@@ -31,6 +31,9 @@
 #define	LINK_MONITOR_PERIOD	(1000 * 1000)
 #define	LM_WAIT_MULTIPLIER	8
 
+#define	SERDES_RDY_WT_INTERVAL	50
+#define	MAX_SERDES_RDY_RETRIES	10
+
 extern uint32_t nxge_no_link_notify;
 extern boolean_t nxge_no_msg;
 extern uint32_t nxge_lb_dbg;
@@ -250,6 +253,10 @@ check_phy_done:
 			nxgep->statsp->mac_stats.xcvr_inuse = PCS_XCVR;
 			nxgep->mac.portmode = PORT_1G_FIBER;
 			NXGE_DEBUG_MSG((nxgep, MAC_CTL, "1G FIBER Xcvr"));
+		} else if (strcmp("xgf", prop_val) == 0) {
+			nxgep->statsp->mac_stats.xcvr_inuse = XPCS_XCVR;
+			nxgep->mac.portmode = PORT_10G_FIBER;
+			NXGE_DEBUG_MSG((nxgep, MAC_CTL, "10G FIBER Xcvr"));
 		}
 
 		(void) ddi_prop_update_string(DDI_DEV_T_NONE, nxgep->dip,
@@ -1370,7 +1377,7 @@ nxge_neptune_10G_serdes_init(p_nxge_t nxgep)
 {
 	npi_handle_t		handle;
 	uint8_t			portn;
-	int			chan;
+	int			chan, i;
 	sr_rx_tx_ctrl_l_t	rx_tx_ctrl_l;
 	sr_rx_tx_ctrl_h_t	rx_tx_ctrl_h;
 	sr_glue_ctrl0_l_t	glue_ctrl0_l;
@@ -1546,27 +1553,51 @@ nxge_neptune_10G_serdes_init(p_nxge_t nxgep)
 		    portn, val16l, val16h));
 	}
 
-	ESR_REG_RD(handle, ESR_INTERNAL_SIGNALS_REG, &val);
-
 	if (portn == 0) {
-		if ((val & ESR_SIG_P0_BITS_MASK) !=
-				(ESR_SIG_SERDES_RDY0_P0 | ESR_SIG_DETECT0_P0 |
-					ESR_SIG_XSERDES_RDY_P0 |
-					ESR_SIG_XDETECT_P0_CH3 |
-					ESR_SIG_XDETECT_P0_CH2 |
-					ESR_SIG_XDETECT_P0_CH1 |
-					ESR_SIG_XDETECT_P0_CH0)) {
-			goto fail;
+		/* Wait for serdes to be ready */
+		for (i = 0; i < MAX_SERDES_RDY_RETRIES; i++) {
+			ESR_REG_RD(handle, ESR_INTERNAL_SIGNALS_REG, &val);
+			if ((val & ESR_SIG_P0_BITS_MASK) !=
+			    (ESR_SIG_SERDES_RDY0_P0 | ESR_SIG_DETECT0_P0 |
+			    ESR_SIG_XSERDES_RDY_P0 |
+			    ESR_SIG_XDETECT_P0_CH3 |
+			    ESR_SIG_XDETECT_P0_CH2 |
+			    ESR_SIG_XDETECT_P0_CH1 |
+			    ESR_SIG_XDETECT_P0_CH0))
+
+				NXGE_DELAY(SERDES_RDY_WT_INTERVAL);
+			else
+				break;
+		}
+
+		if (i == MAX_SERDES_RDY_RETRIES) {
+			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
+			    "nxge_neptune_10G_serdes_init: "
+			    "Serdes/signal for port<%d> not ready", portn));
+				goto fail;
 		}
 	} else if (portn == 1) {
-		if ((val & ESR_SIG_P1_BITS_MASK) !=
-				(ESR_SIG_SERDES_RDY0_P1 | ESR_SIG_DETECT0_P1 |
-					ESR_SIG_XSERDES_RDY_P1 |
-					ESR_SIG_XDETECT_P1_CH3 |
-					ESR_SIG_XDETECT_P1_CH2 |
-					ESR_SIG_XDETECT_P1_CH1 |
-					ESR_SIG_XDETECT_P1_CH0)) {
-			goto fail;
+		/* Wait for serdes to be ready */
+		for (i = 0; i < MAX_SERDES_RDY_RETRIES; i++) {
+			ESR_REG_RD(handle, ESR_INTERNAL_SIGNALS_REG, &val);
+			if ((val & ESR_SIG_P1_BITS_MASK) !=
+			    (ESR_SIG_SERDES_RDY0_P1 | ESR_SIG_DETECT0_P1 |
+			    ESR_SIG_XSERDES_RDY_P1 |
+			    ESR_SIG_XDETECT_P1_CH3 |
+			    ESR_SIG_XDETECT_P1_CH2 |
+			    ESR_SIG_XDETECT_P1_CH1 |
+			    ESR_SIG_XDETECT_P1_CH0))
+
+				NXGE_DELAY(SERDES_RDY_WT_INTERVAL);
+			else
+				break;
+		}
+
+		if (i == MAX_SERDES_RDY_RETRIES) {
+			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
+			    "nxge_neptune_10G_serdes_init: "
+			    "Serdes/signal for port<%d> not ready", portn));
+				goto fail;
 		}
 	}
 
@@ -3539,7 +3570,7 @@ nxge_mii_read(p_nxge_t nxgep, uint8_t xcvr_portn, uint8_t xcvr_reg,
 	NXGE_DEBUG_MSG((nxgep, MIF_CTL, "==> nxge_mii_read: xcvr_port<%d>"
 			"xcvr_reg<%d>", xcvr_portn, xcvr_reg));
 
-	MUTEX_ENTER(&nxgep->nxge_hw_p->nxge_mii_lock);
+	MUTEX_ENTER(&nxgep->nxge_hw_p->nxge_mdio_lock);
 
 	if ((nxgep->mac.portmode == PORT_1G_COPPER) ||
 	    (nxgep->mac.portmode == PORT_1G_RGMII_FIBER)) {
@@ -3554,14 +3585,14 @@ nxge_mii_read(p_nxge_t nxgep, uint8_t xcvr_portn, uint8_t xcvr_reg,
 	} else
 		goto fail;
 
-	MUTEX_EXIT(&nxgep->nxge_hw_p->nxge_mii_lock);
+	MUTEX_EXIT(&nxgep->nxge_hw_p->nxge_mdio_lock);
 
 	NXGE_DEBUG_MSG((nxgep, MIF_CTL, "<== nxge_mii_read: xcvr_port<%d>"
 			"xcvr_reg<%d> value=0x%x",
 			xcvr_portn, xcvr_reg, *value));
 	return (NXGE_OK);
 fail:
-	MUTEX_EXIT(&nxgep->nxge_hw_p->nxge_mii_lock);
+	MUTEX_EXIT(&nxgep->nxge_hw_p->nxge_mdio_lock);
 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
 			"nxge_mii_read: Failed to read mii on xcvr %d",
 			xcvr_portn));
@@ -3581,7 +3612,7 @@ nxge_mii_write(p_nxge_t nxgep, uint8_t xcvr_portn, uint8_t xcvr_reg,
 			"xcvr_reg<%d> value=0x%x", xcvr_portn, xcvr_reg,
 			value));
 
-	MUTEX_ENTER(&nxgep->nxge_hw_p->nxge_mii_lock);
+	MUTEX_ENTER(&nxgep->nxge_hw_p->nxge_mdio_lock);
 
 	if ((nxgep->mac.portmode == PORT_1G_COPPER) ||
 	    (nxgep->mac.portmode == PORT_1G_RGMII_FIBER)) {
@@ -3596,13 +3627,13 @@ nxge_mii_write(p_nxge_t nxgep, uint8_t xcvr_portn, uint8_t xcvr_reg,
 	} else
 		goto fail;
 
-	MUTEX_EXIT(&nxgep->nxge_hw_p->nxge_mii_lock);
+	MUTEX_EXIT(&nxgep->nxge_hw_p->nxge_mdio_lock);
 
 	NXGE_DEBUG_MSG((nxgep, MIF_CTL, "<== nxge_mii_write: xcvr_port<%d>"
 			"xcvr_reg<%d>", xcvr_portn, xcvr_reg));
 	return (NXGE_OK);
 fail:
-	MUTEX_EXIT(&nxgep->nxge_hw_p->nxge_mii_lock);
+	MUTEX_EXIT(&nxgep->nxge_hw_p->nxge_mdio_lock);
 
 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
 			"nxge_mii_write: Failed to write mii on xcvr %d",
@@ -4560,8 +4591,8 @@ nxge_link_monitor(p_nxge_t nxgep, link_mon_enable_t enable)
 		/* stats has not been allocated. */
 		return (NXGE_OK);
 	}
-	/* Don't check link if we're not in internal loopback mode */
-	if (nxgep->statsp->port_stats.lb_mode != nxge_lb_normal)
+	/* Don't check link if we're in internal loopback mode */
+	if (nxgep->statsp->port_stats.lb_mode >= nxge_lb_serdes10g)
 		return (NXGE_OK);
 
 	NXGE_DEBUG_MSG((nxgep, MAC_CTL,
@@ -5241,7 +5272,7 @@ nxge_get_cl22_phy_id(p_nxge_t nxgep, int phy_port)
 	npi_handle_t	handle = NXGE_DEV_NPI_HANDLE(nxgep);
 	npi_status_t	npi_status = NPI_SUCCESS;
 
-	MUTEX_ENTER(&nxgep->nxge_hw_p->nxge_mii_lock);
+	MUTEX_ENTER(&nxgep->nxge_hw_p->nxge_mdio_lock);
 	npi_status = npi_mac_mif_mii_read(handle, phy_port, NXGE_PHY_ID_REG_1,
 	    &val1);
 	if (npi_status != NPI_SUCCESS) {
@@ -5261,7 +5292,7 @@ nxge_get_cl22_phy_id(p_nxge_t nxgep, int phy_port)
 	phy_id |= val2;
 
 exit:
-	MUTEX_EXIT(&nxgep->nxge_hw_p->nxge_mii_lock);
+	MUTEX_EXIT(&nxgep->nxge_hw_p->nxge_mdio_lock);
 	NXGE_DEBUG_MSG((nxgep, MAC_CTL, "port[%d] PHY ID [0x%llx]",
 	    phy_port, phy_id));
 
@@ -5520,9 +5551,25 @@ nxge_scan_ports_phy(p_nxge_t nxgep, p_nxge_hw_list_t hw_p)
 					    "Unsupported neptune type 10"));
 					goto error_exit;
 				}
-
-				hw_p->xcvr_addr[nxgep->function_num] =
-				    port_fd_arr[0];
+				/*
+				 * For GOA, which is a hot swappable PHY, the
+				 * phy address to function number mapping
+				 * should be preserved, i.e., addr 16 is
+				 * assigned to function 0 and 20 to function 1
+				 * But for Huron XAUI, the assignment should
+				 * be by function number, i.e., whichever
+				 * function number attaches should be
+				 * assigned the available PHY (this is required
+				 * primarily to support pre-production Huron
+				 * boards where function 0 is mapped to addr 17
+				 */
+				if (port_fd_arr[0] ==
+				    BCM8706_ALT_GOA_PORT1_ADDR) {
+					hw_p->xcvr_addr[1] = port_fd_arr[0];
+				} else {
+					hw_p->xcvr_addr[nxgep->function_num] =
+					    port_fd_arr[0];
+				}
 			} else {
 				NXGE_DEBUG_MSG((nxgep, MAC_CTL,
 				    "Unsupported neptune type 10 - 1"));
@@ -5717,14 +5764,14 @@ nxge_bcm5464_link_led_off(p_nxge_t nxgep) {
 		break;
 	}
 
-	MUTEX_ENTER(&nxgep->nxge_hw_p->nxge_mii_lock);
+	MUTEX_ENTER(&nxgep->nxge_hw_p->nxge_mdio_lock);
 	rs = npi_mac_mif_mii_write(nxgep->npi_handle,
 	    xcvr_portn, BCM5464R_MISC, 0xb4ee);
 	if (rs != NPI_SUCCESS) {
 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
 		    "<== nxge_bcm5464_link_led_off: npi_mac_mif_mii_write "
 		    "returned error 0x[%x]", rs));
-		MUTEX_EXIT(&nxgep->nxge_hw_p->nxge_mii_lock);
+		MUTEX_EXIT(&nxgep->nxge_hw_p->nxge_mdio_lock);
 		return;
 	}
 
@@ -5736,7 +5783,7 @@ nxge_bcm5464_link_led_off(p_nxge_t nxgep) {
 		    "returned error 0x[%x]", rs));
 	}
 
-	MUTEX_EXIT(&nxgep->nxge_hw_p->nxge_mii_lock);
+	MUTEX_EXIT(&nxgep->nxge_hw_p->nxge_mdio_lock);
 }
 
 static nxge_status_t
diff --git a/usr/src/uts/common/io/nxge/nxge_main.c b/usr/src/uts/common/io/nxge/nxge_main.c
index 7d8f577979..659995b323 100644
--- a/usr/src/uts/common/io/nxge/nxge_main.c
+++ b/usr/src/uts/common/io/nxge/nxge_main.c
@@ -4800,7 +4800,6 @@ nxge_init_common_dev(p_nxge_t nxgep)
 		MUTEX_INIT(&hw_p->nxge_tcam_lock, NULL, MUTEX_DRIVER, NULL);
 		MUTEX_INIT(&hw_p->nxge_vlan_lock, NULL, MUTEX_DRIVER, NULL);
 		MUTEX_INIT(&hw_p->nxge_mdio_lock, NULL, MUTEX_DRIVER, NULL);
-		MUTEX_INIT(&hw_p->nxge_mii_lock, NULL, MUTEX_DRIVER, NULL);
 
 		nxge_hw_list = hw_p;
 
@@ -4863,7 +4862,6 @@ nxge_uninit_common_dev(p_nxge_t nxgep)
 				MUTEX_DESTROY(&hw_p->nxge_tcam_lock);
 				MUTEX_DESTROY(&hw_p->nxge_cfg_lock);
 				MUTEX_DESTROY(&hw_p->nxge_mdio_lock);
-				MUTEX_DESTROY(&hw_p->nxge_mii_lock);
 				NXGE_DEBUG_MSG((nxgep, MOD_CTL,
 					"==> nxge_uninit_common_device: "
 					"func # %d "
diff --git a/usr/src/uts/common/io/nxge/nxge_send.c b/usr/src/uts/common/io/nxge/nxge_send.c
index dc5fcb4965..bccab914ee 100644
--- a/usr/src/uts/common/io/nxge/nxge_send.c
+++ b/usr/src/uts/common/io/nxge/nxge_send.c
@@ -137,13 +137,34 @@ nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp)
 
 	statsp = nxgep->statsp;
 
-	if (nxgep->statsp->port_stats.lb_mode == nxge_lb_normal) {
-		if (!statsp->mac_stats.link_up) {
-			freemsg(mp);
-			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
-				"link not up or LB mode"));
-			goto nxge_start_fail1;
+	switch (nxgep->mac.portmode) {
+	default:
+		if (nxgep->statsp->port_stats.lb_mode == nxge_lb_normal) {
+			if (!statsp->mac_stats.link_up) {
+				freemsg(mp);
+				NXGE_DEBUG_MSG((nxgep, TX_CTL,
+				    "==> nxge_start: "
+				    "link not up"));
+				goto nxge_start_fail1;
+			}
 		}
+		break;
+	case PORT_10G_FIBER:
+		/*
+		 * For the following modes, check the link status
+		 * before sending the packet out:
+		 * nxge_lb_normal, nxge_lb_ext10g, nxge_lb_phy10g
+		 */
+		if (nxgep->statsp->port_stats.lb_mode < nxge_lb_serdes10g) {
+			if (!statsp->mac_stats.link_up) {
+				freemsg(mp);
+				NXGE_DEBUG_MSG((nxgep, TX_CTL,
+				    "==> nxge_start: "
+				    "link not up"));
+				goto nxge_start_fail1;
+			}
+		}
+		break;
 	}
 
 	if (nxgep->soft_lso_enable) {
diff --git a/usr/src/uts/common/sys/nxge/nxge.h b/usr/src/uts/common/sys/nxge/nxge.h
index 54fdc55bfd..c3e27fe6d8 100644
--- a/usr/src/uts/common/sys/nxge/nxge.h
+++ b/usr/src/uts/common/sys/nxge/nxge.h
@@ -261,7 +261,10 @@ typedef struct _nxge_param_t {
 } nxge_param_t, *p_nxge_param_t;
 
 
-
+/*
+ * Do not change the order of the elements of this enum as that will
+ * break the driver code.
+ */
 typedef enum {
 	nxge_lb_normal,
 	nxge_lb_ext10g,
@@ -933,6 +936,7 @@ typedef	struct _nxge_mac_kstat {
 	kstat_named_t	rx_hist4_cnt;
 	kstat_named_t	rx_hist5_cnt;
 	kstat_named_t	rx_hist6_cnt;
+	kstat_named_t	rx_hist7_cnt;
 	kstat_named_t	rx_broadcast_cnt;
 	kstat_named_t	rx_mult_cnt;
 	kstat_named_t	rx_frag_cnt;
diff --git a/usr/src/uts/common/sys/nxge/nxge_common.h b/usr/src/uts/common/sys/nxge/nxge_common.h
index fc4f4bf0f9..78c4e045ca 100644
--- a/usr/src/uts/common/sys/nxge/nxge_common.h
+++ b/usr/src/uts/common/sys/nxge/nxge_common.h
@@ -484,7 +484,6 @@ typedef struct nxge_hw_list {
 	nxge_os_mutex_t 	nxge_tcam_lock;
 	nxge_os_mutex_t 	nxge_vlan_lock;
 	nxge_os_mutex_t 	nxge_mdio_lock;
-	nxge_os_mutex_t 	nxge_mii_lock;
 
 	nxge_dev_info_t		*parent_devp;
 	struct _nxge_t		*nxge_p[NXGE_MAX_PORTS];
diff --git a/usr/src/uts/common/sys/nxge/nxge_impl.h b/usr/src/uts/common/sys/nxge/nxge_impl.h
index 5645d7762c..adbb8d16c6 100644
--- a/usr/src/uts/common/sys/nxge/nxge_impl.h
+++ b/usr/src/uts/common/sys/nxge/nxge_impl.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -658,6 +658,8 @@ typedef struct _nxge_mmac_stats_t {
 #define	NXGE_2XGF_PEM_BM_STR		"501-7626"
 #define	NXGE_ALONSO_BM_STR		"373-0202-01"
 #define	NXGE_ALONSO_MODEL_STR		"SUNW,CP3220"
+#define	NXGE_RFEM_BM_STR		"501-7961-01"
+#define	NXGE_RFEM_MODEL_STR		"SUNW,pcie-rfem"
 #define	NXGE_EROM_LEN			1048576
 
 #endif
@@ -751,7 +753,7 @@ int nxge_m_stat(void *arg, uint_t, uint64_t *);
 void
 nxge_hw_ioctl(p_nxge_t, queue_t *, mblk_t *, struct iocblk *);
 void nxge_loopback_ioctl(p_nxge_t, queue_t *, mblk_t *, struct iocblk *);
-void nxge_global_reset(p_nxge_t);
+nxge_status_t nxge_global_reset(p_nxge_t);
 uint_t nxge_intr(void *, void *);
 void nxge_intr_enable(p_nxge_t);
 void nxge_intr_disable(p_nxge_t);
@@ -761,7 +763,6 @@ void nxge_hw_init_niu_common(p_nxge_t);
 void nxge_intr_hw_enable(p_nxge_t);
 void nxge_intr_hw_disable(p_nxge_t);
 void nxge_hw_stop(p_nxge_t);
-void nxge_global_reset(p_nxge_t);
 void nxge_check_hw_state(p_nxge_t);
 
 void nxge_rxdma_channel_put64(nxge_os_acc_handle_t,
@@ -816,7 +817,7 @@ boolean_t nxge_nd_load(caddr_t *, char *, pfi_t, pfi_t, caddr_t);
 void nxge_nd_free(caddr_t *);
 int nxge_nd_getset(p_nxge_t, queue_t *, caddr_t, p_mblk_t);
 
-void nxge_set_lb_normal(p_nxge_t);
+nxge_status_t nxge_set_lb_normal(p_nxge_t);
 boolean_t nxge_set_lb(p_nxge_t, queue_t *, p_mblk_t);
 
 /* nxge_virtual.c */
diff --git a/usr/src/uts/common/sys/nxge/nxge_mac_hw.h b/usr/src/uts/common/sys/nxge/nxge_mac_hw.h
index 57244710fd..d4e4534d4c 100644
--- a/usr/src/uts/common/sys/nxge/nxge_mac_hw.h
+++ b/usr/src/uts/common/sys/nxge/nxge_mac_hw.h
@@ -1094,6 +1094,7 @@ typedef union _xmac_cfg_t {
 #define	XRXMAC_HIST_CNT4_MASK		0x0007FFFF
 #define	XRXMAC_HIST_CNT5_MASK		0x0003FFFF
 #define	XRXMAC_HIST_CNT6_MASK		0x0001FFFF
+#define	XRXMAC_HIST_CNT7_MASK		0x07FFFFFF
 #define	XRXMAC_BC_FRM_CNT_MASK		0x001FFFFF
 #define	XRXMAC_MC_FRM_CNT_MASK		0x001FFFFF
 #define	XRXMAC_FRAG_CNT_MASK		0x001FFFFF