6426449 Fix for 6387501 causes lock contention from px_dma_allocmp() during I/O load

9 files changed, 89 insertions, 93 deletions

diff --git a/usr/src/uts/common/sys/pcie_impl.h b/usr/src/uts/common/sys/pcie_impl.h
index 82efb5b1a8..51a5c2d8fc 100644
--- a/usr/src/uts/common/sys/pcie_impl.h
+++ b/usr/src/uts/common/sys/pcie_impl.h
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -34,6 +33,12 @@ extern "C" {
 #endif
 
 /*
+ * The following flag is used for Broadcom 5714/5715 bridge prefetch issue.
+ * This flag will be used both by px and px_pci nexus drivers.
+ */
+#define	PX_DMAI_FLAGS_MAP_BUFZONE	0x40000
+
+/*
  * PCI-Express Friendly Functions
  */
 extern int pcie_initchild(dev_info_t *dip);
diff --git a/usr/src/uts/sparc/pxb_bcm/Makefile b/usr/src/uts/sparc/pxb_bcm/Makefile
index 3689495340..b1b12c752b 100644
--- a/usr/src/uts/sparc/pxb_bcm/Makefile
+++ b/usr/src/uts/sparc/pxb_bcm/Makefile
@@ -72,9 +72,10 @@ CFLAGS += $(CCVERBOSE)
 CFLAGS += -dalign
 
 #
-# Enable Broadcom workaround code and lint duplicate symbol avoidance hack
+# Enable Broadcom 5714/5715 workaround code and lint duplicate symbol
+# avoidance hack
 #
-CPPFLAGS += -DBROADCOM_ADDR_LIMIT -DPX_MOD_NAME=pxb_bcm
+CPPFLAGS += -DBCM_SW_WORKAROUNDS -DPX_MOD_NAME=pxb_bcm
 
 #
 #	Dependency
diff --git a/usr/src/uts/sun4/io/px/px_dma.c b/usr/src/uts/sun4/io/px/px_dma.c
index f0709c0f74..7dbd3936cf 100644
--- a/usr/src/uts/sun4/io/px/px_dma.c
+++ b/usr/src/uts/sun4/io/px/px_dma.c
@@ -112,10 +112,6 @@ px_dma_allocmp(dev_info_t *dip, dev_info_t *rdip, int (*waitfp)(caddr_t),
 	mp->dmai_error.err_fep = NULL;
 	mp->dmai_error.err_cf = NULL;
 
-	if (px_child_prefetch(mp->dmai_rdip))
-		mp->dmai_flags |= (PX_DMAI_FLAGS_MAP_BUFZONE |
-		    PX_DMAI_FLAGS_REDZONE);
-
 	return (mp);
 }
 
@@ -376,6 +372,7 @@ px_dma_type(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
 	px_pec_t *pec_p = px_p->px_pec_p;
 	uint32_t offset;
 	pfn_t pfn0;
+	uint_t redzone;
 
 	mp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS | DMP_NOSYNC;
 
@@ -429,9 +426,13 @@ px_dma_type(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
 		cmn_err(CE_WARN, "Bad peer-to-peer req %s%d", NAMEINST(rdip));
 		return (DDI_DMA_NOMAPPING);
 	}
+
+	redzone = (mp->dmai_rflags & DDI_DMA_REDZONE) ||
+	    (mp->dmai_flags & PX_DMAI_FLAGS_MAP_BUFZONE) ?
+	    PX_DMAI_FLAGS_REDZONE : 0;
+
 	mp->dmai_flags |= (mp->dmai_flags & PX_DMAI_FLAGS_BYPASSREQ) ?
-	    PX_DMAI_FLAGS_BYPASS : PX_DMAI_FLAGS_DVMA |
-	    (mp->dmai_rflags & DDI_DMA_REDZONE ? PX_DMAI_FLAGS_REDZONE : 0);
+	    PX_DMAI_FLAGS_BYPASS : (PX_DMAI_FLAGS_DVMA | redzone);
 done:
 	mp->dmai_object	 = *dobj_p;			/* whole object    */
 	mp->dmai_pfn0	 = (void *)pfn0;		/* cache pfn0	   */
diff --git a/usr/src/uts/sun4/io/px/px_dma.h b/usr/src/uts/sun4/io/px/px_dma.h
index 0bd23f9eb2..4a14d3c721 100644
--- a/usr/src/uts/sun4/io/px/px_dma.h
+++ b/usr/src/uts/sun4/io/px/px_dma.h
@@ -124,7 +124,10 @@ struct px_dma_impl { /* forthdebug only, keep in sync with ddi_dma_impl_t */
 #define	PX_DMAI_FLAGS_NOSYNC		0x4000
 #define	PX_DMAI_FLAGS_PTP32		0x10000
 #define	PX_DMAI_FLAGS_PTP64		0x20000
-#define	PX_DMAI_FLAGS_MAP_BUFZONE	0x40000
+/*
+ * #define PX_DMAI_FLAGS_MAP_BUFZONE	0x40000
+ * See pcie_impl.h
+ */
 #define	PX_DMAI_FLAGS_REDZONE		0x80000
 #define	PX_DMAI_FLAGS_PRESERVE	(PX_DMAI_FLAGS_PEER_ONLY | \
 	PX_DMAI_FLAGS_BYPASSREQ | PX_DMAI_FLAGS_NOSYSLIMIT | \
diff --git a/usr/src/uts/sun4/io/px/px_fdvma.c b/usr/src/uts/sun4/io/px/px_fdvma.c
index 24b2a57067..5edfa9e40c 100644
--- a/usr/src/uts/sun4/io/px/px_fdvma.c
+++ b/usr/src/uts/sun4/io/px/px_fdvma.c
@@ -150,14 +150,6 @@ px_fdvma_reserve(dev_info_t *dip, dev_info_t *rdip, px_t *px_p,
 		ddi_driver_name(rdip), ddi_get_instance(rdip));
 
 	/*
-	 * FDVMA feature is not supported for any child device of Broadcom
-	 * PCIe-PCI bridge due to prefetch bug. Return failure immediately,
-	 * so that these drivers will switch to regular DVMA path.
-	 */
-	if (px_child_prefetch(rdip))
-		return (DDI_FAILURE);
-
-	/*
 	 * Check the limit structure.
 	 */
 	if ((lo >= hi) || (hi < mmu_p->mmu_dvma_base))
diff --git a/usr/src/uts/sun4/io/px/px_pci.c b/usr/src/uts/sun4/io/px/px_pci.c
index 1e7783710a..481285e0e0 100644
--- a/usr/src/uts/sun4/io/px/px_pci.c
+++ b/usr/src/uts/sun4/io/px/px_pci.c
@@ -67,9 +67,14 @@ static boolean_t pxb_enable_msi = B_TRUE; /* MSI enabled if TRUE, else INTX */
 
 static int pxb_bus_map(dev_info_t *, dev_info_t *, ddi_map_req_t *,
 	off_t, off_t, caddr_t *);
-static int pxb_ddi_dma_allochdl_limited(dev_info_t *dip, dev_info_t *rdip,
+#ifdef	BCM_SW_WORKAROUNDS
+static int pxb_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
 	ddi_dma_attr_t *attr_p, int (*waitfp)(caddr_t), caddr_t arg,
 	ddi_dma_handle_t *handlep);
+static int pxb_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
+	ddi_dma_handle_t handle, enum ddi_dma_ctlops cmd, off_t *offp,
+	size_t *lenp, caddr_t *objp, uint_t cache_flags);
+#endif	/* BCM_SW_WORKAROUNDS */
 static int pxb_ctlops(dev_info_t *, dev_info_t *, ddi_ctl_enum_t,
 	void *, void *);
 static int pxb_intr_ops(dev_info_t *dip, dev_info_t *rdip,
@@ -98,17 +103,21 @@ static struct bus_ops pxb_bus_ops = {
 	0,
 	i_ddi_map_fault,
 	ddi_dma_map,
-#if defined(PXB_ADDR_LIMIT_HI) || defined(PXB_ADDR_LIMIT_LO)
-	pxb_ddi_dma_allochdl_limited,
+#ifdef	BCM_SW_WORKAROUNDS
+	pxb_dma_allochdl,
 #else
 	ddi_dma_allochdl,
-#endif
+#endif	/* BCM_SW_WORKAROUNDS */
 	ddi_dma_freehdl,
 	ddi_dma_bindhdl,
 	ddi_dma_unbindhdl,
 	ddi_dma_flush,
 	ddi_dma_win,
+#ifdef	BCM_SW_WORKAROUNDS
+	pxb_dma_mctl,
+#else
 	ddi_dma_mctl,
+#endif	/* BCM_SW_WORKAROUNDS */
 	pxb_ctlops,
 	ddi_bus_prop_op,
 	ndi_busop_get_eventcookie,	/* (*bus_get_eventcookie)();	*/
@@ -350,23 +359,6 @@ pxb_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
 	pxb->pxb_rev_id = pci_config_get8(config_handle, PCI_CONF_REVID);
 
 	/*
-	 * This is a software workaround to fix the Broadcom PCIe-PCI bridge
-	 * prefetch bug. Create a boolean property and its existence means
-	 * the px nexus driver has to allocate an extra page & make it valid
-	 * one, for any DVMA request that comes from any of the Broadcom child
-	 * device.
-	 */
-	if (PXB_IS_BCM5714(pxb)) {
-		if (ndi_prop_create_boolean(DDI_DEV_T_NONE, pxb->pxb_dip,
-		    "cross-page-prefetch") != DDI_PROP_SUCCESS) {
-			DBG(DBG_ATTACH, pxb->pxb_dip,
-			    "ndi_prop_create_boolean() failed\n");
-
-			goto fail;
-		}
-	}
-
-	/*
 	 * Power management setup. This also makes sure that switch/bridge
 	 * is at D0 during attach.
 	 */
@@ -499,9 +491,6 @@ pxb_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
 		if (pxb->pxb_init_flags & PXB_INIT_FM)
 			pxb_fm_fini(pxb);
 
-		(void) ndi_prop_remove(DDI_DEV_T_NONE, pxb->pxb_dip,
-		    "cross-page-prefetch");
-
 		if (pxb->pxb_init_flags & PXB_INIT_CONFIG_HANDLE)
 			pci_config_teardown(&pxb->pxb_config_handle);
 
@@ -2032,7 +2021,7 @@ pxb_check_bad_devs(pxb_devstate_t *pxb, int vend)
 	return (ret);
 }
 
-#if defined(PXB_ADDR_LIMIT_HI) || defined(PXB_ADDR_LIMIT_LO)
+#ifdef	BCM_SW_WORKAROUNDS
 
 /*
  * Some PCI-X to PCI-E bridges do not support full 64-bit addressing on the
@@ -2043,27 +2032,57 @@ pxb_check_bad_devs(pxb_devstate_t *pxb, int vend)
  * PCI-E nexus driver from allocating any memory the bridge can't deal
  * with.
  */
-
 static int
-pxb_ddi_dma_allochdl_limited(dev_info_t *dip, dev_info_t *rdip,
+pxb_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
 	ddi_dma_attr_t *attr_p, int (*waitfp)(caddr_t), caddr_t arg,
 	ddi_dma_handle_t *handlep)
 {
+	uint64_t	lim;
+	int		ret;
+
 	/*
-	 * If the leaf device's limits are outside than what the bridge can
-	 * handle, we need to clip the values passed up the chain.
+	 * If the leaf device's limits are outside than what the Broadcom
+	 * bridge can handle, we need to clip the values passed up the chain.
 	 */
-#ifdef PXB_ADDR_LIMIT_LO
-	if (attr_p->dma_attr_addr_lo < PXB_ADDR_LIMIT_LO)
-		attr_p->dma_attr_addr_lo = PXB_ADDR_LIMIT_LO;
-#endif
+	lim = attr_p->dma_attr_addr_lo;
+	attr_p->dma_attr_addr_lo = MAX(lim, PXB_ADDR_LIMIT_LO);
 
-#ifdef PXB_ADDR_LIMIT_HI
-	if (attr_p->dma_attr_addr_hi > PXB_ADDR_LIMIT_HI)
-		attr_p->dma_attr_addr_hi = PXB_ADDR_LIMIT_HI;
-#endif
+	lim = attr_p->dma_attr_addr_hi;
+	attr_p->dma_attr_addr_hi = MIN(lim, PXB_ADDR_LIMIT_HI);
 
-	return (ddi_dma_allochdl(dip, rdip, attr_p, waitfp, arg, handlep));
+	/*
+	 * This is a software workaround to fix the Broadcom 5714/5715 PCIe-PCI
+	 * bridge prefetch bug. Intercept the DMA alloc handle request and set
+	 * PX_DMAI_FLAGS_MAP_BUFZONE flag in the handle. If this flag is set,
+	 * the px nexus driver will allocate an extra page & make it valid one,
+	 * for any DVMA request that comes from any of the Broadcom bridge child
+	 * devices.
+	 */
+	if ((ret = ddi_dma_allochdl(dip, rdip, attr_p, waitfp, arg,
+	    handlep)) == DDI_SUCCESS) {
+		ddi_dma_impl_t	*mp = (ddi_dma_impl_t *)*handlep;
+		mp->dmai_inuse |= PX_DMAI_FLAGS_MAP_BUFZONE;
+	}
+
+	return (ret);
 }
 
-#endif
+/*
+ * FDVMA feature is not supported for any child device of Broadcom 5714/5715
+ * PCIe-PCI bridge due to prefetch bug. Return failure immediately, so that
+ * these drivers will switch to regular DVMA path.
+ */
+/*ARGSUSED*/
+static int
+pxb_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
+	enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp,
+	uint_t cache_flags)
+{
+	if (cmd == DDI_DMA_RESERVE)
+		return (DDI_FAILURE);
+
+	return (ddi_dma_mctl(dip, rdip, handle, cmd, offp, lenp, objp,
+	    cache_flags));
+}
+
+#endif	/* BCM_SW_WORKAROUNDS */
diff --git a/usr/src/uts/sun4/io/px/px_pci.h b/usr/src/uts/sun4/io/px/px_pci.h
index b44d43d292..862a5ee832 100644
--- a/usr/src/uts/sun4/io/px/px_pci.h
+++ b/usr/src/uts/sun4/io/px/px_pci.h
@@ -178,20 +178,6 @@ extern void *pxb_state;
 #define	PXB_DEVICE_PLX_8532		0x8532
 #define	PXB_DEVICE_PLX_8516		0x8516
 
-/* Broadcom bridge Vendor/Device IDs for HW workaround */
-#define	PXB_VENDOR_BCM			0x1166
-#define	PXB_DEVICE_BCM5714		0x0103
-
-/* Simple macro to test if this is a Broadcom bridge */
-#define	PXB_IS_BCM5714(pxb) \
-	((pxb->pxb_vendor_id == PXB_VENDOR_BCM) && \
-	(pxb->pxb_device_id == PXB_DEVICE_BCM5714))
-
-/* Workaround for address space limitation in Broadcom 5708, 5714, 5715 */
-#ifdef BROADCOM_ADDR_LIMIT
-#define	PXB_ADDR_LIMIT_HI		0xFFFFFFFFFFULL
-#endif
-
 #define	PXB_DEVICE_PLX_BAD_MSI_REV	0xAA	/* last known bad rev for MSI */
 
 #define	PXB_VENDOR_SUN			0x108E
@@ -205,6 +191,14 @@ extern void *pxb_state;
 #define	PXB_LINK_INIT			2
 #define	PXB_HOTPLUG_MSGS		3
 
+#ifdef	BCM_SW_WORKAROUNDS
+
+/* Workaround for address space limitation in Broadcom 5714/5715 */
+#define	PXB_ADDR_LIMIT_LO		0ull
+#define	PXB_ADDR_LIMIT_HI		((1ull << 40) - 1)
+
+#endif	/* BCM_SW_WORKAROUNDS */
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/usr/src/uts/sun4/io/px/px_util.c b/usr/src/uts/sun4/io/px/px_util.c
index 074957da06..04b0c23449 100644
--- a/usr/src/uts/sun4/io/px/px_util.c
+++ b/usr/src/uts/sun4/io/px/px_util.c
@@ -652,20 +652,3 @@ px_log_cfg_err(dev_info_t *dip, ushort_t status_reg, char *err_msg)
 
 	return (nerr);
 }
-
-/*
- * This is a software workaround to fix the Broadcom PCIe-PCI bridge
- * prefetch bug. Existence of a "cross-page-prefetch" property in the
- * child or parent node means the px nexus driver has to allocate an
- * extra page and make it valid one, for any DVMA request that comes
- * from any of the Broadcom child device.
- */
-boolean_t
-px_child_prefetch(dev_info_t *child)
-{
-	if (ddi_prop_exists(DDI_DEV_T_ANY, child, DDI_PROP_NOTPROM,
-	    "cross-page-prefetch"))
-		return (B_TRUE);
-
-	return (B_FALSE);
-}
diff --git a/usr/src/uts/sun4/io/px/px_util.h b/usr/src/uts/sun4/io/px/px_util.h
index 78530f8e02..0d186360e9 100644
--- a/usr/src/uts/sun4/io/px/px_util.h
+++ b/usr/src/uts/sun4/io/px/px_util.h
@@ -60,8 +60,6 @@ extern uint_t px_get_nreg_set(dev_info_t *child);
 extern uint_t px_get_nintr(dev_info_t *child);
 extern uint64_t px_get_cfg_pabase(px_t *px_p);
 
-extern boolean_t px_child_prefetch(dev_info_t *child);
-
 #ifdef	__cplusplus
 }
 #endif