6809618 bad mutex panic in vdds_cleanup_hybrid_res on low memory guest

6796579 panic caused by ldm remove-vswitch
6785269 bad mutex panic in vnet'vgen_ldc_rcv_worker routine on low-memory guest domain

12 files changed, 511 insertions, 362 deletions

diff --git a/usr/src/uts/sun4v/io/vio_util.c b/usr/src/uts/sun4v/io/vio_util.c
index 55acacbe4b..9d2dc03ac5 100644
--- a/usr/src/uts/sun4v/io/vio_util.c
+++ b/usr/src/uts/sun4v/io/vio_util.c
@@ -20,13 +20,12 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #include <sys/types.h>
 #include <sys/sysmacros.h>
-#include <sys/cmn_err.h>
 #include <sys/errno.h>
 #include <sys/kmem.h>
 #include <sys/ksynch.h>
@@ -35,14 +34,19 @@
 #include <sys/sunddi.h>
 #include <sys/vio_util.h>
 
+static int vio_pool_cleanup_retries = 10;	/* Max retries to free pool */
+static int vio_pool_cleanup_delay = 10000;	/* 10ms */
+
 /*
  * Create a pool of mblks from which future vio_allocb() requests
  * will be serviced.
  *
  * NOTE: num_mblks has to non-zero and a power-of-2
  *
- * Returns 0 on success or EINVAL if num_mblks is zero or not
- * a power of 2.
+ * Returns
+ *	0 on success
+ *	EINVAL if num_mblks is zero or not a power of 2.
+ *	ENOSPC if the pool could not be created due to alloc failures.
  */
 int
 vio_create_mblks(uint64_t num_mblks, size_t mblk_size, vio_mblk_pool_t **poolp)
@@ -87,8 +91,19 @@ vio_create_mblks(uint64_t num_mblks, size_t mblk_size, vio_mblk_pool_t **poolp)
 		vmp->mp = desballoc(vmp->datap, mblk_size, BPRI_MED,
 		    &vmp->reclaim);
 
-		if (vmp->mp == NULL)
-			continue;
+		if (vmp->mp == NULL) {
+			/* reset tail */
+			vmplp->tail = vmplp->head;
+
+			/*
+			 * vio_destroy_mblks() frees mblks that have been
+			 * allocated so far and then destroys the pool.
+			 */
+			vio_destroy_mblks(vmplp);
+
+			*poolp = NULL;
+			return (ENOSPC);
+		}
 
 		/* put this vmp on the free stack */
 		vmplp->quep[vmplp->tail] = vmp;
@@ -114,9 +129,11 @@ vio_create_mblks(uint64_t num_mblks, size_t mblk_size, vio_mblk_pool_t **poolp)
 int
 vio_destroy_mblks(vio_mblk_pool_t *vmplp)
 {
-	uint64_t i;
-	uint64_t num_mblks;
-	vio_mblk_t *vmp;
+	uint64_t	i;
+	uint64_t	num_mblks;
+	vio_mblk_t	*vmp;
+	int		pool_cleanup_retries = 0;
+
 
 	if (vmplp == NULL)
 		return (EINVAL);
@@ -125,8 +142,16 @@ vio_destroy_mblks(vio_mblk_pool_t *vmplp)
 	 * We can only destroy the pool once all the mblks have
 	 * been reclaimed.
 	 */
-	if (vmplp->head != vmplp->tail) {
+	do {
+		if (vmplp->head == vmplp->tail) {
+			break;
+		}
+
 		/* some mblks still in use */
+		drv_usecwait(vio_pool_cleanup_delay);
+	} while (++pool_cleanup_retries < vio_pool_cleanup_retries);
+
+	if (vmplp->head != vmplp->tail) {
 		return (EBUSY);
 	}
 
@@ -142,7 +167,14 @@ vio_destroy_mblks(vio_mblk_pool_t *vmplp)
 	vmplp->flag |= VMPL_FLAG_DESTROYING;
 	for (i = 0; i < num_mblks; i++) {
 		vmp = &(vmplp->basep[i]);
-		if (vmp->mp)
+		/*
+		 * It is possible that mblks have been allocated only upto
+		 * a certain index and the entire quelen has not been
+		 * initialized. This might happen due to desballoc() failure
+		 * while creating the pool. The below check handles this
+		 * condition.
+		 */
+		if (vmp->mp != NULL)
 			freeb(vmp->mp);
 	}
 	vmplp->flag &= ~(VMPL_FLAG_DESTROYING);
@@ -309,7 +341,8 @@ vio_destroy_multipools(vio_multi_pool_t *vmultip, vio_mblk_pool_t **fvmp)
 			}
 		}
 	}
-	kmem_free(vmultip->bufsz_tbl, vmultip->tbsz);
+	if (vmultip->tbsz != 0)
+		kmem_free(vmultip->bufsz_tbl, vmultip->tbsz);
 	vmultip->bufsz_tbl = NULL;
 	vmultip->nbuf_tbl = NULL;
 	vmultip->vmpp = NULL;
diff --git a/usr/src/uts/sun4v/io/vnet.c b/usr/src/uts/sun4v/io/vnet.c
index 1d7b9cd706..62f8b9d530 100644
--- a/usr/src/uts/sun4v/io/vnet.c
+++ b/usr/src/uts/sun4v/io/vnet.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -73,6 +73,7 @@ static int vnet_m_unicst(void *, const uint8_t *);
 mblk_t *vnet_m_tx(void *, mblk_t *);
 
 /* vnet internal functions */
+static int vnet_unattach(vnet_t *vnetp);
 static int vnet_mac_register(vnet_t *);
 static int vnet_read_mac_address(vnet_t *vnetp);
 
@@ -109,6 +110,9 @@ extern int vgen_init(void *vnetp, uint64_t regprop, dev_info_t *vnetdip,
     const uint8_t *macaddr, void **vgenhdl);
 extern int vgen_uninit(void *arg);
 extern int vgen_dds_tx(void *arg, void *dmsg);
+extern void vgen_mod_init(void);
+extern int vgen_mod_cleanup(void);
+extern void vgen_mod_fini(void);
 
 /* Externs that are imported from vnet_dds */
 extern void vdds_mod_init(void);
@@ -285,6 +289,7 @@ _init(void)
 		mac_fini_ops(&vnetops);
 	}
 	vdds_mod_init();
+	vgen_mod_init();
 	DBG1(NULL, "exit(%d)\n", status);
 	return (status);
 }
@@ -293,14 +298,19 @@ _init(void)
 int
 _fini(void)
 {
-	int status;
+	int		status;
 
 	DBG1(NULL, "enter\n");
 
+	status = vgen_mod_cleanup();
+	if (status != 0)
+		return (status);
+
 	status = mod_remove(&modlinkage);
 	if (status != 0)
 		return (status);
 	mac_fini_ops(&vnetops);
+	vgen_mod_fini();
 	vdds_mod_fini();
 
 	DBG1(NULL, "exit(%d)\n", status);
@@ -321,18 +331,14 @@ _info(struct modinfo *modinfop)
 static int
 vnetattach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 {
-	vnet_t		*vnetp;
-	int		status;
-	int		instance;
-	uint64_t	reg;
-	char		qname[TASKQ_NAMELEN];
-	enum	{ AST_init = 0x0, AST_vnet_alloc = 0x1,
-		AST_mac_alloc = 0x2, AST_read_macaddr = 0x4,
-		AST_vgen_init = 0x8, AST_fdbh_alloc = 0x10,
-		AST_vdds_init = 0x20, AST_taskq_create = 0x40,
-		AST_vnet_list = 0x80 } attach_state;
+	vnet_t			*vnetp;
+	int			status;
+	int			instance;
+	uint64_t		reg;
+	char			qname[TASKQ_NAMELEN];
+	vnet_attach_progress_t	attach_progress;
 
-	attach_state = AST_init;
+	attach_progress = AST_init;
 
 	switch (cmd) {
 	case DDI_ATTACH:
@@ -352,13 +358,13 @@ vnetattach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 	vnetp->instance = instance;
 	rw_init(&vnetp->vrwlock, NULL, RW_DRIVER, NULL);
 	rw_init(&vnetp->vsw_fp_rw, NULL, RW_DRIVER, NULL);
-	attach_state |= AST_vnet_alloc;
+	attach_progress |= AST_vnet_alloc;
 
 	status = vdds_init(vnetp);
 	if (status != 0) {
 		goto vnet_attach_fail;
 	}
-	attach_state |= AST_vdds_init;
+	attach_progress |= AST_vdds_init;
 
 	/* setup links to vnet_t from both devinfo and mac_t */
 	ddi_set_driver_private(dip, (caddr_t)vnetp);
@@ -368,7 +374,7 @@ vnetattach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 	if (status != DDI_SUCCESS) {
 		goto vnet_attach_fail;
 	}
-	attach_state |= AST_read_macaddr;
+	attach_progress |= AST_read_macaddr;
 
 	reg = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 	    DDI_PROP_DONTPASS, "reg", -1);
@@ -378,7 +384,7 @@ vnetattach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 	vnetp->reg = reg;
 
 	vnet_fdb_create(vnetp);
-	attach_state |= AST_fdbh_alloc;
+	attach_progress |= AST_fdbh_alloc;
 
 	(void) snprintf(qname, TASKQ_NAMELEN, "vnet_taskq%d", instance);
 	if ((vnetp->taskqp = ddi_taskq_create(dip, qname, 1,
@@ -387,7 +393,7 @@ vnetattach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 		    instance);
 		goto vnet_attach_fail;
 	}
-	attach_state |= AST_taskq_create;
+	attach_progress |= AST_taskq_create;
 
 	/* add to the list of vnet devices */
 	WRITE_ENTER(&vnet_rw);
@@ -395,7 +401,7 @@ vnetattach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 	vnet_headp = vnetp;
 	RW_EXIT(&vnet_rw);
 
-	attach_state |= AST_vnet_list;
+	attach_progress |= AST_vnet_list;
 
 	/*
 	 * Initialize the generic vnet plugin which provides
@@ -409,7 +415,7 @@ vnetattach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 		DERR(vnetp, "vgen_init() failed\n");
 		goto vnet_attach_fail;
 	}
-	attach_state |= AST_vgen_init;
+	attach_progress |= AST_vgen_init;
 
 	/* register with MAC layer */
 	status = vnet_mac_register(vnetp);
@@ -417,42 +423,16 @@ vnetattach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 		goto vnet_attach_fail;
 	}
 
+	attach_progress |= AST_macreg;
+
+	vnetp->attach_progress = attach_progress;
+
 	DBG1(NULL, "instance(%d) exit\n", instance);
 	return (DDI_SUCCESS);
 
 vnet_attach_fail:
-
-	if (attach_state & AST_vnet_list) {
-		vnet_t		**vnetpp;
-		/* unlink from instance(vnet_t) list */
-		WRITE_ENTER(&vnet_rw);
-		for (vnetpp = &vnet_headp; *vnetpp;
-		    vnetpp = &(*vnetpp)->nextp) {
-			if (*vnetpp == vnetp) {
-				*vnetpp = vnetp->nextp;
-				break;
-			}
-		}
-		RW_EXIT(&vnet_rw);
-	}
-
-	if (attach_state & AST_vdds_init) {
-		vdds_cleanup(vnetp);
-	}
-	if (attach_state & AST_taskq_create) {
-		ddi_taskq_destroy(vnetp->taskqp);
-	}
-	if (attach_state & AST_fdbh_alloc) {
-		vnet_fdb_destroy(vnetp);
-	}
-	if (attach_state & AST_vgen_init) {
-		(void) vgen_uninit(vnetp->vgenhdl);
-	}
-	if (attach_state & AST_vnet_alloc) {
-		rw_destroy(&vnetp->vrwlock);
-		rw_destroy(&vnetp->vsw_fp_rw);
-		KMEM_FREE(vnetp);
-	}
+	vnetp->attach_progress = attach_progress;
+	vnet_unattach(vnetp);
 	return (DDI_FAILURE);
 }
 
@@ -463,9 +443,7 @@ static int
 vnetdetach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 {
 	vnet_t		*vnetp;
-	vnet_t		**vnetpp;
 	int		instance;
-	int		rv;
 
 	instance = ddi_get_instance(dip);
 	DBG1(NULL, "instance(%d) enter\n", instance);
@@ -484,42 +462,98 @@ vnetdetach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 		goto vnet_detach_fail;
 	}
 
-	(void) vdds_cleanup(vnetp);
-	rv = vgen_uninit(vnetp->vgenhdl);
-	if (rv != DDI_SUCCESS) {
+	if (vnet_unattach(vnetp) != 0) {
 		goto vnet_detach_fail;
 	}
 
+	return (DDI_SUCCESS);
+
+vnet_detach_fail:
+	return (DDI_FAILURE);
+}
+
+/*
+ * Common routine to handle vnetattach() failure and vnetdetach(). Note that
+ * the only reason this function could fail is if mac_unregister() fails.
+ * Otherwise, this function must ensure that all resources are freed and return
+ * success.
+ */
+static int
+vnet_unattach(vnet_t *vnetp)
+{
+	vnet_attach_progress_t	attach_progress;
+
+	attach_progress = vnetp->attach_progress;
+
 	/*
-	 * Unregister from the MAC subsystem.  This can fail, in
-	 * particular if there are DLPI style-2 streams still open -
-	 * in which case we just return failure.
+	 * Unregister from the gldv3 subsystem. This can fail, in particular
+	 * if there are still any open references to this mac device; in which
+	 * case we just return failure without continuing to detach further.
 	 */
-	if (mac_unregister(vnetp->mh) != 0)
-		goto vnet_detach_fail;
-
-	/* unlink from instance(vnet_t) list */
-	WRITE_ENTER(&vnet_rw);
-	for (vnetpp = &vnet_headp; *vnetpp; vnetpp = &(*vnetpp)->nextp) {
-		if (*vnetpp == vnetp) {
-			*vnetpp = vnetp->nextp;
-			break;
+	if (attach_progress & AST_macreg) {
+		if (mac_unregister(vnetp->mh) != 0) {
+			return (1);
 		}
+		attach_progress &= ~AST_macreg;
 	}
-	RW_EXIT(&vnet_rw);
 
-	ddi_taskq_destroy(vnetp->taskqp);
-	/* destroy fdb */
-	vnet_fdb_destroy(vnetp);
+	/*
+	 * Now that we have unregistered from gldv3, we must finish all other
+	 * steps and successfully return from this function; otherwise we will
+	 * end up leaving the device in a broken/unusable state.
+	 *
+	 * First, release any hybrid resources assigned to this vnet device.
+	 */
+	if (attach_progress & AST_vdds_init) {
+		vdds_cleanup(vnetp);
+		attach_progress &= ~AST_vdds_init;
+	}
 
-	rw_destroy(&vnetp->vrwlock);
-	rw_destroy(&vnetp->vsw_fp_rw);
-	KMEM_FREE(vnetp);
+	/*
+	 * Uninit vgen. This stops further mdeg callbacks to this vnet
+	 * device and/or its ports; and detaches any existing ports.
+	 */
+	if (attach_progress & AST_vgen_init) {
+		vgen_uninit(vnetp->vgenhdl);
+		attach_progress &= ~AST_vgen_init;
+	}
 
-	return (DDI_SUCCESS);
+	/* Destroy the taskq. */
+	if (attach_progress & AST_taskq_create) {
+		ddi_taskq_destroy(vnetp->taskqp);
+		attach_progress &= ~AST_taskq_create;
+	}
 
-vnet_detach_fail:
-	return (DDI_FAILURE);
+	/* Destroy fdb. */
+	if (attach_progress & AST_fdbh_alloc) {
+		vnet_fdb_destroy(vnetp);
+		attach_progress &= ~AST_fdbh_alloc;
+	}
+
+	/* Remove from the device list */
+	if (attach_progress & AST_vnet_list) {
+		vnet_t		**vnetpp;
+		/* unlink from instance(vnet_t) list */
+		WRITE_ENTER(&vnet_rw);
+		for (vnetpp = &vnet_headp; *vnetpp;
+		    vnetpp = &(*vnetpp)->nextp) {
+			if (*vnetpp == vnetp) {
+				*vnetpp = vnetp->nextp;
+				break;
+			}
+		}
+		RW_EXIT(&vnet_rw);
+		attach_progress &= ~AST_vnet_list;
+	}
+
+	if (attach_progress & AST_vnet_alloc) {
+		rw_destroy(&vnetp->vrwlock);
+		rw_destroy(&vnetp->vsw_fp_rw);
+		attach_progress &= ~AST_vnet_list;
+		KMEM_FREE(vnetp);
+	}
+
+	return (0);
 }
 
 /* enable the device for transmit/receive */
diff --git a/usr/src/uts/sun4v/io/vnet_gen.c b/usr/src/uts/sun4v/io/vnet_gen.c
index 160100de01..54d6a2ebce 100644
--- a/usr/src/uts/sun4v/io/vnet_gen.c
+++ b/usr/src/uts/sun4v/io/vnet_gen.c
@@ -75,6 +75,9 @@ int vgen_init(void *vnetp, uint64_t regprop, dev_info_t *vnetdip,
     const uint8_t *macaddr, void **vgenhdl);
 int vgen_uninit(void *arg);
 int vgen_dds_tx(void *arg, void *dmsg);
+void vgen_mod_init(void);
+int vgen_mod_cleanup(void);
+void vgen_mod_fini(void);
 static int vgen_start(void *arg);
 static void vgen_stop(void *arg);
 static mblk_t *vgen_tx(void *arg, mblk_t *mp);
@@ -332,6 +335,9 @@ uint32_t vgen_tx_delay = 0x30;		/* delay when tx descr not available */
 
 int vgen_rcv_thread_enabled = 1;	/* Enable Recieve thread */
 
+static vio_mblk_pool_t	*vgen_rx_poolp = NULL;
+static krwlock_t	vgen_rw;
+
 /*
  * max # of packets accumulated prior to sending them up. It is best
  * to keep this at 60% of the number of recieve buffers.
@@ -556,9 +562,10 @@ vgen_uninit(void *arg)
 	while (rp != NULL) {
 		nrp = vgenp->rmp = rp->nextp;
 		if (vio_destroy_mblks(rp)) {
-			vgenp->rmp = rp;
-			mutex_exit(&vgenp->lock);
-			return (DDI_FAILURE);
+			WRITE_ENTER(&vgen_rw);
+			rp->nextp = vgen_rx_poolp;
+			vgen_rx_poolp = rp;
+			RW_EXIT(&vgen_rw);
 		}
 		rp = nrp;
 	}
@@ -584,6 +591,52 @@ vgen_uninit(void *arg)
 	return (DDI_SUCCESS);
 }
 
+/*
+ * module specific initialization common to all instances of vnet/vgen.
+ */
+void
+vgen_mod_init(void)
+{
+	rw_init(&vgen_rw, NULL, RW_DRIVER, NULL);
+}
+
+/*
+ * module specific cleanup common to all instances of vnet/vgen.
+ */
+int
+vgen_mod_cleanup(void)
+{
+	vio_mblk_pool_t	*poolp, *npoolp;
+
+	/*
+	 * If any rx mblk pools are still in use, return
+	 * error and stop the module from unloading.
+	 */
+	WRITE_ENTER(&vgen_rw);
+	poolp = vgen_rx_poolp;
+	while (poolp != NULL) {
+		npoolp = vgen_rx_poolp = poolp->nextp;
+		if (vio_destroy_mblks(poolp) != 0) {
+			vgen_rx_poolp = poolp;
+			RW_EXIT(&vgen_rw);
+			return (EBUSY);
+		}
+		poolp = npoolp;
+	}
+	RW_EXIT(&vgen_rw);
+
+	return (0);
+}
+
+/*
+ * module specific uninitialization common to all instances of vnet/vgen.
+ */
+void
+vgen_mod_fini(void)
+{
+	rw_destroy(&vgen_rw);
+}
+
 /* enable transmit/receive for the device */
 int
 vgen_start(void *arg)
@@ -1321,11 +1374,9 @@ vgen_detach_ports(vgen_t *vgenp)
 
 	plistp = &(vgenp->vgenports);
 	WRITE_ENTER(&plistp->rwlock);
-
 	while ((portp = plistp->headp) != NULL) {
 		vgen_port_detach(portp);
 	}
-
 	RW_EXIT(&plistp->rwlock);
 }
 
@@ -2830,9 +2881,18 @@ vgen_ldc_attach(vgen_port_t *portp, uint64_t ldc_id)
 	/* allocate receive resources */
 	status = vgen_init_multipools(ldcp);
 	if (status != 0) {
-		goto ldc_attach_failed;
+		/*
+		 * We do not return failure if receive mblk pools can't be
+		 * allocated; instead allocb(9F) will be used to dynamically
+		 * allocate buffers during receive.
+		 */
+		DWARN(vgenp, ldcp,
+		    "vnet%d: status(%d), failed to allocate rx mblk pools for "
+		    "channel(0x%lx)\n",
+		    vgenp->instance, status, ldcp->ldc_id);
+	} else {
+		attach_state |= AST_create_rxmblks;
 	}
-	attach_state |= AST_create_rxmblks;
 
 	/* Setup kstats for the channel */
 	instance = vgenp->instance;
@@ -2870,7 +2930,6 @@ ldc_attach_failed:
 	}
 	if (attach_state & AST_create_rxmblks) {
 		vio_mblk_pool_t *fvmp = NULL;
-
 		vio_destroy_multipools(&ldcp->vmp, &fvmp);
 		ASSERT(fvmp == NULL);
 	}
@@ -6636,7 +6695,6 @@ vgen_ldc_rcv_worker(void *arg)
 	CALLB_CPR_INIT(&cprinfo, &ldcp->rcv_thr_lock, callb_generic_cpr,
 	    "vnet_rcv_thread");
 	mutex_enter(&ldcp->rcv_thr_lock);
-	ldcp->rcv_thr_flags |= VGEN_WTHR_RUNNING;
 	while (!(ldcp->rcv_thr_flags & VGEN_WTHR_STOP)) {
 
 		CALLB_CPR_SAFE_BEGIN(&cprinfo);
@@ -6670,9 +6728,10 @@ vgen_ldc_rcv_worker(void *arg)
 	 * Update the run status and wakeup the thread that
 	 * has sent the stop request.
 	 */
-	ldcp->rcv_thr_flags &= ~VGEN_WTHR_RUNNING;
-	cv_signal(&ldcp->rcv_thr_cv);
+	ldcp->rcv_thr_flags &= ~VGEN_WTHR_STOP;
+	ldcp->rcv_thread = NULL;
 	CALLB_CPR_EXIT(&cprinfo);
+
 	thread_exit();
 	DBG1(vgenp, ldcp, "exit\n");
 }
@@ -6681,6 +6740,7 @@ vgen_ldc_rcv_worker(void *arg)
 static void
 vgen_stop_rcv_thread(vgen_ldc_t *ldcp)
 {
+	kt_did_t	tid = 0;
 	vgen_t *vgenp = LDC_TO_VGEN(ldcp);
 
 	DBG1(vgenp, ldcp, "enter\n");
@@ -6689,16 +6749,16 @@ vgen_stop_rcv_thread(vgen_ldc_t *ldcp)
 	 * wait until the receive thread stops.
 	 */
 	mutex_enter(&ldcp->rcv_thr_lock);
-	if (ldcp->rcv_thr_flags & VGEN_WTHR_RUNNING) {
+	if (ldcp->rcv_thread != NULL) {
+		tid = ldcp->rcv_thread->t_did;
 		ldcp->rcv_thr_flags |= VGEN_WTHR_STOP;
 		cv_signal(&ldcp->rcv_thr_cv);
-		DBG2(vgenp, ldcp, "waiting...");
-		while (ldcp->rcv_thr_flags & VGEN_WTHR_RUNNING) {
-			cv_wait(&ldcp->rcv_thr_cv, &ldcp->rcv_thr_lock);
-		}
 	}
 	mutex_exit(&ldcp->rcv_thr_lock);
-	ldcp->rcv_thread = NULL;
+
+	if (tid != 0) {
+		thread_join(tid);
+	}
 	DBG1(vgenp, ldcp, "exit\n");
 }
 
diff --git a/usr/src/uts/sun4v/io/vsw.c b/usr/src/uts/sun4v/io/vsw.c
index dc4635aad7..03a2bd0918 100644
--- a/usr/src/uts/sun4v/io/vsw.c
+++ b/usr/src/uts/sun4v/io/vsw.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -77,8 +77,10 @@
  */
 static	int vsw_attach(dev_info_t *, ddi_attach_cmd_t);
 static	int vsw_detach(dev_info_t *, ddi_detach_cmd_t);
+static	int vsw_unattach(vsw_t *vswp);
 static	int vsw_get_md_physname(vsw_t *, md_t *, mde_cookie_t, char *);
 static	int vsw_get_md_smodes(vsw_t *, md_t *, mde_cookie_t, uint8_t *);
+static	int vsw_mod_cleanup(void);
 
 /* MDEG routines */
 static	int vsw_mdeg_register(vsw_t *vswp);
@@ -128,7 +130,7 @@ extern int vsw_add_mcst(vsw_t *, uint8_t, uint64_t, void *);
 extern int vsw_del_mcst(vsw_t *, uint8_t, uint64_t, void *);
 extern void vsw_del_mcst_vsw(vsw_t *);
 extern mcst_addr_t *vsw_del_addr(uint8_t devtype, void *arg, uint64_t addr);
-extern int vsw_detach_ports(vsw_t *vswp);
+extern void vsw_detach_ports(vsw_t *vswp);
 extern int vsw_port_add(vsw_t *vswp, md_t *mdp, mde_cookie_t *node);
 extern int vsw_port_detach(vsw_t *vswp, int p_instance);
 static int vsw_port_update(vsw_t *vswp, md_t *curr_mdp, mde_cookie_t curr_mdex,
@@ -175,6 +177,8 @@ int	vsw_mac_open_retries = 300;	/* max # of mac_open() retries */
 					/* 300*3 = 900sec(15min) of max tmout */
 int	vsw_ldc_tx_delay = 5;		/* delay(ticks) for tx retries */
 int	vsw_ldc_tx_retries = 10;	/* # of ldc tx retries */
+int	vsw_ldc_retries = 5;		/* # of ldc_close() retries */
+int	vsw_ldc_delay = 1000;		/* 1 ms delay for ldc_close() */
 boolean_t vsw_ldc_rxthr_enabled = B_TRUE;	/* LDC Rx thread enabled */
 boolean_t vsw_ldc_txthr_enabled = B_TRUE;	/* LDC Tx thread enabled */
 
@@ -343,6 +347,7 @@ static void	*vsw_state;
  * Linked list of "vsw_t" structures - one per instance.
  */
 vsw_t		*vsw_head = NULL;
+vio_mblk_pool_t	*vsw_rx_poolp = NULL;
 krwlock_t	vsw_rw;
 
 /*
@@ -474,6 +479,10 @@ _fini(void)
 {
 	int status;
 
+	status = vsw_mod_cleanup();
+	if (status != 0)
+		return (status);
+
 	status = mod_remove(&modlinkage);
 	if (status != 0)
 		return (status);
@@ -494,23 +503,12 @@ _info(struct modinfo *modinfop)
 static int
 vsw_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 {
-	vsw_t		*vswp;
-	int		instance;
-	char		hashname[MAXNAMELEN];
-	char		qname[TASKQ_NAMELEN];
-	enum		{ PROG_init = 0x00,
-				PROG_locks = 0x01,
-				PROG_readmd = 0x02,
-				PROG_fdb = 0x04,
-				PROG_mfdb = 0x08,
-				PROG_taskq = 0x10,
-				PROG_swmode = 0x20,
-				PROG_macreg = 0x40,
-				PROG_mdreg = 0x80}
-			progress;
-
-	progress = PROG_init;
-	int		rv;
+	vsw_t			*vswp;
+	int			instance;
+	char			hashname[MAXNAMELEN];
+	char			qname[TASKQ_NAMELEN];
+	vsw_attach_progress_t	progress = PROG_init;
+	int			rv;
 
 	switch (cmd) {
 	case DDI_ATTACH:
@@ -639,6 +637,8 @@ vsw_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 
 	progress |= PROG_mdreg;
 
+	vswp->attach_progress = progress;
+
 	WRITE_ENTER(&vsw_rw);
 	vswp->next = vsw_head;
 	vsw_head = vswp;
@@ -650,52 +650,8 @@ vsw_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 vsw_attach_fail:
 	DERR(NULL, "vsw_attach: failed");
 
-	if (progress & PROG_mdreg) {
-		vsw_mdeg_unregister(vswp);
-		(void) vsw_detach_ports(vswp);
-	}
-
-	if (progress & PROG_macreg)
-		(void) vsw_mac_unregister(vswp);
-
-	if (progress & PROG_swmode) {
-		vsw_setup_switching_stop(vswp);
-		vsw_hio_cleanup(vswp);
-		mutex_enter(&vswp->mac_lock);
-		vsw_mac_close(vswp);
-		mutex_exit(&vswp->mac_lock);
-	}
-
-	if (progress & PROG_taskq)
-		ddi_taskq_destroy(vswp->taskq_p);
-
-	if (progress & PROG_mfdb)
-		mod_hash_destroy_hash(vswp->mfdb);
-
-	if (progress & PROG_fdb) {
-		vsw_destroy_vlans(vswp, VSW_LOCALDEV);
-		mod_hash_destroy_hash(vswp->fdb_hashp);
-	}
-
-	if (progress & PROG_readmd) {
-		if (VSW_PRI_ETH_DEFINED(vswp)) {
-			kmem_free(vswp->pri_types,
-			    sizeof (uint16_t) * vswp->pri_num_types);
-		}
-		(void) vio_destroy_mblks(vswp->pri_tx_vmp);
-	}
-
-	if (progress & PROG_locks) {
-		rw_destroy(&vswp->plist.lockrw);
-		rw_destroy(&vswp->mfdbrw);
-		rw_destroy(&vswp->if_lockrw);
-		rw_destroy(&vswp->maccl_rwlock);
-		cv_destroy(&vswp->sw_thr_cv);
-		mutex_destroy(&vswp->sw_thr_lock);
-		mutex_destroy(&vswp->mca_lock);
-		mutex_destroy(&vswp->mac_lock);
-	}
-
+	vswp->attach_progress = progress;
+	(void) vsw_unattach(vswp);
 	ddi_soft_state_free(vsw_state, instance);
 	return (DDI_FAILURE);
 }
@@ -703,7 +659,6 @@ vsw_attach_fail:
 static int
 vsw_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 {
-	vio_mblk_pool_t		*poolp, *npoolp;
 	vsw_t			**vswpp, *vswp;
 	int 			instance;
 
@@ -725,118 +680,189 @@ vsw_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 
 	D2(vswp, "detaching instance %d", instance);
 
-	/* Stop any pending thread to setup switching mode. */
-	vsw_setup_switching_stop(vswp);
+	if (vsw_unattach(vswp) != 0) {
+		return (DDI_FAILURE);
+	}
 
-	/* Cleanup the interface's mac client */
-	vsw_mac_client_cleanup(vswp, NULL, VSW_LOCALDEV);
+	ddi_remove_minor_node(dip, NULL);
 
-	if (vswp->if_state & VSW_IF_REG) {
-		if (vsw_mac_unregister(vswp) != 0) {
-			cmn_err(CE_WARN, "!vsw%d: Unable to detach from "
-			    "MAC layer", vswp->instance);
-			return (DDI_FAILURE);
+	WRITE_ENTER(&vsw_rw);
+	for (vswpp = &vsw_head; *vswpp; vswpp = &(*vswpp)->next) {
+		if (*vswpp == vswp) {
+			*vswpp = vswp->next;
+			break;
 		}
 	}
+	RW_EXIT(&vsw_rw);
 
-	vsw_mdeg_unregister(vswp);
-
-	/* cleanup HybridIO */
-	vsw_hio_cleanup(vswp);
+	ddi_soft_state_free(vsw_state, instance);
 
-	if (vsw_detach_ports(vswp) != 0) {
-		cmn_err(CE_WARN, "!vsw%d: Unable to unconfigure ports",
-		    vswp->instance);
-		return (DDI_FAILURE);
-	}
+	return (DDI_SUCCESS);
+}
 
-	rw_destroy(&vswp->if_lockrw);
+/*
+ * Common routine to handle vsw_attach() failure and vsw_detach(). Note that
+ * the only reason this function could fail is if mac_unregister() fails.
+ * Otherwise, this function must ensure that all resources are freed and return
+ * success.
+ */
+static int
+vsw_unattach(vsw_t *vswp)
+{
+	vio_mblk_pool_t		*poolp, *npoolp;
+	vsw_attach_progress_t	progress;
 
-	vsw_mac_cleanup_ports(vswp);
+	progress = vswp->attach_progress;
 
 	/*
-	 * Now that the ports have been deleted, stop and close
-	 * the physical device.
+	 * Unregister from the gldv3 subsystem. This can fail, in particular
+	 * if there are still any open references to this mac device; in which
+	 * case we just return failure without continuing to detach further.
 	 */
-	mutex_enter(&vswp->mac_lock);
-	vsw_mac_close(vswp);
-	mutex_exit(&vswp->mac_lock);
-
-	mutex_destroy(&vswp->mac_lock);
-	cv_destroy(&vswp->sw_thr_cv);
-	mutex_destroy(&vswp->sw_thr_lock);
-	rw_destroy(&vswp->maccl_rwlock);
+	if (progress & PROG_macreg) {
+		if (vsw_mac_unregister(vswp) != 0) {
+			cmn_err(CE_WARN, "!vsw%d: Unable to detach from "
+			    "MAC layer", vswp->instance);
+			return (1);
+		}
+		progress &= ~PROG_macreg;
+	}
 
 	/*
-	 * Destroy any free pools that may still exist.
+	 * Now that we have unregistered from gldv3, we must finish all other
+	 * steps and successfully return from this function; otherwise we will
+	 * end up leaving the device in a broken/unusable state.
+	 *
+	 * If we have registered with mdeg, unregister now to stop further
+	 * callbacks to this vsw device and/or its ports. Then, detach any
+	 * existing ports.
 	 */
-	poolp = vswp->rxh;
-	while (poolp != NULL) {
-		npoolp = vswp->rxh = poolp->nextp;
-		if (vio_destroy_mblks(poolp) != 0) {
-			vswp->rxh = poolp;
-			return (DDI_FAILURE);
+	if (progress & PROG_mdreg) {
+		vsw_mdeg_unregister(vswp);
+		vsw_detach_ports(vswp);
+
+		/*
+		 * At this point, we attempt to free receive mblk pools that
+		 * couldn't be destroyed when the ports were detached; if this
+		 * attempt also fails, we hook up the pool(s) to the module so
+		 * they can be cleaned up in _fini().
+		 */
+		poolp = vswp->rxh;
+		while (poolp != NULL) {
+			npoolp = vswp->rxh = poolp->nextp;
+			if (vio_destroy_mblks(poolp) != 0) {
+				WRITE_ENTER(&vsw_rw);
+				poolp->nextp = vsw_rx_poolp;
+				vsw_rx_poolp = poolp;
+				RW_EXIT(&vsw_rw);
+			}
+			poolp = npoolp;
 		}
-		poolp = npoolp;
+		progress &= ~PROG_mdreg;
 	}
 
 	/*
-	 * Remove this instance from any entries it may be on in
-	 * the hash table by using the list of addresses maintained
-	 * in the vsw_t structure.
+	 * If we have started a thread to setup the switching mode, stop it, if
+	 * it is still running. If it has finished setting up the switching
+	 * mode, then we need to clean up some additional things if we are
+	 * running in L2 mode: first free up any hybrid resources; then stop
+	 * and close the underlying physical device. Note that we would have
+	 * already released all per mac_client resources (ucast, mcast addrs,
+	 * hio-shares etc) as all the ports are detached and if the vsw device
+	 * itself was in use as an interface, it has been unplumbed (otherwise
+	 * mac_unregister() above would fail).
 	 */
-	vsw_del_mcst_vsw(vswp);
+	if (progress & PROG_swmode) {
 
-	vswp->mcap = NULL;
-	mutex_destroy(&vswp->mca_lock);
+		vsw_setup_switching_stop(vswp);
+
+		if (vswp->hio_capable == B_TRUE) {
+			vsw_hio_cleanup(vswp);
+			vswp->hio_capable = B_FALSE;
+		}
+
+		mutex_enter(&vswp->mac_lock);
+		vsw_mac_close(vswp);
+		mutex_exit(&vswp->mac_lock);
+
+		progress &= ~PROG_swmode;
+	}
 
 	/*
 	 * By now any pending tasks have finished and the underlying
 	 * ldc's have been destroyed, so its safe to delete the control
 	 * message taskq.
 	 */
-	if (vswp->taskq_p != NULL)
+	if (progress & PROG_taskq) {
 		ddi_taskq_destroy(vswp->taskq_p);
+		progress &= ~PROG_taskq;
+	}
 
-	/*
-	 * At this stage all the data pointers in the hash table
-	 * should be NULL, as all the ports have been removed and will
-	 * have deleted themselves from the port lists which the data
-	 * pointers point to. Hence we can destroy the table using the
-	 * default destructors.
-	 */
-	D2(vswp, "vsw_detach: destroying hash tables..");
-	vsw_destroy_vlans(vswp, VSW_LOCALDEV);
-	mod_hash_destroy_hash(vswp->fdb_hashp);
-	vswp->fdb_hashp = NULL;
+	/* Destroy the multicast hash table */
+	if (progress & PROG_mfdb) {
+		mod_hash_destroy_hash(vswp->mfdb);
+		progress &= ~PROG_mfdb;
+	}
 
-	WRITE_ENTER(&vswp->mfdbrw);
-	mod_hash_destroy_hash(vswp->mfdb);
-	vswp->mfdb = NULL;
-	RW_EXIT(&vswp->mfdbrw);
-	rw_destroy(&vswp->mfdbrw);
+	/* Destroy the vlan hash table and fdb */
+	if (progress & PROG_fdb) {
+		vsw_destroy_vlans(vswp, VSW_LOCALDEV);
+		mod_hash_destroy_hash(vswp->fdb_hashp);
+		progress &= ~PROG_fdb;
+	}
 
-	/* free pri_types table */
-	if (VSW_PRI_ETH_DEFINED(vswp)) {
-		kmem_free(vswp->pri_types,
-		    sizeof (uint16_t) * vswp->pri_num_types);
-		(void) vio_destroy_mblks(vswp->pri_tx_vmp);
+	if (progress & PROG_readmd) {
+		if (VSW_PRI_ETH_DEFINED(vswp)) {
+			kmem_free(vswp->pri_types,
+			    sizeof (uint16_t) * vswp->pri_num_types);
+			(void) vio_destroy_mblks(vswp->pri_tx_vmp);
+		}
+		progress &= ~PROG_readmd;
 	}
 
-	ddi_remove_minor_node(dip, NULL);
+	if (progress & PROG_locks) {
+		rw_destroy(&vswp->plist.lockrw);
+		rw_destroy(&vswp->mfdbrw);
+		rw_destroy(&vswp->if_lockrw);
+		rw_destroy(&vswp->maccl_rwlock);
+		cv_destroy(&vswp->sw_thr_cv);
+		mutex_destroy(&vswp->sw_thr_lock);
+		mutex_destroy(&vswp->mca_lock);
+		mutex_destroy(&vswp->mac_lock);
+		progress &= ~PROG_locks;
+	}
+
+	vswp->attach_progress = progress;
+
+	return (0);
+}
 
-	rw_destroy(&vswp->plist.lockrw);
+/*
+ * one time cleanup.
+ */
+static int
+vsw_mod_cleanup(void)
+{
+	vio_mblk_pool_t		*poolp, *npoolp;
+
+	/*
+	 * If any rx mblk pools are still in use, return
+	 * error and stop the module from unloading.
+	 */
 	WRITE_ENTER(&vsw_rw);
-	for (vswpp = &vsw_head; *vswpp; vswpp = &(*vswpp)->next) {
-		if (*vswpp == vswp) {
-			*vswpp = vswp->next;
-			break;
+	poolp = vsw_rx_poolp;
+	while (poolp != NULL) {
+		npoolp = vsw_rx_poolp = poolp->nextp;
+		if (vio_destroy_mblks(poolp) != 0) {
+			vsw_rx_poolp = poolp;
+			RW_EXIT(&vsw_rw);
+			return (EBUSY);
 		}
+		poolp = npoolp;
 	}
 	RW_EXIT(&vsw_rw);
-	ddi_soft_state_free(vsw_state, instance);
 
-	return (DDI_SUCCESS);
+	return (0);
 }
 
 /*
diff --git a/usr/src/uts/sun4v/io/vsw_hio.c b/usr/src/uts/sun4v/io/vsw_hio.c
index 084c338548..19f02c7d17 100644
--- a/usr/src/uts/sun4v/io/vsw_hio.c
+++ b/usr/src/uts/sun4v/io/vsw_hio.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -313,6 +313,7 @@ vsw_hio_free_share(vsw_share_t *vsharep)
 	RW_EXIT(&portp->maccl_rwlock);
 	vsharep->vs_state = VSW_SHARE_FREE;
 	vsharep->vs_macaddr = 0;
+	vsharep->vs_portp = NULL;
 
 	/* DERR only for printing by default */
 	DERR(vswp, "Share freed for ldc_id=0x%lx Cookie=0x%lX",
diff --git a/usr/src/uts/sun4v/io/vsw_ldc.c b/usr/src/uts/sun4v/io/vsw_ldc.c
index bfd6dde2fb..e68965fd70 100644
--- a/usr/src/uts/sun4v/io/vsw_ldc.c
+++ b/usr/src/uts/sun4v/io/vsw_ldc.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -72,18 +72,18 @@
 #include <sys/vlan.h>
 
 /* Port add/deletion/etc routines */
-static	int vsw_port_delete(vsw_port_t *port);
+static	void vsw_port_delete(vsw_port_t *port);
 static	int vsw_ldc_attach(vsw_port_t *port, uint64_t ldc_id);
-static	int vsw_ldc_detach(vsw_port_t *port, uint64_t ldc_id);
+static	void vsw_ldc_detach(vsw_port_t *port, uint64_t ldc_id);
 static	int vsw_init_ldcs(vsw_port_t *port);
-static	int vsw_uninit_ldcs(vsw_port_t *port);
+static	void vsw_uninit_ldcs(vsw_port_t *port);
 static	int vsw_ldc_init(vsw_ldc_t *ldcp);
-static	int vsw_ldc_uninit(vsw_ldc_t *ldcp);
-static	int vsw_drain_ldcs(vsw_port_t *port);
-static	int vsw_drain_port_taskq(vsw_port_t *port);
+static	void vsw_ldc_uninit(vsw_ldc_t *ldcp);
+static	void vsw_drain_ldcs(vsw_port_t *port);
+static	void vsw_drain_port_taskq(vsw_port_t *port);
 static	void vsw_marker_task(void *);
 static	int vsw_plist_del_node(vsw_t *, vsw_port_t *port);
-int vsw_detach_ports(vsw_t *vswp);
+void vsw_detach_ports(vsw_t *vswp);
 int vsw_port_add(vsw_t *vswp, md_t *mdp, mde_cookie_t *node);
 mcst_addr_t *vsw_del_addr(uint8_t devtype, void *arg, uint64_t addr);
 int vsw_port_detach(vsw_t *vswp, int p_instance);
@@ -218,6 +218,8 @@ extern int vsw_desc_delay;
 extern int vsw_read_attempts;
 extern int vsw_ldc_tx_delay;
 extern int vsw_ldc_tx_retries;
+extern int vsw_ldc_retries;
+extern int vsw_ldc_delay;
 extern boolean_t vsw_ldc_rxthr_enabled;
 extern boolean_t vsw_ldc_txthr_enabled;
 extern uint32_t vsw_ntxds;
@@ -442,9 +444,7 @@ vsw_port_detach(vsw_t *vswp, int p_instance)
 	/* Remove any multicast addresses.. */
 	vsw_del_mcst_port(port);
 
-	if (vsw_port_delete(port)) {
-		return (1);
-	}
+	vsw_port_delete(port);
 
 	D1(vswp, "%s: exit: p_instance(%d)", __func__, p_instance);
 	return (0);
@@ -452,10 +452,8 @@ vsw_port_detach(vsw_t *vswp, int p_instance)
 
 /*
  * Detach all active ports.
- *
- * Returns 0 on success, 1 on failure.
  */
-int
+void
 vsw_detach_ports(vsw_t *vswp)
 {
 	vsw_port_list_t 	*plist = &vswp->plist;
@@ -466,12 +464,10 @@ vsw_detach_ports(vsw_t *vswp)
 	WRITE_ENTER(&plist->lockrw);
 
 	while ((port = plist->head) != NULL) {
-		if (vsw_plist_del_node(vswp, port)) {
-			DERR(vswp, "%s: Error deleting port %d"
-			    " from port list", __func__, port->p_instance);
-			RW_EXIT(&plist->lockrw);
-			return (1);
-		}
+		(void) vsw_plist_del_node(vswp, port);
+
+		/* cleanup any HybridIO for this port */
+		vsw_hio_stop_port(port);
 
 		/* Cleanup and close the mac client */
 		vsw_mac_client_cleanup(vswp, port, VSW_VNETPORT);
@@ -489,26 +485,18 @@ vsw_detach_ports(vsw_t *vswp)
 		 * list.
 		 */
 		RW_EXIT(&plist->lockrw);
-		if (vsw_port_delete(port)) {
-			DERR(vswp, "%s: Error deleting port %d",
-			    __func__, port->p_instance);
-			return (1);
-		}
+		vsw_port_delete(port);
 		WRITE_ENTER(&plist->lockrw);
 	}
 	RW_EXIT(&plist->lockrw);
 
 	D1(vswp, "%s: exit", __func__);
-
-	return (0);
 }
 
 /*
  * Delete the specified port.
- *
- * Returns 0 on success, 1 on failure.
  */
-static int
+static void
 vsw_port_delete(vsw_port_t *port)
 {
 	vsw_ldc_list_t 		*ldcl;
@@ -517,32 +505,24 @@ vsw_port_delete(vsw_port_t *port)
 
 	D1(vswp, "%s: enter : port id %d", __func__, port->p_instance);
 
-	(void) vsw_uninit_ldcs(port);
+	vsw_uninit_ldcs(port);
 
 	/*
 	 * Wait for any pending ctrl msg tasks which reference this
 	 * port to finish.
 	 */
-	if (vsw_drain_port_taskq(port))
-		return (1);
+	vsw_drain_port_taskq(port);
 
 	/*
 	 * Wait for any active callbacks to finish
 	 */
-	if (vsw_drain_ldcs(port))
-		return (1);
+	vsw_drain_ldcs(port);
 
 	ldcl = &port->p_ldclist;
 	num_ldcs = port->num_ldcs;
 	WRITE_ENTER(&ldcl->lockrw);
 	while (num_ldcs > 0) {
-		if (vsw_ldc_detach(port, ldcl->head->ldc_id) != 0) {
-			cmn_err(CE_WARN, "!vsw%d: unable to detach ldc %ld",
-			    vswp->instance, ldcl->head->ldc_id);
-			RW_EXIT(&ldcl->lockrw);
-			port->num_ldcs = num_ldcs;
-			return (1);
-		}
+		vsw_ldc_detach(port, ldcl->head->ldc_id);
 		num_ldcs--;
 	}
 	RW_EXIT(&ldcl->lockrw);
@@ -568,8 +548,6 @@ vsw_port_delete(vsw_port_t *port)
 	kmem_free(port, sizeof (vsw_port_t));
 
 	D1(vswp, "%s: exit", __func__);
-
-	return (0);
 }
 
 static int
@@ -859,16 +837,15 @@ ldc_attach_fail:
 /*
  * Detach a logical domain channel (ldc) belonging to a
  * particular port.
- *
- * Returns 0 on success, 1 on failure.
  */
-static int
+static void
 vsw_ldc_detach(vsw_port_t *port, uint64_t ldc_id)
 {
 	vsw_t 		*vswp = port->p_vswp;
 	vsw_ldc_t 	*ldcp, *prev_ldcp;
 	vsw_ldc_list_t	*ldcl = &port->p_ldclist;
 	int 		rv;
+	int		retries = 0;
 
 	prev_ldcp = ldcl->head;
 	for (; (ldcp = prev_ldcp) != NULL; prev_ldcp = ldcp->ldc_next) {
@@ -878,10 +855,7 @@ vsw_ldc_detach(vsw_port_t *port, uint64_t ldc_id)
 	}
 
 	/* specified ldc id not found */
-	if (ldcp == NULL) {
-		DERR(vswp, "%s: ldcp = NULL", __func__);
-		return (1);
-	}
+	ASSERT(ldcp != NULL);
 
 	D2(vswp, "%s: detaching channel %lld", __func__, ldcp->ldc_id);
 
@@ -916,13 +890,18 @@ vsw_ldc_detach(vsw_port_t *port, uint64_t ldc_id)
 	vsw_free_lane_resources(ldcp, OUTBOUND);
 
 	/*
-	 * If the close fails we are in serious trouble, as won't
-	 * be able to delete the parent port.
+	 * Close the channel, retry on EAAGIN.
 	 */
-	if ((rv = ldc_close(ldcp->ldc_handle)) != 0) {
-		DERR(vswp, "%s: error %d closing channel %lld",
-		    __func__, rv, ldcp->ldc_id);
-		return (1);
+	while ((rv = ldc_close(ldcp->ldc_handle)) == EAGAIN) {
+		if (++retries > vsw_ldc_retries) {
+			break;
+		}
+		drv_usecwait(vsw_ldc_delay);
+	}
+	if (rv != 0) {
+		cmn_err(CE_NOTE,
+		    "!vsw%d: Error(%d) closing the channel(0x%lx)\n",
+		    vswp->instance, rv, ldcp->ldc_id);
 	}
 
 	(void) ldc_fini(ldcp->ldc_handle);
@@ -954,8 +933,6 @@ vsw_ldc_detach(vsw_port_t *port, uint64_t ldc_id)
 	rw_destroy(&ldcp->lane_out.dlistrw);
 
 	kmem_free(ldcp, sizeof (vsw_ldc_t));
-
-	return (0);
 }
 
 /*
@@ -1047,7 +1024,7 @@ vsw_ldc_init(vsw_ldc_t *ldcp)
 }
 
 /* disable callbacks on the channel */
-static int
+static void
 vsw_ldc_uninit(vsw_ldc_t *ldcp)
 {
 	vsw_t	*vswp = ldcp->ldc_vswp;
@@ -1059,10 +1036,8 @@ vsw_ldc_uninit(vsw_ldc_t *ldcp)
 
 	rv = ldc_set_cb_mode(ldcp->ldc_handle, LDC_CB_DISABLE);
 	if (rv != 0) {
-		DERR(vswp, "vsw_ldc_uninit(%lld): error disabling "
+		cmn_err(CE_NOTE, "!vsw_ldc_uninit(%ld): error disabling "
 		    "interrupts (rv = %d)\n", ldcp->ldc_id, rv);
-		LDC_EXIT_LOCK(ldcp);
-		return (1);
 	}
 
 	mutex_enter(&ldcp->status_lock);
@@ -1072,8 +1047,6 @@ vsw_ldc_uninit(vsw_ldc_t *ldcp)
 	LDC_EXIT_LOCK(ldcp);
 
 	D1(vswp, "vsw_ldc_uninit: exit: id(%lx)", ldcp->ldc_id);
-
-	return (0);
 }
 
 static int
@@ -1092,7 +1065,7 @@ vsw_init_ldcs(vsw_port_t *port)
 	return (0);
 }
 
-static int
+static void
 vsw_uninit_ldcs(vsw_port_t *port)
 {
 	vsw_ldc_list_t	*ldcl = &port->p_ldclist;
@@ -1103,13 +1076,11 @@ vsw_uninit_ldcs(vsw_port_t *port)
 	READ_ENTER(&ldcl->lockrw);
 	ldcp =  ldcl->head;
 	for (; ldcp  != NULL; ldcp = ldcp->ldc_next) {
-		(void) vsw_ldc_uninit(ldcp);
+		vsw_ldc_uninit(ldcp);
 	}
 	RW_EXIT(&ldcl->lockrw);
 
 	D1(NULL, "vsw_uninit_ldcs: exit\n");
-
-	return (0);
 }
 
 /*
@@ -1147,7 +1118,7 @@ vsw_uninit_ldcs(vsw_port_t *port)
  * the case where the callback has finished but the ldc framework has not yet
  * cleared the active flag. In this case we would never get a cv_signal.
  */
-static int
+static void
 vsw_drain_ldcs(vsw_port_t *port)
 {
 	vsw_ldc_list_t	*ldcl = &port->p_ldclist;
@@ -1202,7 +1173,6 @@ vsw_drain_ldcs(vsw_port_t *port)
 	RW_EXIT(&ldcl->lockrw);
 
 	D1(vswp, "%s: exit", __func__);
-	return (0);
 }
 
 /*
@@ -1211,7 +1181,7 @@ vsw_drain_ldcs(vsw_port_t *port)
  * Prior to this function being invoked each channel under this port
  * should have been quiesced via ldc_set_cb_mode(DISABLE).
  */
-static int
+static void
 vsw_drain_port_taskq(vsw_port_t *port)
 {
 	vsw_t		*vswp = port->p_vswp;
@@ -1229,10 +1199,10 @@ vsw_drain_port_taskq(vsw_port_t *port)
 	if ((vswp->taskq_p == NULL) ||
 	    (ddi_taskq_dispatch(vswp->taskq_p, vsw_marker_task,
 	    port, DDI_NOSLEEP) != DDI_SUCCESS)) {
-		DERR(vswp, "%s: unable to dispatch marker task",
-		    __func__);
+		cmn_err(CE_NOTE, "!vsw%d: unable to dispatch marker task",
+		    vswp->instance);
 		mutex_exit(&port->state_lock);
-		return (1);
+		return;
 	}
 
 	/*
@@ -1244,8 +1214,6 @@ vsw_drain_port_taskq(vsw_port_t *port)
 	mutex_exit(&port->state_lock);
 
 	D1(vswp, "%s: exit", __func__);
-
-	return (0);
 }
 
 static void
@@ -4943,10 +4911,13 @@ vsw_create_dring_info_pkt(vsw_ldc_t *ldcp)
 	/* Allocate pools of receive mblks */
 	rv = vsw_init_multipools(ldcp, vswp);
 	if (rv) {
+		/*
+		 * We do not return failure if receive mblk pools can't be
+		 * allocated, instead allocb(9F) will be used to dynamically
+		 * allocate buffers during receive.
+		 */
 		DWARN(vswp, "%s: unable to create free mblk pools for"
 		    " channel %ld (rv %d)", __func__, ldcp->ldc_id, rv);
-		vsw_free_lane_resources(ldcp, OUTBOUND);
-		return (NULL);
 	}
 
 	mp = kmem_zalloc(sizeof (vio_dring_reg_msg_t), KM_SLEEP);
@@ -5868,7 +5839,6 @@ vsw_ldc_rx_worker(void *arg)
 	CALLB_CPR_INIT(&cprinfo, &ldcp->rx_thr_lock, callb_generic_cpr,
 	    "vsw_rx_thread");
 	mutex_enter(&ldcp->rx_thr_lock);
-	ldcp->rx_thr_flags |= VSW_WTHR_RUNNING;
 	while (!(ldcp->rx_thr_flags & VSW_WTHR_STOP)) {
 
 		CALLB_CPR_SAFE_BEGIN(&cprinfo);
@@ -5904,8 +5874,8 @@ vsw_ldc_rx_worker(void *arg)
 	 * Update the run status and wakeup the thread that
 	 * has sent the stop request.
 	 */
-	ldcp->rx_thr_flags &= ~VSW_WTHR_RUNNING;
-	cv_signal(&ldcp->rx_thr_cv);
+	ldcp->rx_thr_flags &= ~VSW_WTHR_STOP;
+	ldcp->rx_thread = NULL;
 	CALLB_CPR_EXIT(&cprinfo);
 	D1(vswp, "%s(%lld):exit\n", __func__, ldcp->ldc_id);
 	thread_exit();
@@ -5915,7 +5885,8 @@ vsw_ldc_rx_worker(void *arg)
 static void
 vsw_stop_rx_thread(vsw_ldc_t *ldcp)
 {
-	vsw_t *vswp = ldcp->ldc_vswp;
+	kt_did_t	tid = 0;
+	vsw_t		*vswp = ldcp->ldc_vswp;
 
 	D1(vswp, "%s(%lld):enter\n", __func__, ldcp->ldc_id);
 	/*
@@ -5923,15 +5894,16 @@ vsw_stop_rx_thread(vsw_ldc_t *ldcp)
 	 * wait until the receive thread stops.
 	 */
 	mutex_enter(&ldcp->rx_thr_lock);
-	if (ldcp->rx_thr_flags & VSW_WTHR_RUNNING) {
+	if (ldcp->rx_thread != NULL) {
+		tid = ldcp->rx_thread->t_did;
 		ldcp->rx_thr_flags |= VSW_WTHR_STOP;
 		cv_signal(&ldcp->rx_thr_cv);
-		while (ldcp->rx_thr_flags & VSW_WTHR_RUNNING) {
-			cv_wait(&ldcp->rx_thr_cv, &ldcp->rx_thr_lock);
-		}
 	}
 	mutex_exit(&ldcp->rx_thr_lock);
-	ldcp->rx_thread = NULL;
+
+	if (tid != 0) {
+		thread_join(tid);
+	}
 	D1(vswp, "%s(%lld):exit\n", __func__, ldcp->ldc_id);
 }
 
@@ -5953,7 +5925,6 @@ vsw_ldc_tx_worker(void *arg)
 	CALLB_CPR_INIT(&cprinfo, &ldcp->tx_thr_lock, callb_generic_cpr,
 	    "vnet_tx_thread");
 	mutex_enter(&ldcp->tx_thr_lock);
-	ldcp->tx_thr_flags |= VSW_WTHR_RUNNING;
 	while (!(ldcp->tx_thr_flags & VSW_WTHR_STOP)) {
 
 		CALLB_CPR_SAFE_BEGIN(&cprinfo);
@@ -5994,8 +5965,8 @@ vsw_ldc_tx_worker(void *arg)
 	 * Update the run status and wakeup the thread that
 	 * has sent the stop request.
 	 */
-	ldcp->tx_thr_flags &= ~VSW_WTHR_RUNNING;
-	cv_signal(&ldcp->tx_thr_cv);
+	ldcp->tx_thr_flags &= ~VSW_WTHR_STOP;
+	ldcp->tx_thread = NULL;
 	CALLB_CPR_EXIT(&cprinfo);
 	D1(vswp, "%s(%lld):exit\n", __func__, ldcp->ldc_id);
 	thread_exit();
@@ -6005,7 +5976,8 @@ vsw_ldc_tx_worker(void *arg)
 static void
 vsw_stop_tx_thread(vsw_ldc_t *ldcp)
 {
-	vsw_t *vswp = ldcp->ldc_vswp;
+	kt_did_t	tid = 0;
+	vsw_t		*vswp = ldcp->ldc_vswp;
 
 	D1(vswp, "%s(%lld):enter\n", __func__, ldcp->ldc_id);
 	/*
@@ -6013,15 +5985,17 @@ vsw_stop_tx_thread(vsw_ldc_t *ldcp)
 	 * wait until the receive thread stops.
 	 */
 	mutex_enter(&ldcp->tx_thr_lock);
-	if (ldcp->tx_thr_flags & VSW_WTHR_RUNNING) {
+	if (ldcp->tx_thread != NULL) {
+		tid = ldcp->tx_thread->t_did;
 		ldcp->tx_thr_flags |= VSW_WTHR_STOP;
 		cv_signal(&ldcp->tx_thr_cv);
-		while (ldcp->tx_thr_flags & VSW_WTHR_RUNNING) {
-			cv_wait(&ldcp->tx_thr_cv, &ldcp->tx_thr_lock);
-		}
 	}
 	mutex_exit(&ldcp->tx_thr_lock);
-	ldcp->tx_thread = NULL;
+
+	if (tid != 0) {
+		thread_join(tid);
+	}
+
 	D1(vswp, "%s(%lld):exit\n", __func__, ldcp->ldc_id);
 }
 
diff --git a/usr/src/uts/sun4v/io/vsw_phys.c b/usr/src/uts/sun4v/io/vsw_phys.c
index cd4f355e42..39c55f35c6 100644
--- a/usr/src/uts/sun4v/io/vsw_phys.c
+++ b/usr/src/uts/sun4v/io/vsw_phys.c
@@ -924,11 +924,13 @@ vsw_tx_msg(vsw_t *vswp, mblk_t *mp, int caller, vsw_port_t *port)
 	mch = (caller == VSW_LOCALDEV) ? vswp->mch : port->p_mch;
 	muh = (caller == VSW_LOCALDEV) ? vswp->muh : port->p_muh;
 
-	if ((mch != NULL) && (muh != NULL)) {
-		/* packets are sent or dropped */
-		(void) mac_tx(mch, mp, 0, MAC_DROP_ON_NO_DESC, NULL);
+	if (mch == NULL || muh == NULL) {
+		RW_MACCL_EXIT(vswp, port, caller);
+		return (mp);
 	}
 
+	/* packets are sent or dropped */
+	(void) mac_tx(mch, mp, 0, MAC_DROP_ON_NO_DESC, NULL);
 	RW_MACCL_EXIT(vswp, port, caller);
 	return (NULL);
 }
diff --git a/usr/src/uts/sun4v/sys/vio_util.h b/usr/src/uts/sun4v/sys/vio_util.h
index ce1cdc0efe..a676cc9b6d 100644
--- a/usr/src/uts/sun4v/sys/vio_util.h
+++ b/usr/src/uts/sun4v/sys/vio_util.h
@@ -20,15 +20,13 @@
  */
 
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef	_VIO_UTIL_H
 #define	_VIO_UTIL_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/stream.h>
 #include <sys/vio_mailbox.h>
 
@@ -141,6 +139,7 @@ void vio_freeb(void *arg);
 int vio_init_multipools(vio_multi_pool_t *vmultip, int num_pools, ...);
 void vio_destroy_multipools(vio_multi_pool_t *vmultip, vio_mblk_pool_t **fvmp);
 mblk_t *vio_multipool_allocb(vio_multi_pool_t *vmultip, size_t size);
+int vio_check_pending_pools(vio_multi_pool_t *vmultip);
 
 /* VIO versioning helpers */
 #define	VIO_VER_IS_NEGOTIATED(ver, maj, min)		\
diff --git a/usr/src/uts/sun4v/sys/vnet.h b/usr/src/uts/sun4v/sys/vnet.h
index 3270dcbfeb..40cc712456 100644
--- a/usr/src/uts/sun4v/sys/vnet.h
+++ b/usr/src/uts/sun4v/sys/vnet.h
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -143,6 +143,14 @@ typedef struct vnet_dds_info {
 
 #define	VLAN_ID_KEY(key)	((mod_hash_key_t)(uintptr_t)(key))
 
+typedef enum {
+		AST_init = 0x0, AST_vnet_alloc = 0x1,
+		AST_mac_alloc = 0x2, AST_read_macaddr = 0x4,
+		AST_vgen_init = 0x8, AST_fdbh_alloc = 0x10,
+		AST_vdds_init = 0x20, AST_taskq_create = 0x40,
+		AST_vnet_list = 0x80, AST_macreg = 0x100
+} vnet_attach_progress_t;
+
 /*
  * vnet instance state information
  */
@@ -150,6 +158,7 @@ typedef struct vnet {
 	int			instance;	/* instance # */
 	dev_info_t		*dip;		/* dev_info */
 	uint64_t		reg;		/* reg prop value */
+	vnet_attach_progress_t	attach_progress; /* attach progress flags */
 	struct vnet		*nextp;		/* next in list */
 	mac_handle_t		mh;		/* handle to GLDv3 mac module */
 	uchar_t			vendor_addr[ETHERADDRL]; /* orig macadr */
diff --git a/usr/src/uts/sun4v/sys/vnet_gen.h b/usr/src/uts/sun4v/sys/vnet_gen.h
index 08d4c70178..5e06e097f4 100644
--- a/usr/src/uts/sun4v/sys/vnet_gen.h
+++ b/usr/src/uts/sun4v/sys/vnet_gen.h
@@ -70,10 +70,9 @@ extern "C" {
 #define	LDC_TO_VGEN(ldcp)  ((ldcp)->portp->vgenp)
 
 /* receive thread flags */
-#define	VGEN_WTHR_RUNNING 	0x01	/* worker thread running */
-#define	VGEN_WTHR_DATARCVD 	0x02	/* data received */
-#define	VGEN_WTHR_STOP 		0x04	/* stop worker thread request */
-#define	VGEN_WTHR_PROCESSING	0x08	/* worker thread awake & processing */
+#define	VGEN_WTHR_DATARCVD 	0x01	/* data received */
+#define	VGEN_WTHR_STOP 		0x02	/* stop worker thread request */
+#define	VGEN_WTHR_PROCESSING	0x04	/* worker thread awake & processing */
 
 #define	VGEN_LDC_UP_DELAY	100	/* usec delay between ldc_up retries */
 #define	VGEN_LDC_CLOSE_DELAY	100	/* usec delay between ldc_cl retries */
diff --git a/usr/src/uts/sun4v/sys/vsw.h b/usr/src/uts/sun4v/sys/vsw.h
index 8a30940096..0755b58715 100644
--- a/usr/src/uts/sun4v/sys/vsw.h
+++ b/usr/src/uts/sun4v/sys/vsw.h
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -92,6 +92,18 @@ typedef enum {
 	VSW_SWTHR_STOP = 0x1
 } sw_thr_flags_t;
 
+typedef enum {
+	PROG_init = 0x00,
+	PROG_locks = 0x01,
+	PROG_readmd = 0x02,
+	PROG_fdb = 0x04,
+	PROG_mfdb = 0x08,
+	PROG_taskq = 0x10,
+	PROG_swmode = 0x20,
+	PROG_macreg = 0x40,
+	PROG_mdreg = 0x80
+} vsw_attach_progress_t;
+
 /*
  * vlan-id information.
  */
@@ -108,6 +120,7 @@ typedef struct	vsw {
 	int			instance;	/* instance # */
 	dev_info_t		*dip;		/* associated dev_info */
 	uint64_t		regprop;	/* "reg" property */
+	vsw_attach_progress_t	attach_progress; /* attach progress flags */
 	struct vsw		*next;		/* next in list */
 	char			physname[LIFNAMSIZ];	/* phys-dev */
 	uint8_t			smode;		/* switching mode */
diff --git a/usr/src/uts/sun4v/sys/vsw_ldc.h b/usr/src/uts/sun4v/sys/vsw_ldc.h
index 46d04fac10..ae6144de21 100644
--- a/usr/src/uts/sun4v/sys/vsw_ldc.h
+++ b/usr/src/uts/sun4v/sys/vsw_ldc.h
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -339,9 +339,8 @@ typedef struct vsw_ldc {
 } vsw_ldc_t;
 
 /* worker thread flags */
-#define	VSW_WTHR_RUNNING 	0x01	/* worker thread running */
-#define	VSW_WTHR_DATARCVD 	0x02	/* data received */
-#define	VSW_WTHR_STOP 		0x04	/* stop worker thread request */
+#define	VSW_WTHR_DATARCVD 	0x01	/* data received */
+#define	VSW_WTHR_STOP 		0x02	/* stop worker thread request */
 
 /* list of ldcs per port */
 typedef struct vsw_ldc_list {