6405380 LDoms vSwitch needs to be modified to support network interfaces with multiple DMA channels

6442123 CDDL missing in usr/src/uts/sun4v/sys/mmu.h
6444620 Consoles in a group should not logout user when switched to another console.
6451435 vntsd should be checked by lint during a nightly build
6461268 Ctrl-S hangs ldoms guest console.
6461648 vds should handle requests for absolute disk offsets per FWARC/2006/195
6464824 vnet panics due to ldc_mem_alloc_handle(0x1f) error allocating table memory
6465197 Console connections fail on an active ldom.
6467567 panic[cpu0]/thread=2a10045fcc0: recursive mutex_enter
6469790 ldc call to vmem_free uses incorrect size

16 files changed, 624 insertions, 113 deletions

diff --git a/usr/src/Makefile.lint b/usr/src/Makefile.lint
index cc0ea0206c..6b744020b8 100644
--- a/usr/src/Makefile.lint
+++ b/usr/src/Makefile.lint
@@ -423,6 +423,7 @@ sparc_SUBDIRS= \
 	cmd/prtdscp \
 	cmd/prtfru \
 	cmd/sckmd \
+	cmd/vntsd \
 	cmd/wrsmconf \
 	cmd/wrsmstat \
 	lib/libdscp \
diff --git a/usr/src/cmd/vntsd/chars.h b/usr/src/cmd/vntsd/chars.h
index 66abce66b7..e038170e64 100644
--- a/usr/src/cmd/vntsd/chars.h
+++ b/usr/src/cmd/vntsd/chars.h
@@ -43,8 +43,8 @@ extern "C" {
 #define	    HT	    9	    /* eoln */
 #define	    VT	    11	    /* not support */
 #define	    FF	    12	    /* not support */
-#define	    STOP    18
-#define	    START   19
+#define	    STOP    19
+#define	    START   17
 
 #define	    SE	    240	    /* end of subnegotiation params */
 #define	    NOP	    241
diff --git a/usr/src/cmd/vntsd/cmd.c b/usr/src/cmd/vntsd/cmd.c
index 8bee8417fe..ec32f87e88 100644
--- a/usr/src/cmd/vntsd/cmd.c
+++ b/usr/src/cmd/vntsd/cmd.c
@@ -465,7 +465,6 @@ vntsd_ctrl_cmd(vntsd_client_t *clientp, char c)
 	}
 
 	if (c == START) {
-
 		D3(stderr, "t@%d client restart\n", thr_self());
 
 		/* send resume read */
@@ -475,12 +474,6 @@ vntsd_ctrl_cmd(vntsd_client_t *clientp, char c)
 			return (VNTSD_STATUS_VCC_IO_ERR);
 		}
 
-		/* send resume write */
-		cmd = 3;
-
-		if (ioctl(clientp->cons->vcc_fd, TCXONC, &cmd)) {
-			return (VNTSD_STATUS_VCC_IO_ERR);
-		}
 	}
 
 	if (c == STOP) {
@@ -493,13 +486,6 @@ vntsd_ctrl_cmd(vntsd_client_t *clientp, char c)
 			return (VNTSD_STATUS_VCC_IO_ERR);
 		}
 
-		/* send suspend write */
-		cmd = 2;
-
-		if (ioctl(clientp->cons->vcc_fd, TCXONC, &cmd)) {
-			perror("ioctl TCXONC");
-			return (VNTSD_STATUS_VCC_IO_ERR);
-		}
 	}
 
 	return (VNTSD_STATUS_CONTINUE);
diff --git a/usr/src/cmd/vntsd/common.c b/usr/src/cmd/vntsd/common.c
index 2cbad73309..65f1a5111f 100644
--- a/usr/src/cmd/vntsd/common.c
+++ b/usr/src/cmd/vntsd/common.c
@@ -541,7 +541,7 @@ vntsd_log(vntsd_status_t status, char *msg)
 		break;
 
 	case VNTSD_STATUS_NO_CONS:
-		status_msg = "GROUP HAS NO CONSOLE";
+		status_msg = "All console(s) in the group have been deleted.";
 		break;
 
 	case VNTSD_ERR_NO_MEM:
diff --git a/usr/src/cmd/vntsd/console.c b/usr/src/cmd/vntsd/console.c
index caff15357a..212d9c9415 100644
--- a/usr/src/cmd/vntsd/console.c
+++ b/usr/src/cmd/vntsd/console.c
@@ -382,10 +382,12 @@ connect_cons(vntsd_cons_t *consp, vntsd_client_t *clientp)
 	/* enable daemon cmd */
 	clientp->status &= ~VNTSD_CLIENT_DISABLE_DAEMON_CMD;
 
-	if (consp->clientpq == NULL) {
-		/* first connect to console - a writer */
-		assert(consp->vcc_fd == -1);
-		/* open vcc */
+	if (consp->clientpq == NULL && consp->vcc_fd == -1) {
+
+		/*
+		 *  the first connection to a console - a writer
+		 *  and the console has not opened.
+		 */
 		consp->vcc_fd = vntsd_open_vcc(consp->dev_name, consp->cons_no);
 		if (consp->vcc_fd < 0) {
 			(void) mutex_unlock(&clientp->lock);
@@ -455,10 +457,6 @@ connect_cons(vntsd_cons_t *consp, vntsd_client_t *clientp)
 		/* clean up console since there is no client connected to it */
 		assert(consp->vcc_fd != -1);
 
-		/* close vcc port */
-		(void) close(consp->vcc_fd);
-		consp->vcc_fd = -1;
-
 		/* force write thread to exit */
 		assert(consp->wr_tid != (thread_t)-1);
 		(void) thr_kill(consp->wr_tid, SIGUSR1);
@@ -522,6 +520,43 @@ client_init(vntsd_client_t *clientp)
 	clientp->status = 0;
 	(void) mutex_unlock(&clientp->lock);
 }
+/* is there any connection to a given console? */
+static boolean_t
+is_client_que_empty(vntsd_cons_t *consp)
+{
+	boolean_t  has_client = B_FALSE;
+
+	(void) mutex_lock(&consp->lock);
+
+	if (consp->clientpq != NULL)
+		has_client = B_TRUE;
+
+	(void) mutex_unlock(&consp->lock);
+
+	return (has_client);
+}
+
+/*
+ * close one opened console.
+ * This function is passed to vntsd_que_walk to close one console.
+ * The function returns B_FALSE so that vntsd_que_walk will
+ * continue to apply the function to all consoles in the group.
+ */
+static boolean_t
+close_one_vcc_fd(vntsd_cons_t *consp)
+{
+	(void) mutex_lock(&consp->lock);
+
+	if (consp->vcc_fd != -1) {
+		(void) close(consp->vcc_fd);
+		consp->vcc_fd = -1;
+	}
+
+	(void) mutex_unlock(&consp->lock);
+
+	return (B_FALSE);
+}
+
 
 /* clean up client and exit the thread */
 static void
@@ -536,6 +571,18 @@ client_fini(vntsd_group_t *groupp, vntsd_client_t *clientp)
 	(void) close(clientp->sockfd);
 
 	(void) mutex_lock(&groupp->lock);
+
+	/*
+	 * close all consoles in the group if the client is the
+	 * last one connected to the group
+	 */
+	if (vntsd_que_walk(groupp->conspq, (el_func_t)is_client_que_empty) ==
+	    VNTSD_SUCCESS) {
+		(void) vntsd_que_walk(groupp->conspq,
+		    (el_func_t)close_one_vcc_fd);
+	}
+
+
 	(void) vntsd_que_rm(&groupp->no_cons_clientpq, clientp);
 
 	if ((groupp->no_cons_clientpq == NULL) &&
diff --git a/usr/src/cmd/vntsd/vntsd.c b/usr/src/cmd/vntsd/vntsd.c
index e1ded5dc3b..ea0112016e 100644
--- a/usr/src/cmd/vntsd/vntsd.c
+++ b/usr/src/cmd/vntsd/vntsd.c
@@ -80,15 +80,8 @@ static void
 exit_sig_handler(int sig)
 {
 
-	char err_msg[VNTSD_LINE_LEN];
-
 	D1(stderr, "t@%d exit_sig_handler%d \n", thr_self(), sig);
 
-	(void) snprintf(err_msg, sizeof (err_msg), "exit_sig_handler() sig=%d",
-	    sig);
-
-	vntsd_log(VNTSD_STATUS_EXIT_SIG, err_msg);
-
 	exit(0);
 }
 
@@ -528,9 +521,7 @@ vntsd_vcc_ioctl(int ioctl_code, uint_t portno, void *buf)
 	}
 
 	if (ioctl(vntsdp->ctrl_fd, ioctl_code, (caddr_t)buf)) {
-		/*  control port get error */
-		syslog(LOG_ERR, "vcc control port error! abort vntsd");
-		(void) thr_kill(vntsdp->tid, SIGINT);
+		/*  ioctl request error */
 		return (VNTSD_STATUS_VCC_IO_ERR);
 	}
 
diff --git a/usr/src/cmd/vntsd/vntsdvcc.c b/usr/src/cmd/vntsd/vntsdvcc.c
index 9facdf7c75..b47f37660b 100644
--- a/usr/src/cmd/vntsd/vntsdvcc.c
+++ b/usr/src/cmd/vntsd/vntsdvcc.c
@@ -152,7 +152,9 @@ vntsd_delete_cons(vntsd_t *vntsdp)
 			(void) mutex_unlock(&vntsdp->lock);
 			return;
 		}
+		(void) mutex_lock(&groupp->lock);
 		groupp->status &= ~VNTSD_GROUP_CLEAN_CONS;
+		(void) mutex_unlock(&groupp->lock);
 		(void) mutex_unlock(&vntsdp->lock);
 
 		for (; ; ) {
@@ -207,6 +209,11 @@ vntsd_clean_group(vntsd_group_t *groupp)
 
 	/* prevent from reentry */
 	if (groupp->status & VNTSD_GROUP_CLEANUP) {
+		if (groupp->listen_tid == thr_self()) {
+			/* signal that the listen thread is exiting */
+			groupp->status &= ~VNTSD_GROUP_SIG_WAIT;
+			(void) cond_signal(&groupp->cvp);
+		}
 		(void) mutex_unlock(&groupp->lock);
 		return;
 	}
@@ -561,6 +568,10 @@ vntsd_daemon_wakeup(vntsd_t *vntsdp)
 		do_add_cons(vntsdp, inq_data.cons_no);
 		break;
 
+	case VCC_CONS_MISS_ADDED:
+		/* an added port was deleted before vntsd can process it */
+		return;
+
 	default:
 		DERR(stderr, "t@%d daemon_wakeup:ioctl_unknown %d\n",
 		    thr_self(), inq_data.reason);
diff --git a/usr/src/uts/sun4v/io/cnex.c b/usr/src/uts/sun4v/io/cnex.c
index 293c20e131..afd961373d 100644
--- a/usr/src/uts/sun4v/io/cnex.c
+++ b/usr/src/uts/sun4v/io/cnex.c
@@ -290,6 +290,7 @@ cnex_intr_redist(void *arg)
 				return;
 			}
 			if (intr_state == HV_INTR_NOTVALID) {
+				mutex_exit(&cldcp->lock);
 				cldcp = cldcp->next;
 				continue;
 			}
@@ -350,6 +351,7 @@ cnex_intr_redist(void *arg)
 				return;
 			}
 			if (intr_state == HV_INTR_NOTVALID) {
+				mutex_exit(&cldcp->lock);
 				cldcp = cldcp->next;
 				continue;
 			}
@@ -833,7 +835,7 @@ cnex_clr_intr(dev_info_t *dip, uint64_t id, cnex_intrtype_t itype)
 	} else if (itype == CNEX_RX_INTR) {
 		iinfo = &(cldcp->rx);
 	} else {
-		DWARN("cnex_rem_intr: invalid interrupt type\n");
+		DWARN("cnex_clr_intr: invalid interrupt type\n");
 		mutex_exit(&cldcp->lock);
 		return (EINVAL);
 	}
@@ -850,7 +852,7 @@ cnex_clr_intr(dev_info_t *dip, uint64_t id, cnex_intrtype_t itype)
 	rv = hvldc_intr_setstate(cnex_ssp->cfghdl, iinfo->ino,
 	    HV_INTR_IDLE_STATE);
 	if (rv) {
-		DWARN("cnex_intr_wrapper: cannot clear interrupt state\n");
+		DWARN("cnex_clr_intr: cannot clear interrupt state\n");
 		mutex_exit(&cldcp->lock);
 		return (ENXIO);
 	}
diff --git a/usr/src/uts/sun4v/io/ldc.c b/usr/src/uts/sun4v/io/ldc.c
index 386fe29177..f6481dfcde 100644
--- a/usr/src/uts/sun4v/io/ldc.c
+++ b/usr/src/uts/sun4v/io/ldc.c
@@ -5135,7 +5135,7 @@ ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie, uint32_t ccount,
 
 				/* free kernel virtual space */
 				vmem_free(heap_arena, (void *)memseg->vaddr,
-				    memseg->size);
+				    map_size);
 
 				/* direct map failed - revert to shadow map */
 				mtype = LDC_SHADOW_MAP;
diff --git a/usr/src/uts/sun4v/io/vcc.c b/usr/src/uts/sun4v/io/vcc.c
index 2eab569346..ac57013c23 100644
--- a/usr/src/uts/sun4v/io/vcc.c
+++ b/usr/src/uts/sun4v/io/vcc.c
@@ -88,6 +88,7 @@ static int	i_vcc_cons_tbl(vcc_t *vccp, uint_t num_ports,
 static int	i_vcc_del_cons_ok(vcc_t *vccp, caddr_t buf, int mode);
 static int	i_vcc_close_port(vcc_port_t *vport);
 static int	i_vcc_write_ldc(vcc_port_t *vport, vcc_msg_t *buf);
+static int	i_vcc_read_ldc(vcc_port_t *vport, char *data_buf, size_t *sz);
 
 static void *vcc_ssp;
 
@@ -460,6 +461,7 @@ i_vcc_ldc_fini(vcc_port_t *vport)
 {
 	int 		rv = EIO;
 	vcc_msg_t	buf;
+	size_t		sz;
 
 	D1("i_vcc_ldc_fini: port@%lld, ldc_id%%llx\n", vport->number,
 	    vport->ldc_id);
@@ -485,6 +487,21 @@ i_vcc_ldc_fini(vcc_port_t *vport)
 	i_vcc_set_port_status(vport, &vport->write_cv, VCC_PORT_USE_WRITE_LDC);
 	mutex_enter(&vport->lock);
 
+	/* flush ldc channel */
+	rv = i_vcc_wait_port_status(vport, &vport->read_cv,
+	    VCC_PORT_USE_READ_LDC);
+	if (rv) {
+		return (rv);
+	}
+
+	vport->status &= ~VCC_PORT_USE_READ_LDC;
+	do {
+		sz = sizeof (buf);
+		rv = i_vcc_read_ldc(vport, (char *)&buf, &sz);
+	} while (rv == 0 && sz > 0);
+
+	vport->status |= VCC_PORT_USE_READ_LDC;
+
 	(void) ldc_set_cb_mode(vport->ldc_handle, LDC_CB_DISABLE);
 	if ((rv = ldc_close(vport->ldc_handle)) != 0) {
 		cmn_err(CE_CONT, "i_vcc_ldc_fini: cannot close channel %ld\n",
@@ -1220,7 +1237,6 @@ vcc_close(dev_t dev, int flag, int otyp, cred_t *cred)
 
 	D1("vcc_close: closing virtual-console-concentrator@%d:%d\n",
 	    instance, portno);
-
 	vport = &(vccp->port[portno]);
 
 
@@ -1491,7 +1507,17 @@ i_vcc_inquiry(vcc_t *vccp, caddr_t buf, int mode)
 		}
 	}
 
-	return (EINVAL);
+	/* the added port was deleted before vntsd wakes up */
+	msg.reason = VCC_CONS_MISS_ADDED;
+
+	if (ddi_copyout((void *)&msg, (void *)buf,
+		    sizeof (msg), mode) == -1) {
+		cmn_err(CE_CONT, "i_vcc_find_changed_port: ddi_copyout"
+		    " failed\n");
+		return (EFAULT);
+	}
+
+	return (0);
 }
 
 /* clean up events after vntsd exits */
@@ -1828,38 +1854,30 @@ i_vcc_port_ioctl(vcc_t *vccp, minor_t minor, int portno, int cmd, void *arg,
 		switch (cmd) {
 
 		case 0:
-			vport->status |= VCC_PORT_TERM_WR;
-			cv_broadcast(&vport->write_cv);
+			/* suspend read */
+			vport->status &= ~VCC_PORT_TERM_RD;
 			break;
+
 		case 1:
-			/* get write lock */
-			rv = i_vcc_wait_port_status(vport, &vport->write_cv,
-			    VCC_PORT_USE_WRITE_LDC);
-			if (rv) {
-				mutex_exit(&vport->lock);
-				return (rv);
-			}
-			vport->status &= ~VCC_PORT_TERM_WR;
-			cv_broadcast(&vport->write_cv);
-			break;
-		case 2:
+			/* resume read */
 			vport->status |= VCC_PORT_TERM_RD;
 			cv_broadcast(&vport->read_cv);
 			break;
+
+		case 2:
+			/* suspend write */
+			vport->status &= ~VCC_PORT_TERM_WR;
+			break;
+
 		case 3:
-			/* get read lock */
-			rv = i_vcc_wait_port_status(vport, &vport->write_cv,
-			    VCC_PORT_USE_READ_LDC);
-			if (rv) {
-				mutex_exit(&vport->lock);
-				return (rv);
-			}
-			vport->status &= ~VCC_PORT_TERM_RD;
-			cv_broadcast(&vport->read_cv);
+			/* resume write */
+			vport->status |= VCC_PORT_TERM_WR;
+			cv_broadcast(&vport->write_cv);
 			break;
 
 		default:
-			break;
+			mutex_exit(&vport->lock);
+			return (EINVAL);
 		}
 
 		mutex_exit(&vport->lock);
@@ -1869,7 +1887,7 @@ i_vcc_port_ioctl(vcc_t *vccp, minor_t minor, int portno, int cmd, void *arg,
 		return (0);
 
 	default:
-		return (ENODEV);
+		return (EINVAL);
 	}
 
 }
@@ -1967,7 +1985,6 @@ vcc_read(dev_t dev, struct uio *uiop, cred_t *credp)
 		return (rv);
 	}
 
-
 	rv = i_vcc_wait_port_status(vport, &vport->read_cv,
 		    VCC_PORT_TERM_RD|VCC_PORT_LDC_CHANNEL_READY|
 		    VCC_PORT_USE_READ_LDC);
@@ -2004,11 +2021,20 @@ vcc_read(dev_t dev, struct uio *uiop, cred_t *credp)
 
 		/* wait for data from ldc */
 		vport->status &= ~VCC_PORT_LDC_DATA_READY;
+
+		mutex_exit(&vport->lock);
+		i_vcc_set_port_status(vport, &vport->read_cv,
+		    VCC_PORT_USE_READ_LDC);
+		mutex_enter(&vport->lock);
+
 		rv = i_vcc_wait_port_status(vport, &vport->read_cv,
-			    VCC_PORT_LDC_DATA_READY);
+		    VCC_PORT_TERM_RD|VCC_PORT_LDC_CHANNEL_READY|
+		    VCC_PORT_USE_READ_LDC| VCC_PORT_LDC_DATA_READY);
 		if (rv) {
 			break;
 		}
+
+		vport->status &= ~VCC_PORT_USE_READ_LDC;
 	}
 
 	mutex_exit(&vport->lock);
diff --git a/usr/src/uts/sun4v/io/vds.c b/usr/src/uts/sun4v/io/vds.c
index bb3cec23af..76b3878186 100644
--- a/usr/src/uts/sun4v/io/vds.c
+++ b/usr/src/uts/sun4v/io/vds.c
@@ -975,13 +975,6 @@ vd_process_task(vd_task_t *task)
 	ASSERT(vd != NULL);
 	ASSERT(request != NULL);
 
-	/* Range-check slice */
-	if (request->slice >= vd->nslices) {
-		PRN("Invalid \"slice\" %u (max %u) for virtual disk",
-		    request->slice, (vd->nslices - 1));
-		return (EINVAL);
-	}
-
 	/* Find the requested operation */
 	for (i = 0; i < vds_noperations; i++)
 		if (request->operation == vds_operation[i].operation)
@@ -991,6 +984,18 @@ vd_process_task(vd_task_t *task)
 		return (ENOTSUP);
 	}
 
+	/* Handle client using absolute disk offsets */
+	if ((vd->vdisk_type == VD_DISK_TYPE_DISK) &&
+	    (request->slice == UINT8_MAX))
+		request->slice = VD_ENTIRE_DISK_SLICE;
+
+	/* Range-check slice */
+	if (request->slice >= vd->nslices) {
+		PRN("Invalid \"slice\" %u (max %u) for virtual disk",
+		    request->slice, (vd->nslices - 1));
+		return (EINVAL);
+	}
+
 	/* Start the operation */
 	if ((status = vds_operation[i].start(task)) != EINPROGRESS) {
 		request->status = status;	/* op succeeded or failed */
diff --git a/usr/src/uts/sun4v/io/vnet_gen.c b/usr/src/uts/sun4v/io/vnet_gen.c
index 9e5f522778..238b4d5f8f 100644
--- a/usr/src/uts/sun4v/io/vnet_gen.c
+++ b/usr/src/uts/sun4v/io/vnet_gen.c
@@ -1794,7 +1794,8 @@ vgen_ldc_init(vgen_ldc_t *ldcp)
 	ldc_status_t	istatus;
 	int		rv;
 	enum		{ ST_init = 0x0, ST_init_tbufs = 0x1,
-			    ST_ldc_open = 0x2, ST_dring_bind = 0x4
+			    ST_ldc_open = 0x2, ST_dring_bind = 0x4,
+			    ST_cb_enable = 0x8
 			    }
 			init_state;
 	uint32_t	ncookies = 0;
@@ -1845,6 +1846,15 @@ vgen_ldc_init(vgen_ldc_t *ldcp)
 
 	init_state |= ST_dring_bind;
 
+	rv = ldc_set_cb_mode(ldcp->ldc_handle, LDC_CB_ENABLE);
+	if (rv != 0) {
+		DWARN((vnetp, "vgen_ldc_init: id (%lx) "
+		    "ldc_set_cb_mode failed\n", ldcp->ldc_id));
+		goto ldcinit_failed;
+	}
+
+	init_state |= ST_cb_enable;
+
 	do {
 		rv = ldc_up(ldcp->ldc_handle);
 		if ((rv != 0) && (rv == EWOULDBLOCK)) {
@@ -1874,6 +1884,9 @@ vgen_ldc_init(vgen_ldc_t *ldcp)
 	return (DDI_SUCCESS);
 
 ldcinit_failed:
+	if (init_state & ST_cb_enable) {
+		(void) ldc_set_cb_mode(ldcp->ldc_handle, LDC_CB_DISABLE);
+	}
 	if (init_state & ST_dring_bind) {
 		(void) ldc_mem_dring_unbind(ldcp->tx_dhandle);
 	}
@@ -1911,6 +1924,12 @@ vgen_ldc_uninit(vgen_ldc_t *ldcp)
 		    "ldc_set_cb_mode failed\n", ldcp->ldc_id));
 	}
 
+	rv = ldc_down(ldcp->ldc_handle);
+	if (rv != 0) {
+		DWARN((vnetp, "vgen_ldc_uninit: id (%lx) "
+		    "ldc_down failed\n", ldcp->ldc_id));
+	}
+
 	/* clear handshake done bit and wait for pending tx and cb to finish */
 	ldcp->hphase &= ~(VH_DONE);
 	LDC_UNLOCK(ldcp);
@@ -2046,29 +2065,21 @@ static void
 vgen_uninit_tbufs(vgen_ldc_t *ldcp)
 {
 	vgen_private_desc_t	*tbufp = ldcp->tbufp;
-	vnet_public_desc_t	*txdp;
-	vio_dring_entry_hdr_t		*hdrp;
 	int 			i;
 
 	/* for each tbuf (priv_desc), free ldc mem_handle */
 	for (i = 0; i < ldcp->num_txds; i++) {
 
 		tbufp = &(ldcp->tbufp[i]);
-		txdp = tbufp->descp;
-		hdrp = &txdp->hdr;
 
 		if (tbufp->datap) { /* if bound to a ldc memhandle */
 			(void) ldc_mem_unbind_handle(tbufp->memhandle);
 			tbufp->datap = NULL;
 		}
-		tbufp->flags = VGEN_PRIV_DESC_FREE;
-		hdrp->dstate = VIO_DESC_FREE;
-		hdrp->ack = B_FALSE;
 		if (tbufp->memhandle) {
 			(void) ldc_mem_free_handle(tbufp->memhandle);
 			tbufp->memhandle = 0;
 		}
-		tbufp->descp = NULL;
 	}
 
 	if (ldcp->tx_datap) {
@@ -2077,8 +2088,8 @@ vgen_uninit_tbufs(vgen_ldc_t *ldcp)
 		ldcp->tx_datap = NULL;
 	}
 
-	bzero(ldcp->tbufp, sizeof (*tbufp) * (ldcp->num_txds));
-	bzero(ldcp->txdp, sizeof (*txdp) * (ldcp->num_txds));
+	bzero(ldcp->tbufp, sizeof (vgen_private_desc_t) * (ldcp->num_txds));
+	bzero(ldcp->txdp, sizeof (vnet_public_desc_t) * (ldcp->num_txds));
 }
 
 /* clobber tx descriptor ring */
diff --git a/usr/src/uts/sun4v/io/vsw.c b/usr/src/uts/sun4v/io/vsw.c
index 6b5484704e..89b6ca4c6f 100644
--- a/usr/src/uts/sun4v/io/vsw.c
+++ b/usr/src/uts/sun4v/io/vsw.c
@@ -82,15 +82,27 @@ static	int vsw_get_physaddr(vsw_t *);
 static	int vsw_setup_layer2(vsw_t *);
 static	int vsw_setup_layer3(vsw_t *);
 
+/* MAC Ring table functions. */
+static void vsw_mac_ring_tbl_init(vsw_t *vswp);
+static void vsw_mac_ring_tbl_destroy(vsw_t *vswp);
+static void vsw_queue_worker(vsw_mac_ring_t *rrp);
+static void vsw_queue_stop(vsw_queue_t *vqp);
+static vsw_queue_t *vsw_queue_create();
+static void vsw_queue_destroy(vsw_queue_t *vqp);
+
 /* MAC layer routines */
-static	int vsw_mac_attach(vsw_t *vswp);
-static	void vsw_mac_detach(vsw_t *vswp);
+static mac_resource_handle_t vsw_mac_ring_add_cb(void *arg,
+		mac_resource_t *mrp);
 static	int vsw_get_hw_maddr(vsw_t *);
 static	int vsw_set_hw(vsw_t *, vsw_port_t *);
 static	int vsw_set_hw_promisc(vsw_t *, vsw_port_t *);
 static	int vsw_unset_hw(vsw_t *, vsw_port_t *);
 static	int vsw_unset_hw_promisc(vsw_t *, vsw_port_t *);
 static	int vsw_reconfig_hw(vsw_t *);
+static int vsw_mac_attach(vsw_t *vswp);
+static void vsw_mac_detach(vsw_t *vswp);
+
+static void vsw_rx_queue_cb(void *, mac_resource_handle_t, mblk_t *);
 static void vsw_rx_cb(void *, mac_resource_handle_t, mblk_t *);
 static mblk_t *vsw_tx_msg(vsw_t *, mblk_t *);
 static int vsw_mac_register(vsw_t *);
@@ -344,6 +356,13 @@ static mdeg_prop_spec_t vsw_prop_template[] = {
 #define	VSW_SET_MDEG_PROP_INST(specp, val)	(specp)[1].ps_val = (val);
 
 /*
+ * From /etc/system enable/disable thread per ring. This is a mode
+ * selection that is done a vsw driver attach time.
+ */
+boolean_t vsw_multi_ring_enable = B_FALSE;
+int vsw_mac_rx_rings = VSW_MAC_RX_RINGS;
+
+/*
  * Print debug messages - set to 0x1f to enable all msgs
  * or 0x0 to turn all off.
  */
@@ -496,9 +515,12 @@ vsw_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 	char		hashname[MAXNAMELEN];
 	char		qname[TASKQ_NAMELEN];
 	int		rv = 1;
-	enum		{ PROG_init = 0x0, PROG_if_lock = 0x1,
-				PROG_fdb = 0x2, PROG_mfdb = 0x4,
-				PROG_report_dev = 0x8, PROG_plist = 0x10,
+	enum		{ PROG_init = 0x00,
+				PROG_if_lock = 0x01,
+				PROG_fdb = 0x02,
+				PROG_mfdb = 0x04,
+				PROG_report_dev = 0x08,
+				PROG_plist = 0x10,
 				PROG_taskq = 0x20}
 			progress;
 
@@ -532,7 +554,6 @@ vsw_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 	ddi_set_driver_private(dip, (caddr_t)vswp);
 
 	rw_init(&vswp->if_lockrw, NULL, RW_DRIVER, NULL);
-
 	progress |= PROG_if_lock;
 
 	/*
@@ -737,7 +758,10 @@ vsw_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 	 * the physical device.
 	 */
 	if (vswp->mh != NULL) {
-		mac_stop(vswp->mh);
+		if (vswp->mstarted)
+			mac_stop(vswp->mh);
+		if (vswp->mresources)
+			mac_resource_set(vswp->mh, NULL, NULL);
 		mac_close(vswp->mh);
 
 		vswp->mh = NULL;
@@ -1126,7 +1150,8 @@ vsw_get_hw_maddr(vsw_t *vswp)
 	}
 
 	if (vswp->maddr.maddr_naddrfree == 0) {
-		cmn_err(CE_WARN, "!device %s has no free unicast address slots",
+		cmn_err(CE_WARN,
+			"!device %s has no free unicast address slots",
 			vswp->physname);
 		return (1);
 	}
@@ -1219,10 +1244,12 @@ vsw_mac_attach(vsw_t *vswp)
 	char	drv[LIFNAMSIZ];
 	uint_t	ddi_instance;
 
-	D1(vswp, "vsw_mac_attach: enter");
+	D1(vswp, "%s: enter", __func__);
 
 	vswp->mh = NULL;
 	vswp->mrh = NULL;
+	vswp->mstarted = B_FALSE;
+	vswp->mresources = B_FALSE;
 
 	ASSERT(vswp->mdprops & VSW_MD_PHYSNAME);
 
@@ -1235,12 +1262,40 @@ vsw_mac_attach(vsw_t *vswp)
 		goto mac_fail_exit;
 	}
 
+	ASSERT(vswp->mh != NULL);
+
 	D2(vswp, "vsw_mac_attach: using device %s", vswp->physname);
 
-	/* register our rx callback function */
-	vswp->mrh = mac_rx_add(vswp->mh, vsw_rx_cb, (void *)vswp);
+	if (vsw_multi_ring_enable) {
+		vsw_mac_ring_tbl_init(vswp);
+
+		/*
+		 * Register our receive callback.
+		 */
+		vswp->mrh = mac_rx_add(vswp->mh,
+			vsw_rx_queue_cb, (void *)vswp);
+
+		/*
+		 * Register our mac resource callback.
+		 */
+		mac_resource_set(vswp->mh, vsw_mac_ring_add_cb, (void *)vswp);
+		vswp->mresources = B_TRUE;
+
+		/*
+		 * Get the ring resources available to us from
+		 * the mac below us.
+		 */
+		mac_resources(vswp->mh);
+	} else {
+		/*
+		 * Just register our rx callback function
+		 */
+		vswp->mrh = mac_rx_add(vswp->mh, vsw_rx_cb, (void *)vswp);
+	}
+
+	ASSERT(vswp->mrh != NULL);
 
-	/* get the MAC tx fn */
+	/* Get the MAC tx fn */
 	vswp->txinfo = mac_tx_get(vswp->mh);
 
 	/* start the interface */
@@ -1249,22 +1304,15 @@ vsw_mac_attach(vsw_t *vswp)
 		goto mac_fail_exit;
 	}
 
-	D1(vswp, "vsw_mac_attach: exit");
+	vswp->mstarted = B_TRUE;
+
+	D1(vswp, "%s: exit", __func__);
 	return (0);
 
 mac_fail_exit:
-	if (vswp->mh != NULL) {
-		if (vswp->mrh != NULL)
-			mac_rx_remove(vswp->mh, vswp->mrh);
-
-		mac_close(vswp->mh);
-	}
-
-	vswp->mrh = NULL;
-	vswp->mh = NULL;
-	vswp->txinfo = NULL;
+	vsw_mac_detach(vswp);
 
-	D1(vswp, "vsw_mac_attach: fail exit");
+	D1(vswp, "%s: exit", __func__);
 	return (1);
 }
 
@@ -1273,17 +1321,25 @@ vsw_mac_detach(vsw_t *vswp)
 {
 	D1(vswp, "vsw_mac_detach: enter");
 
-	if (vswp->mh != NULL) {
-		if (vswp->mrh != NULL)
-			mac_rx_remove(vswp->mh, vswp->mrh);
+	ASSERT(vswp != NULL);
+	ASSERT(vswp->mh != NULL);
 
-		mac_stop(vswp->mh);
-		mac_close(vswp->mh);
+	if (vsw_multi_ring_enable) {
+		vsw_mac_ring_tbl_destroy(vswp);
 	}
 
+	if (vswp->mstarted)
+		mac_stop(vswp->mh);
+	if (vswp->mrh != NULL)
+		mac_rx_remove(vswp->mh, vswp->mrh);
+	if (vswp->mresources)
+		mac_resource_set(vswp->mh, NULL, NULL);
+	mac_close(vswp->mh);
+
 	vswp->mrh = NULL;
 	vswp->mh = NULL;
 	vswp->txinfo = NULL;
+	vswp->mstarted = B_FALSE;
 
 	D1(vswp, "vsw_mac_detach: exit");
 }
@@ -1630,6 +1686,305 @@ reconfig_err_exit:
 	return (rv);
 }
 
+static void
+vsw_mac_ring_tbl_entry_init(vsw_t *vswp, vsw_mac_ring_t *ringp)
+{
+	ringp->ring_state = VSW_MAC_RING_FREE;
+	ringp->ring_arg = NULL;
+	ringp->ring_blank = NULL;
+	ringp->ring_vqp = NULL;
+	ringp->ring_vswp = vswp;
+}
+
+static void
+vsw_mac_ring_tbl_init(vsw_t *vswp)
+{
+	int		i;
+
+	mutex_init(&vswp->mac_ring_lock, NULL, MUTEX_DRIVER, NULL);
+
+	vswp->mac_ring_tbl_sz = vsw_mac_rx_rings;
+	vswp->mac_ring_tbl  =
+		kmem_alloc(vsw_mac_rx_rings * sizeof (vsw_mac_ring_t),
+		KM_SLEEP);
+
+	for (i = 0; i < vswp->mac_ring_tbl_sz; i++)
+		vsw_mac_ring_tbl_entry_init(vswp, &vswp->mac_ring_tbl[i]);
+}
+
+static void
+vsw_mac_ring_tbl_destroy(vsw_t *vswp)
+{
+	int	i;
+
+	mutex_enter(&vswp->mac_ring_lock);
+	for (i = 0; i < vswp->mac_ring_tbl_sz; i++) {
+		if (vswp->mac_ring_tbl[i].ring_state != VSW_MAC_RING_FREE) {
+			/*
+			 * Destroy the queue.
+			 */
+			vsw_queue_stop(vswp->mac_ring_tbl[i].ring_vqp);
+			vsw_queue_destroy(vswp->mac_ring_tbl[i].ring_vqp);
+
+			/*
+			 * Re-initialize the structure.
+			 */
+			vsw_mac_ring_tbl_entry_init(vswp,
+				&vswp->mac_ring_tbl[i]);
+		}
+	}
+	mutex_exit(&vswp->mac_ring_lock);
+
+	mutex_destroy(&vswp->mac_ring_lock);
+	kmem_free(vswp->mac_ring_tbl,
+		vswp->mac_ring_tbl_sz * sizeof (vsw_mac_ring_t));
+	vswp->mac_ring_tbl_sz = 0;
+}
+
+/*
+ * Handle resource add callbacks from the driver below.
+ */
+static mac_resource_handle_t
+vsw_mac_ring_add_cb(void *arg, mac_resource_t *mrp)
+{
+	vsw_t		*vswp = (vsw_t *)arg;
+	mac_rx_fifo_t	*mrfp = (mac_rx_fifo_t *)mrp;
+	vsw_mac_ring_t	*ringp;
+	vsw_queue_t	*vqp;
+	int		i;
+
+	ASSERT(vswp != NULL);
+	ASSERT(mrp != NULL);
+	ASSERT(vswp->mac_ring_tbl != NULL);
+
+	D1(vswp, "%s: enter", __func__);
+
+	/*
+	 * Check to make sure we have the correct resource type.
+	 */
+	if (mrp->mr_type != MAC_RX_FIFO)
+		return (NULL);
+
+	/*
+	 * Find a open entry in the ring table.
+	 */
+	mutex_enter(&vswp->mac_ring_lock);
+	for (i = 0; i < vswp->mac_ring_tbl_sz; i++) {
+		ringp = &vswp->mac_ring_tbl[i];
+
+		/*
+		 * Check for an empty slot, if found, then setup queue
+		 * and thread.
+		 */
+		if (ringp->ring_state == VSW_MAC_RING_FREE) {
+			/*
+			 * Create the queue for this ring.
+			 */
+			vqp = vsw_queue_create();
+
+			/*
+			 * Initialize the ring data structure.
+			 */
+			ringp->ring_vqp = vqp;
+			ringp->ring_arg = mrfp->mrf_arg;
+			ringp->ring_blank = mrfp->mrf_blank;
+			ringp->ring_state = VSW_MAC_RING_INUSE;
+
+			/*
+			 * Create the worker thread.
+			 */
+			vqp->vq_worker = thread_create(NULL, 0,
+				vsw_queue_worker, ringp, 0, &p0,
+				TS_RUN, minclsyspri);
+			if (vqp->vq_worker == NULL) {
+				vsw_queue_destroy(vqp);
+				vsw_mac_ring_tbl_entry_init(vswp, ringp);
+				ringp = NULL;
+			}
+
+			mutex_exit(&vswp->mac_ring_lock);
+			D1(vswp, "%s: exit", __func__);
+			return ((mac_resource_handle_t)ringp);
+		}
+	}
+	mutex_exit(&vswp->mac_ring_lock);
+
+	/*
+	 * No slots in the ring table available.
+	 */
+	D1(vswp, "%s: exit", __func__);
+	return (NULL);
+}
+
+static void
+vsw_queue_stop(vsw_queue_t *vqp)
+{
+	mutex_enter(&vqp->vq_lock);
+
+	if (vqp->vq_state == VSW_QUEUE_RUNNING) {
+		vqp->vq_state = VSW_QUEUE_STOP;
+		cv_signal(&vqp->vq_cv);
+
+		while (vqp->vq_state != VSW_QUEUE_DRAINED)
+			cv_wait(&vqp->vq_cv, &vqp->vq_lock);
+	}
+
+	mutex_exit(&vqp->vq_lock);
+}
+
+static vsw_queue_t *
+vsw_queue_create()
+{
+	vsw_queue_t *vqp;
+
+	vqp = kmem_zalloc(sizeof (vsw_queue_t), KM_SLEEP);
+
+	mutex_init(&vqp->vq_lock, NULL, MUTEX_DRIVER, NULL);
+	cv_init(&vqp->vq_cv, NULL, CV_DRIVER, NULL);
+	vqp->vq_first = NULL;
+	vqp->vq_last = NULL;
+	vqp->vq_state = VSW_QUEUE_STOP;
+
+	return (vqp);
+}
+
+static void
+vsw_queue_destroy(vsw_queue_t *vqp)
+{
+	cv_destroy(&vqp->vq_cv);
+	mutex_destroy(&vqp->vq_lock);
+	kmem_free(vqp, sizeof (vsw_queue_t));
+}
+
+static void
+vsw_queue_worker(vsw_mac_ring_t *rrp)
+{
+	mblk_t		*mp;
+	vsw_queue_t	*vqp = rrp->ring_vqp;
+	vsw_t		*vswp = rrp->ring_vswp;
+
+	mutex_enter(&vqp->vq_lock);
+
+	ASSERT(vqp->vq_state == VSW_QUEUE_STOP);
+
+	/*
+	 * Set the state to running, since the thread is now active.
+	 */
+	vqp->vq_state = VSW_QUEUE_RUNNING;
+
+	while (vqp->vq_state == VSW_QUEUE_RUNNING) {
+		/*
+		 * Wait for work to do or the state has changed
+		 * to not running.
+		 */
+		while ((vqp->vq_state == VSW_QUEUE_RUNNING) &&
+				(vqp->vq_first == NULL)) {
+			cv_wait(&vqp->vq_cv, &vqp->vq_lock);
+		}
+
+		/*
+		 * Process packets that we received from the interface.
+		 */
+		if (vqp->vq_first != NULL) {
+			mp = vqp->vq_first;
+
+			vqp->vq_first = NULL;
+			vqp->vq_last = NULL;
+
+			mutex_exit(&vqp->vq_lock);
+
+			/* switch the chain of packets received */
+			vsw_switch_frame(vswp, mp, VSW_PHYSDEV, NULL, NULL);
+
+			mutex_enter(&vqp->vq_lock);
+		}
+	}
+
+	/*
+	 * We are drained and signal we are done.
+	 */
+	vqp->vq_state = VSW_QUEUE_DRAINED;
+	cv_signal(&vqp->vq_cv);
+
+	/*
+	 * Exit lock and drain the remaining packets.
+	 */
+	mutex_exit(&vqp->vq_lock);
+
+	/*
+	 * Exit the thread
+	 */
+	thread_exit();
+}
+
+/*
+ * static void
+ * vsw_rx_queue_cb() - Receive callback routine when
+ *	vsw_multi_ring_enable is non-zero.  Queue the packets
+ *	to a packet queue for a worker thread to process.
+ */
+static void
+vsw_rx_queue_cb(void *arg, mac_resource_handle_t mrh, mblk_t *mp)
+{
+	vsw_mac_ring_t	*ringp = (vsw_mac_ring_t *)mrh;
+	vsw_t		*vswp = (vsw_t *)arg;
+	vsw_queue_t	*vqp;
+	mblk_t		*bp, *last;
+
+	ASSERT(mrh != NULL);
+	ASSERT(vswp != NULL);
+	ASSERT(mp != NULL);
+
+	D1(vswp, "%s: enter", __func__);
+
+	/*
+	 * Find the last element in the mblk chain.
+	 */
+	bp = mp;
+	do {
+		last = bp;
+		bp = bp->b_next;
+	} while (bp != NULL);
+
+	/* Get the queue for the packets */
+	vqp = ringp->ring_vqp;
+
+	/*
+	 * Grab the lock such we can queue the packets.
+	 */
+	mutex_enter(&vqp->vq_lock);
+
+	if (vqp->vq_state != VSW_QUEUE_RUNNING) {
+		freemsg(mp);
+		goto vsw_rx_queue_cb_exit;
+	}
+
+	/*
+	 * Add the mblk chain to the queue.  If there
+	 * is some mblks in the queue, then add the new
+	 * chain to the end.
+	 */
+	if (vqp->vq_first == NULL)
+		vqp->vq_first = mp;
+	else
+		vqp->vq_last->b_next = mp;
+
+	vqp->vq_last = last;
+
+	/*
+	 * Signal the worker thread that there is work to
+	 * do.
+	 */
+	cv_signal(&vqp->vq_cv);
+
+	/*
+	 * Let go of the lock and exit.
+	 */
+vsw_rx_queue_cb_exit:
+	mutex_exit(&vqp->vq_lock);
+	D1(vswp, "%s: exit", __func__);
+}
+
 /*
  * receive callback routine. Invoked by MAC layer when there
  * are pkts being passed up from physical device.
diff --git a/usr/src/uts/sun4v/sys/mmu.h b/usr/src/uts/sun4v/sys/mmu.h
index 61d0812ace..80d1c07f8f 100644
--- a/usr/src/uts/sun4v/sys/mmu.h
+++ b/usr/src/uts/sun4v/sys/mmu.h
@@ -1,4 +1,25 @@
 /*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
diff --git a/usr/src/uts/sun4v/sys/vcc.h b/usr/src/uts/sun4v/sys/vcc.h
index 378fdce8e2..7be5772c10 100644
--- a/usr/src/uts/sun4v/sys/vcc.h
+++ b/usr/src/uts/sun4v/sys/vcc.h
@@ -81,6 +81,7 @@ extern "C" {
 typedef enum {
 	VCC_CONS_ADDED,		    /* a port was added */
 	VCC_CONS_DELETED,	    /* a port was removed */
+	VCC_CONS_MISS_ADDED,	    /* wakeup after an added port was deleted */
 	/* XXX not implemented yet */
 	VCC_CONS_UPDATED	    /* a port configuration was changed */
 } vcc_reason_t;
diff --git a/usr/src/uts/sun4v/sys/vsw.h b/usr/src/uts/sun4v/sys/vsw.h
index 348000127e..eaf0873d9f 100644
--- a/usr/src/uts/sun4v/sys/vsw.h
+++ b/usr/src/uts/sun4v/sys/vsw.h
@@ -385,6 +385,51 @@ typedef struct vsw_ctrl_task {
 } vsw_ctrl_task_t;
 
 /*
+ * Vsw queue -- largely modeled after squeue
+ */
+#define	VSW_QUEUE_RUNNING	0x01
+#define	VSW_QUEUE_STOP		0x02
+#define	VSW_QUEUE_DRAINED	0x04
+
+typedef struct vsw_queue_s {
+	kmutex_t	vq_lock;	/* Lock, before using any member. */
+	kcondvar_t	vq_cv;		/* Async threads block on. */
+	uint32_t	vq_state;	/* State flags. */
+
+	mblk_t		*vq_first;	/* First mblk chain or NULL. */
+	mblk_t		*vq_last;	/* Last mblk chain. */
+
+	processorid_t	vq_bind;	/* Process to bind to */
+	kthread_t	*vq_worker;	/* Queue's thread */
+} vsw_queue_t;
+
+/*
+ * VSW MAC Ring Resources.
+ *	MAC Ring resource is composed of this state structure and
+ *	a kernel thread to perform the processing of the ring.
+ */
+typedef struct vsw_mac_ring_s {
+	uint32_t	ring_state;
+
+	mac_blank_t	ring_blank;
+	void		*ring_arg;
+
+	vsw_queue_t	*ring_vqp;
+	struct vsw	*ring_vswp;
+} vsw_mac_ring_t;
+
+/*
+ * Maximum Ring Resources.
+ */
+#define	VSW_MAC_RX_RINGS	0x40
+
+/*
+ * States for entry in ring table.
+ */
+#define	VSW_MAC_RING_FREE	1
+#define	VSW_MAC_RING_INUSE	2
+
+/*
  * Number of hash chains in the multicast forwarding database.
  */
 #define		VSW_NCHAINS	8
@@ -442,6 +487,15 @@ typedef struct	vsw {
 	mac_rx_handle_t		mrh;
 	multiaddress_capab_t	maddr;		/* Multiple uni addr capable */
 	const mac_txinfo_t	*txinfo;	/* MAC tx routine */
+	boolean_t		mstarted;	/* Mac Started? */
+	boolean_t		mresources;	/* Mac Resources cb? */
+
+	/*
+	 * MAC Ring Resources.
+	 */
+	kmutex_t		mac_ring_lock;	/* Lock for the table. */
+	uint32_t		mac_ring_tbl_sz;
+	vsw_mac_ring_t		*mac_ring_tbl;	/* Mac ring table. */
 
 	boolean_t		recfg_reqd;	/* Reconfig of addrs needed */
 	int			promisc_cnt;