6482166 vntsd dumps core and all console connections get killed.

6490897 vntsd fails on default control domain
6492688 ldc panic under load

11 files changed, 389 insertions, 132 deletions

diff --git a/usr/src/cmd/vntsd/console.c b/usr/src/cmd/vntsd/console.c
index 212d9c9415..f4a50349de 100644
--- a/usr/src/cmd/vntsd/console.c
+++ b/usr/src/cmd/vntsd/console.c
@@ -587,8 +587,11 @@ client_fini(vntsd_group_t *groupp, vntsd_client_t *clientp)
 
 	if ((groupp->no_cons_clientpq == NULL) &&
 	    (groupp->status & VNTSD_GROUP_SIG_WAIT)) {
-		/* group is waiting to be deleted */
-		groupp->status &= ~VNTSD_GROUP_SIG_WAIT;
+		/*
+		 * group is waiting to be deleted. - signal the group's
+		 * listen thread - the VNTSD_GROUP_SIG_WAIT state will
+		 * be cleared when the listen thread exits.
+		 */
 		(void) cond_signal(&groupp->cvp);
 	}
 	(void) mutex_unlock(&groupp->lock);
@@ -612,9 +615,17 @@ console_chk_status(vntsd_group_t *groupp, vntsd_client_t *clientp, int status)
 	(void) snprintf(err_msg, VNTSD_LINE_LEN, "console_chk_status client%d"
 	    " num_cos=%d", clientp->sockfd, groupp->num_cons);
 
+	/*
+	 * obtain group lock to protect groupp->num_cons.
+	 * When groupp->num_cons == 0, close client and exit the tread.
+	 */
+	(void) mutex_lock(&groupp->lock);
+
 	if (groupp->num_cons == 0) {
 		/* no more console in the group */
+		(void) mutex_unlock(&groupp->lock);
 		client_fini(groupp, clientp);
+		return;
 	}
 
 	if (status == VNTSD_STATUS_INTR) {
@@ -625,33 +636,53 @@ console_chk_status(vntsd_group_t *groupp, vntsd_client_t *clientp, int status)
 	switch (status) {
 
 	case VNTSD_STATUS_CLIENT_QUIT:
+		(void) mutex_unlock(&groupp->lock);
 		client_fini(groupp, clientp);
 		return;
 
 	case VNTSD_STATUS_RESELECT_CONS:
-		assert(clientp->cons);
+
+		if (clientp->cons == NULL) {
+			/*
+			 * domain was deleted before client connects to it
+			 * connect to other console in the same group
+			 */
+			(void) mutex_unlock(&groupp->lock);
+			client_init(clientp);
+			return;
+		}
+
 		if ((groupp->num_cons == 1) &&
-		    (groupp->conspq->handle == clientp->cons)) {
+		    ((clientp->status & VNTSD_CLIENT_CONS_DELETED) ||
+		    (groupp->conspq->handle == clientp->cons))) {
 			/* no other selection available */
+			(void) mutex_unlock(&groupp->lock);
 			client_fini(groupp, clientp);
 		} else {
+			(void) mutex_unlock(&groupp->lock);
 			client_init(clientp);
 		}
+
 		return;
 
 	case VNTSD_STATUS_VCC_IO_ERR:
 		if ((clientp->status & VNTSD_CLIENT_CONS_DELETED) == 0) {
 			/* check if console was deleted  */
+			(void) mutex_unlock(&groupp->lock);
 			status = vntsd_vcc_err(clientp->cons);
+			(void) mutex_lock(&groupp->lock);
 		}
 
 		if (status != VNTSD_STATUS_CONTINUE) {
 			/* console was deleted */
-			if (groupp->num_cons == 1) {
+			if (groupp->num_cons <= 1) {
+				(void) mutex_unlock(&groupp->lock);
 				client_fini(groupp, clientp);
+				return;
 			}
 		}
 
+		(void) mutex_unlock(&groupp->lock);
 		/* console is ok */
 		client_init(clientp);
 		return;
@@ -660,28 +691,31 @@ console_chk_status(vntsd_group_t *groupp, vntsd_client_t *clientp, int status)
 	case VNTSD_STATUS_MOV_CONS_BACKWARD:
 		if (groupp->num_cons == 1) {
 			/* same console */
+			(void) mutex_unlock(&groupp->lock);
 			return;
 		}
 
 		/* get selected console */
-		(void) mutex_lock(&(clientp->cons->group->lock));
-		clientp->cons = vntsd_que_pos(clientp->cons->group->conspq,
+		clientp->cons = vntsd_que_pos(groupp->conspq,
 		    clientp->cons,
 		    (status == VNTSD_STATUS_MOV_CONS_FORWARD)?(1):(-1));
-		(void) mutex_unlock(&(clientp->cons->group->lock));
+		(void) mutex_unlock(&groupp->lock);
 		return;
 
 	case VNTSD_SUCCESS:
 	case VNTSD_STATUS_CONTINUE:
 	case VNTSD_STATUS_NO_CONS:
+		(void) mutex_unlock(&groupp->lock);
 		client_init(clientp);
 		return;
 
 	case VNTSD_ERR_INVALID_INPUT:
+		(void) mutex_unlock(&groupp->lock);
 		return;
 
 	default:
 		/* fatal error */
+		(void) mutex_unlock(&groupp->lock);
 		vntsd_log(status, err_msg);
 		client_fini(groupp, clientp);
 		return;
@@ -743,6 +777,12 @@ vntsd_console_thread(vntsd_thr_arg_t *argp)
 			rv = read_cmd(clientp, prompt, &cmd);
 			/* check error and may exit */
 			console_chk_status(groupp, clientp, rv);
+
+			/* any console is removed from group? */
+			num_cons = vntsd_chk_group_total_cons(groupp);
+			if (num_cons <= 1) {
+				cmd = ' ';
+			}
 		}
 
 		switch (cmd) {
@@ -761,6 +801,10 @@ vntsd_console_thread(vntsd_thr_arg_t *argp)
 
 		case ' ':
 
+			if (num_cons == 0)
+				/* no console in the group */
+				break;
+
 			if (clientp->cons == NULL) {
 				if (num_cons == 1) {
 					/* by pass selecting console */
diff --git a/usr/src/cmd/vntsd/listen.c b/usr/src/cmd/vntsd/listen.c
index 358e2665aa..abcc5f1f8b 100644
--- a/usr/src/cmd/vntsd/listen.c
+++ b/usr/src/cmd/vntsd/listen.c
@@ -44,9 +44,11 @@
 #include <syslog.h>
 #include "vntsd.h"
 
+#define	    MAX_BIND_RETRIES		6
 /*
  * check the state of listen thread. exit if there is an fatal error
- * or the group is removed.
+ * or the group is removed. Main thread will call free_group
+ * to close group socket and free group structure.
  */
 static void
 listen_chk_status(vntsd_group_t *groupp, int status)
@@ -69,18 +71,15 @@ listen_chk_status(vntsd_group_t *groupp, int status)
 		return;
 
 	case VNTSD_STATUS_INTR:
+		/* signal for deleting group */
 		assert(groupp->status & VNTSD_GROUP_SIG_WAIT);
-		/* close listen socket */
-		(void) mutex_lock(&groupp->lock);
-		(void) close(groupp->sockfd);
-		groupp->sockfd = -1;
 
-		/* let group know */
+		/* let main thread know  */
+		(void) mutex_lock(&groupp->lock);
 		groupp->status &= ~VNTSD_GROUP_SIG_WAIT;
 		(void) cond_signal(&groupp->cvp);
-
 		(void) mutex_unlock(&groupp->lock);
-		/* exit thread */
+
 		thr_exit(0);
 		break;
 
@@ -89,16 +88,33 @@ listen_chk_status(vntsd_group_t *groupp, int status)
 
 	case VNTSD_STATUS_NO_CONS:
 	default:
-		/* fatal, exit thread */
+		/* fatal error or no console in the group, remove the group. */
 
 		(void) mutex_lock(&groupp->lock);
-		(void) close(groupp->sockfd);
-		groupp->sockfd = -1;
+
+		if (groupp->status & VNTSD_GROUP_SIG_WAIT) {
+			/* group is already in deletion */
+			(void) mutex_unlock(&groupp->lock);
+			return;
+		}
+
+		/*
+		 * if there still is console(s) in the group,
+		 * the console(s) could not be connected any more because of
+		 * a fatal error. Therefore, mark the console and notify
+		 * main thread to delete console and group.
+		 */
+		(void) vntsd_que_walk(groupp->conspq,
+		    (el_func_t)vntsd_mark_deleted_cons);
+		groupp->status |= VNTSD_GROUP_CLEAN_CONS;
+
+		/* signal main thread to delete the group */
+		(void) thr_kill(groupp->vntsd->tid, SIGUSR1);
 		(void) mutex_unlock(&groupp->lock);
-		vntsd_log(status, err_msg);
-		vntsd_clean_group(groupp);
 
-		thr_exit(0);
+		/* log error */
+		if (status != VNTSD_STATUS_NO_CONS)
+			vntsd_log(status, err_msg);
 		break;
 	}
 }
@@ -110,6 +126,7 @@ open_socket(int port_no, int *sockfd)
 
 	struct	    sockaddr_in addr;
 	int	    on;
+	int	    retries = 0;
 
 
 	/* allocate a socket */
@@ -134,11 +151,34 @@ open_socket(int port_no, int *sockfd)
 	addr.sin_port = htons(port_no);
 
 	/* bind socket */
-	if (bind(*sockfd, (struct sockaddr *)&addr, sizeof (addr)) < 0) {
-		if (errno == EINTR) {
-			return (VNTSD_STATUS_INTR);
+
+	for (; ; ) {
+
+		/*
+		 * After a socket is closed, the port
+		 * is transitioned to TIME_WAIT state.
+		 * It may take a few retries to bind
+		 * a just released port.
+		 */
+		if (bind(*sockfd, (struct sockaddr *)&addr,
+			    sizeof (addr)) < 0) {
+
+			if (errno == EINTR) {
+				return (VNTSD_STATUS_INTR);
+			}
+
+			if (errno == EADDRINUSE && retries < MAX_BIND_RETRIES) {
+				/* port may be in TIME_WAIT state, retry */
+				(void) sleep(5);
+				retries++;
+				continue;
+			}
+
+			return (VNTSD_ERR_LISTEN_BIND);
+
 		}
-		return (VNTSD_ERR_LISTEN_BIND);
+
+		break;
 
 	}
 
@@ -265,6 +305,7 @@ vntsd_listen_thread(vntsd_group_t *groupp)
 		if (num_cons == 0) {
 			(void) close(newsockfd);
 			listen_chk_status(groupp, VNTSD_STATUS_NO_CONS);
+			continue;
 		}
 
 		/* a connection is established */
diff --git a/usr/src/cmd/vntsd/svc-vntsd b/usr/src/cmd/vntsd/svc-vntsd
index e573b4ecd5..1ac7d6a9fd 100644
--- a/usr/src/cmd/vntsd/svc-vntsd
+++ b/usr/src/cmd/vntsd/svc-vntsd
@@ -55,7 +55,17 @@ if [ -n "$timeout" ]; then
 fi
 
 if [ -x /usr/lib/ldoms/vntsd ]; then
-    /usr/lib/ldoms/vntsd $args || exit $SMF_EXIT_ERR_CONFIG
+    /usr/lib/ldoms/vntsd $args
+    rc=$?
+    if [ $rc -ne 0 ]; then
+	# if vntsd exited in error with status 1, let SMF restart it
+	# otherwise we want it to go into maintenance.
+	if [ $rc -eq 1 ]; then
+	    exit $SMF_ERR_OTHER
+	else
+	    exit $SMF_ERR_FATAL
+	fi
+    fi
 else
     echo "WARNING: /usr/lib/ldoms/vntsd is missing or not executable" >& 2
     exit $SMF_EXIT_ERR_CONFIG
diff --git a/usr/src/cmd/vntsd/vntsd.c b/usr/src/cmd/vntsd/vntsd.c
index ea0112016e..5ec64a2d6e 100644
--- a/usr/src/cmd/vntsd/vntsd.c
+++ b/usr/src/cmd/vntsd/vntsd.c
@@ -82,7 +82,12 @@ exit_sig_handler(int sig)
 
 	D1(stderr, "t@%d exit_sig_handler%d \n", thr_self(), sig);
 
-	exit(0);
+	if (thr_self() != vntsdp->tid) {
+		/* not main thread, pass to main thread */
+		(void) thr_kill(vntsdp->tid, sig);
+	} else {
+		exit(0);
+	}
 }
 
 /*
@@ -358,20 +363,18 @@ main(int argc, char ** argv)
 	(void) snprintf(path, sz-1, VCC_DEVICE_CTL_PATH, vntsdp->devinst,
 	    sizeof (vntsdp->devinst));
 	vntsdp->ctrl_fd = open(path, O_RDWR);
-	free(path);
 
 	if (vntsdp->ctrl_fd == -1) {
-		/*
-		 * do not print error if device is not present
-		 * the daemon is probably being started incorrectly
-		 */
-		if (errno != ENOENT) {
-			syslog(LOG_ERR,
-			    "Error opening VCC device control port: %s",
-			    strerror(errno));
-		}
-		exit(1);
+		/* print error if device is not present */
+		syslog(LOG_ERR,
+		    "Error opening VCC device control port: %s",
+		    path);
+		/* tell SMF no retry */
+		exit(2);
 	}
+
+	free(path);
+
 	if ((vntsdp->options & VNTSD_OPT_DAEMON_OFF) == 0) {
 		/* daemonize it */
 		pid = fork();
@@ -483,6 +486,11 @@ main(int argc, char ** argv)
 		    poll_drv[0].revents);
 
 		vntsd_daemon_wakeup(vntsdp);
+		/*
+		 * Main thread may miss a console-delete signal when it is
+		 * not polling vcc. check if any console is deleted.
+		 */
+		vntsd_delete_cons(vntsdp);
 
 	}
 
@@ -529,9 +537,8 @@ vntsd_vcc_ioctl(int ioctl_code, uint_t portno, void *buf)
 }
 
 /*
- * check if a vcc i/o error is caused by removal of a console. If so notify
- * all clients connected to the console and wake up main thread to cleanup
- * the console.
+ * check if a vcc i/o error is caused by removal of a console. If so
+ * wake up main thread to cleanup the console.
  */
 int
 vntsd_vcc_err(vntsd_cons_t *consp)
@@ -556,9 +563,10 @@ vntsd_vcc_err(vntsd_cons_t *consp)
 	(void) mutex_lock(&consp->lock);
 	consp->status |= VNTSD_CONS_DELETED;
 
-	/* signal all clients to disconnect from console */
-	(void) vntsd_que_walk(consp->clientpq,
-	    (el_func_t)vntsd_notify_client_cons_del);
+	/*
+	 * main thread will close all clients after receiving console
+	 * delete signal.
+	 */
 	(void) mutex_unlock(&consp->lock);
 
 	/* mark the group */
diff --git a/usr/src/cmd/vntsd/vntsd.h b/usr/src/cmd/vntsd/vntsd.h
index efaf724c15..5c97b66284 100644
--- a/usr/src/cmd/vntsd/vntsd.h
+++ b/usr/src/cmd/vntsd/vntsd.h
@@ -206,20 +206,41 @@ extern "C" {
 /* vntsd options */
 #define	    VNTSD_OPT_DAEMON_OFF	0x1
 
-/* group states */
+/*
+ * group states
+ * When a console is removed or vntsd is exiting, main thread
+ * notifies listen, read and write thread to exit.
+ * After those threads exit, main thread clears up group structurre.
+ *
+ * VNTSD_GROUP_SIG_WAIT
+ * The main thread is waiting for listen thread to exit.
+ * VNTSD_GROUP_CLEAN_CONS
+ * There are console(s) in the group that are being removed.
+ * This is a transition state where the corresponding vcc port has been
+ * removed, but vntsd has not done its clean up yet.
+ * VNTSD_GROUP_IN_CLEANUP
+ * vntsd main thread has started cleaning up the group.
+ */
 
-#define	    VNTSD_GROUP_SIG_WAIT	0x1	/*  waiting for signal */
-#define	    VNTSD_GROUP_CLEAN_CONS	0x2	/*  cons needs to be clean */
-#define	    VNTSD_GROUP_CLEANUP		0x4	/*  waiting for signal */
+#define	    VNTSD_GROUP_SIG_WAIT	0x1
+#define	    VNTSD_GROUP_CLEAN_CONS	0x2
+#define	    VNTSD_GROUP_IN_CLEANUP	0x4
 
 
 
 
 
-/* console status */
+/*
+ * console states
+ * There are two states when a console is removed
+ * VNTSD_CONS_DELETED
+ * the console is being deleted
+ * VNTSD_CONS_SIG_WAIT
+ * console is waiting for all clients to exit.
+ */
 
 #define	    VNTSD_CONS_DELETED		0x1	/* deleted */
-#define	    VNTSD_CONS_SIG_WAIT		0x2	/* waiting fro signal */
+#define	    VNTSD_CONS_SIG_WAIT		0x2	/* waiting for signal */
 
 
 #define	    VNTSD_CLIENT_IO_ERR		    0x1	    /* reader */
@@ -451,6 +472,7 @@ int		vntsd_cons_chk_intr(vntsd_client_t *clientp);
 boolean_t	vntsd_vcc_cons_alive(vntsd_cons_t *consp);
 boolean_t	vntsd_notify_client_cons_del(vntsd_client_t *clientp);
 int		vntsd_chk_group_total_cons(vntsd_group_t *groupp);
+boolean_t	vntsd_mark_deleted_cons(vntsd_cons_t *consp);
 
 
 #ifdef	DEBUG
diff --git a/usr/src/cmd/vntsd/vntsdvcc.c b/usr/src/cmd/vntsd/vntsdvcc.c
index 8466601efc..1ac6bc8cdf 100644
--- a/usr/src/cmd/vntsd/vntsdvcc.c
+++ b/usr/src/cmd/vntsd/vntsdvcc.c
@@ -69,9 +69,23 @@ free_cons(vntsd_cons_t *consp)
 	assert(consp);
 	(void) mutex_destroy(&consp->lock);
 	(void) cond_destroy(&consp->cvp);
+	if (consp->vcc_fd != -1)
+		(void) close(consp->vcc_fd);
 	free(consp);
 }
 
+/* free group structure */
+static void
+free_group(vntsd_group_t *groupp)
+{
+	assert(groupp);
+	(void) mutex_destroy(&groupp->lock);
+	(void) cond_destroy(&groupp->cvp);
+	if (groupp->sockfd != -1)
+		(void) close(groupp->sockfd);
+	free(groupp);
+}
+
 /*
  *  all clients connected to a console must disconnect before
  *  removing a console.
@@ -166,25 +180,28 @@ vntsd_delete_cons(vntsd_t *vntsdp)
 		for (; ; ) {
 			/* get the console to be deleted */
 			(void) mutex_lock(&groupp->lock);
-			assert(groupp->conspq);
-			consp = vntsd_que_walk(groupp->conspq,
-			    (el_func_t)find_clean_cons);
-			if (consp == NULL) {
-				/* no more cons to delete */
-				(void) mutex_unlock(&groupp->lock);
-				break;
-			}
 
-			/* remove console from the group */
-			(void) vntsd_que_rm(&groupp->conspq, consp);
-			(void) mutex_unlock(&groupp->lock);
+			/* clean up any deleted console in the group */
+			if (groupp->conspq != NULL) {
+				consp = vntsd_que_walk(groupp->conspq,
+				    (el_func_t)find_clean_cons);
+				if (consp == NULL) {
+					/* no more cons to delete */
+					(void) mutex_unlock(&groupp->lock);
+					break;
+				}
+
+				/* remove console from the group */
+				(void) vntsd_que_rm(&groupp->conspq, consp);
+				(void) mutex_unlock(&groupp->lock);
 
-			/* clean up the console */
-			cleanup_cons(consp);
+				/* clean up the console */
+				cleanup_cons(consp);
+			}
 
 			/* delete group? */
-			if (groupp->num_cons == 0) {
-				/* no more console delete it */
+			if (groupp->conspq == NULL) {
+				/* no more console in the group delete group */
 				assert(groupp->vntsd);
 
 				(void) mutex_lock(&groupp->vntsd->lock);
@@ -213,23 +230,21 @@ vntsd_clean_group(vntsd_group_t *groupp)
 	(void) mutex_lock(&groupp->lock);
 
 	/* prevent from reentry */
-	if (groupp->status & VNTSD_GROUP_CLEANUP) {
-		if (groupp->listen_tid == thr_self()) {
-			/* signal that the listen thread is exiting */
-			groupp->status &= ~VNTSD_GROUP_SIG_WAIT;
-			(void) cond_signal(&groupp->cvp);
-		}
+	if (groupp->status & VNTSD_GROUP_IN_CLEANUP) {
 		(void) mutex_unlock(&groupp->lock);
 		return;
 	}
-	groupp->status |= VNTSD_GROUP_CLEANUP;
+	groupp->status |= VNTSD_GROUP_IN_CLEANUP;
+
+	/* mark group waiting for listen thread to exits */
+	groupp->status |= VNTSD_GROUP_SIG_WAIT;
 	(void) mutex_unlock(&groupp->lock);
 
 	vntsd_free_que(&groupp->conspq, (clean_func_t)cleanup_cons);
 
+	(void) mutex_lock(&groupp->lock);
 	/* walk through no cons client queue */
 	while (groupp->no_cons_clientpq != NULL) {
-		groupp->status |= VNTSD_GROUP_SIG_WAIT;
 		(void) vntsd_que_walk(groupp->no_cons_clientpq,
 		    (el_func_t)vntsd_notify_client_cons_del);
 		to.tv_sec = VNTSD_CV_WAIT_DELTIME;
@@ -237,23 +252,9 @@ vntsd_clean_group(vntsd_group_t *groupp)
 		(void) cond_reltimedwait(&groupp->cvp, &groupp->lock, &to);
 	}
 
-	if (groupp->listen_tid == thr_self()) {
-		/* listen thread is exiting */
-		(void) mutex_lock(&(groupp->vntsd->lock));
-		(void) vntsd_que_rm(&groupp->vntsd->grouppq, groupp);
-		(void) mutex_unlock(&groupp->vntsd->lock);
-
-		(void) cond_destroy(&groupp->cvp);
-		(void) mutex_unlock(&groupp->lock);
-		(void) mutex_destroy(&groupp->lock);
-		free(groupp);
-		return;
-	}
-
-	/* signal listen thread to exit  */
-	groupp->status |= VNTSD_GROUP_SIG_WAIT;
-
+	/* waiting for listen thread to exit */
 	while (groupp->status & VNTSD_GROUP_SIG_WAIT) {
+		/* signal listen thread to exit  */
 		(void) thr_kill(groupp->listen_tid, SIGUSR1);
 		to.tv_sec = VNTSD_CV_WAIT_DELTIME;
 		to.tv_nsec = 0;
@@ -264,9 +265,7 @@ vntsd_clean_group(vntsd_group_t *groupp)
 	(void) mutex_unlock(&groupp->lock);
 	(void) thr_join(groupp->listen_tid, NULL, NULL);
 	/* free group */
-	(void) cond_destroy(&groupp->cvp);
-	(void) mutex_destroy(&groupp->lock);
-	free(groupp);
+	free_group(groupp);
 }
 
 /* allocate and initialize console structure */
@@ -293,7 +292,7 @@ alloc_cons(vntsd_group_t *groupp, vcc_console_t *consolep)
 	(void) strlcpy(consp->domain_name, consolep->domain_name, MAXPATHLEN);
 	(void) strlcpy(consp->dev_name, consolep->dev_name, MAXPATHLEN);
 	consp->wr_tid = (thread_t)-1;
-	consp->vcc_fd = (thread_t)-1;
+	consp->vcc_fd = -1;
 
 	/* join the group */
 	(void) mutex_lock(&groupp->lock);
@@ -350,7 +349,7 @@ alloc_group(vntsd_t *vntsdp, char *group_name, uint64_t tcp_port)
 
 	groupp->tcp_port = tcp_port;
 	groupp->listen_tid = (thread_t)-1;
-	groupp->sockfd = (thread_t)-1;
+	groupp->sockfd = -1;
 	groupp->vntsd = vntsdp;
 
 	D1(stderr, "t@%d alloc_group@%lld:%s\n", thr_self(), groupp->tcp_port,
@@ -359,6 +358,16 @@ alloc_group(vntsd_t *vntsdp, char *group_name, uint64_t tcp_port)
 	return (groupp);
 }
 
+/* mark a deleted console */
+boolean_t
+vntsd_mark_deleted_cons(vntsd_cons_t *consp)
+{
+	(void) mutex_lock(&consp->lock);
+	consp->status |= VNTSD_CONS_DELETED;
+	(void) mutex_unlock(&consp->lock);
+	return (B_FALSE);
+}
+
 /*
  * Initialize a console, if console is associated with with a
  * new group, intialize the group.
@@ -378,19 +387,55 @@ alloc_cons_with_group(vntsd_t *vntsdp, vcc_console_t *consp,
 	(void) mutex_lock(&vntsdp->lock);
 	groupp = vntsd_que_find(vntsdp->grouppq,
 	    (compare_func_t)grp_by_tcp, (void *)&(consp->tcp_port));
+	if (groupp != NULL)
+		(void) mutex_lock(&groupp->lock);
+
 	(void) mutex_unlock(&vntsdp->lock);
 
 	if (groupp != NULL) {
-		/* group with same tcp port found */
+		/*
+		 *  group with same tcp port found.
+		 *  if there is no console in the group, the
+		 *  group should be removed and the tcp port can
+		 *  be used for tne new group.
+		 *  This is possible, when there is tight loop of
+		 *  creating/deleting domains. When a vcc port is
+		 *  removed, a read thread will have an I/O error because
+		 *  vcc has closed the port. The read thread then marks
+		 *  the console is removed and notify main thread to
+		 *  remove the console.
+		 *  Meanwhile, the same port and its group (with same
+		 *  tcp port and group name) is created. Vcc notify
+		 *  vntsd that new console is added.
+		 *  Main thread now have two events. If main thread polls
+		 *  out vcc notification first, it will find that there is
+		 *  a group has no console.
+		 */
+
+		if (vntsd_chk_group_total_cons(groupp) == 0) {
+
+			/* all consoles in the group have been removed */
+			(void) vntsd_que_walk(groupp->conspq,
+			    (el_func_t)vntsd_mark_deleted_cons);
+			groupp->status |= VNTSD_GROUP_CLEAN_CONS;
+			(void) mutex_unlock(&groupp->lock);
+			groupp = NULL;
 
-		if (strcmp(groupp->group_name, consp->group_name)) {
+		} else if (strcmp(groupp->group_name, consp->group_name)) {
 			/* conflict group name */
 			vntsd_log(VNTSD_ERR_VCC_GRP_NAME,
 			    "group name is different from existing group");
+			(void) mutex_unlock(&groupp->lock);
 			return (VNTSD_ERR_VCC_CTRL_DATA);
+
+		} else {
+			/* group already existed */
+			(void) mutex_unlock(&groupp->lock);
 		}
 
-	} else {
+	}
+
+	if (groupp == NULL) {
 		/* new group */
 		groupp = alloc_group(vntsdp, consp->group_name,
 		    consp->tcp_port);
@@ -416,9 +461,7 @@ alloc_cons_with_group(vntsd_t *vntsdp, vcc_console_t *consp,
 		/* no memory */
 		if (new_groupp != NULL) {
 			/* clean up new group */
-			(void) cond_destroy(&groupp->cvp);
-			(void) mutex_destroy(&groupp->lock);
-			free(groupp);
+			free_group(groupp);
 		}
 
 		return (VNTSD_ERR_NO_MEM);
@@ -463,17 +506,59 @@ create_listen_thread(vntsd_group_t *groupp)
 	return (rv != 0);
 }
 
+/* find deleted console by console no */
+static boolean_t
+deleted_cons_by_consno(vntsd_cons_t *consp, int *cons_no)
+{
+	vntsd_client_t *clientp;
+
+	assert(consp);
+
+	if (consp->cons_no != *cons_no)
+		return (B_FALSE);
+
+	/* has console marked as deleted? */
+	if ((consp->status & VNTSD_CONS_DELETED) == 0)
+		return (B_TRUE);
+
+	/* notify clients of console ? */
+	clientp = (vntsd_client_t *)consp->clientpq->handle;
+
+	if (clientp == NULL)
+		/* therre is no client for this console */
+		return (B_TRUE);
+
+	if (clientp->status & VNTSD_CLIENT_CONS_DELETED)
+		/* clients of console have notified */
+		return (B_FALSE);
+
+	return (B_TRUE);
+}
+
+/* find group structure from console no */
+static boolean_t
+find_cons_group_by_cons_no(vntsd_group_t *groupp, uint_t *cons_no)
+{
+	vntsd_cons_t *consp;
+
+	consp = vntsd_que_find(groupp->conspq,
+	    (compare_func_t)deleted_cons_by_consno, cons_no);
+	return (consp != NULL);
+
+}
+
 /* delete a console if the console exists in the vntsd */
 static void
-delete_cons_before_add(vntsd_t *vntsdp, uint64_t tcp_port, uint_t cons_no)
+delete_cons_before_add(vntsd_t *vntsdp, uint_t cons_no)
 {
 	vntsd_group_t	    *groupp;
 	vntsd_cons_t	    *consp;
 
 	/* group exists? */
 	(void) mutex_lock(&vntsdp->lock);
-	groupp = vntsd_que_find(vntsdp->grouppq, (compare_func_t)grp_by_tcp,
-	    (void *)&(tcp_port));
+	groupp = vntsd_que_find(vntsdp->grouppq,
+	    (compare_func_t)find_cons_group_by_cons_no,
+	    &cons_no);
 	(void) mutex_unlock(&vntsdp->lock);
 
 	if (groupp == NULL) {
@@ -484,7 +569,7 @@ delete_cons_before_add(vntsd_t *vntsdp, uint64_t tcp_port, uint_t cons_no)
 	/* group exists, if console exists? */
 	(void) mutex_lock(&groupp->lock);
 	consp = vntsd_que_find(groupp->conspq,
-	    (compare_func_t)vntsd_cons_by_consno, &cons_no);
+	    (compare_func_t)deleted_cons_by_consno, &cons_no);
 
 	if (consp == NULL) {
 		/* no such console */
@@ -527,7 +612,7 @@ do_add_cons(vntsd_t *vntsdp, int cons_no)
 	}
 
 	/* clean up the console if console was deleted and added again */
-	delete_cons_before_add(vntsdp, console.tcp_port, console.cons_no);
+	delete_cons_before_add(vntsdp, console.cons_no);
 
 	/* initialize console */
 
@@ -543,9 +628,7 @@ do_add_cons(vntsd_t *vntsdp, int cons_no)
 		/* create listen thread for this console */
 		if (create_listen_thread(groupp)) {
 			vntsd_log(VNTSD_ERR_CREATE_LISTEN_THR, err_msg);
-			(void) cond_destroy(&groupp->cvp);
-			(void) mutex_destroy(&groupp->lock);
-			free(groupp);
+			free_group(groupp);
 		}
 
 	}
diff --git a/usr/src/uts/sun4v/io/ldc.c b/usr/src/uts/sun4v/io/ldc.c
index 1ac0d7c3ec..5f32265779 100644
--- a/usr/src/uts/sun4v/io/ldc.c
+++ b/usr/src/uts/sun4v/io/ldc.c
@@ -185,6 +185,12 @@ uint64_t ldc_maptable_entries = LDC_MTBL_ENTRIES;
 int ldc_max_retries = LDC_MAX_RETRIES;
 clock_t ldc_delay = LDC_DELAY;
 
+/*
+ * delay between each retry of channel unregistration in
+ * ldc_close(), to wait for pending interrupts to complete.
+ */
+clock_t ldc_close_delay = LDC_CLOSE_DELAY;
+
 #ifdef DEBUG
 
 /*
@@ -1832,7 +1838,7 @@ i_ldc_tx_hdlr(caddr_t arg1, caddr_t arg2)
 	if (!ldcp->cb_enabled)
 		notify_client = B_FALSE;
 
-	/* Unlock channel */
+	i_ldc_clear_intr(ldcp, CNEX_TX_INTR);
 
 	if (notify_client) {
 		ldcp->cb_inprogress = B_TRUE;
@@ -1847,7 +1853,6 @@ i_ldc_tx_hdlr(caddr_t arg1, caddr_t arg2)
 		ldcp->cb_inprogress = B_FALSE;
 	}
 
-	i_ldc_clear_intr(ldcp, CNEX_TX_INTR);
 	mutex_exit(&ldcp->lock);
 
 	D1(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) exiting handler", ldcp->id);
@@ -2108,17 +2113,22 @@ loop_exit:
 	} else
 		ldcp->rx_intr_state = LDC_INTR_PEND;
 
-	mutex_exit(&ldcp->lock);
 
 	if (notify_client) {
+		ldcp->cb_inprogress = B_TRUE;
+		mutex_exit(&ldcp->lock);
 		rv = ldcp->cb(notify_event, ldcp->cb_arg);
 		if (rv) {
 			DWARN(ldcp->id,
 			    "i_ldc_rx_hdlr: (0x%llx) callback failure",
 			    ldcp->id);
 		}
+		mutex_enter(&ldcp->lock);
+		ldcp->cb_inprogress = B_FALSE;
 	}
 
+	mutex_exit(&ldcp->lock);
+
 	D1(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) exiting handler", ldcp->id);
 	return (DDI_INTR_CLAIMED);
 }
@@ -2656,6 +2666,13 @@ ldc_close(ldc_handle_t handle)
 		return (EBUSY);
 	}
 
+	if (ldcp->cb_inprogress) {
+		DWARN(ldcp->id, "ldc_close: (0x%llx) callback active\n",
+		    ldcp->id);
+		mutex_exit(&ldcp->lock);
+		return (EWOULDBLOCK);
+	}
+
 	/* Obtain Tx lock */
 	mutex_enter(&ldcp->tx_lock);
 
@@ -2723,7 +2740,7 @@ ldc_close(ldc_handle_t handle)
 		 * As there could be pending interrupts we need
 		 * to wait and try again
 		 */
-		drv_usecwait(ldc_delay);
+		drv_usecwait(ldc_close_delay);
 		mutex_enter(&ldcp->lock);
 		mutex_enter(&ldcp->tx_lock);
 		retries++;
diff --git a/usr/src/uts/sun4v/io/vcc.c b/usr/src/uts/sun4v/io/vcc.c
index ac57013c23..739a7899f4 100644
--- a/usr/src/uts/sun4v/io/vcc.c
+++ b/usr/src/uts/sun4v/io/vcc.c
@@ -415,8 +415,8 @@ i_vcc_ldc_init(vcc_t *vccp, vcc_port_t *vport)
 	attr.mode = LDC_MODE_RAW;
 
 	if ((rv = ldc_init(vport->ldc_id, &attr, &(vport->ldc_handle))) != 0) {
-		cmn_err(CE_CONT, "i_vcc_ldc_init: port %d inv channel 0x%lx\n",
-		    vport->number, vport->ldc_id);
+		cmn_err(CE_CONT, "i_vcc_ldc_init: port %d ldc channel %ld"
+		    " failed ldc_init %d \n", vport->number, vport->ldc_id, rv);
 		vport->ldc_id = VCC_INVALID_CHANNEL;
 		return (rv);
 	}
@@ -1133,6 +1133,12 @@ vcc_open(dev_t *devp, int flag, int otyp, cred_t *cred)
 
 	mutex_enter(&vport->lock);
 
+	if ((vport->status & VCC_PORT_AVAIL) == 0) {
+		/* port may be removed */
+		mutex_exit(&vport->lock);
+		return (ENXIO);
+	}
+
 	if (vport->status & VCC_PORT_OPEN) {
 		/* only one open per port */
 		cmn_err(CE_CONT, "vcc_open: virtual-console-concentrator@%d:%d "
@@ -1482,7 +1488,7 @@ i_vcc_inquiry(vcc_t *vccp, caddr_t buf, int mode)
 		return (EINVAL);
 	}
 
-		/* an added port */
+	/* an added port */
 
 	D1("i_vcc_inquiry\n");
 
@@ -1629,6 +1635,8 @@ i_vcc_cons_status(vcc_t *vccp, caddr_t buf, int mode)
 		console.cons_no = -1;
 	} else if (console.tcp_port != vport->tcp_port) {
 		console.cons_no = -1;
+	} else if (vport->ldc_id == VCC_INVALID_CHANNEL) {
+		console.cons_no = -1;
 	}
 
 	D1("i_vcc_cons_status@%d: %s %s %llx\n", console.cons_no,
@@ -2360,6 +2368,7 @@ vcc_mdeg_cb(void *cb_argp, mdeg_result_t *resp)
 		if (rv !=  MDEG_SUCCESS) {
 			return (rv);
 		}
+
 	}
 
 	/*
@@ -2368,7 +2377,6 @@ vcc_mdeg_cb(void *cb_argp, mdeg_result_t *resp)
 	 * for now.
 	 */
 
-
 	return (MDEG_SUCCESS);
 }
 
diff --git a/usr/src/uts/sun4v/io/vldc.c b/usr/src/uts/sun4v/io/vldc.c
index a96b03f435..ee54b14f12 100644
--- a/usr/src/uts/sun4v/io/vldc.c
+++ b/usr/src/uts/sun4v/io/vldc.c
@@ -161,12 +161,10 @@ uint32_t vldc_max_mtu = VLDC_MAX_MTU;
 uint64_t vldc_max_cookie = VLDC_MAX_COOKIE;
 
 /*
- * when calls to LDC return EWOULDBLOCK or EAGAIN the operation is retried
- * up to 'vldc_retries' times with a wait of 'vldc_delay' microseconds
- * between each retry.
+ * when ldc_close() returns EAGAIN, it is retried with a wait
+ * of 'vldc_close_delay' between each retry.
  */
-static clock_t	vldc_delay = 100;
-static int	vldc_retries = 3;
+static clock_t	vldc_close_delay = VLDC_CLOSE_DELAY;
 
 #ifdef DEBUG
 
@@ -270,6 +268,10 @@ i_vldc_cb(uint64_t event, caddr_t arg)
 	/* ensure the port can't be destroyed while we are handling the cb */
 	mutex_enter(&vport->minorp->lock);
 
+	if (vport->status == VLDC_PORT_CLOSED) {
+		return (LDC_SUCCESS);
+	}
+
 	old_status = vport->ldc_status;
 	rv = ldc_status(vport->ldc_handle, &vport->ldc_status);
 	if (rv != 0) {
@@ -290,7 +292,7 @@ i_vldc_cb(uint64_t event, caddr_t arg)
 		 * implies that the port cannot be used until it has
 		 * been closed and reopened.
 		 */
-		if (vport->status != VLDC_PORT_CLOSED && old_status == LDC_UP) {
+		if (old_status == LDC_UP) {
 			vport->status = VLDC_PORT_RESET;
 			vport->hanged_up = B_TRUE;
 			pollevents = POLLHUP;
@@ -759,16 +761,10 @@ i_vldc_remove_port(vldc_t *vldcp, uint_t portno)
 static int
 i_vldc_ldc_close(vldc_port_t *vport)
 {
-	int retries = 0;	/* count of number of retries attempted */
 	int err = 0;
 
 	ASSERT(MUTEX_HELD(&vport->minorp->lock));
 
-	while ((err = ldc_close(vport->ldc_handle)) == EAGAIN) {
-		drv_usecwait(vldc_delay);
-		if (++retries > vldc_retries)
-			break;
-	}
 	/*
 	 * If ldc_close() succeeded or if the channel was already closed[*]
 	 * (possibly by a previously unsuccessful call to this function)
@@ -777,6 +773,7 @@ i_vldc_ldc_close(vldc_port_t *vport)
 	 *
 	 * [*] indicated by ldc_close() returning a value of EFAULT
 	 */
+	err = ldc_close(vport->ldc_handle);
 	if ((err != 0) && (err != EFAULT))
 		return (err);
 
@@ -798,6 +795,7 @@ static int
 i_vldc_close_port(vldc_t *vldcp, uint_t portno)
 {
 	vldc_port_t	*vport;
+	vldc_minor_t	*vminor;
 	int		rv = DDI_SUCCESS;
 
 	vport = &(vldcp->port[portno]);
@@ -807,6 +805,8 @@ i_vldc_close_port(vldc_t *vldcp, uint_t portno)
 	D1("i_vldc_close_port: vldc@%d:%d: closing port\n",
 	    vport->inst, vport->minorp->portno);
 
+	vminor = vport->minorp;
+
 	switch (vport->status) {
 	case VLDC_PORT_CLOSED:
 		/* nothing to do */
@@ -816,9 +816,27 @@ i_vldc_close_port(vldc_t *vldcp, uint_t portno)
 
 	case VLDC_PORT_READY:
 	case VLDC_PORT_RESET:
-		rv = i_vldc_ldc_close(vport);
-		if (rv != 0)
+		do {
+			rv = i_vldc_ldc_close(vport);
+			if (rv != EAGAIN)
+				break;
+
+			/*
+			 * EAGAIN indicates that ldc_close() failed because
+			 * ldc callback thread is active for the channel.
+			 * cv_timedwait() is used to release vminor->lock and
+			 * allow ldc callback thread to complete.
+			 * after waking up, check if the port has been closed
+			 * by another thread in the meantime.
+			 */
+			(void) cv_timedwait(&vminor->cv, &vminor->lock,
+			    ddi_get_lbolt() + drv_usectohz(vldc_close_delay));
+			rv = 0;
+		} while (vport->status != VLDC_PORT_CLOSED);
+
+		if ((rv != 0) || (vport->status == VLDC_PORT_CLOSED))
 			return (rv);
+
 		break;
 
 	case VLDC_PORT_OPEN:
@@ -837,7 +855,7 @@ i_vldc_close_port(vldc_t *vldcp, uint_t portno)
 	kmem_free(vport->send_buf, vport->mtu);
 	kmem_free(vport->recv_buf, vport->mtu);
 
-	if (strcmp(vport->minorp->sname, VLDC_HVCTL_SVCNAME) == 0)
+	if (strcmp(vminor->sname, VLDC_HVCTL_SVCNAME) == 0)
 		kmem_free(vport->cookie_buf, vldc_max_cookie);
 
 	vport->status = VLDC_PORT_CLOSED;
diff --git a/usr/src/uts/sun4v/sys/ldc_impl.h b/usr/src/uts/sun4v/sys/ldc_impl.h
index 16ec881617..b05f30f341 100644
--- a/usr/src/uts/sun4v/sys/ldc_impl.h
+++ b/usr/src/uts/sun4v/sys/ldc_impl.h
@@ -121,6 +121,9 @@ extern "C" {
 #define	LDC_MAX_RETRIES	1000
 #define	LDC_DELAY	1
 
+/* delay(usec) between channel unregister retries in ldc_close() */
+#define	LDC_CLOSE_DELAY	1
+
 /*
  * LDC Version information
  */
diff --git a/usr/src/uts/sun4v/sys/vldc_impl.h b/usr/src/uts/sun4v/sys/vldc_impl.h
index 7f4cb15d69..71c781fa7b 100644
--- a/usr/src/uts/sun4v/sys/vldc_impl.h
+++ b/usr/src/uts/sun4v/sys/vldc_impl.h
@@ -67,6 +67,9 @@ extern "C" {
 /* indicates an invalid port number */
 #define	VLDC_INVALID_PORTNO	((uint_t)-1)
 
+/* delay(in us) used to wait for pending callback to complete */
+#define	VLDC_CLOSE_DELAY	MICROSEC	/* 1sec */
+
 /*
  * Minor node number to port number mapping table.
  *