6623512 NFSv4 client panics when server changes security modes

6655251 Client side dispatching of NFS RPCs is not virtualized

5 files changed, 195 insertions, 159 deletions

diff --git a/usr/src/uts/common/rpc/clnt.h b/usr/src/uts/common/rpc/clnt.h
index f422f039d6..95cfcca9f3 100644
--- a/usr/src/uts/common/rpc/clnt.h
+++ b/usr/src/uts/common/rpc/clnt.h
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
@@ -276,6 +275,7 @@ typedef struct calllist_s {
 #define	call_status call_err.re_status	/* error on reply (rep is invalid) */
 #define	call_reason call_err.re_errno	/* reason code on T_DISCON_IND */
 	queue_t		*call_wq;	/* the write queue the call is using */
+	zoneid_t	call_zoneid;	/* zoneid the call was made from */
 } calllist_t;
 
 /*
diff --git a/usr/src/uts/common/rpc/clnt_clts.c b/usr/src/uts/common/rpc/clnt_clts.c
index e1a40d7df4..c9bb281d9d 100644
--- a/usr/src/uts/common/rpc/clnt_clts.c
+++ b/usr/src/uts/common/rpc/clnt_clts.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -488,6 +488,7 @@ clnt_clts_kcallit_addr(CLIENT *h, rpcproc_t procnum, xdrproc_t xdr_args,
 			endpnt_rele(p->cku_endpnt);
 		p->cku_endpnt = NULL;
 	}
+	call->call_zoneid = rpc_zoneid();
 
 	mpdup = NULL;
 call_again:
@@ -579,7 +580,7 @@ call_again:
 	round_trip = lbolt;
 
 	error = clnt_clts_dispatch_send(p->cku_endpnt->e_wq, mp,
-					&p->cku_addr, call, p->cku_xid);
+	    &p->cku_addr, call, p->cku_xid);
 
 	if (error != 0) {
 		freemsg(mp);
@@ -590,7 +591,7 @@ call_again:
 	}
 
 	RPCLOG(64, "clnt_clts_kcallit_addr: sent call for xid 0x%x\n",
-		p->cku_xid);
+	    p->cku_xid);
 
 	/*
 	 * There are two reasons for which we go back to to tryread.
@@ -629,14 +630,16 @@ tryread:
 
 		if (h->cl_nosignal)
 			while ((cv_wait_ret =
-				cv_timedwait(&call->call_cv,
-				&call->call_lock, cv_timout)) > 0 &&
-				call->call_notified == FALSE);
+			    cv_timedwait(&call->call_cv,
+			    &call->call_lock, cv_timout)) > 0 &&
+			    call->call_notified == FALSE)
+				;
 		else
 			while ((cv_wait_ret =
-				cv_timedwait_sig(&call->call_cv,
-				&call->call_lock, cv_timout)) > 0 &&
-				call->call_notified == FALSE);
+			    cv_timedwait_sig(&call->call_cv,
+			    &call->call_lock, cv_timout)) > 0 &&
+			    call->call_notified == FALSE)
+				;
 
 		if (cv_wait_ret == 0)
 			interrupted = TRUE;
@@ -679,16 +682,16 @@ tryread:
 				call->call_reply = NULL;
 				mutex_exit(&call->call_lock);
 				RPCLOG(8, "clnt_clts_kcallit_addr: "
-				"response received for request "
-				"w/xid 0x%x after timeout\n",
-				p->cku_xid);
+				    "response received for request "
+				    "w/xid 0x%x after timeout\n",
+				    p->cku_xid);
 				goto getresponse;
 			}
 			mutex_exit(&call->call_lock);
 
 			RPCLOG(8, "clnt_clts_kcallit_addr: "
-				"request w/xid 0x%x timedout "
-				"waiting for reply\n", p->cku_xid);
+			    "request w/xid 0x%x timedout "
+			    "waiting for reply\n", p->cku_xid);
 #if 0 /* XXX not yet */
 			/*
 			 * Timeout may be due to a dead gateway. Send
@@ -698,8 +701,8 @@ tryread:
 			 */
 			if (stries == p->cku_retrys/2) {
 				t_kadvise(p->cku_endpnt->e_tiptr,
-					(uchar_t *)p->cku_addr.buf,
-					p->cku_addr.len);
+				    (uchar_t *)p->cku_addr.buf,
+				    p->cku_addr.len);
 			}
 #endif /* not yet */
 			p->cku_err.re_status = RPC_TIMEDOUT;
@@ -737,7 +740,7 @@ getresponse:
 
 		pptr = (union T_primitives *)resp->b_rptr;
 		bcopy(resp->b_rptr + pptr->unitdata_ind.SRC_offset, sin->buf,
-			pptr->unitdata_ind.SRC_length);
+		    pptr->unitdata_ind.SRC_length);
 		sin->len = pptr->unitdata_ind.SRC_length;
 	}
 
@@ -834,7 +837,7 @@ getresponse:
 		 * Reply is good, check auth.
 		 */
 		if (!AUTH_VALIDATE(h->cl_auth,
-				    &reply_msg.acpted_rply.ar_verf)) {
+		    &reply_msg.acpted_rply.ar_verf)) {
 			p->cku_err.re_status = RPC_AUTHERROR;
 			p->cku_err.re_why = AUTH_INVALIDRESP;
 			RCSTAT_INCR(p->cku_stats, rcbadverfs);
@@ -850,8 +853,7 @@ getresponse:
 	}
 	/* set errno in case we can't recover */
 	if (re_status != RPC_VERSMISMATCH &&
-			    re_status != RPC_AUTHERROR &&
-			    re_status != RPC_PROGVERSMISMATCH)
+	    re_status != RPC_AUTHERROR && re_status != RPC_PROGVERSMISMATCH)
 		p->cku_err.re_errno = EIO;
 	/*
 	 * Determine whether or not we're doing an RPC
@@ -974,7 +976,7 @@ done1:
 	}
 	mutex_exit(&call->call_lock);
 	RPCLOG(64, "clnt_clts_kcallit_addr: xid 0x%x taken off dispatch list",
-		p->cku_xid);
+	    p->cku_xid);
 
 done:
 	if (resp != NULL) {
@@ -1037,7 +1039,7 @@ clnt_clts_kcallit(CLIENT *h, rpcproc_t procnum, xdrproc_t xdr_args,
 	struct timeval wait)
 {
 	return (clnt_clts_kcallit_addr(h, procnum, xdr_args, argsp,
-				xdr_results, resultsp, wait, NULL));
+	    xdr_results, resultsp, wait, NULL));
 }
 
 /*
@@ -1300,9 +1302,9 @@ endpnt_type_create(struct knetconfig *config)
 	cv_init(&etype->e_async_cv, NULL, CV_DEFAULT, NULL);
 
 	list_create(&etype->e_pool, sizeof (endpnt_t),
-			offsetof(endpnt_t, e_node));
+	    offsetof(endpnt_t, e_node));
 	list_create(&etype->e_ilist, sizeof (endpnt_t),
-			offsetof(endpnt_t, e_idle));
+	    offsetof(endpnt_t, e_idle));
 
 	/*
 	 * Check to see if we need to create a taskq for endpoint
@@ -1314,7 +1316,7 @@ endpnt_type_create(struct knetconfig *config)
 		mutex_exit(&endpnt_taskq_lock);
 		ASSERT(endpnt_taskq == NULL);
 		endpnt_taskq = taskq_create("clts_endpnt_taskq", 1,
-						minclsyspri, 200, INT_MAX, 0);
+		    minclsyspri, 200, INT_MAX, 0);
 	} else
 		mutex_exit(&endpnt_taskq_lock);
 
@@ -1360,8 +1362,8 @@ check_endpnt(struct endpnt *endp, struct endpnt **newp)
 	 * thread(s) checking this endpoint will move on.
 	 */
 	if ((endp->e_flags & ENDPNT_ESTABLISHED) &&
-		(!(endp->e_flags & ENDPNT_BOUND) ||
-		(endp->e_flags & ENDPNT_STALE))) {
+	    (!(endp->e_flags & ENDPNT_BOUND) ||
+	    (endp->e_flags & ENDPNT_STALE))) {
 		/*
 		 * Clear the flags here since they will be
 		 * set again by this thread.  They need to be
@@ -1369,7 +1371,7 @@ check_endpnt(struct endpnt *endp, struct endpnt **newp)
 		 * the state for ENDPNT_ONIDLE.
 		 */
 		endp->e_flags &= ~(ENDPNT_ESTABLISHED |
-				ENDPNT_WAITING | ENDPNT_BOUND |	ENDPNT_STALE);
+		    ENDPNT_WAITING | ENDPNT_BOUND | ENDPNT_STALE);
 		mutex_exit(&endp->e_lock);
 		return (1);
 	}
@@ -1381,7 +1383,7 @@ check_endpnt(struct endpnt *endp, struct endpnt **newp)
 	 * ENDPNT_STALE.
 	 */
 	while (!(endp->e_flags & ENDPNT_BOUND) &&
-		!(endp->e_flags & ENDPNT_STALE)) {
+	    !(endp->e_flags & ENDPNT_STALE)) {
 		endp->e_flags |= ENDPNT_WAITING;
 		cv_wait(&endp->e_cv, &endp->e_lock);
 	}
@@ -1449,7 +1451,7 @@ top:
 		if ((np->e_zoneid == zoneid) &&
 		    (np->e_rdev == config->knc_rdev) &&
 		    (strcmp(np->e_protofmly,
-			    config->knc_protofmly) == 0))
+		    config->knc_protofmly) == 0))
 			break;
 
 	if (np == NULL && n_etype != NULL) {
@@ -1596,7 +1598,7 @@ top:
 			}
 			endp = np->e_pcurr;
 			if ((next = list_next(&np->e_pool, np->e_pcurr)) !=
-				NULL)
+			    NULL)
 				np->e_pcurr = next;
 			ASSERT(endp != NULL);
 			mutex_enter(&endp->e_lock);
@@ -1685,13 +1687,12 @@ top:
 
 	if (useresvport &&
 	    (strcmp(config->knc_protofmly, NC_INET) == 0 ||
-		strcmp(config->knc_protofmly, NC_INET6) == 0)) {
+	    strcmp(config->knc_protofmly, NC_INET6) == 0)) {
 
 		while ((error =
-			bindresvport(new->e_tiptr, NULL, NULL, FALSE)) != 0) {
+		    bindresvport(new->e_tiptr, NULL, NULL, FALSE)) != 0) {
 			RPCLOG(1,
-				"endpnt_get: bindresvport error %d\n",
-				error);
+			    "endpnt_get: bindresvport error %d\n", error);
 			if (error != EPROTO) {
 				if (rtries-- <= 0)
 					goto bad;
@@ -1804,7 +1805,7 @@ endpnt_reap_settimer(endpnt_type_t *etp)
 {
 	if (etp->e_itimer == (timeout_id_t)0)
 		etp->e_itimer = timeout(endpnt_reap_dispatch, (void *)etp,
-					clnt_clts_taskq_dispatch_interval);
+		    clnt_clts_taskq_dispatch_interval);
 }
 
 static void
@@ -1883,10 +1884,10 @@ endpnt_reclaim(zoneid_t zoneid)
 
 		mutex_enter(&np->e_plock);
 		RPCLOG(1, "endpnt_reclaim: protofmly %s, ",
-			np->e_protofmly);
+		    np->e_protofmly);
 		RPCLOG(1, "rdev %ld\n", np->e_rdev);
 		RPCLOG(1, "endpnt_reclaim: found %d endpoint(s)\n",
-			np->e_cnt);
+		    np->e_cnt);
 
 		if (np->e_cnt == 0) {
 			mutex_exit(&np->e_plock);
@@ -2062,8 +2063,8 @@ clnt_clts_dispatch_send(queue_t *q, mblk_t *mp, struct netbuf *addr,
 	cp->call_status = RPC_TIMEDOUT;
 	cp->call_notified = FALSE;
 	RPCLOG(64,
-		"clnt_clts_dispatch_send: putting xid 0x%x on "
-		"dispatch list\n", xid);
+	    "clnt_clts_dispatch_send: putting xid 0x%x on "
+	    "dispatch list\n", xid);
 	cp->call_hash = call_hash(xid, clnt_clts_hash_size);
 	cp->call_bucket = &clts_call_ht[cp->call_hash];
 	call_table_enter(cp);
@@ -2162,8 +2163,8 @@ clnt_clts_dispatch_notify(mblk_t *mp, int resp_off, zoneid_t zoneid)
 		ASSERT(tmp == NULL && i < sizeof (xid));
 
 		RPCLOG0(1,
-			"clnt_dispatch_notify(clts): message less than "
-			"size of xid\n");
+		    "clnt_dispatch_notify(clts): message less than "
+		    "size of xid\n");
 
 		freemsg(mp);
 		return;
@@ -2185,14 +2186,26 @@ done_xid_copy:
 
 	if (e != NULL) {
 		mutex_enter(&e->call_lock);
+
+		/*
+		 * verify that the reply is coming in on
+		 * the same zone that it was sent from.
+		 */
+		if (e->call_zoneid != zoneid) {
+			mutex_exit(&e->call_lock);
+			mutex_exit(&chtp->ct_lock);
+			freemsg(mp);
+			return;
+		}
+
 		/*
 		 * found thread waiting for this reply.
 		 */
 		if (e->call_reply) {
 			RPCLOG(8,
-				"clnt_dispatch_notify (clts): discarding old "
-				"reply for xid 0x%x\n",
-				xid);
+			    "clnt_dispatch_notify (clts): discarding old "
+			    "reply for xid 0x%x\n",
+			    xid);
 			freemsg(e->call_reply);
 		}
 		e->call_notified = TRUE;
@@ -2207,7 +2220,7 @@ done_xid_copy:
 
 		mutex_exit(&chtp->ct_lock);
 		RPCLOG(8, "clnt_dispatch_notify (clts): no caller for reply "
-			"0x%x\n", xid);
+		    "0x%x\n", xid);
 		freemsg(mp);
 		/*
 		 * This is unfortunate, but we need to lookup the zone so we
@@ -2276,15 +2289,14 @@ clnt_clts_init(void)
 
 	if (clnt_clts_endpoint_reap_interval < DEFAULT_ENDPOINT_REAP_INTERVAL)
 		clnt_clts_endpoint_reap_interval =
-			DEFAULT_ENDPOINT_REAP_INTERVAL;
+		    DEFAULT_ENDPOINT_REAP_INTERVAL;
 
 	/*
 	 * Dispatch the taskq at an interval which is offset from the
 	 * interval that the endpoints should be reaped.
 	 */
 	clnt_clts_taskq_dispatch_interval =
-		(clnt_clts_endpoint_reap_interval + DEFAULT_INTERVAL_SHIFT)
-		* hz;
+	    (clnt_clts_endpoint_reap_interval + DEFAULT_INTERVAL_SHIFT) * hz;
 
 	/*
 	 * Initialize the completion queue
diff --git a/usr/src/uts/common/rpc/clnt_cots.c b/usr/src/uts/common/rpc/clnt_cots.c
index b6b9bc34c5..f9456d62af 100644
--- a/usr/src/uts/common/rpc/clnt_cots.c
+++ b/usr/src/uts/common/rpc/clnt_cots.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -812,6 +812,8 @@ call_again:
 	 */
 	if (p->cku_xid == 0) {
 		p->cku_xid = alloc_xid();
+		call->call_zoneid = rpc_zoneid();
+
 		/*
 		 * We need to ASSERT here that our xid != 0 because this
 		 * determines whether or not our call record gets placed on
@@ -1080,10 +1082,10 @@ call_again:
 
 	wq = cm_entry->x_wq;
 	clnt_dispatch_send(wq, mp, call, p->cku_xid,
-				(p->cku_flags & CKU_ONQUEUE));
+	    (p->cku_flags & CKU_ONQUEUE));
 
 	RPCLOG(64, "clnt_cots_kcallit: sent call for xid 0x%x\n",
-		(uint_t)p->cku_xid);
+	    (uint_t)p->cku_xid);
 	p->cku_flags = (CKU_ONQUEUE|CKU_SENT);
 	p->cku_recv_attempts = 1;
 
@@ -1128,12 +1130,14 @@ read_again:
 		if (h->cl_nosignal)
 			while ((cv_wait_ret = cv_timedwait(&call->call_cv,
 			    &call->call_lock, timout)) > 0 &&
-			    call->call_status == RPC_TIMEDOUT);
+			    call->call_status == RPC_TIMEDOUT)
+				;
 		else
 			while ((cv_wait_ret = cv_timedwait_sig(
 			    &call->call_cv,
 			    &call->call_lock, timout)) > 0 &&
-			    call->call_status == RPC_TIMEDOUT);
+			    call->call_status == RPC_TIMEDOUT)
+				;
 
 		switch (cv_wait_ret) {
 		case 0:
@@ -1290,10 +1294,10 @@ read_again:
 			 * Reply is good, check auth.
 			 */
 			if (!AUTH_VALIDATE(h->cl_auth,
-					&reply_msg.acpted_rply.ar_verf)) {
+			    &reply_msg.acpted_rply.ar_verf)) {
 				COTSRCSTAT_INCR(p->cku_stats, rcbadverfs);
 				RPCLOG0(1, "clnt_cots_kcallit: validation "
-					"failure\n");
+				    "failure\n");
 				freemsg(mp);
 				(void) xdr_rpc_free_verifier(xdrs, &reply_msg);
 				mutex_enter(&call->call_lock);
@@ -1302,9 +1306,9 @@ read_again:
 				mutex_exit(&call->call_lock);
 				goto read_again;
 			} else if (!AUTH_UNWRAP(h->cl_auth, xdrs,
-						xdr_results, resultsp)) {
+			    xdr_results, resultsp)) {
 				RPCLOG0(1, "clnt_cots_kcallit: validation "
-					"failure (unwrap)\n");
+				    "failure (unwrap)\n");
 				p->cku_err.re_status = RPC_CANTDECODERES;
 				p->cku_err.re_errno = EIO;
 			}
@@ -1333,10 +1337,10 @@ read_again:
 
 				if ((refreshes > 0) &&
 				    AUTH_REFRESH(h->cl_auth, &reply_msg,
-						p->cku_cred)) {
+				    p->cku_cred)) {
 					refreshes--;
 					(void) xdr_rpc_free_verifier(xdrs,
-								&reply_msg);
+					    &reply_msg);
 					freemsg(mp);
 					mp = NULL;
 
@@ -1355,9 +1359,9 @@ read_again:
 					}
 
 					COTSRCSTAT_INCR(p->cku_stats,
-							rcbadcalls);
+					    rcbadcalls);
 					COTSRCSTAT_INCR(p->cku_stats,
-							rcnewcreds);
+					    rcnewcreds);
 					goto call_again;
 				}
 
@@ -1387,7 +1391,7 @@ read_again:
 						p->cku_useresvport = 1;
 						p->cku_xid = 0;
 						(void) xdr_rpc_free_verifier
-							    (xdrs, &reply_msg);
+						    (xdrs, &reply_msg);
 						freemsg(mp);
 						goto call_again;
 					}
@@ -1596,7 +1600,7 @@ conn_kstat_update(kstat_t *ksp, int rw)
 			    b[2] & 0xFF, b[3] & 0xFF);
 		}
 		KSTAT_NAMED_STR_BUFLEN(&cm_ksp_data->x_server) =
-			strlen(fbuf) + 1;
+		    strlen(fbuf) + 1;
 	}
 
 	return (0);
@@ -1886,7 +1890,7 @@ use_new_conn:
 			    device != cm_entry->x_rdev ||
 			    retryaddr->len != cm_entry->x_src.len ||
 			    bcmp(retryaddr->buf, cm_entry->x_src.buf,
-				    retryaddr->len) != 0) {
+			    retryaddr->len) != 0) {
 				cmp = &cm_entry->x_next;
 				continue;
 			}
@@ -1901,8 +1905,8 @@ use_new_conn:
 			 * since that port may never be released.
 			 */
 			if (destaddr->len != cm_entry->x_server.len ||
-				bcmp(destaddr->buf, cm_entry->x_server.buf,
-					destaddr->len) != 0) {
+			    bcmp(destaddr->buf, cm_entry->x_server.buf,
+			    destaddr->len) != 0) {
 				RPCLOG(1, "connmgr_get: tiptr %p"
 				    " is going to a different server"
 				    " with the port that belongs"
@@ -2012,7 +2016,7 @@ use_new_conn:
 	rpc_poptimod(tiptr->fp->f_vnode);
 
 	if (i = strioctl(tiptr->fp->f_vnode, I_PUSH, (intptr_t)"rpcmod", 0,
-			K_TO_K, kcred, &retval)) {
+	    K_TO_K, kcred, &retval)) {
 		RPCLOG(1, "connmgr_get: can't push cots module, %d\n", i);
 		(void) t_kclose(tiptr, 1);
 		rpcerr->re_errno = i;
@@ -2021,7 +2025,7 @@ use_new_conn:
 	}
 
 	if (i = strioctl(tiptr->fp->f_vnode, RPC_CLIENT, 0, 0, K_TO_K,
-		kcred, &retval)) {
+	    kcred, &retval)) {
 		RPCLOG(1, "connmgr_get: can't set client status with cots "
 		    "module, %d\n", i);
 		(void) t_kclose(tiptr, 1);
@@ -2038,7 +2042,7 @@ use_new_conn:
 	mutex_exit(&connmgr_lock);
 
 	if (i = strioctl(tiptr->fp->f_vnode, I_PUSH, (intptr_t)"timod", 0,
-			K_TO_K, kcred, &retval)) {
+	    K_TO_K, kcred, &retval)) {
 		RPCLOG(1, "connmgr_get: can't push timod, %d\n", i);
 		(void) t_kclose(tiptr, 1);
 		rpcerr->re_errno = i;
@@ -2068,7 +2072,7 @@ use_new_conn:
 		if ((i = bindresvport(tiptr, retryaddr, srcaddr, TRUE)) != 0) {
 			(void) t_kclose(tiptr, 1);
 			RPCLOG(1, "connmgr_get: couldn't bind, retryaddr: "
-				"%p\n", (void *)retryaddr);
+			    "%p\n", (void *)retryaddr);
 
 			/*
 			 * 1225408: If we allocated a source address, then it
@@ -2109,8 +2113,7 @@ use_new_conn:
 		 * This is a bound end-point so don't close it's stream.
 		 */
 		connected = connmgr_connect(cm_entry, wq, destaddr, addrfmly,
-						&call, &tidu_size, FALSE, waitp,
-						nosignal);
+		    &call, &tidu_size, FALSE, waitp, nosignal);
 		*rpcerr = call.call_err;
 		cv_destroy(&call.call_cv);
 
@@ -2243,9 +2246,8 @@ connmgr_wrapconnect(
 		cv_init(&call.call_cv, NULL, CV_DEFAULT, NULL);
 
 		connected = connmgr_connect(cm_entry, cm_entry->x_wq,
-					    destaddr, addrfmly, &call,
-					    &cm_entry->x_tidu_size,
-					    reconnect, waitp, nosignal);
+		    destaddr, addrfmly, &call, &cm_entry->x_tidu_size,
+		    reconnect, waitp, nosignal);
 
 		*rpcerr = call.call_err;
 		cv_destroy(&call.call_cv);
@@ -2327,8 +2329,8 @@ connmgr_dis_and_wait(struct cm_xprt *cm_entry)
 	for (;;) {
 		while (cm_entry->x_needdis == TRUE) {
 			RPCLOG(8, "connmgr_dis_and_wait: need "
-				"T_DISCON_REQ for connection 0x%p\n",
-				(void *)cm_entry);
+			    "T_DISCON_REQ for connection 0x%p\n",
+			    (void *)cm_entry);
 			cm_entry->x_needdis = FALSE;
 			cm_entry->x_waitdis = TRUE;
 
@@ -2342,12 +2344,12 @@ connmgr_dis_and_wait(struct cm_xprt *cm_entry)
 			clock_t timout;
 
 			RPCLOG(8, "connmgr_dis_and_wait waiting for "
-				"T_DISCON_REQ's ACK for connection %p\n",
-				(void *)cm_entry);
+			    "T_DISCON_REQ's ACK for connection %p\n",
+			    (void *)cm_entry);
 			curlbolt = ddi_get_lbolt();
 
 			timout = clnt_cots_min_conntout *
-				drv_usectohz(1000000) + curlbolt;
+			    drv_usectohz(1000000) + curlbolt;
 
 			/*
 			 * The TPI spec says that the T_DISCON_REQ
@@ -2356,7 +2358,7 @@ connmgr_dis_and_wait(struct cm_xprt *cm_entry)
 			 * block forever.
 			 */
 			(void) cv_timedwait(&cm_entry->x_dis_cv,
-					    &connmgr_lock, timout);
+			    &connmgr_lock, timout);
 		}
 		/*
 		 * If we got the ACK, break. If we didn't,
@@ -2366,8 +2368,8 @@ connmgr_dis_and_wait(struct cm_xprt *cm_entry)
 			break;
 		} else {
 			RPCLOG(8, "connmgr_dis_and_wait: did"
-				"not get T_DISCON_REQ's ACK for "
-				"connection  %p\n", (void *)cm_entry);
+			    "not get T_DISCON_REQ's ACK for "
+			    "connection  %p\n", (void *)cm_entry);
 			cm_entry->x_needdis = TRUE;
 		}
 	}
@@ -2422,9 +2424,9 @@ connmgr_close(struct cm_xprt *cm_entry)
 		    x_server.value.str.addr.ptr != NULL)
 			kmem_free(((struct cm_kstat_xprt *)(cm_entry->x_ksp->
 			    ks_data))->x_server.value.str.addr.ptr,
-				    INET6_ADDRSTRLEN);
+			    INET6_ADDRSTRLEN);
 		kmem_free(cm_entry->x_ksp->ks_data,
-			    cm_entry->x_ksp->ks_data_size);
+		    cm_entry->x_ksp->ks_data_size);
 		kstat_delete(cm_entry->x_ksp);
 	}
 
@@ -2630,19 +2632,19 @@ connmgr_connect(
 	    (uint_t)(sizeof (cm_kstat_xprt_t) / sizeof (kstat_named_t)),
 	    KSTAT_FLAG_VIRTUAL, cm_entry->x_zoneid)) == NULL) {
 		return (TRUE);
-	    }
+	}
 
 	cm_entry->x_ksp->ks_lock = &connmgr_lock;
 	cm_entry->x_ksp->ks_private = cm_entry;
 	cm_entry->x_ksp->ks_data_size = ((INET6_ADDRSTRLEN * sizeof (char))
-					    + sizeof (cm_kstat_template));
+	    + sizeof (cm_kstat_template));
 	cm_entry->x_ksp->ks_data = kmem_alloc(cm_entry->x_ksp->ks_data_size,
-					    KM_SLEEP);
+	    KM_SLEEP);
 	bcopy(&cm_kstat_template, cm_entry->x_ksp->ks_data,
 	    cm_entry->x_ksp->ks_data_size);
 	((struct cm_kstat_xprt *)(cm_entry->x_ksp->ks_data))->
-		    x_server.value.str.addr.ptr =
-		    kmem_alloc(INET6_ADDRSTRLEN, KM_SLEEP);
+	    x_server.value.str.addr.ptr =
+	    kmem_alloc(INET6_ADDRSTRLEN, KM_SLEEP);
 
 	cm_entry->x_ksp->ks_update = conn_kstat_update;
 	kstat_install(cm_entry->x_ksp);
@@ -2748,7 +2750,7 @@ connmgr_sndrel(struct cm_xprt *cm_entry)
 		cm_entry->x_needrel = TRUE;
 		mutex_exit(&connmgr_lock);
 		RPCLOG(1, "connmgr_sndrel: cannot alloc mp for sending ordrel "
-			"to queue %p\n", (void *)q);
+		    "to queue %p\n", (void *)q);
 		return;
 	}
 	mutex_exit(&connmgr_lock);
@@ -2825,7 +2827,7 @@ clnt_dispatch_send(queue_t *q, mblk_t *mp, calllist_t *e, uint_t xid,
 	 */
 	if (xid != 0) {
 		RPCLOG(64, "clnt_dispatch_send: putting xid 0x%x on "
-			"dispatch list\n", xid);
+		    "dispatch list\n", xid);
 		e->call_hash = call_hash(xid, clnt_cots_hash_size);
 		e->call_bucket = &cots_call_ht[e->call_hash];
 		call_table_enter(e);
@@ -2900,6 +2902,17 @@ done_xid_copy:
 		 * Found thread waiting for this reply
 		 */
 		mutex_enter(&e->call_lock);
+
+		/*
+		 * verify that the reply is coming in on
+		 * the same zone that it was sent from.
+		 */
+		if (e->call_zoneid != zoneid) {
+			mutex_exit(&e->call_lock);
+			mutex_exit(&chtp->ct_lock);
+			return (FALSE);
+		}
+
 		if (e->call_reply)
 			/*
 			 * This can happen under the following scenario:
@@ -3004,7 +3017,7 @@ clnt_dispatch_notifyconn(queue_t *q, mblk_t *mp)
 		 */
 		mutex_exit(&clnt_pending_lock);
 		ASSERT(mp->b_datap->db_lim - mp->b_datap->db_base >=
-			sizeof (struct T_info_req));
+		    sizeof (struct T_info_req));
 		mp->b_rptr = mp->b_datap->db_base;
 		((union T_primitives *)mp->b_rptr)->type = T_INFO_REQ;
 		mp->b_wptr = mp->b_rptr + sizeof (struct T_info_req);
@@ -3097,32 +3110,37 @@ clnt_dispatch_notifyall(queue_t *q, int32_t msg_type, int32_t reason)
 				 */
 				if (cm_entry->x_connected ||
 				    cm_entry->x_doomed) {
-				    if (cm_entry->x_ordrel) {
-					if (cm_entry->x_closing == TRUE) {
-					/*
-					 * The connection is obviously
-					 * wedged due to a bug or problem
-					 * with the transport. Mark it
-					 * as dead. Otherwise we can leak
-					 * connections.
-					 */
-					    cm_entry->x_dead = TRUE;
-					    mutex_exit(&connmgr_lock);
-					    have_connmgr_lock = 0;
-					    if (clnt_stop_idle != NULL)
-						(*clnt_stop_idle)(q);
-					    break;
+					if (cm_entry->x_ordrel) {
+						if (cm_entry->x_closing ==
+						    TRUE) {
+							/*
+							 * The connection is
+							 * obviously wedged due
+							 * to a bug or problem
+							 * with the transport.
+							 * Mark it as dead.
+							 * Otherwise we can
+							 * leak connections.
+							 */
+							cm_entry->x_dead = TRUE;
+							mutex_exit(
+							    &connmgr_lock);
+							have_connmgr_lock = 0;
+							if (clnt_stop_idle !=
+							    NULL)
+							(*clnt_stop_idle)(q);
+							break;
+						}
+						cm_entry->x_closing = TRUE;
+						connmgr_sndrel(cm_entry);
+						have_connmgr_lock = 0;
+					} else {
+						cm_entry->x_dead = TRUE;
+						mutex_exit(&connmgr_lock);
+						have_connmgr_lock = 0;
+						if (clnt_stop_idle != NULL)
+							(*clnt_stop_idle)(q);
 					}
-					cm_entry->x_closing = TRUE;
-					connmgr_sndrel(cm_entry);
-					have_connmgr_lock = 0;
-				    } else {
-					cm_entry->x_dead = TRUE;
-					mutex_exit(&connmgr_lock);
-					have_connmgr_lock = 0;
-					if (clnt_stop_idle != NULL)
-						(*clnt_stop_idle)(q);
-				    }
 				} else {
 					/*
 					 * We don't mark the connection
@@ -3235,14 +3253,14 @@ clnt_dispatch_notifyall(queue_t *q, int32_t msg_type, int32_t reason)
 		ctp = &cots_call_ht[i];
 		mutex_enter(&ctp->ct_lock);
 		for (e = ctp->ct_call_next;
-			e != (calllist_t *)ctp;
-			e = e->call_next) {
+		    e != (calllist_t *)ctp;
+		    e = e->call_next) {
 			if (e->call_wq == q && e->call_notified == FALSE) {
 				RPCLOG(1,
-				"clnt_dispatch_notifyall for queue %p ",
-					(void *)q);
+				    "clnt_dispatch_notifyall for queue %p ",
+				    (void *)q);
 				RPCLOG(1, "aborting clnt_pending call %p\n",
-					(void *)e);
+				    (void *)e);
 
 				if (msg_type == T_DISCON_IND)
 					e->call_reason = reason;
@@ -3322,7 +3340,7 @@ connmgr_cpr_reset(void *arg, int code)
 		return (B_FALSE);
 	for (cxp = cm_hd; cxp; cxp = cxp->x_next) {
 		if ((cxp->x_family == AF_INET || cxp->x_family == AF_INET6) &&
-			cxp->x_connected == TRUE) {
+		    cxp->x_connected == TRUE) {
 			if (cxp->x_thread)
 				cxp->x_early_disc = TRUE;
 			else
diff --git a/usr/src/uts/common/rpc/mt_rpcinit.c b/usr/src/uts/common/rpc/mt_rpcinit.c
index 849315ddd0..8ef5111d4d 100644
--- a/usr/src/uts/common/rpc/mt_rpcinit.c
+++ b/usr/src/uts/common/rpc/mt_rpcinit.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -171,6 +171,11 @@ alloc_xid(void)
  * These functions are temporary and designed for the upgrade-workaround only.
  * They cannot be used for general zone-crossing RPC client support, and will
  * be removed shortly.
+ *
+ * Currently these functions route all nfs global clients to the global zone.
+ * When this upgrade-workaround is removed these function should return the
+ * correct zone or their calls should be changed (rpc_zone() to curproc->p_zone
+ * and rpc_zoneid() to getzoneid()).
  */
 struct zone *
 rpc_zone(void)
diff --git a/usr/src/uts/common/rpc/sec_gss/rpcsec_gss.c b/usr/src/uts/common/rpc/sec_gss/rpcsec_gss.c
index e9495d7137..4f36a8a1ab 100644
--- a/usr/src/uts/common/rpc/sec_gss/rpcsec_gss.c
+++ b/usr/src/uts/common/rpc/sec_gss/rpcsec_gss.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -150,8 +150,8 @@ gssauth_init(void)
 	 * Allocate gss auth cache handle
 	 */
 	ga_cache_handle = kmem_cache_create("ga_cache_handle",
-			sizeof (struct ga_cache_entry), 0, NULL, NULL,
-			gssauth_cache_reclaim, NULL, NULL, 0);
+	    sizeof (struct ga_cache_entry), 0, NULL, NULL,
+	    gssauth_cache_reclaim, NULL, NULL, 0);
 	zone_key_create(&gssauth_zone_key, NULL, NULL, gssauth_zone_fini);
 }
 
@@ -199,9 +199,9 @@ gssauth_zone_fini(zoneid_t zoneid, void *unused)
 				 */
 				now = gethrestime_sec();
 				if ((p->ref_time + rpc_gss_cache_time >
-					    now) || p->in_use) {
+				    now) || p->in_use) {
 					if ((p->ref_time + rpc_gss_cache_time <=
-						    now) && p->in_use) {
+					    now) && p->in_use) {
 						RPCGSS_LOG0(2, "gssauth_cache_"
 						    "reclaim: in_use\n");
 					}
@@ -217,7 +217,7 @@ gssauth_zone_fini(zoneid_t zoneid, void *unused)
 			}
 
 			RPCGSS_LOG(2, "gssauth_cache_reclaim: destroy auth "
-				"%p\n", (void *)p->auth);
+			    "%p\n", (void *)p->auth);
 			rpc_gss_destroy(p->auth);
 			kmem_cache_free(ga_cache_handle, (void *)p);
 			if (prev == NULL) {
@@ -404,10 +404,10 @@ rpc_gss_secfree(AUTH *auth)
 			next = cur->next;
 			NOT_DEAD(next);
 			if (cur->auth == auth) {
-			    ASSERT(cur->in_use == TRUE);
-			    cur->in_use = FALSE;
-			    rw_exit(&ga_cache_table_lock);
-			    return;
+				ASSERT(cur->in_use == TRUE);
+				cur->in_use = FALSE;
+				rw_exit(&ga_cache_table_lock);
+				return;
 			}
 		}
 	}
@@ -448,11 +448,11 @@ rpc_gss_seccreate(CLIENT *clnt,
 	input_name.length = strlen(principal);
 
 	gssstat = gss_import_name(&minor_stat, &input_name,
-			(gss_OID)GSS_C_NT_HOSTBASED_SERVICE, &target_name);
+	    (gss_OID)GSS_C_NT_HOSTBASED_SERVICE, &target_name);
 
 	if (gssstat != GSS_S_COMPLETE) {
 		RPCGSS_LOG0(1,
-			"rpc_gss_seccreate: unable to import gss name\n");
+		    "rpc_gss_seccreate: unable to import gss name\n");
 		return (ENOMEM);
 	}
 
@@ -494,14 +494,14 @@ rpc_gss_seccreate(CLIENT *clnt,
 	 * the information stashed away in the private data.
 	 */
 	if (error = rpc_gss_seccreate_pvt(&gssstat, &minor_stat, auth, ap,
-		    mechanism, &ap->mech_type, &ret_flags, &time_rec, cr, 0)) {
+	    mechanism, &ap->mech_type, &ret_flags, &time_rec, cr, 0)) {
 		if (ap->target_name) {
 			(void) gss_release_name(&minor_stat, &ap->target_name);
 		}
 		kmem_free((char *)ap, sizeof (*ap));
 		kmem_free((char *)auth, sizeof (*auth));
 		RPCGSS_LOG(1, "rpc_gss_seccreate: init context failed"
-				" errno=%d\n", error);
+		    " errno=%d\n", error);
 		return (error);
 	}
 
@@ -510,8 +510,8 @@ rpc_gss_seccreate(CLIENT *clnt,
 	 * cases, integrity service must be available.
 	 */
 	if ((ap->service == rpc_gss_svc_privacy &&
-			!(ret_flags & GSS_C_CONF_FLAG)) ||
-			!(ret_flags & GSS_C_INTEG_FLAG)) {
+	    !(ret_flags & GSS_C_CONF_FLAG)) ||
+	    !(ret_flags & GSS_C_INTEG_FLAG)) {
 		rpc_gss_destroy(auth);
 		RPCGSS_LOG0(1, "rpc_gss_seccreate: service not supported\n");
 		return (EPROTONOSUPPORT);
@@ -532,7 +532,7 @@ rpc_gss_seccreate(CLIENT *clnt,
 		 */
 		NOT_NULL(ap->mech_type);
 		__rpc_gss_dup_oid(ap->mech_type,
-			(gss_OID *)&options_ret->actual_mechanism);
+		    (gss_OID *)&options_ret->actual_mechanism);
 	}
 
 	*retauth = auth;
@@ -996,15 +996,15 @@ validate_seqwin(rpc_gss_data *ap)
 	tok_buf.length = ap->verifier->oa_length;
 	tok_buf.value = ap->verifier->oa_base;
 	major = kgss_verify(&minor, ap->context, &msg_buf, &tok_buf,
-				&qop_state);
+	    &qop_state);
 
 	if (major != GSS_S_COMPLETE) {
-	    RPCGSS_LOG1(1,
-		"validate_seqwin: kgss_verify failed GSS Major %x Minor %x\n",
-		major, minor);
-	    RPCGSS_LOG1(1, "seq_window %d, verf len %d ", ap->seq_window,
-				ap->verifier->oa_length);
-	    return (FALSE);
+		RPCGSS_LOG1(1,
+		    "validate_seqwin: kgss_verify failed GSS Major "
+		    "%x Minor %x\n", major, minor);
+		RPCGSS_LOG1(1, "seq_window %d, verf len %d ", ap->seq_window,
+		    ap->verifier->oa_length);
+		return (FALSE);
 	}
 	return (TRUE);
 }
@@ -1464,14 +1464,14 @@ rpc_gss_revauth(uid_t uid, rpc_gss_OID mech)
 			    (cur->zoneid == zoneid)) {
 				if (cur->in_use) {
 					RPCGSS_LOG(2, "rpc_gss_revauth:invalid "
-						"auth %p\n", (void *)cur->auth);
+					    "auth %p\n", (void *)cur->auth);
 					ap->invalid = TRUE;
 				} else {
 					RPCGSS_LOG(2, "rpc_gss_revauth:destroy "
-						"auth %p\n", (void *)cur->auth);
+					    "auth %p\n", (void *)cur->auth);
 					rpc_gss_destroy(cur->auth);
 					kmem_cache_free(ga_cache_handle,
-							(void *)cur);
+					    (void *)cur);
 				}
 				if (prev == NULL) {
 					ga_cache_table[i] = next;
@@ -1515,8 +1515,9 @@ rpc_gss_secpurge(void *cache_key)
 			NOT_DEAD(next);
 			if (cache_key == cur->cache_key) {
 				RPCGSS_LOG(2, "rpc_gss_secpurge: destroy auth "
-					"%p\n", (void *)cur->auth);
-				rpc_gss_destroy(cur->auth);
+				    "%p\n", (void *)cur->auth);
+				if (cur->in_use == FALSE)
+					rpc_gss_destroy(cur->auth);
 				kmem_cache_free(ga_cache_handle, (void *)cur);
 				if (prev == NULL) {
 					ga_cache_table[i] = next;