6844040 NFSv2 WRITEs over RDMA fail to unregister data buffers

6845259 nfsd should honor the '-p proto' and '-t provider'options
6197327 Client mounts should fail gracefully when RDMA is disabled

4 files changed, 99 insertions, 35 deletions

diff --git a/usr/src/cmd/fs.d/nfs/nfsd/nfsd.c b/usr/src/cmd/fs.d/nfs/nfsd/nfsd.c
index 9c86d18a54..e786d5d868 100644
--- a/usr/src/cmd/fs.d/nfs/nfsd/nfsd.c
+++ b/usr/src/cmd/fs.d/nfs/nfsd/nfsd.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -39,8 +39,6 @@
 /* LINTLIBRARY */
 /* PROTOLIB1 */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 /* NFS server */
 
 #include <sys/param.h>
@@ -497,11 +495,15 @@ main(int ac, char *av[])
 	 * destructor thread.
 	 *
 	 * start rdma services and block in the kernel.
+	 * (only if proto or provider is not set to TCP or UDP)
 	 */
-	if (svcrdma(NFS_SVCPOOL_ID, nfs_server_vers_min, nfs_server_vers_max,
-	    nfs_server_delegation)) {
-		fprintf(stderr, "Can't set up RDMA creator thread : %s",
-		    strerror(errno));
+	if ((proto == NULL) && (provider == NULL)) {
+		if (svcrdma(NFS_SVCPOOL_ID, nfs_server_vers_min,
+		    nfs_server_vers_max, nfs_server_delegation)) {
+			fprintf(stderr,
+			    "Can't set up RDMA creator thread : %s",
+			    strerror(errno));
+		}
 	}
 
 	/*
diff --git a/usr/src/uts/common/fs/nfs/nfs_xdr.c b/usr/src/uts/common/fs/nfs/nfs_xdr.c
index 2f461a1d8a..4e37903f5c 100644
--- a/usr/src/uts/common/fs/nfs/nfs_xdr.c
+++ b/usr/src/uts/common/fs/nfs/nfs_xdr.c
@@ -144,9 +144,30 @@ xdr_writeargs(XDR *xdrs, struct nfswriteargs *wa)
 			wa->wa_begoff = IXDR_GET_U_INT32(ptr);
 			wa->wa_offset = IXDR_GET_U_INT32(ptr);
 			wa->wa_totcount = IXDR_GET_U_INT32(ptr);
-			if (xdrs->x_ops == &xdrmblk_ops)
+			wa->wa_mblk = NULL;
+			wa->wa_data = NULL;
+			wa->wa_rlist = NULL;
+			wa->wa_conn = NULL;
+			if (xdrs->x_ops == &xdrmblk_ops) {
 				return (xdrmblk_getmblk(xdrs, &wa->wa_mblk,
 				    &wa->wa_count));
+			} else {
+				if (xdrs->x_ops == &xdrrdmablk_ops) {
+					if (xdrrdma_getrdmablk(xdrs,
+					    &wa->wa_rlist,
+					    &wa->wa_count,
+					    &wa->wa_conn,
+					    NFS_MAXDATA) == TRUE)
+					return (xdrrdma_read_from_client(
+					    wa->wa_rlist,
+					    &wa->wa_conn,
+					    wa->wa_count));
+
+					wa->wa_rlist = NULL;
+					wa->wa_conn = NULL;
+				}
+			}
+
 			/*
 			 * It is just as efficient to xdr_bytes
 			 * an array of unknown length as to inline copy it.
diff --git a/usr/src/uts/common/rpc/rpcib.c b/usr/src/uts/common/rpc/rpcib.c
index 88f1a87e4c..22facca568 100644
--- a/usr/src/uts/common/rpc/rpcib.c
+++ b/usr/src/uts/common/rpc/rpcib.c
@@ -302,6 +302,8 @@ static rdma_stat rib_ping_srv(int addr_type, struct netbuf *, rpcib_ping_t *);
 static rdma_stat rib_conn_get(struct netbuf *, struct netbuf *,
 	int addr_type, void *, CONN **);
 static rdma_stat rib_conn_release(CONN *conn);
+static rdma_stat rib_connect(struct netbuf *, struct netbuf *, int,
+	rpcib_ping_t *, CONN **);
 static rdma_stat rib_getinfo(rdma_info_t *info);
 
 static rib_lrc_entry_t *rib_get_cache_buf(CONN *conn, uint32_t len);
@@ -1436,7 +1438,7 @@ static void
 rib_attach_hca()
 {
 	mutex_enter(&rib_stat->open_hca_lock);
-	rpcib_open_hcas(rib_stat);
+	(void) rpcib_open_hcas(rib_stat);
 	rib_listen(NULL);
 	mutex_exit(&rib_stat->open_hca_lock);
 }
@@ -1548,12 +1550,16 @@ rib_reachable(int addr_type, struct netbuf *raddr, void **handle)
 {
 	rdma_stat	status;
 	rpcib_ping_t	rpt;
+	struct netbuf	saddr;
+	CONN		*conn;
 
-	bzero(&rpt, sizeof (rpcib_ping_t));
-	status = rib_ping_srv(addr_type, raddr, &rpt);
+	bzero(&saddr, sizeof (struct netbuf));
+	status = rib_connect(&saddr, raddr, addr_type, &rpt, &conn);
 
 	if (status == RDMA_SUCCESS) {
 		*handle = (void *)rpt.hca;
+		/* release the reference */
+		(void) rib_conn_release(conn);
 		return (RDMA_SUCCESS);
 	} else {
 		*handle = NULL;
@@ -4134,6 +4140,18 @@ rib_rm_conn(CONN *cn, rib_conn_list_t *connlist)
 	return (RDMA_SUCCESS);
 }
 
+/* ARGSUSED */
+static rdma_stat
+rib_conn_get(struct netbuf *s_svcaddr, struct netbuf *d_svcaddr,
+    int addr_type, void *handle, CONN **conn)
+{
+	rdma_stat status;
+	rpcib_ping_t rpt;
+
+	status = rib_connect(s_svcaddr, d_svcaddr, addr_type, &rpt, conn);
+	return (status);
+}
+
 /*
  * rib_find_hca_connection
  *
@@ -4260,14 +4278,13 @@ again:
  */
 /* ARGSUSED */
 static rdma_stat
-rib_conn_get(struct netbuf *s_svcaddr, struct netbuf *d_svcaddr,
-    int addr_type, void *handle, CONN **conn)
+rib_connect(struct netbuf *s_svcaddr, struct netbuf *d_svcaddr,
+    int addr_type, rpcib_ping_t *rpt, CONN **conn)
 {
 	CONN *cn;
 	int status;
 	rib_hca_t *hca;
 	rib_qp_t *qp;
-	rpcib_ping_t rpt;
 	int s_addr_len;
 	char *s_addr_buf;
 
@@ -4290,22 +4307,23 @@ rib_conn_get(struct netbuf *s_svcaddr, struct netbuf *d_svcaddr,
 	/*
 	 * No existing connection found, establish a new connection.
 	 */
-	bzero(&rpt, sizeof (rpcib_ping_t));
+	bzero(rpt, sizeof (rpcib_ping_t));
 
-	status = rib_ping_srv(addr_type, d_svcaddr, &rpt);
+	status = rib_ping_srv(addr_type, d_svcaddr, rpt);
 	if (status != RDMA_SUCCESS) {
 		return (RDMA_FAILED);
 	}
-	hca = rpt.hca;
+	hca = rpt->hca;
 
-	if (rpt.srcip.family == AF_INET) {
-		s_addr_len = sizeof (rpt.srcip.un.ip4addr);
-		s_addr_buf = (char *)&rpt.srcip.un.ip4addr;
-	} else if (rpt.srcip.family == AF_INET6) {
-		s_addr_len = sizeof (rpt.srcip.un.ip6addr);
-		s_addr_buf = (char *)&rpt.srcip.un.ip6addr;
-	} else
+	if (rpt->srcip.family == AF_INET) {
+		s_addr_len = sizeof (rpt->srcip.un.ip4addr);
+		s_addr_buf = (char *)&rpt->srcip.un.ip4addr;
+	} else if (rpt->srcip.family == AF_INET6) {
+		s_addr_len = sizeof (rpt->srcip.un.ip6addr);
+		s_addr_buf = (char *)&rpt->srcip.un.ip6addr;
+	} else {
 		return (RDMA_FAILED);
+	}
 
 	/*
 	 * Channel to server doesn't exist yet, create one.
@@ -4332,7 +4350,7 @@ rib_conn_get(struct netbuf *s_svcaddr, struct netbuf *d_svcaddr,
 	 * WRITER lock.
 	 */
 	(void) rib_add_connlist(cn, &hca->cl_conn_list);
-	status = rib_conn_to_srv(hca, qp, &rpt);
+	status = rib_conn_to_srv(hca, qp, rpt);
 	mutex_enter(&cn->c_lock);
 	if (status == RDMA_SUCCESS) {
 		cn->c_state = C_CONNECTED;
diff --git a/usr/src/uts/common/rpc/xdr_rdma.c b/usr/src/uts/common/rpc/xdr_rdma.c
index 5e507bdcb2..6e6436ac00 100644
--- a/usr/src/uts/common/rpc/xdr_rdma.c
+++ b/usr/src/uts/common/rpc/xdr_rdma.c
@@ -430,8 +430,8 @@ xdrrdma_getbytes(XDR *xdrs, caddr_t addr, int len)
 	uint32_t	cur_offset = 0;
 	uint32_t	total_segments = 0;
 	uint32_t	actual_segments = 0;
-	uint32_t	status;
-	uint32_t	alen;
+	uint32_t	status = RDMA_SUCCESS;
+	uint32_t	alen = 0;
 	uint32_t	xpoff;
 
 	while (cle) {
@@ -456,8 +456,13 @@ xdrrdma_getbytes(XDR *xdrs, caddr_t addr, int len)
 	if (cle != NULL && cle->c_xdroff == xpoff) {
 		for (actual_segments = 0;
 		    actual_segments < total_segments; actual_segments++) {
+
 			if (total_len <= 0)
 				break;
+
+			if (status != RDMA_SUCCESS)
+				goto out;
+
 			cle->u.c_daddr = (uint64)(uintptr_t)addr + cur_offset;
 			alen = 0;
 			if (cle->c_len > total_len) {
@@ -495,10 +500,17 @@ xdrrdma_getbytes(XDR *xdrs, caddr_t addr, int len)
 			}
 			cl = *cle;
 			cl.c_next = NULL;
-			if (clist_register(xdrp->xp_conn, &cl, CLIST_REG_DST)
-			    != RDMA_SUCCESS) {
-				return (FALSE);
+			status = clist_register(xdrp->xp_conn, &cl,
+			    CLIST_REG_DST);
+			if (status != RDMA_SUCCESS) {
+				retval = FALSE;
+				/*
+				 * Deregister the previous chunks
+				 * before return
+				 */
+				goto out;
 			}
+
 			cle->c_dmemhandle = cl.c_dmemhandle;
 			cle->c_dsynchandle = cl.c_dsynchandle;
 
@@ -516,9 +528,10 @@ xdrrdma_getbytes(XDR *xdrs, caddr_t addr, int len)
 				    krpc__i__xdrrdma_getblk_readfailed,
 				    int, status);
 				retval = FALSE;
-				goto out;
 			}
+
 			cle = cle->c_next;
+
 		}
 
 		/*
@@ -532,19 +545,29 @@ xdrrdma_getbytes(XDR *xdrs, caddr_t addr, int len)
 			retval = FALSE;
 		}
 out:
+
 		/*
 		 * Deregister the chunks
 		 */
 		cle = cls;
-		cl = *cle;
-		cl.c_next = NULL;
-		cl.c_len = cur_offset;
-		(void) clist_deregister(xdrp->xp_conn, &cl);
+		while (actual_segments != 0) {
+			cl = *cle;
+			cl.c_next = NULL;
+
+			cl.c_regtype = CLIST_REG_DST;
+			(void) clist_deregister(xdrp->xp_conn, &cl);
+
+			cle = cle->c_next;
+			actual_segments--;
+		}
+
 		if (alen) {
+			cle = *(xdrp->xp_rcl_next);
 			cle->w.c_saddr =
 			    (uint64)(uintptr_t)cle->w.c_saddr + cle->c_len;
 			cle->c_len = alen - cle->c_len;
 		}
+
 		return (retval);
 	}