8250 libnsl: Raw RPC client sends unlimited data

Reviewed by: Yuri Pankov <yuripv@gmx.com>
Reviewed by: Jason King <jason.brian.king+illumos@gmail.com>
Approved by: Dan McDonald <danmcd@joyent.com>

3 files changed, 170 insertions, 143 deletions

diff --git a/usr/src/lib/libnsl/common/mapfile-vers b/usr/src/lib/libnsl/common/mapfile-vers
index c8fa67b625..3f31af2f5e 100644
--- a/usr/src/lib/libnsl/common/mapfile-vers
+++ b/usr/src/lib/libnsl/common/mapfile-vers
@@ -633,7 +633,6 @@ SYMBOL_VERSION SUNWprivate_1.1 {
 	pagf;
 	passwd2des;
 	passwd2des_g;
-	_rawcombuf;
 	__rpcbind_is_up;
 	__rpc_bindresvport;
 	rpcb_taddr2uaddr;
diff --git a/usr/src/lib/libnsl/rpc/clnt_raw.c b/usr/src/lib/libnsl/rpc/clnt_raw.c
index 8f1766cd73..805e41c361 100644
--- a/usr/src/lib/libnsl/rpc/clnt_raw.c
+++ b/usr/src/lib/libnsl/rpc/clnt_raw.c
@@ -32,46 +32,43 @@
  * California.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 /*
  * clnt_raw.c
  *
  * Memory based rpc for simple testing and timing.
  * Interface to create an rpc client and server in the same process.
- * This lets us similate rpc and get round trip overhead, without
+ * This lets us simulate rpc and get round trip overhead, without
  * any interference from the kernel.
  */
 #include "mt.h"
 #include "rpc_mt.h"
 #include <stdlib.h>
 #include <rpc/rpc.h>
-#include <rpc/raw.h>
 #include <syslog.h>
 
 extern mutex_t	clntraw_lock;
 #define	MCALL_MSG_SIZE 24
-#ifndef UDPMSGSIZE
-#define	UDPMSGSIZE 8800
-#endif
 
 /*
  * This is the "network" we will be moving stuff over.
  */
 static struct clnt_raw_private {
 	CLIENT	client_object;
-	XDR	xdr_stream;
-	char	*raw_buf;	/* should be shared with server handle */
+	struct netbuf	*raw_netbuf;
 	char	mashl_callmsg[MCALL_MSG_SIZE];
 	uint_t	mcnt;
 } *clnt_raw_private;
 
 static struct clnt_ops *clnt_raw_ops();
 
-extern void svc_getreq_common(int);
 extern bool_t xdr_opaque_auth();
 
 /*
+ * This netbuf is shared with the raw server.
+ */
+extern struct netbuf _rawcomnetbuf;
+
+/*
  * Create a client handle for memory based rpc.
  */
 CLIENT *
@@ -79,35 +76,26 @@ clnt_raw_create(const rpcprog_t prog, const rpcvers_t vers)
 {
 	struct clnt_raw_private *clp;
 	struct rpc_msg call_msg;
-	XDR *xdrs;
+	XDR xdrs;
 	CLIENT *client;
+	uint_t start;
 
 /* VARIABLES PROTECTED BY clntraw_lock: clp */
 
 	(void) mutex_lock(&clntraw_lock);
 	clp = clnt_raw_private;
+	if (clp != NULL) {
+		(void) mutex_unlock(&clntraw_lock);
+		return (&clp->client_object);
+	}
+
+	clp = calloc(1, sizeof (*clp));
 	if (clp == NULL) {
-		clp = calloc(1, sizeof (*clp));
-		if (clp == NULL) {
-			(void) mutex_unlock(&clntraw_lock);
-			return (NULL);
-		}
-		if (_rawcombuf == NULL) {
-			_rawcombuf = calloc(UDPMSGSIZE, sizeof (char));
-			if (_rawcombuf == NULL) {
-				syslog(LOG_ERR, "clnt_raw_create: "
-					"out of memory.");
-				if (clp)
-					free(clp);
-				(void) mutex_unlock(&clntraw_lock);
-				return (NULL);
-			}
-		}
-		clp->raw_buf = _rawcombuf; /* Share it with the server */
-		clnt_raw_private = clp;
+		(void) mutex_unlock(&clntraw_lock);
+		return (NULL);
 	}
-	xdrs = &clp->xdr_stream;
-	client = &clp->client_object;
+
+	clp->raw_netbuf = &_rawcomnetbuf;
 
 	/*
 	 * pre-serialize the static part of the call msg and stash it away
@@ -116,25 +104,30 @@ clnt_raw_create(const rpcprog_t prog, const rpcvers_t vers)
 	call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
 	call_msg.rm_call.cb_prog = prog;
 	call_msg.rm_call.cb_vers = vers;
-	xdrmem_create(xdrs, clp->mashl_callmsg, MCALL_MSG_SIZE, XDR_ENCODE);
-	if (!xdr_callhdr(xdrs, &call_msg))
-		(void) syslog(LOG_ERR,
-			(const char *) "clnt_raw_create :  \
-			Fatal header serialization error.");
 
-	clp->mcnt = XDR_GETPOS(xdrs);
-	XDR_DESTROY(xdrs);
+	xdrmem_create(&xdrs, clp->mashl_callmsg, sizeof (clp->mashl_callmsg),
+	    XDR_ENCODE);
+	start = XDR_GETPOS(&xdrs);
+	if (!xdr_callhdr(&xdrs, &call_msg)) {
+		free(clp);
+		(void) syslog(LOG_ERR,
+		    "clnt_raw_create: Fatal header serialization error");
 
-	/*
-	 * Set xdrmem for client/server shared buffer
-	 */
-	xdrmem_create(xdrs, clp->raw_buf, UDPMSGSIZE, XDR_FREE);
+		(void) mutex_unlock(&clntraw_lock);
+		return (NULL);
+	}
+	clp->mcnt = XDR_GETPOS(&xdrs) - start;
+	XDR_DESTROY(&xdrs);
 
 	/*
 	 * create client handle
 	 */
+	client = &clp->client_object;
 	client->cl_ops = clnt_raw_ops();
 	client->cl_auth = authnone_create();
+
+	clnt_raw_private = clp;
+
 	(void) mutex_unlock(&clntraw_lock);
 	return (client);
 }
@@ -142,20 +135,21 @@ clnt_raw_create(const rpcprog_t prog, const rpcvers_t vers)
 /*ARGSUSED*/
 static enum clnt_stat
 clnt_raw_call(CLIENT *h, rpcproc_t proc, xdrproc_t xargs, caddr_t argsp,
-	xdrproc_t xresults, caddr_t resultsp, struct timeval timeout)
+    xdrproc_t xresults, caddr_t resultsp, struct timeval timeout)
 {
 	struct clnt_raw_private *clp;
-	XDR *xdrs;
+	XDR xdrs;
 	struct rpc_msg msg;
-	enum clnt_stat status;
-	struct rpc_err error;
+	uint_t start;
+
+	rpc_callerr.re_errno = 0;
+	rpc_callerr.re_terrno = 0;
 
 	(void) mutex_lock(&clntraw_lock);
 	clp = clnt_raw_private;
-	xdrs = &clp->xdr_stream;
 	if (clp == NULL) {
 		(void) mutex_unlock(&clntraw_lock);
-		return (RPC_FAILED);
+		return (rpc_callerr.re_status = RPC_FAILED);
 	}
 	(void) mutex_unlock(&clntraw_lock);
 
@@ -163,45 +157,55 @@ call_again:
 	/*
 	 * send request
 	 */
-	xdrs->x_op = XDR_ENCODE;
-	XDR_SETPOS(xdrs, 0);
+	xdrmem_create(&xdrs, clp->raw_netbuf->buf, clp->raw_netbuf->maxlen,
+	    XDR_ENCODE);
+	start = XDR_GETPOS(&xdrs);
 /* LINTED pointer alignment */
 	((struct rpc_msg *)clp->mashl_callmsg)->rm_xid++;
-	if ((!XDR_PUTBYTES(xdrs, clp->mashl_callmsg, clp->mcnt)) ||
-	    (!XDR_PUTINT32(xdrs, (int32_t *)&proc)) ||
-	    (!AUTH_MARSHALL(h->cl_auth, xdrs)) ||
-	    (!(*xargs)(xdrs, argsp)))
-		return (RPC_CANTENCODEARGS);
-	(void) XDR_GETPOS(xdrs);  /* called just to cause overhead */
+	if ((!XDR_PUTBYTES(&xdrs, clp->mashl_callmsg, clp->mcnt)) ||
+	    (!XDR_PUTINT32(&xdrs, (int32_t *)&proc)) ||
+	    (!AUTH_MARSHALL(h->cl_auth, &xdrs)) ||
+	    (!(*xargs)(&xdrs, argsp))) {
+		XDR_DESTROY(&xdrs);
+		return (rpc_callerr.re_status = RPC_CANTENCODEARGS);
+	}
+	clp->raw_netbuf->len = XDR_GETPOS(&xdrs) - start;
+	XDR_DESTROY(&xdrs);
 
 	/*
 	 * We have to call server input routine here because this is
 	 * all going on in one process.
-	 * By convention using FD_SETSIZE as the psuedo file descriptor.
+	 * By convention using FD_SETSIZE as the pseudo file descriptor.
 	 */
 	svc_getreq_common(FD_SETSIZE);
 
 	/*
 	 * get results
 	 */
-	xdrs->x_op = XDR_DECODE;
-	XDR_SETPOS(xdrs, 0);
+	xdrmem_create(&xdrs, clp->raw_netbuf->buf, clp->raw_netbuf->len,
+	    XDR_DECODE);
 	msg.acpted_rply.ar_verf = _null_auth;
 	msg.acpted_rply.ar_results.where = resultsp;
 	msg.acpted_rply.ar_results.proc = xresults;
-	if (!xdr_replymsg(xdrs, &msg))
-		return (RPC_CANTDECODERES);
-	if ((msg.rm_reply.rp_stat == MSG_ACCEPTED) &&
-		    (msg.acpted_rply.ar_stat == SUCCESS))
-		status = RPC_SUCCESS;
-	else {
-		__seterr_reply(&msg, &error);
-		status = error.re_status;
+	if (!xdr_replymsg(&xdrs, &msg)) {
+		XDR_DESTROY(&xdrs);
+		return (rpc_callerr.re_status = RPC_CANTDECODERES);
 	}
+	XDR_DESTROY(&xdrs);
+	if ((msg.rm_reply.rp_stat == MSG_ACCEPTED) &&
+	    (msg.acpted_rply.ar_stat == SUCCESS))
+		rpc_callerr.re_status = RPC_SUCCESS;
+	else
+		__seterr_reply(&msg, &rpc_callerr);
 
-	if (status == RPC_SUCCESS) {
+	if (rpc_callerr.re_status == RPC_SUCCESS) {
 		if (!AUTH_VALIDATE(h->cl_auth, &msg.acpted_rply.ar_verf)) {
-			status = RPC_AUTHERROR;
+			rpc_callerr.re_status = RPC_AUTHERROR;
+			rpc_callerr.re_why = AUTH_INVALIDRESP;
+		}
+		if (msg.acpted_rply.ar_verf.oa_base != NULL) {
+			xdr_free(xdr_opaque_auth,
+			    (char *)&(msg.acpted_rply.ar_verf));
 		}
 		/* end successful completion */
 	} else {
@@ -210,17 +214,7 @@ call_again:
 		/* end of unsuccessful completion */
 	}
 
-	if (status == RPC_SUCCESS) {
-		if (!AUTH_VALIDATE(h->cl_auth, &msg.acpted_rply.ar_verf)) {
-			status = RPC_AUTHERROR;
-		}
-		if (msg.acpted_rply.ar_verf.oa_base != NULL) {
-			xdrs->x_op = XDR_FREE;
-			(void) xdr_opaque_auth(xdrs,
-					&(msg.acpted_rply.ar_verf));
-		}
-	}
-	return (status);
+	return (rpc_callerr.re_status);
 }
 
 /*ARGSUSED*/
@@ -228,45 +222,53 @@ static enum clnt_stat
 clnt_raw_send(CLIENT *h, rpcproc_t proc, xdrproc_t xargs, caddr_t argsp)
 {
 	struct clnt_raw_private *clp;
-	XDR *xdrs;
+	XDR xdrs;
+	uint_t start;
+
+	rpc_callerr.re_errno = 0;
+	rpc_callerr.re_terrno = 0;
 
 	(void) mutex_lock(&clntraw_lock);
 	clp = clnt_raw_private;
-	xdrs = &clp->xdr_stream;
 	if (clp == NULL) {
 		(void) mutex_unlock(&clntraw_lock);
-		return (RPC_FAILED);
+		return (rpc_callerr.re_status = RPC_FAILED);
 	}
 	(void) mutex_unlock(&clntraw_lock);
 
 	/*
 	 * send request
 	 */
-	xdrs->x_op = XDR_ENCODE;
-	XDR_SETPOS(xdrs, 0);
+	xdrmem_create(&xdrs, clp->raw_netbuf->buf, clp->raw_netbuf->maxlen,
+	    XDR_ENCODE);
+	start = XDR_GETPOS(&xdrs);
 /* LINTED pointer alignment */
 	((struct rpc_msg *)clp->mashl_callmsg)->rm_xid++;
-	if ((!XDR_PUTBYTES(xdrs, clp->mashl_callmsg, clp->mcnt)) ||
-	    (!XDR_PUTINT32(xdrs, (int32_t *)&proc)) ||
-	    (!AUTH_MARSHALL(h->cl_auth, xdrs)) ||
-	    (!(*xargs)(xdrs, argsp)))
-		return (RPC_CANTENCODEARGS);
-	(void) XDR_GETPOS(xdrs);  /* called just to cause overhead */
+	if ((!XDR_PUTBYTES(&xdrs, clp->mashl_callmsg, clp->mcnt)) ||
+	    (!XDR_PUTINT32(&xdrs, (int32_t *)&proc)) ||
+	    (!AUTH_MARSHALL(h->cl_auth, &xdrs)) ||
+	    (!(*xargs)(&xdrs, argsp))) {
+		XDR_DESTROY(&xdrs);
+		return (rpc_callerr.re_status = RPC_CANTENCODEARGS);
+	}
+	clp->raw_netbuf->len = XDR_GETPOS(&xdrs) - start;
+	XDR_DESTROY(&xdrs);
 
 	/*
 	 * We have to call server input routine here because this is
 	 * all going on in one process.
-	 * By convention using FD_SETSIZE as the psuedo file descriptor.
+	 * By convention using FD_SETSIZE as the pseudo file descriptor.
 	 */
 	svc_getreq_common(FD_SETSIZE);
 
-	return (RPC_SUCCESS);
+	return (rpc_callerr.re_status = RPC_SUCCESS);
 }
 
 /*ARGSUSED*/
 static void
 clnt_raw_geterr(CLIENT *cl, struct rpc_err *errp)
 {
+	*errp = rpc_callerr;
 }
 
 /*ARGSUSED*/
@@ -274,18 +276,18 @@ static bool_t
 clnt_raw_freeres(CLIENT *cl, xdrproc_t xdr_res, caddr_t res_ptr)
 {
 	struct clnt_raw_private *clp;
-	XDR *xdrs;
 
 	(void) mutex_lock(&clntraw_lock);
 	clp = clnt_raw_private;
-	xdrs = &clp->xdr_stream;
 	if (clp == NULL) {
 		(void) mutex_unlock(&clntraw_lock);
 		return (FALSE);
 	}
 	(void) mutex_unlock(&clntraw_lock);
-	xdrs->x_op = XDR_FREE;
-	return ((*xdr_res)(xdrs, res_ptr));
+
+	xdr_free(xdr_res, res_ptr);
+
+	return (TRUE);
 }
 
 /*ARGSUSED*/
diff --git a/usr/src/lib/libnsl/rpc/svc_raw.c b/usr/src/lib/libnsl/rpc/svc_raw.c
index a3e9e9c7c6..1fe2ad7c6a 100644
--- a/usr/src/lib/libnsl/rpc/svc_raw.c
+++ b/usr/src/lib/libnsl/rpc/svc_raw.c
@@ -32,14 +32,11 @@
  * California.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 /*
- * svc_raw.c,   This a toy for simple testing and timing.
+ * svc_raw.c,   This is a toy for simple testing and timing.
  * Interface to create an rpc client and server in the same UNIX process.
- * This lets us similate rpc and get rpc (round trip) overhead, without
+ * This lets us simulate rpc and get rpc (round trip) overhead, without
  * any interference from the kernal.
- *
  */
 
 #include "mt.h"
@@ -47,7 +44,6 @@
 #include <stdlib.h>
 #include <rpc/rpc.h>
 #include <sys/types.h>
-#include <rpc/raw.h>
 #include <syslog.h>
 
 #ifndef UDPMSGSIZE
@@ -58,7 +54,7 @@
  * This is the "network" that we will be moving data over
  */
 static struct svc_raw_private {
-	char	*raw_buf;	/* should be shared with the cl handle */
+	struct netbuf	*raw_netbuf;
 	SVCXPRT	*server;
 	XDR	xdr_stream;
 	char	verf_body[MAX_AUTH_BYTES];
@@ -67,56 +63,66 @@ static struct svc_raw_private {
 static struct xp_ops *svc_raw_ops();
 extern mutex_t	svcraw_lock;
 
-
+/*
+ * This netbuf is shared with the raw client.
+ */
+struct netbuf _rawcomnetbuf;
 
 SVCXPRT *
 svc_raw_create(void)
 {
 	struct svc_raw_private *srp;
-	bool_t flag1 = FALSE, flag2 = FALSE;
 
 /* VARIABLES PROTECTED BY svcraw_lock: svc_raw_private, srp */
 	(void) mutex_lock(&svcraw_lock);
 	srp = svc_raw_private;
+	if (srp != NULL) {
+		(void) mutex_unlock(&svcraw_lock);
+		return (srp->server);
+	}
+
+	srp = calloc(1, sizeof (*srp));
 	if (srp == NULL) {
-		srp = calloc(1, sizeof (*srp));
-		if (srp == NULL) {
-			syslog(LOG_ERR, "svc_raw_create: out of memory");
-			(void) mutex_unlock(&svcraw_lock);
-			return (NULL);
-		}
-		flag1 = TRUE;
-		if (_rawcombuf == NULL) {
-			_rawcombuf = calloc(UDPMSGSIZE, sizeof (char));
-			if (_rawcombuf == NULL) {
-				free(srp);
-				syslog(LOG_ERR, "svc_raw_create: "
-					"out of memory");
-				(void) mutex_unlock(&svcraw_lock);
-				return (NULL);
-			}
-			flag2 = TRUE;
-		}
-		srp->raw_buf = _rawcombuf; /* Share it with the client */
-		svc_raw_private = srp;
+		syslog(LOG_ERR, "svc_raw_create: out of memory");
+
+		(void) mutex_unlock(&svcraw_lock);
+		return (NULL);
 	}
+
+	srp->raw_netbuf = &_rawcomnetbuf;
+	srp->raw_netbuf->buf = malloc(UDPMSGSIZE);
+	if (srp->raw_netbuf->buf == NULL) {
+		free(srp);
+		syslog(LOG_ERR, "svc_raw_create: out of memory");
+
+		(void) mutex_unlock(&svcraw_lock);
+		return (NULL);
+	}
+	srp->raw_netbuf->maxlen = UDPMSGSIZE;
+	srp->raw_netbuf->len = 0;
+
 	if ((srp->server = svc_xprt_alloc()) == NULL) {
-		if (flag2)
-			free(svc_raw_private->raw_buf);
-		if (flag1)
-			free(svc_raw_private);
+		free(srp->raw_netbuf->buf);
+		srp->raw_netbuf->buf = NULL;
+		srp->raw_netbuf->maxlen = 0;
+
+		free(srp);
+
 		(void) mutex_unlock(&svcraw_lock);
 		return (NULL);
 	}
+
 	/*
-	 * By convention, using FD_SETSIZE as the psuedo file descriptor
+	 * By convention, using FD_SETSIZE as the pseudo file descriptor
 	 */
 	srp->server->xp_fd = FD_SETSIZE;
 	srp->server->xp_port = 0;
 	srp->server->xp_ops = svc_raw_ops();
 	srp->server->xp_verf.oa_base = srp->verf_body;
-	xdrmem_create(&srp->xdr_stream, srp->raw_buf, UDPMSGSIZE, XDR_DECODE);
 	xprt_register(srp->server);
+
+	svc_raw_private = srp;
+
 	(void) mutex_unlock(&svcraw_lock);
 	return (srp->server);
 }
@@ -144,9 +150,16 @@ svc_raw_recv(SVCXPRT *xprt, struct rpc_msg *msg)
 	(void) mutex_unlock(&svcraw_lock);
 
 	xdrs = &srp->xdr_stream;
-	xdrs->x_op = XDR_DECODE;
-	(void) XDR_SETPOS(xdrs, 0);
-	return (xdr_callmsg(xdrs, msg));
+
+	xdrmem_create(xdrs, srp->raw_netbuf->buf, srp->raw_netbuf->len,
+	    XDR_DECODE);
+
+	if (!xdr_callmsg(xdrs, msg)) {
+		XDR_DESTROY(xdrs);
+		return (FALSE);
+	}
+
+	return (TRUE);
 }
 
 /*ARGSUSED*/
@@ -155,6 +168,7 @@ svc_raw_reply(SVCXPRT *xprt, struct rpc_msg *msg)
 {
 	struct svc_raw_private *srp;
 	XDR *xdrs;
+	uint_t start;
 
 	(void) mutex_lock(&svcraw_lock);
 	srp = svc_raw_private;
@@ -165,9 +179,19 @@ svc_raw_reply(SVCXPRT *xprt, struct rpc_msg *msg)
 	(void) mutex_unlock(&svcraw_lock);
 
 	xdrs = &srp->xdr_stream;
-	xdrs->x_op = XDR_ENCODE;
-	(void) XDR_SETPOS(xdrs, 0);
-	return (xdr_replymsg(xdrs, msg));
+
+	XDR_DESTROY(xdrs);
+	xdrmem_create(xdrs, srp->raw_netbuf->buf, srp->raw_netbuf->maxlen,
+	    XDR_ENCODE);
+
+	start = XDR_GETPOS(xdrs);
+	if (!xdr_replymsg(xdrs, msg)) {
+		XDR_DESTROY(xdrs);
+		return (FALSE);
+	}
+	srp->raw_netbuf->len = XDR_GETPOS(xdrs) - start;
+
+	return (TRUE);
 }
 
 /*ARGSUSED*/
@@ -183,6 +207,7 @@ svc_raw_getargs(SVCXPRT *xprt, xdrproc_t xdr_args, caddr_t args_ptr)
 		return (FALSE);
 	}
 	(void) mutex_unlock(&svcraw_lock);
+
 	return ((*xdr_args)(&srp->xdr_stream, args_ptr));
 }
 
@@ -191,7 +216,6 @@ static bool_t
 svc_raw_freeargs(SVCXPRT *xprt, xdrproc_t xdr_args, caddr_t args_ptr)
 {
 	struct svc_raw_private *srp;
-	XDR *xdrs;
 
 	(void) mutex_lock(&svcraw_lock);
 	srp = svc_raw_private;
@@ -201,9 +225,11 @@ svc_raw_freeargs(SVCXPRT *xprt, xdrproc_t xdr_args, caddr_t args_ptr)
 	}
 	(void) mutex_unlock(&svcraw_lock);
 
-	xdrs = &srp->xdr_stream;
-	xdrs->x_op = XDR_FREE;
-	return ((*xdr_args)(xdrs, args_ptr));
+	XDR_DESTROY(&srp->xdr_stream);
+
+	xdr_free(xdr_args, args_ptr);
+
+	return (TRUE);
 }
 
 /*ARGSUSED*/