[illumos-gate merge]

commit bce01b59de50fe66a0267f4aa23e1d6e60d973d9
    5995 RPC over SMB named pipes should use AF_UNIX sockets (missed bits)

3 files changed, 50 insertions, 21 deletions

diff --git a/usr/src/uts/common/fs/smbsrv/smb_common_transact.c b/usr/src/uts/common/fs/smbsrv/smb_common_transact.c
index 6fe650735e..936d8b98c7 100644
--- a/usr/src/uts/common/fs/smbsrv/smb_common_transact.c
+++ b/usr/src/uts/common/fs/smbsrv/smb_common_transact.c
@@ -1433,16 +1433,8 @@ smb_trans_nmpipe(smb_request_t *sr, smb_xa_t *xa)
 	mb = smb_mbuf_allocate(&vdb.vdb_uio);
 
 	rc = smb_opipe_read(sr, &vdb.vdb_uio);
-	if (rc == E2BIG) {
-		/*
-		 * Note: E2BIG is not a real error.  It just
-		 * tells us there's more data to be read.
-		 */
-		smbsr_status(sr, NT_STATUS_BUFFER_OVERFLOW,
-		    ERRDOS, ERROR_MORE_DATA);
-		rc = 0;
-	}
 	if (rc != 0) {
+		m_freem(mb);
 		smbsr_errno(sr, rc);
 		return (SDRC_ERROR);
 	}
@@ -1450,6 +1442,25 @@ smb_trans_nmpipe(smb_request_t *sr, smb_xa_t *xa)
 	smb_mbuf_trim(mb, xa->smb_mdrcnt - vdb.vdb_uio.uio_resid);
 	MBC_ATTACH_MBUF(&xa->rep_data_mb, mb);
 
+	/*
+	 * If the output buffer holds a partial pipe message,
+	 * we're supposed to return NT_STATUS_BUFFER_OVERFLOW.
+	 * As we don't have message boundary markers, the best
+	 * we can do is return that status when we have ALL of:
+	 *	Output buffer was < SMB_PIPE_MAX_MSGSIZE
+	 *	We filled the output buffer (resid==0)
+	 *	There's more data (ioctl FIONREAD)
+	 */
+	if (xa->smb_mdrcnt < SMB_PIPE_MAX_MSGSIZE &&
+	    vdb.vdb_uio.uio_resid == 0) {
+		int nread = 0;
+		rc = smb_opipe_get_nread(sr, &nread);
+		if (rc == 0 && nread != 0) {
+			smbsr_status(sr, NT_STATUS_BUFFER_OVERFLOW,
+			    ERRDOS, ERROR_MORE_DATA);
+		}
+	}
+
 	return (SDRC_SUCCESS);
 }
 
diff --git a/usr/src/uts/common/fs/smbsrv/smb_opipe.c b/usr/src/uts/common/fs/smbsrv/smb_opipe.c
index 5eacc82a60..a77535e2f7 100644
--- a/usr/src/uts/common/fs/smbsrv/smb_opipe.c
+++ b/usr/src/uts/common/fs/smbsrv/smb_opipe.c
@@ -390,20 +390,37 @@ smb_opipe_read(smb_request_t *sr, struct uio *uio)
 	}
 	uio->uio_resid -= recvcnt;
 
-	/*
-	 * If we filled the user's buffer,
-	 * find out if there's more data.
-	 */
-	if (uio->uio_resid == 0) {
-		int rc2, nread, trval;
-		rc2 = ksocket_ioctl(sock, FIONREAD, (intptr_t)&nread,
-		    &trval, ofile->f_cr);
-		if (rc2 == 0 && nread != 0)
-			rc = E2BIG;	/* more data */
-	}
-
 out:
 	ksocket_rele(sock);
 
 	return (rc);
 }
+
+int
+smb_opipe_get_nread(smb_request_t *sr, int *nread)
+{
+	smb_ofile_t *ofile;
+	smb_opipe_t *opipe;
+	ksocket_t sock;
+	int rc, trval;
+
+	ofile = sr->fid_ofile;
+	ASSERT(ofile->f_ftype == SMB_FTYPE_MESG_PIPE);
+	opipe = ofile->f_pipe;
+	SMB_OPIPE_VALID(opipe);
+
+	mutex_enter(&opipe->p_mutex);
+	sock = opipe->p_socket;
+	if (sock != NULL)
+		ksocket_hold(sock);
+	mutex_exit(&opipe->p_mutex);
+	if (sock == NULL)
+		return (EBADF);
+
+	rc = ksocket_ioctl(sock, FIONREAD, (intptr_t)nread, &trval,
+	    ofile->f_cr);
+
+	ksocket_rele(sock);
+
+	return (rc);
+}
diff --git a/usr/src/uts/common/smbsrv/smb_kproto.h b/usr/src/uts/common/smbsrv/smb_kproto.h
index 122f841a5e..f19dd9926a 100644
--- a/usr/src/uts/common/smbsrv/smb_kproto.h
+++ b/usr/src/uts/common/smbsrv/smb_kproto.h
@@ -380,6 +380,7 @@ int smb_opipe_open(smb_request_t *, uint32_t);
 void smb_opipe_close(smb_ofile_t *);
 int smb_opipe_read(smb_request_t *, struct uio *);
 int smb_opipe_write(smb_request_t *, struct uio *);
+int smb_opipe_get_nread(smb_request_t *, int *);
 
 void smb_opipe_door_init(smb_server_t *);
 void smb_opipe_door_fini(smb_server_t *);