12824 recvmsg(): adjust final cmsg->cmsg_len upon MSG_CTRUNC

Reviewed by: Robert Mustacchi <rm@fingolfin.org>
Reviewed by: Igor Kozhukhov <igor@dilos.org>
Approved by: Dan McDonald <danmcd@joyent.com>

7 files changed, 810 insertions, 30 deletions

diff --git a/usr/src/pkg/manifests/system-test-ostest.mf b/usr/src/pkg/manifests/system-test-ostest.mf
index 9ce0c51432..106e99c690 100644
--- a/usr/src/pkg/manifests/system-test-ostest.mf
+++ b/usr/src/pkg/manifests/system-test-ostest.mf
@@ -13,6 +13,7 @@
 # Copyright (c) 2012, 2016 by Delphix. All rights reserved.
 # Copyright 2014, OmniTI Computer Consulting, Inc. All rights reserved.
 # Copyright 2020 Joyent, Inc.
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 #
 
 set name=pkg.fmri value=pkg:/system/test/ostest@$(PKGVERS)
@@ -87,6 +88,8 @@ file path=opt/os-tests/tests/sockfs/conn mode=0555
 file path=opt/os-tests/tests/sockfs/dgram mode=0555
 file path=opt/os-tests/tests/sockfs/drop_priv mode=0555
 file path=opt/os-tests/tests/sockfs/nosignal mode=0555
+file path=opt/os-tests/tests/sockfs/rights.32 mode=0555
+file path=opt/os-tests/tests/sockfs/rights.64 mode=0555
 file path=opt/os-tests/tests/sockfs/sockpair mode=0555
 file path=opt/os-tests/tests/spoof-ras mode=0555
 file path=opt/os-tests/tests/stress/dladm-kstat mode=0555
diff --git a/usr/src/test/os-tests/runfiles/default.run b/usr/src/test/os-tests/runfiles/default.run
index 0cb915fd6a..d3b0be3920 100644
--- a/usr/src/test/os-tests/runfiles/default.run
+++ b/usr/src/test/os-tests/runfiles/default.run
@@ -12,6 +12,7 @@
 #
 # Copyright (c) 2012 by Delphix. All rights reserved.
 # Copyright 2020 Joyent, Inc.
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 #
 
 [DEFAULT]
@@ -64,7 +65,8 @@ tests = ['runtests.32', 'runtests.64']
 
 [/opt/os-tests/tests/sockfs]
 user = root
-tests = ['conn', 'dgram', 'drop_priv', 'nosignal', 'sockpair']
+tests = ['conn', 'dgram', 'drop_priv', 'nosignal', 'rights.32', 'rights.64',
+         'sockpair']
 
 [/opt/os-tests/tests/pf_key]
 user = root
diff --git a/usr/src/test/os-tests/tests/sockfs/Makefile b/usr/src/test/os-tests/tests/sockfs/Makefile
index 638250a400..267c8bbe4e 100644
--- a/usr/src/test/os-tests/tests/sockfs/Makefile
+++ b/usr/src/test/os-tests/tests/sockfs/Makefile
@@ -13,16 +13,18 @@
 # Copyright (c) 2012 by Delphix. All rights reserved.
 # Copyright 2017 Gordon W. Ross
 # Copyright (c) 2018, Joyent, Inc.
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 #
 
 include $(SRC)/cmd/Makefile.cmd
 include $(SRC)/test/Makefile.com
 
-PROG = conn dgram drop_priv nosignal sockpair
-
-LINTS = $(PROGS:%=%.ln)
+PROG =	conn dgram drop_priv nosignal sockpair \
+	rights.32 rights.64
 
 LDLIBS += -lsocket
+LDLIBS64 += -lsocket
+
 CSTD = $(CSTD_GNU99)
 CPPFLAGS += -D_XOPEN_SOURCE=600 -D__EXTENSIONS__
 
@@ -30,7 +32,8 @@ CPPFLAGS += -D_XOPEN_SOURCE=600 -D__EXTENSIONS__
 SMOFF += all_func_returns
 
 nosignal := LDLIBS += -lnsl
-nosignal.ln := LDLIBS += -lnsl
+rights.32 := LDLIBS += -lproc
+rights.64 := LDLIBS64 += -lproc
 
 ROOTOPTPKG = $(ROOT)/opt/os-tests
 TESTDIR = $(ROOTOPTPKG)/tests/sockfs
@@ -42,8 +45,6 @@ all: $(PROG)
 
 install: $(CMDS)
 
-lint: $(LINTS)
-
 clobber: clean
 	-$(RM) $(PROG)
 
@@ -51,11 +52,16 @@ clean:
 
 $(CMDS): $(TESTDIR) $(PROG)
 
-%.ln : %.c
-	$(LINT.c) $*.c $(UTILS) $(LDLIBS)
-
 $(TESTDIR):
 	$(INS.dir)
 
 $(TESTDIR)/%: %
 	$(INS.file)
+
+%.64: %.c
+	$(LINK64.c) -o $@ $< $(LDLIBS64)
+	$(POST_PROCESS)
+
+%.32: %.c
+	$(LINK.c) -o $@ $< $(LDLIBS)
+	$(POST_PROCESS)
diff --git a/usr/src/test/os-tests/tests/sockfs/rights.c b/usr/src/test/os-tests/tests/sockfs/rights.c
new file mode 100644
index 0000000000..97987d62c5
--- /dev/null
+++ b/usr/src/test/os-tests/tests/sockfs/rights.c
@@ -0,0 +1,700 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
+ */
+
+/*
+ * Test file descriptor passing via SCM_RIGHTS, and in particular what happens
+ * on message truncation in terms of the represented size of the data in the
+ * control message. Ensure that no file descriptors are leaked - the kernel
+ * must close any that would not fit in the available buffer space and the
+ * userland application must close the rest.
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <unistd.h>
+#include <libproc.h>
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/un.h>
+#include <sys/wait.h>
+#include <assert.h>
+#include <alloca.h>
+#include <err.h>
+
+static boolean_t debug;
+
+typedef struct cmsg_test {
+	char *name;		/* Name of the test */
+	uint_t send;		/* Number of FDs to send */
+	uint_t recv;		/* Size receive buffer for this number of FDs */
+	size_t predata;		/* Prepend dummy cmsg of this size */
+	int bufsize;		/* Explicitly set receive buffer size. */
+				/* Overrides 'recv' if non-zero */
+	uint_t x_controllen;	/* Expected received msg_controllen */
+	uint_t x_cmsg_datalen;	/* Expected received cmsg data length */
+	uint32_t x_flags;	/* Expected received msf_flags */
+} cmsg_test_t;
+
+static cmsg_test_t tests[] = {
+	{
+		.name = "send 1, recv 1",
+		.send = 1,
+		.recv = 1,
+		.predata = 0,
+		.bufsize = 0,
+		.x_controllen = 16,
+		.x_cmsg_datalen = 4,
+		.x_flags = 0,
+	},
+	{
+		.name = "send 10, recv 10",
+		.send = 10,
+		.recv = 10,
+		.predata = 0,
+		.bufsize = 0,
+		.x_controllen = 52,
+		.x_cmsg_datalen = 40,
+		.x_flags = 0,
+	},
+	{
+		.name = "send 2, recv 1",
+		.send = 2,
+		.recv = 1,
+		.predata = 0,
+		.bufsize = 0,
+		.x_controllen = 16,
+		.x_cmsg_datalen = 4,
+		.x_flags = MSG_CTRUNC,
+	},
+	{
+		.name = "send 2, recv 1, buffer 5",
+		.send = 2,
+		.recv = 1,
+		.predata = 0,
+		.bufsize = sizeof (int) * 2 - 3,
+		.x_controllen = 17,
+		.x_cmsg_datalen = 5,
+		.x_flags = MSG_CTRUNC,
+	},
+	{
+		.name = "send 2, recv 1, buffer 6",
+		.send = 2,
+		.recv = 1,
+		.predata = 0,
+		.bufsize = sizeof (int) * 2 - 2,
+		.x_controllen = 18,
+		.x_cmsg_datalen = 6,
+		.x_flags = MSG_CTRUNC,
+	},
+	{
+		.name = "send 2, recv 1, buffer 7",
+		.send = 2,
+		.recv = 1,
+		.predata = 0,
+		.bufsize = sizeof (int) * 2 - 1,
+		.x_controllen = 19,
+		.x_cmsg_datalen = 7,
+		.x_flags = MSG_CTRUNC,
+	},
+
+	/* Tests where there is no room allowed for data */
+
+	{
+		.name = "send 2, recv 0, hdronly",
+		.send = 2,
+		.recv = 0,
+		.predata = 0,
+		.bufsize = 0,
+		.x_controllen = 12,
+		.x_cmsg_datalen = 0,
+		.x_flags = MSG_CTRUNC,
+	},
+
+	{
+		.name = "send 2, recv 0, hdr - 1",
+		.send = 2,
+		.recv = 0,
+		.predata = 0,
+		.bufsize = -1,
+		.x_controllen = 11,
+		.x_cmsg_datalen = 0,
+		.x_flags = MSG_CTRUNC,
+	},
+
+	{
+		.name = "send 2, recv 0, hdr - 5",
+		.send = 2,
+		.recv = 0,
+		.predata = 0,
+		.bufsize = -5,
+		.x_controllen = 7,
+		.x_cmsg_datalen = 0,
+		.x_flags = MSG_CTRUNC,
+	},
+
+	/* Tests where SCM_RIGHTS is not the first message */
+
+	{
+		.name = "send 1, recv 1, pre 8",
+		.send = 1,
+		.recv = 1,
+		.predata = 8,
+		.bufsize = 0,
+		.x_controllen = 36,
+		.x_cmsg_datalen = 4,
+		.x_flags = 0,
+	},
+	{
+		.name = "send 1, recv 1, pre 7",
+		.send = 1,
+		.recv = 1,
+		.predata = 7,
+		.bufsize = 0,
+		.x_controllen = 35,
+		.x_cmsg_datalen = 4,
+		.x_flags = 0,
+	},
+	{
+		.name = "send 1, recv 1, pre 6",
+		.send = 1,
+		.recv = 1,
+		.predata = 6,
+		.bufsize = 0,
+		.x_controllen = 34,
+		.x_cmsg_datalen = 4,
+		.x_flags = 0,
+	},
+	{
+		.name = "send 1, recv 1, pre 5",
+		.send = 1,
+		.recv = 1,
+		.predata = 5,
+		.bufsize = 0,
+		.x_controllen = 33,
+		.x_cmsg_datalen = 4,
+		.x_flags = 0,
+	},
+
+	{
+		.name = "send 2, recv 1, pre 8",
+		.send = 2,
+		.recv = 1,
+		.predata = 8,
+		.bufsize = 0,
+		.x_controllen = 36,
+		.x_cmsg_datalen = 8,
+		.x_flags = MSG_CTRUNC,
+	},
+	{
+		.name = "send 2, recv 1, pre 7",
+		.send = 2,
+		.recv = 1,
+		.predata = 7,
+		.bufsize = 0,
+		.x_controllen = 36,
+		.x_cmsg_datalen = 8,
+		.x_flags = MSG_CTRUNC,
+	},
+	{
+		.name = "send 2, recv 1, pre 6",
+		.send = 2,
+		.recv = 1,
+		.predata = 6,
+		.bufsize = 0,
+		.x_controllen = 36,
+		.x_cmsg_datalen = 8,
+		.x_flags = MSG_CTRUNC,
+	},
+	{
+		.name = "send 2, recv 1, pre 5",
+		.send = 2,
+		.recv = 1,
+		.predata = 5,
+		.bufsize = 0,
+		.x_controllen = 36,
+		.x_cmsg_datalen = 8,
+		.x_flags = MSG_CTRUNC,
+	},
+	{
+		.name = "send 2, recv 1, pre 4",
+		.send = 2,
+		.recv = 1,
+		.predata = 4,
+		.bufsize = 0,
+		.x_controllen = 32,
+		.x_cmsg_datalen = 8,
+		.x_flags = MSG_CTRUNC,
+	},
+	{
+		.name = "send 2, recv 1, pre 3",
+		.send = 2,
+		.recv = 1,
+		.predata = 3,
+		.bufsize = 0,
+		.x_controllen = 32,
+		.x_cmsg_datalen = 8,
+		.x_flags = MSG_CTRUNC,
+	},
+	{
+		.name = "send 2, recv 1, pre 2",
+		.send = 2,
+		.recv = 1,
+		.predata = 2,
+		.bufsize = 0,
+		.x_controllen = 32,
+		.x_cmsg_datalen = 8,
+		.x_flags = MSG_CTRUNC,
+	},
+	{
+		.name = "send 2, recv 1, pre 1",
+		.send = 2,
+		.recv = 1,
+		.predata = 1,
+		.bufsize = 0,
+		.x_controllen = 32,
+		.x_cmsg_datalen = 8,
+		.x_flags = MSG_CTRUNC,
+	},
+
+	{
+		.name = "send 2, recv 1, pre 8, buffer 5",
+		.send = 2,
+		.recv = 1,
+		.predata = 8,
+		.bufsize = sizeof (int) * 2 - 3,
+		.x_controllen = 37,
+		.x_cmsg_datalen = 8,
+		.x_flags = MSG_CTRUNC,
+	},
+	{
+		.name = "send 2, recv 1, pre 8, buffer 6",
+		.send = 2,
+		.recv = 1,
+		.predata = 8,
+		.bufsize = sizeof (int) * 2 - 2,
+		.x_controllen = 38,
+		.x_cmsg_datalen = 8,
+		.x_flags = MSG_CTRUNC,
+	},
+	{
+		.name = "send 2, recv 1, pre 8, buffer 7",
+		.send = 2,
+		.recv = 1,
+		.predata = 8,
+		.bufsize = sizeof (int) * 2 - 1,
+		.x_controllen = 39,
+		.x_cmsg_datalen = 8,
+		.x_flags = MSG_CTRUNC,
+	},
+	{
+		.name = "send 10, recv 1, pre 8",
+		.send = 10,
+		.recv = 1,
+		.predata = 8,
+		.bufsize = 0,
+		.x_controllen = 36,
+		.x_cmsg_datalen = 24,
+		.x_flags = MSG_CTRUNC,
+	},
+
+	/* End of tests */
+
+	{
+		.name = NULL
+	}
+};
+
+static int sock = -1, testfd = -1, cfd = -1;
+static int fdcount;
+
+static int
+fdwalkcb(const prfdinfo_t *info, void *arg)
+{
+	if (!S_ISDIR(info->pr_mode) && info->pr_fd > 2 &&
+	    info->pr_fd != sock && info->pr_fd != testfd &&
+	    info->pr_fd != cfd) {
+		if (debug) {
+			fprintf(stderr, "%s: unexpected fd: %d\n",
+			    (char *)arg, info->pr_fd);
+		}
+		fdcount++;
+	}
+
+	return (0);
+
+}
+
+static void
+check_fds(char *tag)
+{
+	fdcount = 0;
+	proc_fdwalk(getpid(), fdwalkcb, tag);
+}
+
+static void
+send_and_wait(pid_t pid, sigset_t *set, int osig, int isig)
+{
+	int sig;
+
+	if (osig > 0)
+		kill(pid, osig);
+
+	if (isig > 0) {
+		if (sigwait(set, &sig) != 0) {
+			err(EXIT_FAILURE,
+			    "sigwait failed waiting for %d", isig);
+		}
+		if (sig == SIGINT) {
+			exit(1);
+		}
+		if (sig != isig) {
+			err(EXIT_FAILURE,
+			    "sigwait returned unexpected signal %d", sig);
+		}
+	}
+}
+
+static void
+sendtest(cmsg_test_t *t)
+{
+	struct msghdr msg;
+	struct cmsghdr *cm;
+	struct iovec iov;
+	ssize_t nbytes;
+	char c = '*';
+	int i, *p;
+
+	bzero(&msg, sizeof (msg));
+
+	msg.msg_name = NULL;
+	msg.msg_namelen = 0;
+
+	iov.iov_base = &c;
+	iov.iov_len = sizeof (c);
+	msg.msg_iov = &iov;
+	msg.msg_iovlen = 1;
+
+	msg.msg_flags = 0;
+
+	msg.msg_controllen = CMSG_SPACE(sizeof (int) * t->send);
+
+	if (t->predata > 0) {
+		/* A dummy cmsg will be inserted at the head of the data */
+		msg.msg_controllen += CMSG_SPACE(t->predata);
+	}
+
+	msg.msg_control = alloca(msg.msg_controllen);
+	bzero(msg.msg_control, msg.msg_controllen);
+
+	cm = CMSG_FIRSTHDR(&msg);
+
+	if (t->predata > 0) {
+		/* Insert the dummy cmsg */
+		cm->cmsg_len = CMSG_LEN(t->predata);
+		cm->cmsg_level = SOL_SOCKET;
+		cm->cmsg_type = 0;
+		cm = CMSG_NXTHDR(&msg, cm);
+	}
+
+	cm->cmsg_len = CMSG_LEN(sizeof (int) * t->send);
+	cm->cmsg_level = SOL_SOCKET;
+	cm->cmsg_type = SCM_RIGHTS;
+
+	p = (int *)CMSG_DATA(cm);
+	for (i = 0; i < t->send; i++) {
+		int s = dup(testfd);
+		if (s == -1)
+			err(EXIT_FAILURE, "dup()");
+		*p++ = s;
+	}
+
+	if (debug)
+		printf("Sending: controllen=%u\n", msg.msg_controllen);
+
+	nbytes = sendmsg(cfd, &msg, 0);
+	if (nbytes == -1)
+		err(EXIT_FAILURE, "sendmsg()");
+
+	p = (int *)CMSG_DATA(cm);
+	for (i = 0; i < t->send; i++)
+		(void) close(*p++);
+}
+
+static int
+server(const char *sockpath, pid_t pid)
+{
+	struct sockaddr_un addr;
+	sigset_t set;
+	cmsg_test_t *t;
+
+	sigemptyset(&set);
+	sigaddset(&set, SIGUSR2);
+	sigaddset(&set, SIGINT);
+
+	sock = socket(PF_LOCAL, SOCK_STREAM, 0);
+	if (sock == -1)
+		err(EXIT_FAILURE, "failed to create socket");
+	addr.sun_family = AF_UNIX;
+	strlcpy(addr.sun_path, sockpath, sizeof (addr.sun_path));
+	if (bind(sock, (struct sockaddr *)&addr, sizeof (addr)) == -1)
+		err(EXIT_FAILURE, "bind failed");
+	if (listen(sock, 0) == -1)
+		err(EXIT_FAILURE, "listen failed");
+
+	if ((testfd = open("/dev/null", O_RDONLY)) == -1)
+		err(EXIT_FAILURE, "/dev/null");
+
+	check_fds("server");
+
+	/* Signal the child to connect to the socket */
+	send_and_wait(pid, &set, SIGUSR1, SIGUSR2);
+
+	if ((cfd = accept(sock, NULL, 0)) == -1)
+		err(EXIT_FAILURE, "accept failed");
+
+	for (t = tests; t->name != NULL; t++) {
+		if (debug)
+			printf("\n>>> Starting test %s\n", t->name);
+
+		sendtest(t);
+		check_fds("server");
+
+		send_and_wait(pid, &set, SIGUSR1, SIGUSR2);
+	}
+
+	close(cfd);
+	close(testfd);
+	close(sock);
+
+	return (0);
+}
+
+static boolean_t pass;
+
+static void
+check(uint_t actual, uint_t expected, char *tag)
+{
+	if (actual != expected) {
+		fprintf(stderr, "    !!!: "
+		    "%1$s = %2$u(%2$#x) (expected %3$u(%3$#x))\n",
+		    tag, actual, expected);
+		pass = _B_FALSE;
+	} else if (debug) {
+		fprintf(stderr, "       : "
+		    "%1$s = %2$u(%2$#x)\n",
+		    tag, actual);
+	}
+}
+
+static boolean_t
+recvtest(cmsg_test_t *t)
+{
+	struct msghdr msg;
+	struct cmsghdr *cm;
+	struct iovec iov;
+	size_t bufsize;
+	ssize_t nbytes;
+	char c = '*';
+
+	bzero(&msg, sizeof (msg));
+
+	msg.msg_name = NULL;
+	msg.msg_namelen = 0;
+
+	iov.iov_base = &c;
+	iov.iov_len = sizeof (c);
+	msg.msg_iov = &iov;
+	msg.msg_iovlen = 1;
+
+	msg.msg_flags = 0;
+
+	/*
+	 * If the test does not specify a receive buffer size, calculate one
+	 * from the number of file descriptors to receive.
+	 */
+	if (t->bufsize == 0) {
+		bufsize = sizeof (int) * t->recv;
+		bufsize = CMSG_SPACE(bufsize);
+	} else {
+		/*
+		 * Use the specific buffer size provided but add in
+		 * space for the header
+		 */
+		bufsize = t->bufsize + CMSG_LEN(0);
+	}
+
+	if (t->predata > 0) {
+		/* A dummy cmsg will be found at the head of the data */
+		bufsize += CMSG_SPACE(t->predata);
+	}
+
+	msg.msg_controllen = bufsize;
+	msg.msg_control = alloca(bufsize);
+	bzero(msg.msg_control, msg.msg_controllen);
+
+	pass = _B_TRUE;
+
+	if (debug)
+		printf("Receiving: controllen=%u, \n", msg.msg_controllen);
+
+	nbytes = recvmsg(sock, &msg, 0);
+
+	if (nbytes == -1) {
+		pass = _B_FALSE;
+		fprintf(stderr, "recvmsg() failed: %s\n", strerror(errno));
+		goto out;
+	}
+
+	if (debug) {
+		printf("Received: controllen=%u, flags=%#x\n",
+		    msg.msg_controllen, msg.msg_flags);
+	}
+
+	check(msg.msg_flags, t->x_flags, "msg_flags");
+	check(msg.msg_controllen, t->x_controllen, "msg_controllen");
+
+	for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) {
+		void *data, *end;
+
+		if (debug) {
+			printf("    >> : Got cmsg %x/%x - %u\n", cm->cmsg_level,
+			    cm->cmsg_type, cm->cmsg_len);
+		}
+
+		if (cm->cmsg_type != SCM_RIGHTS) {
+			if (debug)
+				printf("       : skipping cmsg\n");
+			continue;
+		}
+
+		check(cm->cmsg_len - CMSG_LEN(0),
+		    t->x_cmsg_datalen, "cmsg_len");
+
+		/* Close any received file descriptors */
+		data = CMSG_DATA(cm);
+
+		if ((msg.msg_flags & MSG_CTRUNC) &&
+		    CMSG_NXTHDR(&msg, cm) == NULL) {
+			/*
+			 * illumos did not previously adjust cmsg_len on
+			 * truncation. This is the last cmsg, derive the
+			 * length from msg_controllen
+			 */
+			end = msg.msg_control + msg.msg_controllen;
+		} else {
+			end = data + cm->cmsg_len - CMSG_LEN(0);
+		}
+
+		while (data <= end - sizeof (int)) {
+			int *a = (int *)data;
+			if (debug)
+				printf("       : close(%d)\n", *a);
+			if (close(*a) == -1) {
+				pass = _B_FALSE;
+				fprintf(stderr, "    !!!: "
+				    "failed to close fd %d - %s\n", *a,
+				    strerror(errno));
+			}
+			data += sizeof (int);
+		}
+	}
+
+out:
+
+	check_fds("client");
+	check(fdcount, 0, "client descriptors");
+	printf("     + : %s %s\n", pass ? "PASS" : "FAIL", t->name);
+
+	return (pass);
+}
+
+static int
+client(const char *sockpath, pid_t pid)
+{
+	struct sockaddr_un addr;
+	sigset_t set;
+	cmsg_test_t *t;
+	int ret = 0;
+
+	sigemptyset(&set);
+	sigaddset(&set, SIGUSR1);
+	sigaddset(&set, SIGINT);
+
+	send_and_wait(pid, &set, 0, SIGUSR1);
+
+	sock = socket(PF_LOCAL, SOCK_STREAM, 0);
+	if (sock == -1)
+		err(EXIT_FAILURE, "failed to create socket");
+	addr.sun_family = AF_UNIX;
+	strlcpy(addr.sun_path, sockpath, sizeof (addr.sun_path));
+	if (connect(sock, (struct sockaddr *)&addr, sizeof (addr)) == -1)
+		err(EXIT_FAILURE, "could not connect to server socket");
+
+	for (t = tests; t->name != NULL; t++) {
+		send_and_wait(pid, &set, SIGUSR2, SIGUSR1);
+		if (!recvtest(t))
+			ret = 1;
+	}
+
+	close(sock);
+
+	return (ret);
+}
+
+int
+main(int argc, const char **argv)
+{
+	char sockpath[] = "/tmp/cmsg.testsock.XXXXXX";
+	pid_t pid, ppid;
+	sigset_t set;
+	int ret = 0;
+
+	if (argc > 1 && strcmp(argv[1], "-d") == 0)
+		debug = _B_TRUE;
+
+	sigfillset(&set);
+	sigdelset(&set, SIGINT);
+	sigdelset(&set, SIGTSTP);
+	sigprocmask(SIG_BLOCK, &set, NULL);
+
+	if (mktemp(sockpath) == NULL)
+		err(EXIT_FAILURE, "Failed to make temporary socket path");
+
+	ppid = getpid();
+	pid = fork();
+	switch (pid) {
+	case -1:
+		err(EXIT_FAILURE, "fork failed");
+	case 0:
+		return (server(sockpath, ppid));
+	default:
+		break;
+	}
+
+	ret = client(sockpath, pid);
+	kill(pid, SIGINT);
+
+	unlink(sockpath);
+
+	return (ret);
+}
diff --git a/usr/src/uts/common/fs/sockfs/socksubr.c b/usr/src/uts/common/fs/sockfs/socksubr.c
index 9efc808190..2fc51eba4a 100644
--- a/usr/src/uts/common/fs/sockfs/socksubr.c
+++ b/usr/src/uts/common/fs/sockfs/socksubr.c
@@ -22,8 +22,8 @@
 /*
  * Copyright (c) 1995, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2016 Nexenta Systems, Inc.  All rights reserved.
- * Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
  * Copyright 2015, Joyent, Inc. All rights reserved.
+ * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #include <sys/types.h>
@@ -959,7 +959,46 @@ so_closefds(void *control, t_uscalar_t controllen, int oldflg,
 			    (int)CMSG_CONTENTLEN(cmsg),
 			    startoff - (int)sizeof (struct cmsghdr));
 		}
-		startoff -= cmsg->cmsg_len;
+		startoff -= ROUNDUP_cmsglen(cmsg->cmsg_len);
+	}
+}
+
+/*
+ * Handle truncation of a cmsg when the receive buffer is not big enough.
+ * Adjust the cmsg_len header field in the last cmsg that will be included in
+ * the buffer to reflect the number of bytes included.
+ */
+void
+so_truncatecmsg(void *control, t_uscalar_t controllen, uint_t maxlen)
+{
+	struct cmsghdr *cmsg;
+	uint_t len = 0;
+
+	if (control == NULL)
+		return;
+
+	for (cmsg = control;
+	    CMSG_VALID(cmsg, control, (uintptr_t)control + controllen);
+	    cmsg = CMSG_NEXT(cmsg)) {
+
+		len += ROUNDUP_cmsglen(cmsg->cmsg_len);
+
+		if (len > maxlen) {
+			/*
+			 * This cmsg is the last one that will be included in
+			 * the truncated buffer.
+			 */
+			socklen_t diff = len - maxlen;
+
+			if (diff < CMSG_CONTENTLEN(cmsg)) {
+				dprint(1, ("so_truncatecmsg: %d -> %d\n",
+				    cmsg->cmsg_len, cmsg->cmsg_len - diff));
+				cmsg->cmsg_len -= diff;
+			} else {
+				cmsg->cmsg_len = sizeof (struct cmsghdr);
+			}
+			break;
+		}
 	}
 }
 
diff --git a/usr/src/uts/common/fs/sockfs/socksyscalls.c b/usr/src/uts/common/fs/sockfs/socksyscalls.c
index 6a049b1828..30666f73ca 100644
--- a/usr/src/uts/common/fs/sockfs/socksyscalls.c
+++ b/usr/src/uts/common/fs/sockfs/socksyscalls.c
@@ -24,6 +24,7 @@
  * Copyright 2015, Joyent, Inc.  All rights reserved.
  * Copyright (c) 2013, OmniTI Computer Consulting, Inc. All rights reserved.
  * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
+ * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #include <sys/types.h>
@@ -831,7 +832,7 @@ recvit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags,
 	void *name;
 	socklen_t namelen;
 	void *control;
-	socklen_t controllen;
+	socklen_t controllen, free_controllen;
 	ssize_t len;
 	int error;
 
@@ -858,6 +859,8 @@ recvit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags,
 	lwp_stat_update(LWP_STAT_MSGRCV, 1);
 	releasef(sock);
 
+	free_controllen = msg->msg_controllen;
+
 	error = copyout_name(name, namelen, namelenp,
 	    msg->msg_name, msg->msg_namelen);
 	if (error)
@@ -887,11 +890,7 @@ recvit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags,
 			goto err;
 		}
 	}
-	/*
-	 * Note: This MUST be done last. There can be no "goto err" after this
-	 * point since it could make so_closefds run twice on some part
-	 * of the file descriptor array.
-	 */
+
 	if (controllen != 0) {
 		if (!(flags & MSG_XPG4_2)) {
 			/*
@@ -900,36 +899,65 @@ recvit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags,
 			 */
 			controllen &= ~((int)sizeof (uint32_t) - 1);
 		}
+
+		if (msg->msg_controllen > controllen || control == NULL) {
+			/*
+			 * If the truncated part contains file descriptors,
+			 * then they must be closed in the kernel as they
+			 * will not be included in the data returned to
+			 * user space. Close them now so that the header size
+			 * can be safely adjusted prior to copyout. In case of
+			 * an error during copyout, the remaining file
+			 * descriptors will be closed in the error handler
+			 * below.
+			 */
+			so_closefds(msg->msg_control, msg->msg_controllen,
+			    !(flags & MSG_XPG4_2),
+			    control == NULL ? 0 : controllen);
+
+			/*
+			 * In the case of a truncated control message, the last
+			 * cmsg header that fits into the available buffer
+			 * space must be adjusted to reflect the actual amount
+			 * of associated data that will be returned. This only
+			 * needs to be done for XPG4 messages as non-XPG4
+			 * messages are not structured (they are just a
+			 * buffer and a length - msg_accrights(len)).
+			 */
+			if (control != NULL && (flags & MSG_XPG4_2)) {
+				so_truncatecmsg(msg->msg_control,
+				    msg->msg_controllen, controllen);
+				msg->msg_controllen = controllen;
+			}
+		}
+
 		error = copyout_arg(control, controllen, controllenp,
 		    msg->msg_control, msg->msg_controllen);
+
 		if (error)
 			goto err;
 
-		if (msg->msg_controllen > controllen || control == NULL) {
-			if (control == NULL)
-				controllen = 0;
-			so_closefds(msg->msg_control, msg->msg_controllen,
-			    !(flags & MSG_XPG4_2), controllen);
-		}
 	}
 	if (msg->msg_namelen != 0)
 		kmem_free(msg->msg_name, (size_t)msg->msg_namelen);
-	if (msg->msg_controllen != 0)
-		kmem_free(msg->msg_control, (size_t)msg->msg_controllen);
+	if (free_controllen != 0)
+		kmem_free(msg->msg_control, (size_t)free_controllen);
 	return (len - uiop->uio_resid);
 
 err:
 	/*
 	 * If we fail and the control part contains file descriptors
-	 * we have to close the fd's.
+	 * we have to close them. For a truncated control message, the
+	 * descriptors which were cut off have already been closed and the
+	 * length adjusted so that they will not be closed again.
 	 */
 	if (msg->msg_controllen != 0)
 		so_closefds(msg->msg_control, msg->msg_controllen,
 		    !(flags & MSG_XPG4_2), 0);
 	if (msg->msg_namelen != 0)
 		kmem_free(msg->msg_name, (size_t)msg->msg_namelen);
-	if (msg->msg_controllen != 0)
-		kmem_free(msg->msg_control, (size_t)msg->msg_controllen);
+	if (free_controllen != 0)
+		kmem_free(msg->msg_control, (size_t)free_controllen);
 	return (set_errno(error));
 }
 
diff --git a/usr/src/uts/common/sys/socketvar.h b/usr/src/uts/common/sys/socketvar.h
index f5c4d801de..ef2bc77f74 100644
--- a/usr/src/uts/common/sys/socketvar.h
+++ b/usr/src/uts/common/sys/socketvar.h
@@ -37,7 +37,7 @@
  */
 /*
  * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
- * Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
+ * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #ifndef _SYS_SOCKETVAR_H
@@ -903,6 +903,8 @@ extern void	fdbuf_free(struct fdbuf *);
 extern mblk_t	*fdbuf_allocmsg(int, struct fdbuf *);
 extern int	fdbuf_create(void *, int, struct fdbuf **);
 extern void	so_closefds(void *, t_uscalar_t, int, int);
+extern void	so_truncatecmsg(void *, t_uscalar_t, uint_t);
+
 extern int	so_getfdopt(void *, t_uscalar_t, int, void **, int *);
 t_uscalar_t	so_optlen(void *, t_uscalar_t, int);
 extern void	so_cmsg2opt(void *, t_uscalar_t, int, mblk_t *);