OS-3046 lx brand unsupported socket flags silently ignored

OS-3047 lx brand recvmsg always assumes it gets a control msg

2 files changed, 95 insertions, 24 deletions

diff --git a/usr/src/lib/brand/lx/lx_brand/common/socket.c b/usr/src/lib/brand/lx/lx_brand/common/socket.c
index 7e77c6c8e4..101ecdc937 100644
--- a/usr/src/lib/brand/lx/lx_brand/common/socket.c
+++ b/usr/src/lib/brand/lx/lx_brand/common/socket.c
@@ -551,47 +551,113 @@ convert_sockflags(int lx_flags)
 {
 	int solaris_flags = 0;
 
-	if (lx_flags & LX_MSG_OOB)
+	if (lx_flags & LX_MSG_OOB) {
 		solaris_flags |= MSG_OOB;
+		lx_flags &= ~LX_MSG_OOB;
+	}
 
-	if (lx_flags & LX_MSG_PEEK)
+	if (lx_flags & LX_MSG_PEEK) {
 		solaris_flags |= MSG_PEEK;
+		lx_flags &= ~LX_MSG_PEEK;
+	}
 
-	if (lx_flags & LX_MSG_DONTROUTE)
+	if (lx_flags & LX_MSG_DONTROUTE) {
 		solaris_flags |= MSG_DONTROUTE;
+		lx_flags &= ~LX_MSG_DONTROUTE;
+	}
 
-	if (lx_flags & LX_MSG_CTRUNC)
+	if (lx_flags & LX_MSG_CTRUNC) {
 		solaris_flags |= MSG_CTRUNC;
+		lx_flags &= ~LX_MSG_CTRUNC;
+	}
 
-	if (lx_flags & LX_MSG_TRUNC)
-		solaris_flags |= MSG_TRUNC;
+	if (lx_flags & LX_MSG_PROXY) {
+		lx_unsupported("Unsupported "
+		    "socket operation with MSG_PROXY flag set");
+		lx_flags &= ~LX_MSG_PROXY;
+	}
 
-	if (lx_flags & LX_MSG_WAITALL)
-		solaris_flags |= MSG_WAITALL;
+	if (lx_flags & LX_MSG_TRUNC) {
+		solaris_flags |= MSG_TRUNC;
+		lx_flags &= ~LX_MSG_TRUNC;
+	}
 
-	if (lx_flags & LX_MSG_DONTWAIT)
+	if (lx_flags & LX_MSG_DONTWAIT) {
 		solaris_flags |= MSG_DONTWAIT;
+		lx_flags &= ~LX_MSG_DONTWAIT;
+	}
 
-	if (lx_flags & LX_MSG_EOR)
+	if (lx_flags & LX_MSG_EOR) {
 		solaris_flags |= MSG_EOR;
+		lx_flags &= ~LX_MSG_EOR;
+	}
+
+	if (lx_flags & LX_MSG_WAITALL) {
+		solaris_flags |= MSG_WAITALL;
+		lx_flags &= ~LX_MSG_WAITALL;
+	}
+
+	if (lx_flags & LX_MSG_FIN) {
+		lx_unsupported("Unsupported "
+		    "socket operation with MSG_FIN flag set");
+		lx_flags &= ~LX_MSG_FIN;
+	}
 
-	if (lx_flags & LX_MSG_PROXY)
-		lx_unsupported("socket operation with MSG_PROXY flag set");
+	if (lx_flags & LX_MSG_SYN) {
+		lx_unsupported("Unsupported "
+		    "socket operation with MSG_SYN flag set");
+		lx_flags &= ~LX_MSG_SYN;
+	}
 
-	if (lx_flags & LX_MSG_FIN)
-		lx_unsupported("socket operation with MSG_FIN flag set");
+	if (lx_flags & LX_MSG_CONFIRM) {
+		lx_unsupported("Unsupported "
+		    "socket operation with MSG_CONFIRM set");
+		lx_flags &= ~LX_MSG_CONFIRM;
+	}
 
-	if (lx_flags & LX_MSG_SYN)
-		lx_unsupported("socket operation with MSG_SYN flag set");
+	if (lx_flags & LX_MSG_RST) {
+		lx_unsupported("Unsupported "
+		    "socket operation with MSG_RST flag set");
+		lx_flags &= ~LX_MSG_RST;
+	}
 
-	if (lx_flags & LX_MSG_CONFIRM)
-		lx_unsupported("socket operation with MSG_CONFIRM set");
+	if (lx_flags & LX_MSG_ERRQUEUE) {
+		lx_unsupported("Unsupported "
+		    "socket operation with MSG_ERRQUEUE flag set");
+		lx_flags &= ~LX_MSG_ERRQUEUE;
+	}
 
-	if (lx_flags & LX_MSG_RST)
-		lx_unsupported("socket operation with MSG_RST flag set");
+	if (lx_flags & LX_MSG_NOSIGNAL) {
+		/* MSG_NOSIGNAL handled within each caller */
+		lx_flags &= ~LX_MSG_NOSIGNAL;
+	}
 
-	if (lx_flags & LX_MSG_MORE)
-		lx_unsupported("socket operation with MSG_MORE flag set");
+	if (lx_flags & LX_MSG_MORE) {
+		lx_unsupported("Unsupported "
+		    "socket operation with MSG_MORE flag set");
+		lx_flags &= ~LX_MSG_MORE;
+	}
+
+	if (lx_flags & LX_MSG_WAITFORONE) {
+		lx_unsupported("Unsupported "
+		    "socket operation with MSG_WAITFORONE flag set");
+		lx_flags &= ~LX_MSG_WAITFORONE;
+	}
+
+	if (lx_flags & LX_MSG_FASTOPEN) {
+		lx_unsupported("Unsupported "
+		    "socket operation with MSG_FASTOPEN flag set");
+		lx_flags &= ~LX_MSG_FASTOPEN;
+	}
+
+	if (lx_flags & LX_MSG_CMSG_CLOEXEC) {
+		lx_unsupported("Unsupported "
+		    "socket operation with MSG_CMSG_CLOEXEC flag set");
+		lx_flags &= ~LX_MSG_CMSG_CLOEXEC;
+	}
+
+	if (lx_flags != 0)
+		lx_unsupported("unknown socket flag(s) set 0x%x", lx_flags);
 
 	return (solaris_flags);
 }
@@ -1502,7 +1568,7 @@ lx_recvmsg(ulong_t *args)
 	int nosigpipe = flags & LX_MSG_NOSIGNAL;
 	struct sigaction newact, oact;
 
-	lx_debug("\trecvmsg(%d, 0x%p, 0x%x)", sockfd, (void *)args[1], flags);
+	lx_debug("\trecvmsg(%d, 0x%p, 0x%x)", sockfd, msgp, flags);
 
 	flags = convert_sockflags(flags);
 
@@ -1553,7 +1619,7 @@ lx_recvmsg(ulong_t *args)
 		    gettext("%s: could not reset SIGPIPE handler to "
 		    "emulate LX_MSG_NOSIGNAL"), "recvmsg()");
 
-	if (r >= 0 && msg.msg_control != NULL) {
+	if (r >= 0 && msg.msg_controllen >= sizeof (struct cmsghdr)) {
 		/*
 		 * If there are control messages bundled in this message,
 		 * we need to convert them from Linux to Solaris.
@@ -1565,6 +1631,8 @@ lx_recvmsg(ulong_t *args)
 			return (-errno);
 	}
 
+	msg.msg_control = cmsg;
+
 	/*
 	 * A handful of the values in the msghdr are set by the recvmsg()
 	 * call, so copy their values back to the caller.  Rather than iterate,
diff --git a/usr/src/lib/brand/lx/lx_brand/sys/lx_socket.h b/usr/src/lib/brand/lx/lx_brand/sys/lx_socket.h
index 9cac0dec0c..5bde3af407 100644
--- a/usr/src/lib/brand/lx/lx_brand/sys/lx_socket.h
+++ b/usr/src/lib/brand/lx/lx_brand/sys/lx_socket.h
@@ -246,6 +246,9 @@ extern "C" {
 #define	LX_MSG_ERRQUEUE		0x2000
 #define	LX_MSG_NOSIGNAL		0x4000
 #define	LX_MSG_MORE		0x8000
+#define	LX_MSG_WAITFORONE	0x10000
+#define	LX_MSG_FASTOPEN		0x20000000
+#define	LX_MSG_CMSG_CLOEXEC	0x40000000
 
 struct lx_msghdr {
 	void		*msg_name;		/* optional address */