[illumos-gate merge]

commit 7d8deab2c421c563ab11a55e623ed48109e237af
    12306 XPG4v2 slave pty behaviour should generally be disabled

	Conflicts:
	usr/src/uts/common/os/streamio.c

3 files changed, 57 insertions, 120 deletions

diff --git a/usr/src/lib/libc/port/gen/pt.c b/usr/src/lib/libc/port/gen/pt.c
index 26dd4dd376..3372f7133c 100644
--- a/usr/src/lib/libc/port/gen/pt.c
+++ b/usr/src/lib/libc/port/gen/pt.c
@@ -24,10 +24,12 @@
  * Use is subject to license terms.
  */
 
-/*	Copyright (c) 1988 AT&T	*/
-/*	  All Rights Reserved  	*/
+/*
+ * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
+ */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
+/*	Copyright (c) 1988 AT&T	*/
+/*	  All Rights Reserved	*/
 
 #pragma weak _ptsname = ptsname
 #pragma weak _grantpt = grantpt
@@ -133,6 +135,45 @@ unlockpt(int fd)
 	return (0);
 }
 
+/*
+ * XPG4v2 requires that open of a slave pseudo terminal device
+ * provides the process with an interface that is identical to
+ * the terminal interface.
+ *
+ * To satisfy this, in strict XPG4v2 mode, this routine also sends
+ * a message down the stream that sets a flag in the kernel module
+ * so that additional actions are performed when opening an
+ * associated slave PTY device. When this happens, modules are
+ * automatically pushed onto the stream to provide terminal
+ * semantics and those modules are then informed that they should
+ * behave in strict XPG4v2 mode which modifies their behaviour. In
+ * particular, in strict XPG4v2 mode, empty blocks will be sent up
+ * the master side of the stream rather than being suppressed.
+ *
+ * Most applications do not expect this behaviour so it is only
+ * enabled for programs compiled in strict XPG4v2 mode (see
+ * stdlib.h).
+ */
+int
+__unlockpt_xpg4(int fd)
+{
+	int ret;
+
+	if ((ret = unlockpt(fd)) == 0) {
+		struct strioctl istr;
+
+		istr.ic_cmd = PTSSTTY;
+		istr.ic_len = 0;
+		istr.ic_timout = 0;
+		istr.ic_dp = NULL;
+
+		if (ioctl(fd, I_STR, &istr) < 0)
+			ret = -1;
+	}
+
+	return (ret);
+}
+
 int
 grantpt(int fd)
 {
diff --git a/usr/src/lib/libc/port/mapfile-vers b/usr/src/lib/libc/port/mapfile-vers
index f2c12c1b4b..dbe33eb9d1 100644
--- a/usr/src/lib/libc/port/mapfile-vers
+++ b/usr/src/lib/libc/port/mapfile-vers
@@ -27,7 +27,7 @@
 # Copyright (c) 2013, OmniTI Computer Consulting, Inc. All rights reserved.
 # Copyright (c) 2013 Gary Mills
 # Copyright 2014 Garrett D'Amore <garrett@damore.org>
-# Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 #
 
 #
@@ -78,6 +78,11 @@ $if _x86 && _ELF64
 $add amd64
 $endif
 
+SYMBOL_VERSION ILLUMOS_0.31 {
+    protected:
+	__unlockpt_xpg4;
+} ILLUMOS_0.30;
+
 SYMBOL_VERSION ILLUMOS_0.30 {
     protected:
 	reallocf;
diff --git a/usr/src/lib/libc/port/sys/open.c b/usr/src/lib/libc/port/sys/open.c
index 067bb72d69..2ea6c567f4 100644
--- a/usr/src/lib/libc/port/sys/open.c
+++ b/usr/src/lib/libc/port/sys/open.c
@@ -22,44 +22,32 @@
 /*
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ *
+ * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
  */
 
 /*	Copyright (c) 1988 AT&T	*/
-/*	  All Rights Reserved  	*/
+/*	  All Rights Reserved	*/
 
 #include "lint.h"
-#include <sys/mkdev.h>
 #include <limits.h>
 #include <stdarg.h>
 #include <unistd.h>
-#include <strings.h>
-#include <errno.h>
-#include <sys/stat.h>
 #include <sys/fcntl.h>
-#include <sys/stropts.h>
-#include <sys/stream.h>
-#include <sys/ptms.h>
 #include <sys/syscall.h>
 #include "libc.h"
 
-static int xpg4_fixup(int fd);
-static void push_module(int fd);
-static int isptsfd(int fd);
-static void itoa(int i, char *ptr);
-
 int
 __openat(int dfd, const char *path, int oflag, mode_t mode)
 {
-	int fd = syscall(SYS_openat, dfd, path, oflag, mode);
-	return (xpg4_fixup(fd));
+	return (syscall(SYS_openat, dfd, path, oflag, mode));
 }
 
 int
 __open(const char *path, int oflag, mode_t mode)
 {
 #if defined(_RETAIN_OLD_SYSCALLS)
-	int fd = syscall(SYS_open, path, oflag, mode);
-	return (xpg4_fixup(fd));
+	return (syscall(SYS_open, path, oflag, mode));
 #else
 	return (__openat(AT_FDCWD, path, oflag, mode));
 #endif
@@ -70,114 +58,17 @@ __open(const char *path, int oflag, mode_t mode)
 int
 __openat64(int dfd, const char *path, int oflag, mode_t mode)
 {
-	int fd = syscall(SYS_openat64, dfd, path, oflag, mode);
-	return (xpg4_fixup(fd));
+	return (syscall(SYS_openat64, dfd, path, oflag, mode));
 }
 
 int
 __open64(const char *path, int oflag, mode_t mode)
 {
 #if defined(_RETAIN_OLD_SYSCALLS)
-	int fd = syscall(SYS_open64, path, oflag, mode);
-	return (xpg4_fixup(fd));
+	return (syscall(SYS_open64, path, oflag, mode));
 #else
 	return (__openat64(AT_FDCWD, path, oflag, mode));
 #endif
 }
 
 #endif	/* !_LP64 */
-
-/*
- * XPG4v2 requires that open of a slave pseudo terminal device
- * provides the process with an interface that is identical to
- * the terminal interface. For a more detailed discussion,
- * see bugid 4025044.
- */
-static int
-xpg4_fixup(int fd)
-{
-	if (libc__xpg4 != 0 && fd >= 0 && isptsfd(fd))
-		push_module(fd);
-	return (fd);
-}
-
-/*
- * Check if the file matches an entry in the /dev/pts directory.
- * Be careful to preserve errno.
- */
-static int
-isptsfd(int fd)
-{
-	char buf[TTYNAME_MAX];
-	char *str1 = buf;
-	const char *str2 = "/dev/pts/";
-	struct stat64 fsb, stb;
-	int oerrno = errno;
-	int rval = 0;
-
-	if (fstat64(fd, &fsb) == 0 && S_ISCHR(fsb.st_mode)) {
-		/*
-		 * Do this without strcpy() or strlen(),
-		 * to avoid invoking the dynamic linker.
-		 */
-		while (*str2 != '\0')
-			*str1++ = *str2++;
-		/*
-		 * Inline version of minor(dev), to avoid the dynamic linker.
-		 */
-		itoa(fsb.st_rdev & MAXMIN, str1);
-		if (stat64(buf, &stb) == 0)
-			rval = (stb.st_rdev == fsb.st_rdev);
-	}
-	errno = oerrno;
-	return (rval);
-}
-
-/*
- * Converts a number to a string (null terminated).
- */
-static void
-itoa(int i, char *ptr)
-{
-	int dig = 0;
-	int tempi;
-
-	tempi = i;
-	do {
-		dig++;
-		tempi /= 10;
-	} while (tempi);
-
-	ptr += dig;
-	*ptr = '\0';
-	while (--dig >= 0) {
-		*(--ptr) = i % 10 + '0';
-		i /= 10;
-	}
-}
-
-/*
- * Push modules to provide tty semantics
- */
-static void
-push_module(int fd)
-{
-	struct strioctl istr;
-	int oerrno = errno;
-
-	istr.ic_cmd = PTSSTTY;
-	istr.ic_len = 0;
-	istr.ic_timout = 0;
-	istr.ic_dp = NULL;
-	if (ioctl(fd, I_STR, &istr) != -1) {
-		(void) ioctl(fd, __I_PUSH_NOCTTY, "ptem");
-		(void) ioctl(fd, __I_PUSH_NOCTTY, "ldterm");
-		(void) ioctl(fd, __I_PUSH_NOCTTY, "ttcompat");
-		istr.ic_cmd = PTSSTTY;
-		istr.ic_len = 0;
-		istr.ic_timout = 0;
-		istr.ic_dp = NULL;
-		(void) ioctl(fd, I_STR, &istr);
-	}
-	errno = oerrno;
-}