9042 multiples of tty streams modules cause weirdness

Reviewed by: Randy Fishel <randyf@sibernet.com>
Reviewed by: Carlos Neira <cneirabustos@gmail.com>
Approved by: Hans Rosenfeld <rosenfeld@grumpf.hope-2000.org>

7 files changed, 52 insertions, 6 deletions

diff --git a/usr/src/uts/common/io/ldterm.c b/usr/src/uts/common/io/ldterm.c
index 6c396db601..1af97c86ea 100644
--- a/usr/src/uts/common/io/ldterm.c
+++ b/usr/src/uts/common/io/ldterm.c
@@ -22,6 +22,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  * Copyright (c) 2014, Joyent, Inc.  All rights reserved.
+ * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
  */
 
 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
@@ -381,7 +382,7 @@ static struct streamtab ldtrinfo;
 static struct fmodsw fsw = {
 	"ldterm",
 	&ldtrinfo,
-	D_MTQPAIR | D_MP
+	D_MTQPAIR | D_MP | _D_SINGLE_INSTANCE
 };
 
 static struct modlstrmod modlstrmod = {
diff --git a/usr/src/uts/common/io/ptem.c b/usr/src/uts/common/io/ptem.c
index 481169daf3..e4dc15a3ac 100644
--- a/usr/src/uts/common/io/ptem.c
+++ b/usr/src/uts/common/io/ptem.c
@@ -26,6 +26,7 @@
 /*
  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
  */
 
 /*
@@ -62,7 +63,7 @@ extern struct streamtab pteminfo;
 static struct fmodsw fsw = {
 	"ptem",
 	&pteminfo,
-	D_MTQPAIR | D_MP
+	D_MTQPAIR | D_MP | _D_SINGLE_INSTANCE
 };
 
 static struct modlstrmod modlstrmod = {
diff --git a/usr/src/uts/common/io/ttcompat.c b/usr/src/uts/common/io/ttcompat.c
index ab420c82e7..22b1442e50 100644
--- a/usr/src/uts/common/io/ttcompat.c
+++ b/usr/src/uts/common/io/ttcompat.c
@@ -21,6 +21,7 @@
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
@@ -73,7 +74,7 @@ static struct streamtab ttcoinfo;
 static struct fmodsw fsw = {
 	"ttcompat",
 	&ttcoinfo,
-	D_MTQPAIR | D_MP
+	D_MTQPAIR | D_MP | _D_SINGLE_INSTANCE
 };
 
 /*
diff --git a/usr/src/uts/common/os/streamio.c b/usr/src/uts/common/os/streamio.c
index 62569eefed..509a3cfd40 100644
--- a/usr/src/uts/common/os/streamio.c
+++ b/usr/src/uts/common/os/streamio.c
@@ -25,6 +25,7 @@
 /*
  * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2017 Joyent, Inc.
+ * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #include <sys/types.h>
@@ -3783,6 +3784,30 @@ strioctl(struct vnode *vp, int cmd, intptr_t arg, int flag, int copyflag,
 		TRACE_2(TR_FAC_STREAMS_FR, TR_I_PUSH,
 		    "I_PUSH:fp %p stp %p", fp, stp);
 
+		/*
+		 * If the module is flagged as single-instance, then check
+		 * to see if the module is already pushed. If it is, return
+		 * as if the push was successful.
+		 */
+		if (fp->f_qflag & _QSINGLE_INSTANCE) {
+			queue_t *q;
+
+			claimstr(stp->sd_wrq);
+			for (q = stp->sd_wrq->q_next; q; q = q->q_next) {
+				if (q->q_flag & QREADR) {
+					q = NULL;
+					break;
+				}
+				if (strcmp(mname, Q2NAME(q)) == 0)
+					break;
+			}
+			releasestr(stp->sd_wrq);
+			if (q != NULL) {
+				fmodsw_rele(fp);
+				return (0);
+			}
+		}
+
 		if (error = strstartplumb(stp, flag, cmd)) {
 			fmodsw_rele(fp);
 			return (error);
diff --git a/usr/src/uts/common/os/strsubr.c b/usr/src/uts/common/os/strsubr.c
index 87e0c5511d..8cc27df4eb 100644
--- a/usr/src/uts/common/os/strsubr.c
+++ b/usr/src/uts/common/os/strsubr.c
@@ -26,6 +26,7 @@
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  * Copyright (c) 2016 by Delphix. All rights reserved.
+ * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #include <sys/types.h>
@@ -950,7 +951,7 @@ str_sendsig(vnode_t *vp, int event, uchar_t band, int error)
  */
 static void
 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info,
-	uchar_t band, int error)
+    uchar_t band, int error)
 {
 	ASSERT(MUTEX_HELD(&proc->p_lock));
 
@@ -2350,7 +2351,7 @@ mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss)
  */
 int
 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp,
-	uint32_t *sqtypep)
+    uint32_t *sqtypep)
 {
 	uint32_t qflag = 0;
 	uint32_t sqtype = 0;
@@ -2464,6 +2465,17 @@ devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp,
 		qflag |= _QDIRECT;
 	}
 
+	/*
+	 * Private flag used to indicate that a streams module should only
+	 * be pushed once. The TTY streams modules have this flag since if
+	 * libc believes itself to be an xpg4 process then it will
+	 * automatically and unconditionally push them when a PTS device is
+	 * opened. If an application is not aware of this then without this
+	 * flag we would end up with duplicate modules.
+	 */
+	if (devflag & _D_SINGLE_INSTANCE)
+		qflag |= _QSINGLE_INSTANCE;
+
 	*qflagp = qflag;
 	*sqtypep = sqtype;
 	return (0);
@@ -8087,7 +8099,7 @@ strflushrq(vnode_t *vp, int flag)
 
 void
 strsetrputhooks(vnode_t *vp, uint_t flags,
-		msgfunc_t protofunc, msgfunc_t miscfunc)
+    msgfunc_t protofunc, msgfunc_t miscfunc)
 {
 	struct stdata *stp = vp->v_stream;
 
diff --git a/usr/src/uts/common/sys/conf.h b/usr/src/uts/common/sys/conf.h
index 7447db408f..4bf3d5c7e3 100644
--- a/usr/src/uts/common/sys/conf.h
+++ b/usr/src/uts/common/sys/conf.h
@@ -24,6 +24,7 @@
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #ifndef _SYS_CONF_H
@@ -221,6 +222,8 @@ extern int cdev_prop_op(dev_t, dev_info_t *, ddi_prop_op_t,
 
 #define	D_OPEN_RETURNS_EINTR	0x100000 /* EINTR expected from open(9E) */
 
+#define	_D_SINGLE_INSTANCE	0x200000 /* Module may only be pushed once */
+
 #endif /* !defined(_XPG4_2) || defined(__EXTENSIONS__) */
 
 #ifdef	__cplusplus
diff --git a/usr/src/uts/common/sys/stream.h b/usr/src/uts/common/sys/stream.h
index 4114e5b13e..7a2f3c9305 100644
--- a/usr/src/uts/common/sys/stream.h
+++ b/usr/src/uts/common/sys/stream.h
@@ -21,6 +21,7 @@
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
@@ -189,6 +190,8 @@ typedef struct queue {
 #define	_QASSOCIATED	0x10000000	/* queue is associated with a device */
 #define	_QDIRECT	0x20000000	/* Private; transport module uses */
 					/* direct interface to/from sockfs */
+#define	_QSINGLE_INSTANCE	0x40000000	/* Private; module may only */
+						/* be pushed once */
 
 /* queue sqflags (protected by SQLOCK). */
 #define	Q_SQQUEUED	0x01		/* Queue is in the syncq list */