3 files changed, 109 insertions, 90 deletions

diff --git a/usr/src/uts/common/os/driver_lyr.c b/usr/src/uts/common/os/driver_lyr.c
index 9e5eb33dd6..9456507440 100644
--- a/usr/src/uts/common/os/driver_lyr.c
+++ b/usr/src/uts/common/os/driver_lyr.c
@@ -1258,7 +1258,7 @@ ldi_mlink_lh(vnode_t *vp, int cmd, intptr_t arg, cred_t *crp, int *rvalp)
  * in its internal state so that the devinfo snapshot code has some
  * observability into streams device linkage information.
  */
-void
+int
 ldi_mlink_fp(struct stdata *stp, file_t *fpdown, int lhlink, int type)
 {
 	vnode_t			*vp = fpdown->f_vnode;
@@ -1269,7 +1269,7 @@ ldi_mlink_fp(struct stdata *stp, file_t *fpdown, int lhlink, int type)
 
 	/* if the lower stream is not a device then return */
 	if (!vn_matchops(vp, spec_getvnodeops()))
-		return;
+		return (EINVAL);
 
 	ASSERT(!servicing_interrupt());
 
@@ -1280,6 +1280,41 @@ ldi_mlink_fp(struct stdata *stp, file_t *fpdown, int lhlink, int type)
 	sp = VTOS(vp);
 	csp = VTOS(sp->s_commonvp);
 
+	/* get a layered ident for the upper stream */
+	if (type == LINKNORMAL) {
+		/*
+		 * if the link is not persistant then we can associate
+		 * the upper stream with a dev_t.  this is because the
+		 * upper stream is associated with a vnode, which is
+		 * associated with a dev_t and this binding can't change
+		 * during the life of the stream.  since the link isn't
+		 * persistant once the stream is destroyed the link is
+		 * destroyed.  so the dev_t will be valid for the life
+		 * of the link.
+		 */
+		ret = ldi_ident_from_stream(getendq(stp->sd_wrq), &li);
+	} else {
+		/*
+		 * if the link is persistant we can only associate the
+		 * link with a driver (and not a dev_t.)  this is
+		 * because subsequent opens of the upper device may result
+		 * in a different stream (and dev_t) having access to
+		 * the lower stream.
+		 *
+		 * for example, if the upper stream is closed after the
+		 * persistant link operation is completed, a subsequent
+		 * open of the upper device will create a new stream which
+		 * may have a different dev_t and an unlink operation
+		 * can be performed using this new upper stream.
+		 */
+		VERIFY3S(type, ==, LINKPERSIST);
+		major = getmajor(stp->sd_vnode->v_rdev);
+		ret = ldi_ident_from_major(major, &li);
+	}
+
+	if (ret != 0)
+		return (ret);
+
 	/* check if this was a plink via a layered handle */
 	if (lhlink) {
 		/*
@@ -1303,8 +1338,10 @@ ldi_mlink_fp(struct stdata *stp, file_t *fpdown, int lhlink, int type)
 		 * while there may still be valid layered handles
 		 * pointing to it.
 		 */
+		VERIFY3S(type, ==, LINKPERSIST);
+
 		mutex_enter(&csp->s_lock);
-		ASSERT(csp->s_count >= 1);
+		VERIFY(csp->s_count >= 1);
 		csp->s_count++;
 		mutex_exit(&csp->s_lock);
 
@@ -1330,48 +1367,17 @@ ldi_mlink_fp(struct stdata *stp, file_t *fpdown, int lhlink, int type)
 	 * mark the snode/stream as multiplexed
 	 */
 	mutex_enter(&sp->s_lock);
-	ASSERT(!(sp->s_flag & SMUXED));
+	VERIFY(!(sp->s_flag & SMUXED));
 	sp->s_flag |= SMUXED;
 	mutex_exit(&sp->s_lock);
 
-	/* get a layered ident for the upper stream */
-	if (type == LINKNORMAL) {
-		/*
-		 * if the link is not persistant then we can associate
-		 * the upper stream with a dev_t.  this is because the
-		 * upper stream is associated with a vnode, which is
-		 * associated with a dev_t and this binding can't change
-		 * during the life of the stream.  since the link isn't
-		 * persistant once the stream is destroyed the link is
-		 * destroyed.  so the dev_t will be valid for the life
-		 * of the link.
-		 */
-		ret = ldi_ident_from_stream(getendq(stp->sd_wrq), &li);
-	} else {
-		/*
-		 * if the link is persistant we can only associate the
-		 * link with a driver (and not a dev_t.)  this is
-		 * because subsequent opens of the upper device may result
-		 * in a different stream (and dev_t) having access to
-		 * the lower stream.
-		 *
-		 * for example, if the upper stream is closed after the
-		 * persistant link operation is compleated, a subsequent
-		 * open of the upper device will create a new stream which
-		 * may have a different dev_t and an unlink operation
-		 * can be performed using this new upper stream.
-		 */
-		ASSERT(type == LINKPERSIST);
-		major = getmajor(stp->sd_vnode->v_rdev);
-		ret = ldi_ident_from_major(major, &li);
-	}
-
-	ASSERT(ret == 0);
 	(void) handle_alloc(vp, (struct ldi_ident *)li);
 	ldi_ident_release(li);
+
+	return (0);
 }
 
-void
+int
 ldi_munlink_fp(struct stdata *stp, file_t *fpdown, int type)
 {
 	struct ldi_handle	*lhp;
@@ -1383,29 +1389,15 @@ ldi_munlink_fp(struct stdata *stp, file_t *fpdown, int type)
 
 	/* if the lower stream is not a device then return */
 	if (!vn_matchops(vp, spec_getvnodeops()))
-		return;
+		return (EINVAL);
 
 	ASSERT(!servicing_interrupt());
-	ASSERT((type == LINKNORMAL) || (type == LINKPERSIST));
 
 	LDI_STREAMS_LNK((CE_NOTE, "%s: unlinking streams "
 	    "stp=0x%p, fpdown=0x%p", "ldi_munlink_fp",
 	    (void *)stp, (void *)fpdown));
 
 	/*
-	 * NOTE: here we rely on the streams subsystem not allowing
-	 * a stream to be multiplexed more than once.  if this
-	 * changes, we break.
-	 *
-	 * mark the snode/stream as not multiplexed
-	 */
-	sp = VTOS(vp);
-	mutex_enter(&sp->s_lock);
-	ASSERT(sp->s_flag & SMUXED);
-	sp->s_flag &= ~SMUXED;
-	mutex_exit(&sp->s_lock);
-
-	/*
 	 * clear the owner for this snode
 	 * see the comment in ldi_mlink_fp() for information about how
 	 * the ident is allocated
@@ -1413,15 +1405,32 @@ ldi_munlink_fp(struct stdata *stp, file_t *fpdown, int type)
 	if (type == LINKNORMAL) {
 		ret = ldi_ident_from_stream(getendq(stp->sd_wrq), &li);
 	} else {
-		ASSERT(type == LINKPERSIST);
+		VERIFY3S(type, ==, LINKPERSIST);
 		major = getmajor(stp->sd_vnode->v_rdev);
 		ret = ldi_ident_from_major(major, &li);
 	}
 
-	ASSERT(ret == 0);
+	if (ret != 0)
+		return (ret);
+
+	/*
+	 * NOTE: here we rely on the streams subsystem not allowing
+	 * a stream to be multiplexed more than once.  if this
+	 * changes, we break.
+	 *
+	 * mark the snode/stream as not multiplexed
+	 */
+	sp = VTOS(vp);
+	mutex_enter(&sp->s_lock);
+	VERIFY(sp->s_flag & SMUXED);
+	sp->s_flag &= ~SMUXED;
+	mutex_exit(&sp->s_lock);
+
 	lhp = handle_find(vp, (struct ldi_ident *)li);
 	handle_release(lhp);
 	ldi_ident_release(li);
+
+	return (0);
 }
 
 /*
diff --git a/usr/src/uts/common/os/strsubr.c b/usr/src/uts/common/os/strsubr.c
index 9357e6a234..796f89dca2 100644
--- a/usr/src/uts/common/os/strsubr.c
+++ b/usr/src/uts/common/os/strsubr.c
@@ -1902,36 +1902,9 @@ mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp,
 	 */
 	error = strdoioctl(stp, &strioc, FNATIVE,
 	    K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
-	if (error != 0) {
-		lbfree(linkp);
-
-		if (!(passyncq->sq_flags & SQ_BLOCKED))
-			blocksq(passyncq, SQ_BLOCKED, 0);
-		/*
-		 * Restore the stream head queue and then remove
-		 * the passq. Turn off STPLEX before we turn on
-		 * the stream by removing the passq.
-		 */
-		rq->q_ptr = _WR(rq)->q_ptr = stpdown;
-		setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO,
-		    B_TRUE);
-
-		mutex_enter(&stpdown->sd_lock);
-		stpdown->sd_flag &= ~STPLEX;
-		mutex_exit(&stpdown->sd_lock);
-
-		link_rempassthru(passq);
-
-		mutex_enter(&stpdown->sd_lock);
-		stpdown->sd_flag &= ~STRPLUMB;
-		/* Wakeup anyone waiting for STRPLUMB to clear. */
-		cv_broadcast(&stpdown->sd_monitor);
-		mutex_exit(&stpdown->sd_lock);
+	if (error != 0)
+		goto cleanup;
 
-		mutex_exit(&muxifier);
-		netstack_rele(ss->ss_netstack);
-		return (error);
-	}
 	mutex_enter(&fpdown->f_tlock);
 	fpdown->f_count++;
 	mutex_exit(&fpdown->f_tlock);
@@ -1943,9 +1916,16 @@ mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp,
 
 	ASSERT((cmd == I_LINK) || (cmd == I_PLINK));
 	if (cmd == I_LINK) {
-		ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL);
+		error = ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL);
 	} else {
-		ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST);
+		error = ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST);
+	}
+
+	if (error != 0) {
+		mutex_enter(&fpdown->f_tlock);
+		fpdown->f_count--;
+		mutex_exit(&fpdown->f_tlock);
+		goto cleanup;
 	}
 
 	link_rempassthru(passq);
@@ -1977,6 +1957,36 @@ mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp,
 	*rvalp = linkp->li_lblk.l_index;
 	netstack_rele(ss->ss_netstack);
 	return (0);
+
+cleanup:
+	lbfree(linkp);
+
+	if (!(passyncq->sq_flags & SQ_BLOCKED))
+		blocksq(passyncq, SQ_BLOCKED, 0);
+	/*
+	 * Restore the stream head queue and then remove
+	 * the passq. Turn off STPLEX before we turn on
+	 * the stream by removing the passq.
+	 */
+	rq->q_ptr = _WR(rq)->q_ptr = stpdown;
+	setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO,
+	    B_TRUE);
+
+	mutex_enter(&stpdown->sd_lock);
+	stpdown->sd_flag &= ~STPLEX;
+	mutex_exit(&stpdown->sd_lock);
+
+	link_rempassthru(passq);
+
+	mutex_enter(&stpdown->sd_lock);
+	stpdown->sd_flag &= ~STRPLUMB;
+	/* Wakeup anyone waiting for STRPLUMB to clear. */
+	cv_broadcast(&stpdown->sd_monitor);
+	mutex_exit(&stpdown->sd_lock);
+
+	mutex_exit(&muxifier);
+	netstack_rele(ss->ss_netstack);
+	return (error);
 }
 
 int
@@ -2233,9 +2243,9 @@ munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp,
 
 	/* clean up the layered driver linkages */
 	if ((flag & LINKTYPEMASK) == LINKNORMAL) {
-		ldi_munlink_fp(stp, fpdown, LINKNORMAL);
+		VERIFY0(ldi_munlink_fp(stp, fpdown, LINKNORMAL));
 	} else {
-		ldi_munlink_fp(stp, fpdown, LINKPERSIST);
+		VERIFY0(ldi_munlink_fp(stp, fpdown, LINKPERSIST));
 	}
 
 	link_rempassthru(passq);
diff --git a/usr/src/uts/common/sys/sunldi_impl.h b/usr/src/uts/common/sys/sunldi_impl.h
index e9fdb431bf..3d04078a53 100644
--- a/usr/src/uts/common/sys/sunldi_impl.h
+++ b/usr/src/uts/common/sys/sunldi_impl.h
@@ -72,8 +72,8 @@ void ldi_init(void);
  * LDI streams linking interfaces
  */
 extern int ldi_mlink_lh(vnode_t *, int, intptr_t, cred_t *, int *);
-extern void ldi_mlink_fp(struct stdata *, struct file *, int, int);
-extern void ldi_munlink_fp(struct stdata *, struct file *, int);
+extern int ldi_mlink_fp(struct stdata *, struct file *, int, int);
+extern int ldi_munlink_fp(struct stdata *, struct file *, int);
 
 /*
  * LDI module identifier