6368753 Need a patch solution to 4522909 automountd hangs with executeable maps

6509943 Apparent deadlock between fork()'s use of lwp_suspend and NFS locking (Backout 4522909)

6 files changed, 43 insertions, 89 deletions

diff --git a/usr/src/uts/common/os/condvar.c b/usr/src/uts/common/os/condvar.c
index 71f20c6b67..4ac03cbddb 100644
--- a/usr/src/uts/common/os/condvar.c
+++ b/usr/src/uts/common/os/condvar.c
@@ -555,49 +555,45 @@ cv_wait_stop(kcondvar_t *cvp, kmutex_t *mp, int wakeup_time)
 	(void) untimeout(id);
 
 	/*
-	 * Check for reasons to stop, and stop if lwp_nostop is zero.
+	 * Check for reasons to stop, if lwp_nostop is not true.
 	 * See issig_forreal() for explanations of the various stops.
-	 * Like issig_forreal(), we allow a PR_SUSPENDED/SUSPEND_NORMAL
-	 * to occur even if lwp_nostop is set.
 	 */
 	mutex_enter(&p->p_lock);
-	while (!(p->p_flag & SEXITLWPS)) {
+	while (lwp->lwp_nostop == 0 && !(p->p_flag & SEXITLWPS)) {
 		/*
 		 * Hold the lwp here for watchpoint manipulation.
 		 */
-		if ((t->t_proc_flag & TP_PAUSE) && !lwp->lwp_nostop) {
+		if (t->t_proc_flag & TP_PAUSE) {
 			stop(PR_SUSPENDED, SUSPEND_PAUSE);
 			continue;
 		}
 		/*
 		 * System checkpoint.
 		 */
-		if ((t->t_proc_flag & TP_CHKPT) && !lwp->lwp_nostop) {
+		if (t->t_proc_flag & TP_CHKPT) {
 			stop(PR_CHECKPOINT, 0);
 			continue;
 		}
 		/*
 		 * Honor fork1(), watchpoint activity (remapping a page),
-		 * and lwp_suspend() regardless of whether lwp_nostop is
-		 * set but not if lwp_nostop_r is set (to avoid a recursive
-		 * call to prstop()).
+		 * and lwp_suspend() requests.
 		 */
-		if (((p->p_flag & (SHOLDFORK1|SHOLDWATCH)) ||
-		    (t->t_proc_flag & TP_HOLDLWP)) && !lwp->lwp_nostop_r) {
+		if ((p->p_flag & (SHOLDFORK1|SHOLDWATCH)) ||
+		    (t->t_proc_flag & TP_HOLDLWP)) {
 			stop(PR_SUSPENDED, SUSPEND_NORMAL);
 			continue;
 		}
 		/*
 		 * Honor /proc requested stop.
 		 */
-		if ((t->t_proc_flag & TP_PRSTOP) && !lwp->lwp_nostop) {
+		if (t->t_proc_flag & TP_PRSTOP) {
 			stop(PR_REQUESTED, 0);
 		}
 		/*
 		 * If some lwp in the process has already stopped
 		 * showing PR_JOBCONTROL, stop in sympathy with it.
 		 */
-		if (p->p_stopsig && !lwp->lwp_nostop && (t != p->p_agenttp)) {
+		if (p->p_stopsig && t != p->p_agenttp) {
 			stop(PR_JOBCONTROL, p->p_stopsig);
 			continue;
 		}
diff --git a/usr/src/uts/common/os/sig.c b/usr/src/uts/common/os/sig.c
index 5c72fb749b..69c1074816 100644
--- a/usr/src/uts/common/os/sig.c
+++ b/usr/src/uts/common/os/sig.c
@@ -447,10 +447,10 @@ issig_justlooking(void)
 
 	if ((lwp->lwp_asleep && MUSTRETURN(p, t)) ||
 	    (p->p_flag & (SEXITLWPS|SKILLED)) ||
-	    (!lwp->lwp_nostop_r && ((p->p_flag & (SHOLDFORK1|SHOLDWATCH)) |
-		(t->t_proc_flag & TP_HOLDLWP))) ||
-	    (!lwp->lwp_nostop && (p->p_stopsig | (t->t_proc_flag &
-		(TP_PRSTOP|TP_CHKPT|TP_PAUSE)))) ||
+	    (lwp->lwp_nostop == 0 &&
+	    (p->p_stopsig | (p->p_flag & (SHOLDFORK1|SHOLDWATCH)) |
+	    (t->t_proc_flag &
+	    (TP_PRSTOP|TP_HOLDLWP|TP_CHKPT|TP_PAUSE)))) ||
 	    lwp->lwp_cursig)
 		return (1);
 
@@ -588,15 +588,9 @@ issig_forreal(void)
 		 * or is executing lwp_suspend() on this lwp.
 		 * Again, go back to top of loop to check if an exit
 		 * or hold event has occurred while stopped.
-		 * We explicitly allow this form of stopping of one
-		 * lwp in a process by another lwp in the same process,
-		 * even if lwp->lwp_nostop is set, because otherwise a
-		 * process can become deadlocked on a fork1().
-		 * Allow this only if lwp_nostop_r is not set,
-		 * to avoid a recursive call to prstop().
 		 */
 		if (((p->p_flag & (SHOLDFORK1|SHOLDWATCH)) ||
-		    (t->t_proc_flag & TP_HOLDLWP)) && !lwp->lwp_nostop_r) {
+		    (t->t_proc_flag & TP_HOLDLWP)) && !lwp->lwp_nostop) {
 			stop(PR_SUSPENDED, SUSPEND_NORMAL);
 			continue;
 		}
@@ -863,11 +857,9 @@ stop(int why, int what)
 
 	/*
 	 * Make sure we don't deadlock on a recursive call to prstop().
-	 * prstop() sets the lwp_nostop_r flag and increments lwp_nostop.
+	 * prstop() sets the lwp_nostop flag.
 	 */
-	if (lwp->lwp_nostop_r ||
-	    (lwp->lwp_nostop &&
-	    (why != PR_SUSPENDED || what != SUSPEND_NORMAL)))
+	if (lwp->lwp_nostop)
 		return;
 
 	/*
diff --git a/usr/src/uts/common/sys/klwp.h b/usr/src/uts/common/sys/klwp.h
index 7dea5b4941..93d1e9816f 100644
--- a/usr/src/uts/common/sys/klwp.h
+++ b/usr/src/uts/common/sys/klwp.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -160,34 +160,12 @@ typedef struct _klwp {
 	struct itimerval lwp_timer[3];
 
 	/*
-	 * There are a number of places where you do not wish an lwp to
-	 * be stopped due to some interaction with other lwps in the process.
-	 * In these cases the lwp_nostop value is incremented. At places where
-	 * the lwp would normally be stopped the stop is allowed if lwp_nostop
-	 * is zero. There are a very few cases where even if lwp_nostop is set
-	 * we need to allow the lwp to stop. In those cases the lwp is
-	 * stopped if lwp_nostop_r is not set regardless of the state of
-	 * lwp_nostop. These conditions are:
-	 *
-	 * 1. In issig_forreal() when another lwp is undergoing fork1()
-	 * or watchpoint activity (p_flag contains either SHOLDFORK1 or
-	 * SHOLDWATCH or t_proc_flag contains TP_HOLDLWP)
-	 *
-	 * 2. In stop() when the why argument is not PR_SUSPENDED or the what
-	 * argument is not SUSPEND_NORMAL.
-	 *
-	 * 3. In cv_wait_stop() when another lwp is undergoing fork1() or
-	 * watchpoint activity (p_flag contains either SHOLDFORK1 or
-	 * SHOLDWATCH or t_proc_flag contains TP_HOLDLWP)
-	 *
-	 * lwp_nostop_r is set in prstop(). ie we honour the presence of
-	 * SHOLDFORK1 or SHOLDWATCH or TP_HOLDLWP in the case of
-	 * stop(PR_SUSPENDED, SUSPEND_NORMAL)
+	 * used to stop/alert lwps
 	 */
 	char	lwp_unused;
 	char	lwp_state;	/* Running in User/Kernel mode (no lock req) */
-	ushort_t lwp_nostop;	/* Don't stop this lwp except SUSPEND_NORMAL */
-	ushort_t lwp_nostop_r;	/* Don't stop this lwp (avoid recursion) */
+	ushort_t lwp_nostop;	/* Don't stop this lwp (no lock required) */
+	ushort_t lwp_pad;	/* Reserved for future use */
 
 	/*
 	 * Last failed privilege.
diff --git a/usr/src/uts/common/vm/vm_as.c b/usr/src/uts/common/vm/vm_as.c
index 801e55659c..42df064cc2 100644
--- a/usr/src/uts/common/vm/vm_as.c
+++ b/usr/src/uts/common/vm/vm_as.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -878,10 +878,8 @@ retry:
 		 * for a pagefault.  This is to avoid deadlock while debugging
 		 * a process via /proc over NFS (in particular).
 		 */
-		if (lwp != NULL) {
+		if (lwp != NULL)
 			lwp->lwp_nostop++;
-			lwp->lwp_nostop_r++;
-		}
 
 		/*
 		 * same length must be used when we softlock and softunlock.
@@ -959,10 +957,8 @@ retry:
 		seg = as_segat(as, raddr);
 		if (seg == NULL) {
 			AS_LOCK_EXIT(as, &as->a_lock);
-			if ((lwp != NULL) && (!is_xhat)) {
+			if ((lwp != NULL) && (!is_xhat))
 				lwp->lwp_nostop--;
-				lwp->lwp_nostop_r--;
-			}
 			return (FC_NOMAP);
 		}
 
@@ -1042,10 +1038,9 @@ retry:
 	}
 	if (as_lock_held)
 		AS_LOCK_EXIT(as, &as->a_lock);
-	if ((lwp != NULL) && (!is_xhat)) {
+	if ((lwp != NULL) && (!is_xhat))
 		lwp->lwp_nostop--;
-		lwp->lwp_nostop_r--;
-	}
+
 	/*
 	 * If the lower levels returned EDEADLK for a fault,
 	 * It means that we should retry the fault.  Let's wait
@@ -1083,10 +1078,8 @@ retry:
 	 * for a pagefault.  This is to avoid deadlock while debugging
 	 * a process via /proc over NFS (in particular).
 	 */
-	if (lwp != NULL) {
+	if (lwp != NULL)
 		lwp->lwp_nostop++;
-		lwp->lwp_nostop_r++;
-	}
 
 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
@@ -1096,10 +1089,8 @@ retry:
 	seg = as_segat(as, raddr);
 	if (seg == NULL) {
 		AS_LOCK_EXIT(as, &as->a_lock);
-		if (lwp != NULL) {
+		if (lwp != NULL)
 			lwp->lwp_nostop--;
-			lwp->lwp_nostop_r--;
-		}
 		return (FC_NOMAP);
 	}
 
@@ -1116,10 +1107,8 @@ retry:
 			break;
 	}
 	AS_LOCK_EXIT(as, &as->a_lock);
-	if (lwp != NULL) {
+	if (lwp != NULL)
 		lwp->lwp_nostop--;
-		lwp->lwp_nostop_r--;
-	}
 	/*
 	 * If the lower levels returned EDEADLK for a fault,
 	 * It means that we should retry the fault.  Let's wait
diff --git a/usr/src/uts/intel/fs/proc/prmachdep.c b/usr/src/uts/intel/fs/proc/prmachdep.c
index f9a9fc3247..caebfab916 100644
--- a/usr/src/uts/intel/fs/proc/prmachdep.c
+++ b/usr/src/uts/intel/fs/proc/prmachdep.c
@@ -433,11 +433,11 @@ prstop(int why, int what)
 	struct regs *r = lwptoregs(lwp);
 
 	/*
-	 * Make sure we don't deadlock on a recursive call to prstop().
-	 * stop() tests the lwp_nostop_r and lwp_nostop flags.
+	 * Make sure we don't deadlock on a recursive call
+	 * to prstop().  stop() tests the lwp_nostop flag.
 	 */
-	lwp->lwp_nostop_r++;
-	lwp->lwp_nostop++;
+	ASSERT(lwp->lwp_nostop == 0);
+	lwp->lwp_nostop = 1;
 
 	if (copyin_nowatch((caddr_t)r->r_pc, &lwp->lwp_pcb.pcb_instr,
 		    sizeof (lwp->lwp_pcb.pcb_instr)) == 0)
@@ -448,8 +448,8 @@ prstop(int why, int what)
 	}
 
 	(void) save_syscall_args();
-	lwp->lwp_nostop--;
-	lwp->lwp_nostop_r--;
+	ASSERT(lwp->lwp_nostop == 1);
+	lwp->lwp_nostop = 0;
 }
 
 /*
diff --git a/usr/src/uts/sparc/fs/proc/prmachdep.c b/usr/src/uts/sparc/fs/proc/prmachdep.c
index 3d27ecff08..bca9d0bb2d 100644
--- a/usr/src/uts/sparc/fs/proc/prmachdep.c
+++ b/usr/src/uts/sparc/fs/proc/prmachdep.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -21,7 +20,7 @@
  */
 
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -881,11 +880,11 @@ prstop(int why, int what)
 	extern void fp_prsave(kfpu_t *);
 
 	/*
-	 * Make sure we don't deadlock on a recursive call to prstop().
-	 * stop() tests the lwp_nostop_r and lwp_nostop flags.
+	 * Make sure we don't deadlock on a recursive call
+	 * to prstop().  stop() tests the lwp_nostop flag.
 	 */
-	lwp->lwp_nostop_r++;
-	lwp->lwp_nostop++;
+	ASSERT(lwp->lwp_nostop == 0);
+	lwp->lwp_nostop = 1;
 	(void) flush_user_windows_to_stack(NULL);
 	if (lwp->lwp_pcb.pcb_step != STEP_NONE)
 		(void) prundostep();
@@ -968,8 +967,8 @@ prstop(int why, int what)
 	}
 
 	(void) save_syscall_args();
-	lwp->lwp_nostop--;
-	lwp->lwp_nostop_r--;
+	ASSERT(lwp->lwp_nostop == 1);
+	lwp->lwp_nostop = 0;
 }
 
 /*