6338698 Application deadlocks on an lwpchan lock when a watchpoint is enabled on a thread stack

10 files changed, 95 insertions, 46 deletions

diff --git a/usr/src/uts/common/sys/prsystm.h b/usr/src/uts/common/sys/prsystm.h
index 38ee6566c7..c6f474743a 100644
--- a/usr/src/uts/common/sys/prsystm.h
+++ b/usr/src/uts/common/sys/prsystm.h
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -31,8 +31,6 @@
 #ifndef _SYS_PRSYSTM_H
 #define	_SYS_PRSYSTM_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/isa_defs.h>
 #include <sys/zone.h>
 
@@ -93,6 +91,7 @@ extern void prexecend(void);
 extern void prrelvm(void);
 extern void prbarrier(proc_t *);
 extern void prstop(int, int);
+extern void prunstop(void);
 extern void prnotify(struct vnode *);
 extern void prstep(klwp_t *, int);
 extern void prnostep(klwp_t *);
diff --git a/usr/src/uts/common/syscall/lwp_sobj.c b/usr/src/uts/common/syscall/lwp_sobj.c
index 459b757aef..552f95eeeb 100644
--- a/usr/src/uts/common/syscall/lwp_sobj.c
+++ b/usr/src/uts/common/syscall/lwp_sobj.c
@@ -790,11 +790,6 @@ retry:
 	}
 	/*
 	 * Block for the lock.
-	 * Put the lwp in an orderly state for debugging.
-	 * Calling prstop() has to be done here, and not in
-	 * turnstile_block(), since the preceding call to
-	 * turnstile_lookup() raises the PIL to a level
-	 * at which calls to prstop() should not be made.
 	 */
 	if ((error = lwptp->lwpt_time_error) != 0) {
 		/*
@@ -805,7 +800,6 @@ retry:
 		mutex_exit(&upibp->upib_lock);
 		goto out;
 	}
-	prstop(PR_REQUESTED, 0);
 	if (lwptp->lwpt_tsp != NULL) {
 		/*
 		 * Unlike the protocol for other lwp timedwait operations,
@@ -1147,6 +1141,12 @@ lwp_mutex_timedlock(lwp_mutex_t *lp, timespec_t *tsp)
 	if ((caddr_t)lp >= p->p_as->a_userlimit)
 		return (set_errno(EFAULT));
 
+	/*
+	 * Put the lwp in an orderly state for debugging,
+	 * in case we are stopped while sleeping, below.
+	 */
+	prstop(PR_REQUESTED, 0);
+
 	timedwait = (caddr_t)tsp;
 	if ((time_error = lwp_timer_copyin(&lwpt, tsp)) == 0 &&
 	    lwpt.lwpt_imm_timeout) {
@@ -1230,10 +1230,6 @@ lwp_mutex_timedlock(lwp_mutex_t *lp, timespec_t *tsp)
 			watched = 0;
 		}
 
-		/*
-		 * Put the lwp in an orderly state for debugging.
-		 */
-		prstop(PR_REQUESTED, 0);
 		if (timedwait) {
 			/*
 			 * If we successfully queue the timeout,
@@ -1582,6 +1578,12 @@ lwp_cond_wait(lwp_cond_t *cv, lwp_mutex_t *mp, timespec_t *tsp, int check_park)
 	    (caddr_t)mp >= p->p_as->a_userlimit)
 		return (set_errno(EFAULT));
 
+	/*
+	 * Put the lwp in an orderly state for debugging,
+	 * in case we are stopped while sleeping, below.
+	 */
+	prstop(PR_REQUESTED, 0);
+
 	timedwait = (caddr_t)tsp;
 	if ((error = lwp_timer_copyin(&lwpt, tsp)) != 0)
 		return (set_errno(error));
@@ -1713,10 +1715,6 @@ lwp_cond_wait(lwp_cond_t *cv, lwp_mutex_t *mp, timespec_t *tsp, int check_park)
 		cvwatched = 0;
 	}
 
-	/*
-	 * Put the lwp in an orderly state for debugging.
-	 */
-	prstop(PR_REQUESTED, 0);
 	if (check_park && (!schedctl_is_park() || t->t_unpark)) {
 		/*
 		 * We received a signal at user-level before calling here
@@ -2028,6 +2026,12 @@ lwp_sema_timedwait(lwp_sema_t *sp, timespec_t *tsp, int check_park)
 	if ((caddr_t)sp >= p->p_as->a_userlimit)
 		return (set_errno(EFAULT));
 
+	/*
+	 * Put the lwp in an orderly state for debugging,
+	 * in case we are stopped while sleeping, below.
+	 */
+	prstop(PR_REQUESTED, 0);
+
 	timedwait = (caddr_t)tsp;
 	if ((time_error = lwp_timer_copyin(&lwpt, tsp)) == 0 &&
 	    lwpt.lwpt_imm_timeout) {
@@ -2071,10 +2075,6 @@ lwp_sema_timedwait(lwp_sema_t *sp, timespec_t *tsp, int check_park)
 		suword8_noerr(&sp->sema_waiters, 1);
 		if (watched)
 			watch_enable_addr((caddr_t)sp, sizeof (*sp), S_WRITE);
-		/*
-		 * Put the lwp in an orderly state for debugging.
-		 */
-		prstop(PR_REQUESTED, 0);
 		if (check_park && (!schedctl_is_park() || t->t_unpark)) {
 			/*
 			 * We received a signal at user-level before calling
@@ -2353,6 +2353,12 @@ lwp_rwlock_lock(lwp_rwlock_t *rw, timespec_t *tsp, int rd_wr)
 	if ((caddr_t)rw >= p->p_as->a_userlimit)
 		return (set_errno(EFAULT));
 
+	/*
+	 * Put the lwp in an orderly state for debugging,
+	 * in case we are stopped while sleeping, below.
+	 */
+	prstop(PR_REQUESTED, 0);
+
 	/* We must only report this error if we are about to sleep (later). */
 	timedwait = (caddr_t)tsp;
 	if ((time_error = lwp_timer_copyin(&lwpt, tsp)) == 0 &&
@@ -2594,10 +2600,6 @@ lwp_rwlock_lock(lwp_rwlock_t *rw, timespec_t *tsp, int rd_wr)
 		watched = 0;
 	}
 
-	/*
-	 * Put the LWP in an orderly state for debugging.
-	 */
-	prstop(PR_REQUESTED, 0);
 	if (timedwait) {
 		/*
 		 * If we successfully queue the timeout,
diff --git a/usr/src/uts/i86pc/os/trap.c b/usr/src/uts/i86pc/os/trap.c
index 39bea090b7..12520c4522 100644
--- a/usr/src/uts/i86pc/os/trap.c
+++ b/usr/src/uts/i86pc/os/trap.c
@@ -1522,6 +1522,7 @@ out:	/* We can't get here from a system trap */
 	}
 	if (CPU->cpu_runrun || curthread->t_schedflag & TS_ANYWAITQ)
 		preempt();
+	prunstop();
 	(void) new_mstate(ct, mstate);
 
 	/* Kernel probe */
diff --git a/usr/src/uts/intel/fs/proc/prmachdep.c b/usr/src/uts/intel/fs/proc/prmachdep.c
index 267f6423a9..0511a50fa7 100644
--- a/usr/src/uts/intel/fs/proc/prmachdep.c
+++ b/usr/src/uts/intel/fs/proc/prmachdep.c
@@ -18,17 +18,15 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
 /*	  All Rights Reserved  	*/
 
-
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/types.h>
 #include <sys/t_lock.h>
 #include <sys/param.h>
@@ -133,7 +131,7 @@ prgregset_32ton(klwp_t *lwp, prgregset32_t src, prgregset_t dst)
 	dst[REG_SS] = (uint16_t)src[SS];
 	dst[REG_RSP] = (uint32_t)src[UESP];
 	dst[REG_RFL] =
-		(rp->r_ps & ~PSL_USERMASK) | (src[EFL] & PSL_USERMASK);
+	    (rp->r_ps & ~PSL_USERMASK) | (src[EFL] & PSL_USERMASK);
 	dst[REG_CS] = (uint16_t)src[CS];
 	dst[REG_RIP] = (uint32_t)src[EIP];
 	dst[REG_ERR] = (uint32_t)src[ERR];
@@ -349,7 +347,7 @@ prisstep(klwp_t *lwp)
 	ASSERT(MUTEX_NOT_HELD(&lwptoproc(lwp)->p_lock));
 
 	return ((lwp->lwp_pcb.pcb_flags &
-		(NORMAL_STEP|WATCH_STEP|DEBUG_PENDING)) != 0);
+	    (NORMAL_STEP|WATCH_STEP|DEBUG_PENDING)) != 0);
 }
 
 /*
@@ -423,7 +421,14 @@ prmapout(struct as *as, caddr_t addr, caddr_t vaddr, int writing)
 /*
  * Make sure the lwp is in an orderly state
  * for inspection by a debugger through /proc.
- * Called from stop() and from syslwp_create().
+ *
+ * This needs to be called only once while the current thread remains in the
+ * kernel and needs to be called while holding no resources (mutex locks, etc).
+ *
+ * As a hedge against these conditions, if prstop() is called repeatedly
+ * before prunstop() is called, it does nothing and just returns.
+ *
+ * prunstop() must be called before the thread returns to user level.
  */
 /* ARGSUSED */
 void
@@ -432,6 +437,9 @@ prstop(int why, int what)
 	klwp_t *lwp = ttolwp(curthread);
 	struct regs *r = lwptoregs(lwp);
 
+	if (lwp->lwp_pcb.pcb_flags & PRSTOP_CALLED)
+		return;
+
 	/*
 	 * Make sure we don't deadlock on a recursive call
 	 * to prstop().  stop() tests the lwp_nostop flag.
@@ -440,7 +448,7 @@ prstop(int why, int what)
 	lwp->lwp_nostop = 1;
 
 	if (copyin_nowatch((caddr_t)r->r_pc, &lwp->lwp_pcb.pcb_instr,
-		    sizeof (lwp->lwp_pcb.pcb_instr)) == 0)
+	    sizeof (lwp->lwp_pcb.pcb_instr)) == 0)
 		lwp->lwp_pcb.pcb_flags |= INSTR_VALID;
 	else {
 		lwp->lwp_pcb.pcb_flags &= ~INSTR_VALID;
@@ -450,6 +458,19 @@ prstop(int why, int what)
 	(void) save_syscall_args();
 	ASSERT(lwp->lwp_nostop == 1);
 	lwp->lwp_nostop = 0;
+
+	lwp->lwp_pcb.pcb_flags |= PRSTOP_CALLED;
+	aston(curthread);	/* so prunstop() will be called */
+}
+
+/*
+ * Inform prstop() that it should do its work again
+ * the next time it is called.
+ */
+void
+prunstop(void)
+{
+	ttolwp(curthread)->lwp_pcb.pcb_flags &= ~PRSTOP_CALLED;
 }
 
 /*
diff --git a/usr/src/uts/intel/ia32/os/syscall.c b/usr/src/uts/intel/ia32/os/syscall.c
index 1ec901e231..0f670853a1 100644
--- a/usr/src/uts/intel/ia32/os/syscall.c
+++ b/usr/src/uts/intel/ia32/os/syscall.c
@@ -50,6 +50,7 @@
 #include <sys/vtrace.h>
 #include <sys/sysinfo.h>
 #include <sys/procfs.h>
+#include <sys/prsystm.h>
 #include <c2/audit.h>
 #include <sys/modctl.h>
 #include <sys/aio_impl.h>
@@ -719,6 +720,7 @@ sig_check:
 	}
 	if (CPU->cpu_runrun || t->t_schedflag & TS_ANYWAITQ)
 		preempt();
+	prunstop();
 
 	lwp->lwp_errno = 0;		/* clear error for next time */
 
diff --git a/usr/src/uts/intel/sys/pcb.h b/usr/src/uts/intel/sys/pcb.h
index ebc3b4e222..ec5dea501c 100644
--- a/usr/src/uts/intel/sys/pcb.h
+++ b/usr/src/uts/intel/sys/pcb.h
@@ -18,16 +18,15 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef _SYS_PCB_H
 #define	_SYS_PCB_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/regset.h>
 #include <sys/segments.h>
 
@@ -63,6 +62,7 @@ typedef struct pcb {
 
 /* pcb_flags */
 #define	DEBUG_PENDING	0x02	/* single-step of lcall for a sys call */
+#define	PRSTOP_CALLED	0x04	/* prstop() has been called for this lwp */
 #define	INSTR_VALID	0x08	/* value in pcb_instr is valid (/proc) */
 #define	NORMAL_STEP	0x10	/* normal debugger-requested single-step */
 #define	WATCH_STEP	0x20	/* single-stepping in watchpoint emulation */
diff --git a/usr/src/uts/sparc/fs/proc/prmachdep.c b/usr/src/uts/sparc/fs/proc/prmachdep.c
index 3e059d72b7..734f36610f 100644
--- a/usr/src/uts/sparc/fs/proc/prmachdep.c
+++ b/usr/src/uts/sparc/fs/proc/prmachdep.c
@@ -861,7 +861,14 @@ prundostep(void)
 /*
  * Make sure the lwp is in an orderly state
  * for inspection by a debugger through /proc.
- * Called from stop() and from syslwp_create().
+ *
+ * This needs to be called only once while the current thread remains in the
+ * kernel and needs to be called while holding no resources (mutex locks, etc).
+ *
+ * As a hedge against these conditions, if prstop() is called repeatedly
+ * before prunstop() is called, it does nothing and just returns.
+ *
+ * prunstop() must be called before the thread returns to user level.
  */
 /* ARGSUSED */
 void
@@ -876,6 +883,9 @@ prstop(int why, int what)
 	int watched;
 	extern void fp_prsave(kfpu_t *);
 
+	if (lwp->lwp_pcb.pcb_flags & PRSTOP_CALLED)
+		return;
+
 	/*
 	 * Make sure we don't deadlock on a recursive call
 	 * to prstop().  stop() tests the lwp_nostop flag.
@@ -966,6 +976,19 @@ prstop(int why, int what)
 	(void) save_syscall_args();
 	ASSERT(lwp->lwp_nostop == 1);
 	lwp->lwp_nostop = 0;
+
+	lwp->lwp_pcb.pcb_flags |= PRSTOP_CALLED;
+	aston(curthread);	/* so prunstop() will be called */
+}
+
+/*
+ * Inform prstop() that it should do its work again
+ * the next time it is called.
+ */
+void
+prunstop(void)
+{
+	ttolwp(curthread)->lwp_pcb.pcb_flags &= ~PRSTOP_CALLED;
 }
 
 /*
diff --git a/usr/src/uts/sparc/os/syscall.c b/usr/src/uts/sparc/os/syscall.c
index ea6b4bdf38..4bb8851d9e 100644
--- a/usr/src/uts/sparc/os/syscall.c
+++ b/usr/src/uts/sparc/os/syscall.c
@@ -814,6 +814,7 @@ sig_check:
 	}
 	if (CPU->cpu_runrun || t->t_schedflag & TS_ANYWAITQ)
 		preempt();
+	prunstop();
 
 	/*
 	 * t_post_sys will be set if pcb_step is active.
diff --git a/usr/src/uts/sparc/sys/pcb.h b/usr/src/uts/sparc/sys/pcb.h
index 7ac9a68504..67653707c1 100644
--- a/usr/src/uts/sparc/sys/pcb.h
+++ b/usr/src/uts/sparc/sys/pcb.h
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,16 +18,15 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 1990-2002 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef _SYS_PCB_H
 #define	_SYS_PCB_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/regset.h>
 
 #ifdef	__cplusplus
@@ -55,6 +53,7 @@ typedef struct pcb {
 #endif /* ! _ASM */
 
 /* pcb_flags */
+#define	PRSTOP_CALLED	0x01	/* prstop() has been called for this lwp */
 #define	INSTR_VALID	0x02	/* value in pcb_instr is valid (/proc) */
 #define	NORMAL_STEP	0x04	/* normal debugger requested single-step */
 #define	WATCH_STEP	0x08	/* single-stepping in watchpoint emulation */
diff --git a/usr/src/uts/sun4/os/trap.c b/usr/src/uts/sun4/os/trap.c
index 0d80f1e8d6..d3a2df1e09 100644
--- a/usr/src/uts/sun4/os/trap.c
+++ b/usr/src/uts/sun4/os/trap.c
@@ -1566,6 +1566,7 @@ trap_rtt(void)
 	}
 	if (CPU->cpu_runrun || curthread->t_schedflag & TS_ANYWAITQ)
 		preempt();
+	prunstop();
 	if (lwp->lwp_pcb.pcb_step != STEP_NONE)
 		prdostep();