4852369 x86 panic due to redzone violation

4 files changed, 54 insertions, 9 deletions

diff --git a/usr/src/uts/i86pc/os/trap.c b/usr/src/uts/i86pc/os/trap.c
index 37a38b9a80..74aab9de45 100644
--- a/usr/src/uts/i86pc/os/trap.c
+++ b/usr/src/uts/i86pc/os/trap.c
@@ -1316,6 +1316,22 @@ trap(struct regs *rp, caddr_t addr, processorid_t cpuid)
 			realsigprof(0, 0);
 			cur_thread->t_sig_check = 1;
 		}
+
+		/*
+		 * /proc can't enable/disable the trace bit itself
+		 * because that could race with the call gate used by
+		 * system calls via "lcall". If that happened, an
+		 * invalid EFLAGS would result. prstep()/prnostep()
+		 * therefore schedule an AST for the purpose.
+		 */
+		if (lwp->lwp_pcb.pcb_flags & REQUEST_STEP) {
+			lwp->lwp_pcb.pcb_flags &= ~REQUEST_STEP;
+			rp->r_ps |= PS_T;
+		}
+		if (lwp->lwp_pcb.pcb_flags & REQUEST_NOSTEP) {
+			lwp->lwp_pcb.pcb_flags &= ~REQUEST_NOSTEP;
+			rp->r_ps &= ~PS_T;
+		}
 	}
 
 out:	/* We can't get here from a system trap */
diff --git a/usr/src/uts/intel/fs/proc/prmachdep.c b/usr/src/uts/intel/fs/proc/prmachdep.c
index e0788476c4..12f0a52da4 100644
--- a/usr/src/uts/intel/fs/proc/prmachdep.c
+++ b/usr/src/uts/intel/fs/proc/prmachdep.c
@@ -20,7 +20,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -305,16 +305,21 @@ prgetpsaddr(proc_t *p)
 void
 prstep(klwp_t *lwp, int watchstep)
 {
-	struct regs *r = lwptoregs(lwp);
-
 	ASSERT(MUTEX_NOT_HELD(&lwptoproc(lwp)->p_lock));
 
+	/*
+	 * flag LWP so that its r_efl trace bit (PS_T) will be set on
+	 * next return to usermode.
+	 */
+	lwp->lwp_pcb.pcb_flags |= REQUEST_STEP;
+	lwp->lwp_pcb.pcb_flags &= ~REQUEST_NOSTEP;
+
 	if (watchstep)
 		lwp->lwp_pcb.pcb_flags |= WATCH_STEP;
 	else
 		lwp->lwp_pcb.pcb_flags |= NORMAL_STEP;
 
-	r->r_ps |= PS_T;	/* set the trace flag in PSW */
+	aston(lwptot(lwp));	/* let trap() set PS_T in rp->r_efl */
 }
 
 /*
@@ -323,13 +328,19 @@ prstep(klwp_t *lwp, int watchstep)
 void
 prnostep(klwp_t *lwp)
 {
-	struct regs *r = lwptoregs(lwp);
-
 	ASSERT(ttolwp(curthread) == lwp ||
 	    MUTEX_NOT_HELD(&lwptoproc(lwp)->p_lock));
 
-	r->r_ps &= ~PS_T;	/* turn off trace flag in PSW */
-	lwp->lwp_pcb.pcb_flags &= ~(NORMAL_STEP|WATCH_STEP|DEBUG_PENDING);
+	/*
+	 * flag LWP so that its r_efl trace bit (PS_T) will be cleared on
+	 * next return to usermode.
+	 */
+	lwp->lwp_pcb.pcb_flags |= REQUEST_NOSTEP;
+
+	lwp->lwp_pcb.pcb_flags &=
+	    ~(REQUEST_STEP|NORMAL_STEP|WATCH_STEP|DEBUG_PENDING);
+
+	aston(lwptot(lwp));	/* let trap() clear PS_T in rp->r_efl */
 }
 
 /*
diff --git a/usr/src/uts/intel/ia32/os/syscall.c b/usr/src/uts/intel/ia32/os/syscall.c
index 8176ddb908..cac6cff7ee 100644
--- a/usr/src/uts/intel/ia32/os/syscall.c
+++ b/usr/src/uts/intel/ia32/os/syscall.c
@@ -659,6 +659,22 @@ sig_check:
 		 */
 		if (lwp->lwp_pcb.pcb_flags & CPC_OVERFLOW)
 			aston(t);
+
+		/*
+		 * /proc can't enable/disable the trace bit itself
+		 * because that could race with the call gate used by
+		 * system calls via "lcall". If that happened, an
+		 * invalid EFLAGS would result. prstep()/prnostep()
+		 * therefore schedule an AST for the purpose.
+		 */
+		if (lwp->lwp_pcb.pcb_flags & REQUEST_STEP) {
+			lwp->lwp_pcb.pcb_flags &= ~REQUEST_STEP;
+			rp->r_ps |= PS_T;
+		}
+		if (lwp->lwp_pcb.pcb_flags & REQUEST_NOSTEP) {
+			lwp->lwp_pcb.pcb_flags &= ~REQUEST_NOSTEP;
+			rp->r_ps &= ~PS_T;
+		}
 	}
 
 	lwp->lwp_errno = 0;		/* clear error for next time */
diff --git a/usr/src/uts/intel/sys/pcb.h b/usr/src/uts/intel/sys/pcb.h
index f9f1f782a3..54ac91a787 100644
--- a/usr/src/uts/intel/sys/pcb.h
+++ b/usr/src/uts/intel/sys/pcb.h
@@ -20,7 +20,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -68,6 +68,8 @@ typedef struct pcb {
 #define	WATCH_STEP	0x20	/* single-stepping in watchpoint emulation */
 #define	CPC_OVERFLOW	0x40	/* performance counters overflowed */
 #define	RUPDATE_PENDING	0x80	/* new register values in the pcb -> regs */
+#define	REQUEST_STEP	0x100	/* request pending to single-step this lwp */
+#define	REQUEST_NOSTEP	0x200	/* request pending to disable single-step */
 
 /* fpu_flags */
 #define	FPU_EN		0x1	/* flag signifying fpu in use */