6265036 cv_waituntil_sig() often returns early

6 files changed, 167 insertions, 95 deletions

diff --git a/usr/src/uts/common/os/aio.c b/usr/src/uts/common/os/aio.c
index 7dc72c0d2a..631fb7b0e5 100644
--- a/usr/src/uts/common/os/aio.c
+++ b/usr/src/uts/common/os/aio.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -529,7 +529,6 @@ aiowait(
 	aio_req_t	*reqp;
 	clock_t		status;
 	int		blocking;
-	int		timecheck;
 	timestruc_t	rqtime;
 	timestruc_t	*rqtp;
 
@@ -545,7 +544,6 @@ aiowait(
 		return (error);
 	if (rqtp) {
 		timestruc_t now;
-		timecheck = timechanged;
 		gethrestime(&now);
 		timespecadd(rqtp, &now);
 	}
@@ -575,7 +573,7 @@ aiowait(
 		}
 		if (blocking) {
 			status = cv_waituntil_sig(&aiop->aio_waitcv,
-			    &aiop->aio_mutex, rqtp, timecheck);
+			    &aiop->aio_mutex, rqtp);
 
 			if (status > 0)		/* check done queue again */
 				continue;
@@ -619,7 +617,6 @@ aiowaitn(void *uiocb, uint_t nent, uint_t *nwait, timespec_t *timout)
 	int		iocb_index = 0;
 	model_t		model = get_udatamodel();
 	int		blocking = 1;
-	int		timecheck;
 	timestruc_t	rqtime;
 	timestruc_t	*rqtp;
 
@@ -690,7 +687,6 @@ aiowaitn(void *uiocb, uint_t nent, uint_t *nwait, timespec_t *timout)
 	 */
 	if (rqtp) {
 		timestruc_t now;
-		timecheck = timechanged;
 		gethrestime(&now);
 		timespecadd(rqtp, &now);
 	}
@@ -752,7 +748,7 @@ aiowaitn(void *uiocb, uint_t nent, uint_t *nwait, timespec_t *timout)
 
 		if ((cnt < waitcnt) && blocking) {
 			int rval = cv_waituntil_sig(&aiop->aio_waitcv,
-				&aiop->aio_mutex, rqtp, timecheck);
+				&aiop->aio_mutex, rqtp);
 			if (rval > 0)
 				continue;
 			if (rval < 0) {
@@ -920,7 +916,6 @@ aiosuspend(
 	size_t		ssize;
 	model_t		model = get_udatamodel();
 	int		blocking;
-	int		timecheck;
 	timestruc_t	rqtime;
 	timestruc_t	*rqtp;
 
@@ -936,7 +931,6 @@ aiosuspend(
 		return (error);
 	if (rqtp) {
 		timestruc_t now;
-		timecheck = timechanged;
 		gethrestime(&now);
 		timespecadd(rqtp, &now);
 	}
@@ -1056,7 +1050,7 @@ aiosuspend(
 			 */
 			mutex_exit(&aiop->aio_cleanupq_mutex);
 			rv = cv_waituntil_sig(&aiop->aio_waitcv,
-				&aiop->aio_mutex, rqtp, timecheck);
+				&aiop->aio_mutex, rqtp);
 			/*
 			 * we have to drop aio_mutex and
 			 * grab it in the right order.
diff --git a/usr/src/uts/common/os/callout.c b/usr/src/uts/common/os/callout.c
index 1d13f7d179..92b52e27af 100644
--- a/usr/src/uts/common/os/callout.c
+++ b/usr/src/uts/common/os/callout.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -79,6 +78,49 @@ static callout_table_t *callout_table[CALLOUT_TABLES];
 	CALLOUT_HASH_##INSDEL(ct->ct_lbhash[CALLOUT_LBHASH(runtime)],	\
 	cp, c_lbnext, c_lbprev)
 
+#define	CALLOUT_HRES_INSERT(ct, cp, cnext, cprev, hresms)		\
+{									\
+	callout_t *nextp = ct->ct_hresq;				\
+	callout_t *prevp;						\
+									\
+	if (nextp == NULL || hresms <= nextp->c_hresms) {		\
+		cp->cnext = ct->ct_hresq;				\
+		ct->ct_hresq = cp;					\
+		cp->cprev = NULL;					\
+		if (cp->cnext != NULL)					\
+			cp->cnext->cprev = cp;				\
+	} else {							\
+		do {							\
+			prevp = nextp;					\
+			nextp = nextp->cnext;				\
+		} while (nextp != NULL && hresms > nextp->c_hresms);	\
+		prevp->cnext = cp;					\
+		cp->cprev = prevp;					\
+		cp->cnext = nextp;					\
+		if (nextp != NULL) 					\
+			nextp->cprev = cp;				\
+	}								\
+}
+
+#define	CALLOUT_HRES_DELETE(ct, cp, cnext, cprev, hresms)	\
+{								\
+	if (cp == ct->ct_hresq) {				\
+		ct->ct_hresq = cp->cnext;			\
+		if (cp->cnext != NULL)				\
+			cp->cnext->cprev = NULL;		\
+	} else {						\
+		cp->cprev->cnext = cp->cnext;			\
+		if (cp->cnext != NULL)				\
+			cp->cnext->cprev = cp->cprev;		\
+	}							\
+}
+
+#define	CALLOUT_HRES_UPDATE(INSDEL, ct, cp, id, hresms)		\
+	ASSERT(MUTEX_HELD(&ct->ct_lock));			\
+	ASSERT(cp->c_xid == id);				\
+	CALLOUT_HRES_##INSDEL(ct, cp, c_hrnext,			\
+	c_hrprev, hresms)
+
 /*
  * Allocate a callout structure.  We try quite hard because we
  * can't sleep, and if we can't do the allocation, we're toast.
@@ -106,9 +148,13 @@ static timeout_id_t
 timeout_common(void (*func)(void *), void *arg, clock_t delta,
     callout_table_t *ct)
 {
-	callout_t *cp;
-	callout_id_t id;
-	clock_t runtime;
+	callout_t	*cp;
+	callout_id_t	id;
+	clock_t		runtime;
+	timestruc_t	now;
+	int64_t		hresms;
+
+	gethrestime(&now);
 
 	mutex_enter(&ct->ct_lock);
 
@@ -127,6 +173,11 @@ timeout_common(void (*func)(void *), void *arg, clock_t delta,
 		delta = 1;
 	cp->c_runtime = runtime = lbolt + delta;
 
+	/* Calculate the future time in milli-second */
+	hresms = now.tv_sec * MILLISEC + now.tv_nsec / MICROSEC +
+	    TICK_TO_MSEC(delta);
+	cp->c_hresms = hresms;
+
 	/*
 	 * Assign an ID to this callout
 	 */
@@ -140,6 +191,7 @@ timeout_common(void (*func)(void *), void *arg, clock_t delta,
 	cp->c_xid = id;
 
 	CALLOUT_HASH_UPDATE(INSERT, ct, cp, id, runtime);
+	CALLOUT_HRES_UPDATE(INSERT, ct, cp, id, hresms);
 
 	mutex_exit(&ct->ct_lock);
 
@@ -184,6 +236,7 @@ untimeout(timeout_id_t id_arg)
 			clock_t time_left = runtime - lbolt;
 
 			CALLOUT_HASH_UPDATE(DELETE, ct, cp, id, runtime);
+			CALLOUT_HRES_UPDATE(DELETE, ct, cp, id, 0);
 			cp->c_idnext = ct->ct_freelist;
 			ct->ct_freelist = cp;
 			mutex_exit(&ct->ct_lock);
@@ -245,9 +298,11 @@ untimeout(timeout_id_t id_arg)
 static void
 callout_execute(callout_table_t *ct)
 {
-	callout_t *cp;
-	callout_id_t xid;
-	clock_t runtime;
+	callout_t	*cp;
+	callout_id_t	xid;
+	clock_t		runtime;
+	timestruc_t	now;
+	int64_t		hresms;
 
 	mutex_enter(&ct->ct_lock);
 
@@ -267,14 +322,16 @@ callout_execute(callout_table_t *ct)
 			mutex_enter(&ct->ct_lock);
 
 			/*
-			 * Delete callout from hash tables, return to freelist,
-			 * and tell anyone who cares that we're done.
+			 * Delete callout from both the hash tables and the
+			 * hres queue, return it to freelist, and tell anyone
+			 * who cares that we're done.
 			 * Even though we dropped and reacquired ct->ct_lock,
 			 * it's OK to pick up where we left off because only
 			 * newly-created timeouts can precede cp on ct_lbhash,
 			 * and those timeouts cannot be due on this tick.
 			 */
 			CALLOUT_HASH_UPDATE(DELETE, ct, cp, xid, runtime);
+			CALLOUT_HRES_UPDATE(DELETE, ct, cp, xid, hresms);
 			cp->c_idnext = ct->ct_freelist;
 			ct->ct_freelist = cp;
 			cp->c_xid = 0;	/* Indicate completion for c_done */
@@ -289,6 +346,44 @@ callout_execute(callout_table_t *ct)
 		if (ct->ct_runtime == runtime)
 			ct->ct_runtime = runtime + 1;
 	}
+
+	gethrestime(&now);
+
+	/* Calculate the current time in milli-second */
+	hresms = now.tv_sec * MILLISEC + now.tv_nsec / MICROSEC;
+
+	cp = ct->ct_hresq;
+	while (cp != NULL && hresms >= cp->c_hresms) {
+		xid = cp->c_xid;
+		if (xid & CALLOUT_EXECUTING) {
+			cp = cp->c_hrnext;
+			continue;
+		}
+		cp->c_executor = curthread;
+		cp->c_xid = xid |= CALLOUT_EXECUTING;
+		runtime = cp->c_runtime;
+		mutex_exit(&ct->ct_lock);
+		DTRACE_PROBE1(callout__start, callout_t *, cp);
+		(*cp->c_func)(cp->c_arg);
+		DTRACE_PROBE1(callout__end, callout_t *, cp);
+		mutex_enter(&ct->ct_lock);
+
+		/*
+		 * See comments above.
+		 */
+		CALLOUT_HASH_UPDATE(DELETE, ct, cp, xid, runtime);
+		CALLOUT_HRES_UPDATE(DELETE, ct, cp, xid, hresms);
+		cp->c_idnext = ct->ct_freelist;
+		ct->ct_freelist = cp;
+		cp->c_xid = 0;	/* Indicate completion for c_done */
+		cv_broadcast(&cp->c_done);
+
+		/*
+		 * Start over from the head of the list, see if
+		 * any timeout bearing an earlier hres time.
+		 */
+		cp = ct->ct_hresq;
+	}
 	mutex_exit(&ct->ct_lock);
 }
 
@@ -298,8 +393,10 @@ callout_execute(callout_table_t *ct)
 static void
 callout_schedule_1(callout_table_t *ct)
 {
-	callout_t *cp;
-	clock_t curtime, runtime;
+	callout_t	*cp;
+	clock_t		curtime, runtime;
+	timestruc_t	now;
+	int64_t		hresms;
 
 	mutex_enter(&ct->ct_lock);
 	ct->ct_curtime = curtime = lbolt;
@@ -320,6 +417,26 @@ callout_schedule_1(callout_table_t *ct)
 		}
 		ct->ct_runtime++;
 	}
+
+	gethrestime(&now);
+
+	/* Calculate the current time in milli-second */
+	hresms = now.tv_sec * MILLISEC + now.tv_nsec / MICROSEC;
+
+	cp = ct->ct_hresq;
+	while (cp != NULL && hresms >= cp->c_hresms) {
+		if (cp->c_xid & CALLOUT_EXECUTING) {
+			cp = cp->c_hrnext;
+			continue;
+		}
+		mutex_exit(&ct->ct_lock);
+		if (ct->ct_taskq == NULL)
+			softcall((void (*)(void *))callout_execute, ct);
+		else
+			(void) taskq_dispatch(ct->ct_taskq,
+			    (task_func_t *)callout_execute, ct, KM_NOSLEEP);
+		return;
+	}
 	mutex_exit(&ct->ct_lock);
 }
 
diff --git a/usr/src/uts/common/os/clock.c b/usr/src/uts/common/os/clock.c
index 4eff816e48..0a8a99f0af 100644
--- a/usr/src/uts/common/os/clock.c
+++ b/usr/src/uts/common/os/clock.c
@@ -856,7 +856,7 @@ clock(void)
 					hrestime = tod;
 					membar_enter();	/* hrestime visible */
 					timedelta = 0;
-					timechanged++;
+					hrestime_isvalid = 1;
 					tod_needsync = 0;
 					hr_clock_unlock(s);
 				}
@@ -1787,16 +1787,20 @@ clkset(time_t approx)
 	mutex_exit(&tod_lock);
 }
 
-int	timechanged;	/* for testing if the system time has been reset */
+int	hrestime_isvalid = 0;
 
 void
 set_hrestime(timestruc_t *ts)
 {
 	int spl = hr_clock_lock();
 	hrestime = *ts;
-	membar_enter();	/* hrestime must be visible before timechanged++ */
+	/*
+	 * hrestime must be visible before hrestime_isvalid
+	 * is set to 1
+	 */
+	membar_enter();
 	timedelta = 0;
-	timechanged++;
+	hrestime_isvalid = 1;
 	hr_clock_unlock(spl);
 }
 
diff --git a/usr/src/uts/common/os/condvar.c b/usr/src/uts/common/os/condvar.c
index c27227a20a..71f20c6b67 100644
--- a/usr/src/uts/common/os/condvar.c
+++ b/usr/src/uts/common/os/condvar.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -619,19 +619,13 @@ cv_wait_stop(kcondvar_t *cvp, kmutex_t *mp, int wakeup_time)
  *	        >0 if awakened via cv_signal() or cv_broadcast()
  *		   or by a spurious wakeup.
  *		   (might return time remaining)
- * As a special test, if someone abruptly resets the system time
- * (but not through adjtime(2); drifting of the clock is allowed and
- * expected [see timespectohz_adj()]), then we force a return of -1
- * so the caller can return a premature timeout to the calling process
- * so it can reevaluate the situation in light of the new system time.
- * (The system clock has been reset if timecheck != timechanged.)
  */
 int
-cv_waituntil_sig(kcondvar_t *cvp, kmutex_t *mp,
-	timestruc_t *when, int timecheck)
+cv_waituntil_sig(kcondvar_t *cvp, kmutex_t *mp, timestruc_t *when)
 {
 	timestruc_t now;
 	timestruc_t delta;
+	clock_t ticks;
 	int rval;
 
 	if (when == NULL)
@@ -648,29 +642,20 @@ cv_waituntil_sig(kcondvar_t *cvp, kmutex_t *mp,
 		 */
 		rval = cv_timedwait_sig(cvp, mp, lbolt);
 	} else {
-		if (timecheck == timechanged) {
-			rval = cv_timedwait_sig(cvp, mp,
-				lbolt + timespectohz_adj(when, now));
-		} else {
-			/*
-			 * Someone reset the system time;
-			 * just force an immediate timeout.
-			 */
-			rval = -1;
-		}
-		if (rval == -1 && timecheck == timechanged) {
-			/*
-			 * Even though cv_timedwait_sig() returned showing a
-			 * timeout, the future time may not have passed yet.
-			 * If not, change rval to indicate a normal wakeup.
-			 */
-			gethrestime(&now);
-			delta = *when;
-			timespecsub(&delta, &now);
-			if (delta.tv_sec > 0 || (delta.tv_sec == 0 &&
-			    delta.tv_nsec > 0))
+		ticks = lbolt + timespectohz(when, now);
+		rval = cv_timedwait_sig(cvp, mp, ticks);
+
+		gethrestime(&now);
+		delta = *when;
+		timespecsub(&delta, &now);
+
+		/*
+		 * timeout is premature iff
+		 *	ticks >= lbolt  and  when > now
+		 */
+		if (rval == -1 && ticks >= lbolt && (delta.tv_sec > 0 ||
+		    (delta.tv_sec == 0 && delta.tv_nsec > 0)))
 				rval = 1;
-		}
 	}
 	return (rval);
 }
diff --git a/usr/src/uts/common/os/logsubr.c b/usr/src/uts/common/os/logsubr.c
index d8c513586c..1f5efdebee 100644
--- a/usr/src/uts/common/os/logsubr.c
+++ b/usr/src/uts/common/os/logsubr.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -327,7 +327,7 @@ log_conswitch(log_t *src, log_t *dst)
 		lc->flags |= SL_LOGONLY;
 
 		/*
-		 * The ttime is written with 0 in log_sensmsg() only when
+		 * The ttime is written with 0 in log_sendmsg() only when
 		 * good gethrestime_sec() data is not available to store in
 		 * the log_ctl_t in the early boot phase.
 		 */
@@ -605,16 +605,12 @@ log_sendmsg(mblk_t *mp, zoneid_t zoneid)
 	log_enter();
 
 	/*
-	 * In the early boot phase hrestime is invalid, then timechanged is 0.
-	 * If hrestime is not valid, the ttime is set to 0 here and the correct
-	 * ttime is calculated in log_conswitch() later. The log_conswitch()
-	 * calculation to determine the correct ttime does not use ttime data
-	 * from these log_ctl_t structures; it only uses ttime from log_ctl_t's
-	 * that contain good data.
-	 *
+	 * If we are still in the early boot phase and the hrestime is invalid,
+	 * we set ttime to 0 so that log_conswitch() can determine the correct
+	 * ttime with a log_ctl_t structure which contains a valid ttime stamp.
 	 */
 	lc->ltime = lbolt;
-	if (timechanged) {
+	if (hrestime_isvalid) {
 		lc->ttime = gethrestime_sec();
 	} else {
 		lc->ttime = 0;
diff --git a/usr/src/uts/common/os/timers.c b/usr/src/uts/common/os/timers.c
index 9cb07a948e..ddaa2adff4 100644
--- a/usr/src/uts/common/os/timers.c
+++ b/usr/src/uts/common/os/timers.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -849,28 +849,6 @@ timespectohz(timespec_t *tv, timespec_t now)
 }
 
 /*
- * Same as timespectohz() except that we adjust the clock ticks down a bit.
- * If we will be waiting for a long time, we may encounter skewing problems
- * due to adjtime() system calls.  Since we can skew up to 1/16 lbolt rate
- * if adjtime is going crazy, we reduce the time delta since timeout() takes
- * clock ticks rather than wallclock elapsed time.  This may cause the caller
- * (who calls timeout()) to return with a timeout prematurely and callers
- * must accommodate this.  See lwp_timeout(), queue_lwptimer() and
- * cv_waituntil_sig(), currently the only callers of this function.
- */
-clock_t
-timespectohz_adj(timespec_t *tv, timespec_t now)
-{
-	timespec_t wait_time = *tv;
-
-	timespecsub(&wait_time, &now);
-	wait_time.tv_sec -= wait_time.tv_sec >> 4;
-	wait_time.tv_nsec -= wait_time.tv_nsec >> 4;
-	timespecadd(&wait_time, &now);
-	return (timespectohz(&wait_time, now));
-}
-
-/*
  * hrt2ts(): convert from hrtime_t to timestruc_t.
  *
  * All this routine really does is:
@@ -1183,7 +1161,6 @@ nanosleep(timespec_t *rqtp, timespec_t *rmtp)
 	timespec_t rqtime;
 	timespec_t rmtime;
 	timespec_t now;
-	int timecheck;
 	int ret = 1;
 	model_t datamodel = get_udatamodel();
 
@@ -1203,12 +1180,11 @@ nanosleep(timespec_t *rqtp, timespec_t *rmtp)
 		return (set_errno(EINVAL));
 
 	if (timerspecisset(&rqtime)) {
-		timecheck = timechanged;
 		gethrestime(&now);
 		timespecadd(&rqtime, &now);
 		mutex_enter(&curthread->t_delay_lock);
 		while ((ret = cv_waituntil_sig(&curthread->t_delay_cv,
-		    &curthread->t_delay_lock, &rqtime, timecheck)) > 0)
+		    &curthread->t_delay_lock, &rqtime)) > 0)
 			continue;
 		mutex_exit(&curthread->t_delay_lock);
 	}