6784948 Bug fixes to the Callout implementation putback in SNV 103

7 files changed, 121 insertions, 66 deletions

diff --git a/usr/src/uts/common/os/callout.c b/usr/src/uts/common/os/callout.c
index a3eccfa518..65e94f0f59 100644
--- a/usr/src/uts/common/os/callout.c
+++ b/usr/src/uts/common/os/callout.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -43,6 +43,7 @@
 static hrtime_t callout_debug_hrtime;		/* debugger entry time */
 static int callout_min_resolution;		/* Minimum resolution */
 static callout_table_t *callout_boot_ct;	/* Boot CPU's callout tables */
+static clock_t callout_max_ticks;		/* max interval */
 static hrtime_t callout_longterm;		/* longterm nanoseconds */
 static ulong_t callout_counter_low;		/* callout ID increment */
 static ulong_t callout_table_bits;		/* number of table bits in ID */
@@ -377,7 +378,7 @@ callout_heap_insert(callout_table_t *ct, hrtime_t expiration)
 	 * entered, the cyclic will be programmed for the earliest expiration
 	 * in the heap.
 	 */
-	if (callout_upheap(ct) && !(ct->ct_flags & CALLOUT_TABLE_SUSPENDED))
+	if (callout_upheap(ct) && (ct->ct_suspend == 0))
 		(void) cyclic_reprogram(ct->ct_cyclic, expiration);
 }
 
@@ -521,12 +522,22 @@ callout_heap_delete(callout_table_t *ct)
 	 * by CPR, just return. The cyclic has already been programmed to
 	 * infinity by the cyclic subsystem.
 	 */
-	if ((ct->ct_heap_num == 0) || (ct->ct_flags & CALLOUT_TABLE_SUSPENDED))
+	if ((ct->ct_heap_num == 0) || (ct->ct_suspend > 0))
 		return;
 
 	(void) cyclic_reprogram(ct->ct_cyclic, expiration);
 }
 
+/*
+ * Common function used to create normal and realtime callouts.
+ *
+ * Realtime callouts are handled at CY_LOW_PIL by a cyclic handler. So,
+ * there is one restriction on a realtime callout handler - it should not
+ * directly or indirectly acquire cpu_lock. CPU offline waits for pending
+ * cyclic handlers to complete while holding cpu_lock. So, if a realtime
+ * callout handler were to try to get cpu_lock, there would be a deadlock
+ * during CPU offline.
+ */
 callout_id_t
 timeout_generic(int type, void (*func)(void *), void *arg,
 	hrtime_t expiration, hrtime_t resolution, int flags)
@@ -596,6 +607,13 @@ timeout_generic(int type, void (*func)(void *), void *arg,
 	if (flags & CALLOUT_FLAG_ROUNDUP)
 		expiration += resolution - 1;
 	expiration = (expiration / resolution) * resolution;
+	if (expiration <= 0) {
+		/*
+		 * expiration hrtime overflow has occurred. Just set the
+		 * expiration to infinity.
+		 */
+		expiration = CY_INFINITY;
+	}
 
 	/*
 	 * Assign an ID to this callout
@@ -705,6 +723,8 @@ timeout(void (*func)(void *), void *arg, clock_t delta)
 	 */
 	if (delta <= 0)
 		delta = 1;
+	else if (delta > callout_max_ticks)
+		delta = callout_max_ticks;
 
 	id =  (ulong_t)timeout_generic(CALLOUT_NORMAL, func, arg,
 	    TICK_TO_NSEC(delta), nsec_per_tick, CALLOUT_LEGACY);
@@ -726,6 +746,8 @@ timeout_default(void (*func)(void *), void *arg, clock_t delta)
 	 */
 	if (delta <= 0)
 		delta = 1;
+	else if (delta > callout_max_ticks)
+		delta = callout_max_ticks;
 
 	id = timeout_generic(CALLOUT_NORMAL, func, arg, TICK_TO_NSEC(delta),
 	    nsec_per_tick, 0);
@@ -743,6 +765,8 @@ realtime_timeout(void (*func)(void *), void *arg, clock_t delta)
 	 */
 	if (delta <= 0)
 		delta = 1;
+	else if (delta > callout_max_ticks)
+		delta = callout_max_ticks;
 
 	id =  (ulong_t)timeout_generic(CALLOUT_REALTIME, func, arg,
 	    TICK_TO_NSEC(delta), nsec_per_tick, CALLOUT_LEGACY);
@@ -764,6 +788,8 @@ realtime_timeout_default(void (*func)(void *), void *arg, clock_t delta)
 	 */
 	if (delta <= 0)
 		delta = 1;
+	else if (delta > callout_max_ticks)
+		delta = callout_max_ticks;
 
 	id = timeout_generic(CALLOUT_REALTIME, func, arg, TICK_TO_NSEC(delta),
 	    nsec_per_tick, 0);
@@ -1092,12 +1118,14 @@ callout_suspend(void)
 			ct = &callout_table[CALLOUT_TABLE(t, f)];
 
 			mutex_enter(&ct->ct_mutex);
-			ct->ct_flags |= CALLOUT_TABLE_SUSPENDED;
+			ct->ct_suspend++;
 			if (ct->ct_cyclic == CYCLIC_NONE) {
 				mutex_exit(&ct->ct_mutex);
 				continue;
 			}
-			(void) cyclic_reprogram(ct->ct_cyclic, CY_INFINITY);
+			if (ct->ct_suspend == 1)
+				(void) cyclic_reprogram(ct->ct_cyclic,
+				    CY_INFINITY);
 			mutex_exit(&ct->ct_mutex);
 		}
 	}
@@ -1172,7 +1200,7 @@ callout_resume(hrtime_t delta)
 
 			mutex_enter(&ct->ct_mutex);
 			if (ct->ct_cyclic == CYCLIC_NONE) {
-				ct->ct_flags &= ~CALLOUT_TABLE_SUSPENDED;
+				ct->ct_suspend--;
 				mutex_exit(&ct->ct_mutex);
 				continue;
 			}
@@ -1180,21 +1208,23 @@ callout_resume(hrtime_t delta)
 			if (delta)
 				callout_adjust(ct, delta);
 
-			ct->ct_flags &= ~CALLOUT_TABLE_SUSPENDED;
-
-			/*
-			 * If the expired list is non-empty, then have the
-			 * cyclic expire immediately. Else, program the
-			 * cyclic based on the heap.
-			 */
-			if (ct->ct_expired.ch_head != NULL)
-				exp = gethrtime();
-			else if (ct->ct_heap_num > 0)
-				exp = ct->ct_heap[0];
-			else
-				exp = 0;
-			if (exp != 0)
-				(void) cyclic_reprogram(ct->ct_cyclic, exp);
+			ct->ct_suspend--;
+			if (ct->ct_suspend == 0) {
+				/*
+				 * If the expired list is non-empty, then have
+				 * the cyclic expire immediately. Else, program
+				 * the cyclic based on the heap.
+				 */
+				if (ct->ct_expired.ch_head != NULL)
+					exp = gethrtime();
+				else if (ct->ct_heap_num > 0)
+					exp = ct->ct_heap[0];
+				else
+					exp = 0;
+				if (exp != 0)
+					(void) cyclic_reprogram(ct->ct_cyclic,
+					    exp);
+			}
 			mutex_exit(&ct->ct_mutex);
 		}
 	}
@@ -1280,7 +1310,7 @@ callout_hrestime_one(callout_table_t *ct)
 
 	if (ecl->cl_callouts.ch_head != NULL) {
 		CALLOUT_LIST_APPEND(ct->ct_expired, ecl);
-		if (!(ct->ct_flags & CALLOUT_TABLE_SUSPENDED))
+		if (ct->ct_suspend == 0)
 			(void) cyclic_reprogram(ct->ct_cyclic, gethrtime());
 	} else {
 		ecl->cl_next = ct->ct_lfree;
@@ -1412,16 +1442,8 @@ callout_cyclic_init(callout_table_t *ct)
 	 */
 	ASSERT(ct->ct_cyclic == CYCLIC_NONE);
 
-	/*
-	 * Ideally, the handlers for CALLOUT_REALTIME and CALLOUT_NORMAL should
-	 * be run at CY_LOW_LEVEL. But there are some callers of the delay(9F)
-	 * function that call delay(9F) illegally from PIL > 0. delay(9F) uses
-	 * normal callouts. In order to avoid a deadlock, we run the normal
-	 * handler from LOCK level. When the delay(9F) issue is fixed, this
-	 * should be fixed as well.
-	 */
 	hdlr.cyh_func = (cyc_func_t)CALLOUT_CYCLIC_HANDLER(t);
-	hdlr.cyh_level = (t == CALLOUT_REALTIME) ? CY_LOW_LEVEL : CY_LOCK_LEVEL;
+	hdlr.cyh_level = CY_LOW_LEVEL;
 	hdlr.cyh_arg = ct;
 	when.cyt_when = CY_INFINITY;
 	when.cyt_interval = CY_INFINITY;
@@ -1498,11 +1520,30 @@ callout_cpu_online(cpu_t *cp)
 		mutex_exit(&ct->ct_mutex);
 
 		/*
-		 * Move the cyclic to this CPU by doing a bind. Then unbind
-		 * the cyclic. This will allow the cyclic subsystem to juggle
-		 * the cyclic during CPU offline.
+		 * Move the cyclic to this CPU by doing a bind.
 		 */
 		cyclic_bind(ct->ct_cyclic, cp, NULL);
+	}
+}
+
+void
+callout_cpu_offline(cpu_t *cp)
+{
+	callout_table_t *ct;
+	processorid_t seqid;
+	int t;
+
+	ASSERT(MUTEX_HELD(&cpu_lock));
+
+	seqid = cp->cpu_seqid;
+
+	for (t = 0; t < CALLOUT_NTYPES; t++) {
+		ct = &callout_table[CALLOUT_TABLE(t, seqid)];
+
+		/*
+		 * Unbind the cyclic. This will allow the cyclic subsystem
+		 * to juggle the cyclic during CPU offline.
+		 */
 		cyclic_bind(ct->ct_cyclic, NULL, NULL);
 	}
 }
@@ -1549,6 +1590,7 @@ callout_init(void)
 	callout_table_mask = (1 << callout_table_bits) - 1;
 	callout_counter_low = 1 << CALLOUT_COUNTER_SHIFT;
 	callout_longterm = TICK_TO_NSEC(CALLOUT_LONGTERM_TICKS);
+	callout_max_ticks = CALLOUT_MAX_TICKS;
 
 	/*
 	 * Because of the variability in timing behavior across systems with
@@ -1603,6 +1645,7 @@ callout_init(void)
 		for (t = 0; t < CALLOUT_NTYPES; t++) {
 			table_id = CALLOUT_TABLE(t, f);
 			ct = &callout_table[table_id];
+			ct->ct_type = t;
 			mutex_init(&ct->ct_mutex, NULL, MUTEX_DEFAULT, NULL);
 			/*
 			 * Precompute the base IDs for long and short-term
diff --git a/usr/src/uts/common/os/clock.c b/usr/src/uts/common/os/clock.c
index 9f6ba19769..45f7f53e39 100644
--- a/usr/src/uts/common/os/clock.c
+++ b/usr/src/uts/common/os/clock.c
@@ -23,7 +23,7 @@
 
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -1591,8 +1591,9 @@ delay(clock_t ticks)
 	timeout_id_t id;
 	extern hrtime_t volatile devinfo_freeze;
 
-	if ((panicstr || devinfo_freeze) && ticks > 0) {
+	if ((getpil() > 0 || panicstr || devinfo_freeze) && ticks > 0) {
 		/*
+		 * Either the caller's PIL is not safe for timeout wait or
 		 * Timeouts aren't running, so all we can do is spin.
 		 */
 		drv_usecwait(TICK_TO_USEC(ticks));
diff --git a/usr/src/uts/common/os/condvar.c b/usr/src/uts/common/os/condvar.c
index 344746063a..cb1543e767 100644
--- a/usr/src/uts/common/os/condvar.c
+++ b/usr/src/uts/common/os/condvar.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -206,10 +206,9 @@ cv_wakeup(void *arg)
 	 * This mutex is acquired and released in order to make sure that
 	 * the wakeup does not happen before the block itself happens.
 	 */
-	mutex_enter(t->t_wait_mp);
-	mutex_exit(t->t_wait_mp);
+	mutex_enter(&t->t_wait_mutex);
+	mutex_exit(&t->t_wait_mutex);
 	setrun(t);
-	t->t_wait_mp = NULL;
 }
 
 /*
@@ -233,11 +232,12 @@ cv_timedwait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim)
 	timeleft = tim - lbolt;
 	if (timeleft <= 0)
 		return (-1);
-	t->t_wait_mp = mp;
+	mutex_enter(&t->t_wait_mutex);
 	id = realtime_timeout_default((void (*)(void *))cv_wakeup, t, timeleft);
 	thread_lock(t);		/* lock the thread */
 	cv_block((condvar_impl_t *)cvp);
 	thread_unlock_nopreempt(t);
+	mutex_exit(&t->t_wait_mutex);
 	mutex_exit(mp);
 	swtch();
 	signalled = (t->t_schedflag & TS_SIGNALLED);
@@ -248,7 +248,7 @@ cv_timedwait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim)
 	 * we called untimeout.  We will treat this as if the timeout
 	 * has occured and set timeleft to -1.
 	 */
-	timeleft = (t->t_wait_mp == NULL) ? -1 : untimeout_default(id, 0);
+	timeleft = untimeout_default(id, 0);
 	mutex_enter(mp);
 	if (timeleft <= 0) {
 		timeleft = -1;
@@ -363,7 +363,7 @@ cv_timedwait_sig_internal(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int flag)
 	 * Set the timeout and wait.
 	 */
 	cancel_pending = schedctl_cancel_pending();
-	t->t_wait_mp = mp;
+	mutex_enter(&t->t_wait_mutex);
 	id = timeout_generic(CALLOUT_REALTIME, (void (*)(void *))cv_wakeup, t,
 	    TICK_TO_NSEC(timeleft), nsec_per_tick, flag);
 	lwp->lwp_asleep = 1;
@@ -371,6 +371,7 @@ cv_timedwait_sig_internal(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int flag)
 	thread_lock(t);
 	cv_block_sig(t, (condvar_impl_t *)cvp);
 	thread_unlock_nopreempt(t);
+	mutex_exit(&t->t_wait_mutex);
 	mutex_exit(mp);
 	if (ISSIG(t, JUSTLOOKING) || MUSTRETURN(p, t) || cancel_pending)
 		setrun(t);
@@ -386,7 +387,7 @@ cv_timedwait_sig_internal(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int flag)
 	 * we called untimeout.  We will treat this as if the timeout
 	 * has occured and set rval to -1.
 	 */
-	rval = (t->t_wait_mp == NULL) ? -1 : untimeout_default(id, 0);
+	rval = untimeout_default(id, 0);
 	mutex_enter(mp);
 	if (rval <= 0)
 		rval = -1;
@@ -593,17 +594,17 @@ cv_wait_stop(kcondvar_t *cvp, kmutex_t *mp, int wakeup_time)
 	 * Wakeup in wakeup_time milliseconds, i.e., human time.
 	 */
 	tim = lbolt + MSEC_TO_TICK(wakeup_time);
-	t->t_wait_mp = mp;
+	mutex_enter(&t->t_wait_mutex);
 	id = realtime_timeout_default((void (*)(void *))cv_wakeup, t,
 	    tim - lbolt);
 	thread_lock(t);			/* lock the thread */
 	cv_block((condvar_impl_t *)cvp);
 	thread_unlock_nopreempt(t);
+	mutex_exit(&t->t_wait_mutex);
 	mutex_exit(mp);
 	/* ASSERT(no locks are held); */
 	swtch();
-	if (t->t_wait_mp != NULL)
-		(void) untimeout_default(id, 0);
+	(void) untimeout_default(id, 0);
 
 	/*
 	 * Check for reasons to stop, if lwp_nostop is not true.
diff --git a/usr/src/uts/common/os/cpu.c b/usr/src/uts/common/os/cpu.c
index d07e926084..74dd46fa17 100644
--- a/usr/src/uts/common/os/cpu.c
+++ b/usr/src/uts/common/os/cpu.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -1231,6 +1231,7 @@ cpu_offline(cpu_t *cp, int flags)
 	cpu_t	*ncp;
 	int	intr_enable;
 	int	cyclic_off = 0;
+	int	callout_off = 0;
 	int	loop_count;
 	int	no_quiesce = 0;
 	int	(*bound_func)(struct cpu *, int);
@@ -1356,6 +1357,11 @@ again:	for (loop_count = 0; (*bound_func)(cp, 0); loop_count++) {
 		delay(hz/100);
 	}
 
+	if (error == 0 && callout_off == 0) {
+		callout_cpu_offline(cp);
+		callout_off = 1;
+	}
+
 	if (error == 0 && cyclic_off == 0) {
 		if (!cyclic_offline(cp)) {
 			/*
@@ -1535,6 +1541,13 @@ out:
 		cyclic_online(cp);
 
 	/*
+	 * If we failed, but managed to offline callouts on this CPU,
+	 * bring it back online.
+	 */
+	if (error && callout_off)
+		callout_cpu_online(cp);
+
+	/*
 	 * If we failed, tell the PG subsystem that the CPU is back
 	 */
 	pg_cpupart_in(cp, pp);
diff --git a/usr/src/uts/common/sys/callo.h b/usr/src/uts/common/sys/callo.h
index 9402cfa050..95ec7e769c 100644
--- a/usr/src/uts/common/sys/callo.h
+++ b/usr/src/uts/common/sys/callo.h
@@ -23,7 +23,7 @@
 
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -300,14 +300,6 @@ typedef enum callout_stat_type {
 #endif
 
 /*
- * Callout table flags:
- *
- * CALLOUT_TABLE_SUSPENDED
- *	Callout processing on this table has been suspended.
- */
-#define	CALLOUT_TABLE_SUSPENDED		0x1UL
-
-/*
  * All of the state information associated with a callout table.
  * The fields are ordered with cache performance in mind.
  */
@@ -321,7 +313,8 @@ typedef struct callout_table {
 	callout_hash_t 	*ct_clhash;	/* callout list hash */
 	kstat_named_t	*ct_kstat_data;	/* callout kstat data */
 
-	ulong_t		ct_flags;	/* flags */
+	uint_t		ct_type;	/* callout table type */
+	uint_t		ct_suspend;	/* suspend count */
 	cyclic_id_t	ct_cyclic;	/* cyclic for this table */
 	hrtime_t	*ct_heap;	/* callout expiration heap */
 	ulong_t		ct_heap_num;	/* occupied slots in the heap */
@@ -336,7 +329,7 @@ typedef struct callout_table {
 #ifdef _LP64
 	ulong_t		ct_pad[4];	/* cache alignment */
 #else
-	ulong_t		ct_pad[8];	/* cache alignment */
+	ulong_t		ct_pad[7];	/* cache alignment */
 #endif
 } callout_table_t;
 
@@ -377,12 +370,18 @@ typedef struct callout_table {
 
 #define	CALLOUT_ALIGN	64	/* cache line size */
 
+#ifdef _LP64
+#define	CALLOUT_MAX_TICKS	NSEC_TO_TICK(CY_INFINITY);
+#else
+#define	CALLOUT_MAX_TICKS	LONG_MAX
+#endif
+
 extern void		callout_init(void);
 extern void		membar_sync(void);
-extern void		callout_cpu_configure(cpu_t *);
 extern void		callout_cpu_online(cpu_t *);
-extern void		callout_cpu_init(cpu_t *);
+extern void		callout_cpu_offline(cpu_t *);
 extern void		callout_hrestime(void);
+
 #endif
 
 #ifdef	__cplusplus
diff --git a/usr/src/uts/common/sys/thread.h b/usr/src/uts/common/sys/thread.h
index 85198171d7..6d112ef065 100644
--- a/usr/src/uts/common/sys/thread.h
+++ b/usr/src/uts/common/sys/thread.h
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -341,7 +341,7 @@ typedef struct _kthread {
 	hrtime_t	t_hrtime;	/* high-res last time on cpu */
 	kmutex_t	t_ctx_lock;	/* protects t_ctx in removectx() */
 	struct waitq	*t_waitq;	/* wait queue */
-	kmutex_t	*t_wait_mp;	/* used in CV wait functions */
+	kmutex_t	t_wait_mutex;	/* used in CV wait functions */
 } kthread_t;
 
 /*
diff --git a/usr/src/uts/i86pc/io/cbe.c b/usr/src/uts/i86pc/io/cbe.c
index 29f0176cc2..b1163e1ba1 100644
--- a/usr/src/uts/i86pc/io/cbe.c
+++ b/usr/src/uts/i86pc/io/cbe.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -201,8 +201,6 @@ cbe_xcall(void *arg, cpu_t *dest, cyc_func_t func, void *farg)
 	mutex_exit(&cbe_xcall_lock);
 
 	kpreempt_enable();
-
-	ASSERT(cbe_xcall_func == NULL && cbe_xcall_cpu == NULL);
 }
 
 void *