9234 reduce apic calibration error by taking multiple measurements

Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed by: Igor Kozhukhov <igor@dilos.org>
Reviewed by: Hans Rosenfeld <rosenfeld@grumpf.hope-2000.org>
Approved by: Dan McDonald <danmcd@joyent.com>

4 files changed, 105 insertions, 42 deletions

diff --git a/usr/src/uts/i86pc/io/pcplusmp/apic_common.c b/usr/src/uts/i86pc/io/pcplusmp/apic_common.c
index 3892bc3f85..f5f5fcfe1a 100644
--- a/usr/src/uts/i86pc/io/pcplusmp/apic_common.c
+++ b/usr/src/uts/i86pc/io/pcplusmp/apic_common.c
@@ -24,7 +24,7 @@
  */
 /*
  * Copyright (c) 2017, Joyent, Inc.  All rights reserved.
- * Copyright (c) 2016 by Delphix. All rights reserved.
+ * Copyright (c) 2016, 2017 by Delphix. All rights reserved.
  */
 
 /*
@@ -1059,19 +1059,20 @@ apic_cpu_remove(psm_cpu_request_t *reqp)
 }
 
 /*
- * Return the number of APIC clock ticks elapsed for 8245 to decrement
- * (APIC_TIME_COUNT + pit_ticks_adj) ticks.
+ * Return the number of ticks the APIC decrements in SF nanoseconds.
+ * The fixed-frequency PIT (aka 8254) is used for the measurement.
  */
-uint_t
-apic_calibrate(volatile uint32_t *addr, uint16_t *pit_ticks_adj)
+static uint64_t
+apic_calibrate_impl()
 {
 	uint8_t		pit_tick_lo;
-	uint16_t	pit_tick, target_pit_tick;
-	uint32_t	start_apic_tick, end_apic_tick;
+	uint16_t	pit_tick, target_pit_tick, pit_ticks_adj;
+	uint32_t	pit_ticks;
+	uint32_t	start_apic_tick, end_apic_tick, apic_ticks;
 	ulong_t		iflag;
-	uint32_t	reg;
 
-	reg = addr + APIC_CURR_COUNT - apicadr;
+	apic_reg_ops->apic_write(APIC_DIVIDE_REG, apic_divide_reg_init);
+	apic_reg_ops->apic_write(APIC_INIT_COUNT, APIC_MAXVAL);
 
 	iflag = intr_clear();
 
@@ -1082,7 +1083,7 @@ apic_calibrate(volatile uint32_t *addr, uint16_t *pit_ticks_adj)
 	    pit_tick_lo <= APIC_LB_MIN || pit_tick_lo >= APIC_LB_MAX);
 
 	/*
-	 * Wait for the 8254 to decrement by 5 ticks to ensure
+	 * Wait for the PIT to decrement by 5 ticks to ensure
 	 * we didn't start in the middle of a tick.
 	 * Compare with 0x10 for the wrap around case.
 	 */
@@ -1092,11 +1093,10 @@ apic_calibrate(volatile uint32_t *addr, uint16_t *pit_ticks_adj)
 		pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
 	} while (pit_tick > target_pit_tick || pit_tick_lo < 0x10);
 
-	start_apic_tick = apic_reg_ops->apic_read(reg);
+	start_apic_tick = apic_reg_ops->apic_read(APIC_CURR_COUNT);
 
 	/*
-	 * Wait for the 8254 to decrement by
-	 * (APIC_TIME_COUNT + pit_ticks_adj) ticks
+	 * Wait for the PIT to decrement by APIC_TIME_COUNT ticks
 	 */
 	target_pit_tick = pit_tick - APIC_TIME_COUNT;
 	do {
@@ -1104,13 +1104,95 @@ apic_calibrate(volatile uint32_t *addr, uint16_t *pit_ticks_adj)
 		pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
 	} while (pit_tick > target_pit_tick || pit_tick_lo < 0x10);
 
-	end_apic_tick = apic_reg_ops->apic_read(reg);
-
-	*pit_ticks_adj = target_pit_tick - pit_tick;
+	end_apic_tick = apic_reg_ops->apic_read(APIC_CURR_COUNT);
 
 	intr_restore(iflag);
 
-	return (start_apic_tick - end_apic_tick);
+	apic_ticks = start_apic_tick - end_apic_tick;
+
+	/* The PIT might have decremented by more ticks than planned */
+	pit_ticks_adj = target_pit_tick - pit_tick;
+	/* total number of PIT ticks corresponding to apic_ticks */
+	pit_ticks = APIC_TIME_COUNT + pit_ticks_adj;
+
+	/*
+	 * Determine the number of nanoseconds per APIC clock tick
+	 * and then determine how many APIC ticks to interrupt at the
+	 * desired frequency
+	 * apic_ticks / (pitticks / PIT_HZ) = apic_ticks_per_s
+	 * (apic_ticks * PIT_HZ) / pitticks = apic_ticks_per_s
+	 * apic_ticks_per_ns = (apic_ticks * PIT_HZ) / (pitticks * 10^9)
+	 * apic_ticks_per_SFns =
+	 * (SF * apic_ticks * PIT_HZ) / (pitticks * 10^9)
+	 */
+	return ((SF * apic_ticks * PIT_HZ) / ((uint64_t)pit_ticks * NANOSEC));
+}
+
+/*
+ * It was found empirically that 5 measurements seem sufficient to give a good
+ * accuracy. Most spurious measurements are higher than the target value thus
+ * we eliminate up to 2/5 spurious measurements.
+ */
+#define	APIC_CALIBRATE_MEASUREMENTS		5
+
+#define	APIC_CALIBRATE_PERCENT_OFF_WARNING	10
+
+/*
+ * Return the number of ticks the APIC decrements in SF nanoseconds.
+ * Several measurements are taken to filter out outliers.
+ */
+uint64_t
+apic_calibrate()
+{
+	uint64_t	measurements[APIC_CALIBRATE_MEASUREMENTS];
+	int		median_idx;
+	uint64_t	median;
+
+	/*
+	 * When running under a virtual machine, the emulated PIT and APIC
+	 * counters do not always return the right values and can roll over.
+	 * Those spurious measurements are relatively rare but could
+	 * significantly affect the calibration.
+	 * Therefore we take several measurements and then keep the median.
+	 * The median is preferred to the average here as we only want to
+	 * discard outliers.
+	 */
+	for (int i = 0; i < APIC_CALIBRATE_MEASUREMENTS; i++)
+		measurements[i] = apic_calibrate_impl();
+
+	/*
+	 * sort results and retrieve median.
+	 */
+	for (int i = 0; i < APIC_CALIBRATE_MEASUREMENTS; i++) {
+		for (int j = i + 1; j < APIC_CALIBRATE_MEASUREMENTS; j++) {
+			if (measurements[j] < measurements[i]) {
+				uint64_t tmp = measurements[i];
+				measurements[i] = measurements[j];
+				measurements[j] = tmp;
+			}
+		}
+	}
+	median_idx = APIC_CALIBRATE_MEASUREMENTS / 2;
+	median = measurements[median_idx];
+
+#if (APIC_CALIBRATE_MEASUREMENTS >= 3)
+	/*
+	 * Check that measurements are consistent. Post a warning
+	 * if the three middle values are not close to each other.
+	 */
+	uint64_t delta_warn = median *
+	    APIC_CALIBRATE_PERCENT_OFF_WARNING / 100;
+	if ((median - measurements[median_idx - 1]) > delta_warn ||
+	    (measurements[median_idx + 1] - median) > delta_warn) {
+		cmn_err(CE_WARN, "apic_calibrate measurements lack "
+		    "precision: %llu, %llu, %llu.",
+		    (u_longlong_t)measurements[median_idx - 1],
+		    (u_longlong_t)median,
+		    (u_longlong_t)measurements[median_idx + 1]);
+	}
+#endif
+
+	return (median);
 }
 
 /*
diff --git a/usr/src/uts/i86pc/io/pcplusmp/apic_timer.c b/usr/src/uts/i86pc/io/pcplusmp/apic_timer.c
index 348f5034fc..bc61c114c2 100644
--- a/usr/src/uts/i86pc/io/pcplusmp/apic_timer.c
+++ b/usr/src/uts/i86pc/io/pcplusmp/apic_timer.c
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017 by Delphix. All rights reserved.
  */
 /*
  * Copyright (c) 2010, Intel Corporation.
@@ -90,34 +91,12 @@ static apic_timer_t	apic_timer;
 int
 apic_timer_init(int hertz)
 {
-	uint_t		apic_ticks = 0;
-	uint_t		pit_ticks;
 	int		ret, timer_mode;
-	uint16_t	pit_ticks_adj;
 	static int	firsttime = 1;
 
 	if (firsttime) {
 		/* first time calibrate on CPU0 only */
-
-		apic_reg_ops->apic_write(APIC_DIVIDE_REG, apic_divide_reg_init);
-		apic_reg_ops->apic_write(APIC_INIT_COUNT, APIC_MAXVAL);
-		apic_ticks = apic_calibrate(apicadr, &pit_ticks_adj);
-
-		/* total number of PIT ticks corresponding to apic_ticks */
-		pit_ticks = APIC_TIME_COUNT + pit_ticks_adj;
-
-		/*
-		 * Determine the number of nanoseconds per APIC clock tick
-		 * and then determine how many APIC ticks to interrupt at the
-		 * desired frequency
-		 * apic_ticks / (pitticks / PIT_HZ) = apic_ticks_per_s
-		 * (apic_ticks * PIT_HZ) / pitticks = apic_ticks_per_s
-		 * apic_ticks_per_ns = (apic_ticks * PIT_HZ) / (pitticks * 10^9)
-		 * pic_ticks_per_SFns =
-		 * (SF * apic_ticks * PIT_HZ) / (pitticks * 10^9)
-		 */
-		apic_ticks_per_SFnsecs = ((SF * apic_ticks * PIT_HZ) /
-		    ((uint64_t)pit_ticks * NANOSEC));
+		apic_ticks_per_SFnsecs = apic_calibrate();
 
 		/* the interval timer initial count is 32 bit max */
 		apic_nsec_max = APIC_TICKS_TO_NSECS(APIC_MAXVAL);
diff --git a/usr/src/uts/i86pc/sys/apic.h b/usr/src/uts/i86pc/sys/apic.h
index 6877579cae..0ad325ac49 100644
--- a/usr/src/uts/i86pc/sys/apic.h
+++ b/usr/src/uts/i86pc/sys/apic.h
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2017 Joyent, Inc.
+ * Copyright (c) 2017 by Delphix. All rights reserved.
  */
 /*
  * Copyright (c) 2010, Intel Corporation.
@@ -830,7 +831,7 @@ extern int apic_local_mode();
 extern void apic_change_eoi();
 extern void apic_send_EOI(uint32_t);
 extern void apic_send_directed_EOI(uint32_t);
-extern uint_t apic_calibrate(volatile uint32_t *, uint16_t *);
+extern uint64_t apic_calibrate();
 
 extern volatile uint32_t *apicadr;	/* virtual addr of local APIC   */
 extern int apic_forceload;
diff --git a/usr/src/uts/i86pc/sys/apic_common.h b/usr/src/uts/i86pc/sys/apic_common.h
index efbf04f1ba..2eb7bc2597 100644
--- a/usr/src/uts/i86pc/sys/apic_common.h
+++ b/usr/src/uts/i86pc/sys/apic_common.h
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017 by Delphix. All rights reserved.
  */
 /*
  * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
@@ -181,7 +182,7 @@ extern void	apic_unset_idlecpu(processorid_t cpun);
 extern void	apic_shutdown(int cmd, int fcn);
 extern void	apic_preshutdown(int cmd, int fcn);
 extern processorid_t	apic_get_next_processorid(processorid_t cpun);
-extern uint_t	apic_calibrate(volatile uint32_t *, uint16_t *);
+extern uint64_t	apic_calibrate();
 
 extern int apic_error_intr();
 extern void apic_cpcovf_mask_clear(void);