[illumos-gate merge]

commit 21bcbe6e4903d8521ec66863bf0c21d9ed378cff
    13961 Add HPET as a TSC calibration source

7 files changed, 242 insertions, 45 deletions

diff --git a/usr/src/uts/i86pc/Makefile.files b/usr/src/uts/i86pc/Makefile.files
index 6d5c539308..58118145e7 100644
--- a/usr/src/uts/i86pc/Makefile.files
+++ b/usr/src/uts/i86pc/Makefile.files
@@ -71,6 +71,7 @@ CORE_OBJS +=			\
 	hma_fpu.o		\
 	hment.o			\
 	hold_page.o		\
+	hpet_acpi.o		\
 	hrtimers.o		\
 	htable.o		\
 	hypercall.o		\
@@ -121,6 +122,7 @@ CORE_OBJS +=			\
 	startup.o		\
 	timestamp.o		\
 	todpc_subr.o		\
+	tscc_hpet.o		\
 	tscc_pit.o		\
 	trap.o			\
 	turbo.o			\
@@ -198,9 +200,9 @@ PCI_E_NEXUS_OBJS += pci_common.o pci_kstats.o pci_tools.o
 PCINEXUS_OBJS += pci.o pci_common.o pci_kstats.o pci_tools.o
 PCPLUSMP_OBJS += apic.o apic_regops.o psm_common.o apic_introp.o	\
 			mp_platform_common.o mp_platform_misc.o		\
-			hpet_acpi.o apic_common.o apic_timer.o
+			apic_common.o apic_timer.o
 APIX_OBJS += apix.o apic_regops.o psm_common.o apix_intr.o apix_utils.o \
-		apix_irm.o mp_platform_common.o hpet_acpi.o apic_common.o \
+		apix_irm.o mp_platform_common.o apic_common.o \
 		apic_timer.o apix_regops.o
 
 
diff --git a/usr/src/uts/i86pc/io/hpet_acpi.c b/usr/src/uts/i86pc/io/hpet_acpi.c
index aace99b18b..caef4ca966 100644
--- a/usr/src/uts/i86pc/io/hpet_acpi.c
+++ b/usr/src/uts/i86pc/io/hpet_acpi.c
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2020 Oxide Computer Company
+ * Copyright 2020 Joyent, Inc.
  */
 
 #include <sys/hpet_acpi.h>
@@ -37,6 +38,8 @@
 #include <sys/cpupart.h>
 #include <sys/x86_archext.h>
 #include <sys/prom_debug.h>
+#include <sys/psm.h>
+#include <sys/bootconf.h>
 
 static int hpet_init_proxy(int *hpet_vect, iflag_t *hpet_flags);
 static boolean_t hpet_install_proxy(void);
@@ -100,10 +103,14 @@ static kmutex_t		hpet_proxy_lock;	/* lock for lAPIC proxy data */
  */
 static hpet_proxy_t	*hpet_proxy_users;	/* one per CPU */
 
+static boolean_t	hpet_early_init_failed;
 
 ACPI_TABLE_HPET		*hpet_table;		/* ACPI HPET table */
 hpet_info_t		hpet_info;		/* Human readable Information */
 
+static hrtime_t (*apic_timer_stop_count_fn)(void);
+static void (*apic_timer_restart_fn)(hrtime_t);
+
 /*
  * Provide HPET access from unix.so.
  * Set up pointers to access symbols in pcplusmp.
@@ -118,31 +125,44 @@ hpet_establish_hooks(void)
 }
 
 /*
- * Get the ACPI "HPET" table.
- * acpi_probe() calls this function from mp_startup before drivers are loaded.
- * acpi_probe() verified the system is using ACPI before calling this.
+ * Initialize the HPET early in the boot process if it is both present
+ * and needed to calibrate the TSC. This initializes the HPET enough to
+ * allow the main counter to be read for calibration purposes.
  *
- * There may be more than one ACPI HPET table (Itanium only?).
- * Intel's HPET spec defines each timer block to have up to 32 counters and
- * be 1024 bytes long.  There can be more than one timer block of 32 counters.
- * Each timer block would have an additional ACPI HPET table.
- * Typical x86 systems today only have 1 HPET with 3 counters.
- * On x86 we only consume HPET table "1" for now.
+ * If the HPET is not needed early in the boot process, but is needed later
+ * by ACPI, this will be called at that time to start the initialization
+ * process.
  */
 int
-hpet_acpi_init(int *hpet_vect, iflag_t *hpet_flags)
+hpet_early_init(void)
 {
 	extern hrtime_t tsc_read(void);
-	extern int	idle_cpu_no_deep_c;
-	extern int	cpuid_deep_cstates_supported(void);
 	void		*la;
 	uint64_t	ret;
 	uint_t		num_timers;
 	uint_t		ti;
 
+	PRM_POINT("Initializing the HPET...");
+
+	/* If we tried and failed, don't try again. */
+	if (hpet_early_init_failed) {
+		PRM_POINT("Prior HPET initialization failed, aborting...");
+		return (DDI_FAILURE);
+	}
+
+	/* No need to initialize again if we already succeeded */
+	if (hpet.supported >= HPET_TIMER_SUPPORT)
+		return (DDI_SUCCESS);
+
 	(void) memset(&hpet_info, 0, sizeof (hpet_info));
 	hpet.supported = HPET_NO_SUPPORT;
 
+	/*
+	 * Once called, we assume initialization fails unless we complete all
+	 * the early init tasks.
+	 */
+	hpet_early_init_failed = B_TRUE;
+
 	if ((get_hwenv() & HW_XEN_HVM) != 0) {
 		/*
 		 * In some AWS EC2 guests, though the HPET is advertised via
@@ -156,25 +176,20 @@ hpet_acpi_init(int *hpet_vect, iflag_t *hpet_flags)
 		return (DDI_FAILURE);
 	}
 
-	if (idle_cpu_no_deep_c ||
-	    !cpuid_deep_cstates_supported()) {
-		/*
-		 * If Deep C-States are disabled or not supported, then we do
-		 * not need to program the HPET at all as it will not
-		 * subsequently be used.
-		 */
-		PRM_POINT("no need to program the HPET");
-		return (DDI_FAILURE);
-	}
-
-	hpet_establish_hooks();
-
 	/*
-	 * Get HPET ACPI table 1.
+	 * If there are any HPET tables, we should have mapped and stored
+	 * the address of the first table while building up the boot
+	 * properties.
+	 *
+	 * Systems with a large numbers of HPET timer blocks may have
+	 * multiple HPET tables (each HPET table can contain at most 32 timer
+	 * blocks). Most x86 systems have 1 HPET table with 3 counters (it
+	 * appears multiple HPET timers was largely seen on Itanium systems).
+	 * illumos currently only uses the first HPET table, so we do not need
+	 * to be concerned about additional tables.
 	 */
-	PRM_POINT("AcpiGetTable() HPET #1");
-	if (ACPI_FAILURE(AcpiGetTable(ACPI_SIG_HPET, HPET_TABLE_1,
-	    (ACPI_TABLE_HEADER **)&hpet_table))) {
+	if (BOP_GETPROPLEN(bootops, "hpet-table") != 8 ||
+	    BOP_GETPROP(bootops, "hpet-table", (void *)&hpet_table) != 0) {
 		cmn_err(CE_NOTE, "!hpet_acpi: unable to get ACPI HPET table");
 		return (DDI_FAILURE);
 	}
@@ -303,6 +318,49 @@ hpet_acpi_init(int *hpet_vect, iflag_t *hpet_flags)
 	 * HPET main counter reads are supported now.
 	 */
 	hpet.supported = HPET_TIMER_SUPPORT;
+	hpet_early_init_failed = B_FALSE;
+
+	PRM_POINT("HPET main counter configured for reading...");
+	return (DDI_SUCCESS);
+}
+
+/*
+ * Called by acpi_init() to set up HPET interrupts and fully initialize the
+ * HPET.
+ */
+int
+hpet_acpi_init(int *hpet_vect, iflag_t *hpet_flags, hrtime_t (*stop_fn)(void),
+    void (*restart_fn)(hrtime_t))
+{
+	extern int	idle_cpu_no_deep_c;
+	extern int	cpuid_deep_cstates_supported(void);
+
+	PRM_POINT("Completing HPET initialization...");
+
+	if (hpet_early_init() != DDI_SUCCESS) {
+		PRM_POINT("Early HPET initialization failed; aborting...");
+		return (DDI_FAILURE);
+	}
+
+	/*
+	 * These functions reside in either pcplusmp or apix, and allow
+	 * the HPET to proxy the LAPIC.
+	 */
+	apic_timer_stop_count_fn = stop_fn;
+	apic_timer_restart_fn = restart_fn;
+
+	hpet_establish_hooks();
+
+	if (idle_cpu_no_deep_c ||
+	    !cpuid_deep_cstates_supported()) {
+		/*
+		 * If Deep C-States are disabled or not supported, then we do
+		 * not need to program the HPET at all as it will not
+		 * subsequently be used.
+		 */
+		PRM_POINT("no need to program the HPET");
+		return (DDI_FAILURE);
+	}
 
 	return (hpet_init_proxy(hpet_vect, hpet_flags));
 }
@@ -449,7 +507,8 @@ hpet_checksum_table(unsigned char *table, unsigned int length)
 static void *
 hpet_memory_map(ACPI_TABLE_HPET *hpet_table)
 {
-	return (AcpiOsMapMemory(hpet_table->Address.Address, HPET_SIZE));
+	return (psm_map_new(hpet_table->Address.Address, (size_t)HPET_SIZE,
+	    PSM_PROT_WRITE | PSM_PROT_READ));
 }
 
 static int
@@ -484,6 +543,18 @@ hpet_stop_main_counter(hpet_info_t *hip)
 	return (gcr & HPET_GCFR_ENABLE_CNF ? ~AE_OK : AE_OK);
 }
 
+boolean_t
+hpet_timer_is_readable(void)
+{
+	return ((hpet.supported >= HPET_TIMER_SUPPORT) ? B_TRUE : B_FALSE);
+}
+
+uint64_t
+hpet_read_timer(void)
+{
+	return (hpet_read_main_counter_value(&hpet_info));
+}
+
 /*
  * Set the Legacy Replacement Route bit.
  * This should be called before setting up timers.
@@ -1274,8 +1345,6 @@ hpet_guaranteed_schedule(hrtime_t required_wakeup_time)
 static boolean_t
 hpet_use_hpet_timer(hrtime_t *lapic_expire)
 {
-	extern hrtime_t	apic_timer_stop_count(void);
-	extern void	apic_timer_restart(hrtime_t);
 	hrtime_t	now, expire, dead;
 	uint64_t	lapic_count, dead_count;
 	cpupart_t	*cpu_part;
@@ -1301,7 +1370,7 @@ hpet_use_hpet_timer(hrtime_t *lapic_expire)
 	 * idle thread acquires the mutex but before it clears interrupts.
 	 */
 	ASSERT(!interrupts_enabled());
-	lapic_count = apic_timer_stop_count();
+	lapic_count = apic_timer_stop_count_fn();
 	now = gethrtime();
 	dead = now + hpet_idle_spin_timeout;
 	*lapic_expire = expire = now + lapic_count;
@@ -1322,7 +1391,7 @@ hpet_use_hpet_timer(hrtime_t *lapic_expire)
 		/*
 		 * spin
 		 */
-		apic_timer_restart(expire);
+		apic_timer_restart_fn(expire);
 		sti();
 		cli();
 
@@ -1337,7 +1406,7 @@ hpet_use_hpet_timer(hrtime_t *lapic_expire)
 			}
 		}
 
-		lapic_count = apic_timer_stop_count();
+		lapic_count = apic_timer_stop_count_fn();
 		now = gethrtime();
 		*lapic_expire = expire = now + lapic_count;
 		if (lapic_count == (hrtime_t)-1) {
@@ -1349,7 +1418,7 @@ hpet_use_hpet_timer(hrtime_t *lapic_expire)
 			return (B_TRUE);
 		}
 		if (now > dead) {
-			apic_timer_restart(expire);
+			apic_timer_restart_fn(expire);
 			*lapic_expire = (hrtime_t)HPET_INFINITY;
 			return (B_FALSE);
 		}
@@ -1360,7 +1429,7 @@ hpet_use_hpet_timer(hrtime_t *lapic_expire)
 	    (hpet_state.proxy_installed == B_FALSE) ||
 	    (hpet_state.uni_cstate == B_TRUE)) {
 		mutex_exit(&hpet_proxy_lock);
-		apic_timer_restart(expire);
+		apic_timer_restart_fn(expire);
 		*lapic_expire = (hrtime_t)HPET_INFINITY;
 		return (B_FALSE);
 	}
@@ -1387,7 +1456,7 @@ hpet_use_hpet_timer(hrtime_t *lapic_expire)
 	mutex_exit(&hpet_proxy_lock);
 
 	if (rslt == B_FALSE) {
-		apic_timer_restart(expire);
+		apic_timer_restart_fn(expire);
 		*lapic_expire = (hrtime_t)HPET_INFINITY;
 	}
 
@@ -1409,7 +1478,6 @@ hpet_use_hpet_timer(hrtime_t *lapic_expire)
 static void
 hpet_use_lapic_timer(hrtime_t expire)
 {
-	extern void	apic_timer_restart(hrtime_t);
 	processorid_t	cpu_id = CPU->cpu_id;
 
 	ASSERT(CPU->cpu_thread == CPU->cpu_idle_thread);
@@ -1421,7 +1489,7 @@ hpet_use_lapic_timer(hrtime_t expire)
 	 * Do not enable a LAPIC Timer that was initially disabled.
 	 */
 	if (expire != HPET_INFINITY)
-		apic_timer_restart(expire);
+		apic_timer_restart_fn(expire);
 }
 
 /*
diff --git a/usr/src/uts/i86pc/io/mp_platform_common.c b/usr/src/uts/i86pc/io/mp_platform_common.c
index 9b9944fbd0..54a0ac3506 100644
--- a/usr/src/uts/i86pc/io/mp_platform_common.c
+++ b/usr/src/uts/i86pc/io/mp_platform_common.c
@@ -23,7 +23,7 @@
  * Copyright 2016 Nexenta Systems, Inc.
  * Copyright 2017 Joyent, Inc.
  * Copyright (c) 2017 by Delphix. All rights reserved.
- * Copyright (c) 2019, Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  * Copyright 2020 RackTop Systems, Inc.
  * Copyright 2020 Oxide Computer Company
  */
@@ -1040,7 +1040,8 @@ acpi_probe(char *modname)
 		 * failure message it will be logged by the routine itself.
 		 */
 		PRM_POINT("hpet_acpi_init()");
-		(void) hpet_acpi_init(&apic_hpet_vect, &apic_hpet_flags);
+		(void) hpet_acpi_init(&apic_hpet_vect, &apic_hpet_flags,
+		    apic_timer_stop_count, apic_timer_restart);
 #endif
 
 		kmem_free(local_ids, NCPU * sizeof (uint32_t));
diff --git a/usr/src/uts/i86pc/os/fakebop.c b/usr/src/uts/i86pc/os/fakebop.c
index 933411606e..527e7cee93 100644
--- a/usr/src/uts/i86pc/os/fakebop.c
+++ b/usr/src/uts/i86pc/os/fakebop.c
@@ -2935,6 +2935,15 @@ build_firmware_properties(struct xboot_info *xbp)
 #endif /* __xpv */
 	if (tp != NULL)
 		process_mcfg((ACPI_TABLE_MCFG *)tp);
+
+	/*
+	 * Map the first HPET table (if it exists) and save the address.
+	 * If the HPET is required to calibrate the TSC, we require the
+	 * HPET table prior to being able to load modules, so we cannot use
+	 * the acpica module (and thus AcpiGetTable()) to locate it.
+	 */
+	if ((tp = find_fw_table(rsdp, ACPI_SIG_HPET)) != NULL)
+		bsetprop64("hpet-table", (uint64_t)(uintptr_t)tp);
 }
 
 /*
diff --git a/usr/src/uts/i86pc/os/tscc_hpet.c b/usr/src/uts/i86pc/os/tscc_hpet.c
new file mode 100644
index 0000000000..dc18bb8a90
--- /dev/null
+++ b/usr/src/uts/i86pc/os/tscc_hpet.c
@@ -0,0 +1,106 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2020 Joyent, Inc.
+ */
+
+#include <sys/tsc.h>
+#include <sys/prom_debug.h>
+#include <sys/hpet.h>
+#include <sys/clock.h>
+
+/*
+ * The amount of time (in microseconds) between tsc samples. This is
+ * somewhat arbitrary, but seems reasonable. A frequency of 1GHz is
+ * 1,000,000,000 ticks / sec (10^9). 100us is 10^(-6) * 10^2 => 10^(-4), so
+ * 100us would represent 10^5 (100,000) ticks.
+ */
+#define	HPET_SAMPLE_INTERVAL_US (100)
+
+/*
+ * The same as above, but in nanoseconds (for ease in converting to HPET
+ * ticks)
+ */
+#define	HPET_SAMPLE_INTERVAL_NS (USEC2NSEC(HPET_SAMPLE_INTERVAL_US))
+
+/* The amount of HPET sample ticks to wait */
+#define	HPET_SAMPLE_TICKS (HRTIME_TO_HPET_TICKS(HPET_SAMPLE_INTERVAL_NS))
+
+#define	TSC_NUM_SAMPLES 10
+
+static boolean_t
+tsc_calibrate_hpet(uint64_t *freqp)
+{
+	uint64_t hpet_sum = 0;
+	uint64_t tsc_sum = 0;
+	uint_t i;
+
+	PRM_POINT("Attempting to use HPET for TSC calibration...");
+
+	if (hpet_early_init() != DDI_SUCCESS)
+		return (B_FALSE);
+
+	/*
+	 * The expansion of HPET_SAMPLE_TICKS (specifically
+	 * HRTIME_TO_HPET_TICKS) uses the HPET period to calculate the number
+	 * of HPET ticks for the given time period. Therefore, we cannot
+	 * set hpet_num_ticks until after the early HPET initialization has
+	 * been performed by hpet_early_init() (and the HPET period is known).
+	 */
+	const uint64_t hpet_num_ticks = HPET_SAMPLE_TICKS;
+
+	for (i = 0; i < TSC_NUM_SAMPLES; i++) {
+		uint64_t hpet_now, hpet_end;
+		uint64_t tsc_start, tsc_end;
+
+		hpet_now = hpet_read_timer();
+		hpet_end = hpet_now + hpet_num_ticks;
+
+		tsc_start = tsc_read();
+		while (hpet_now < hpet_end)
+			hpet_now = hpet_read_timer();
+
+		tsc_end = tsc_read();
+
+		/*
+		 * If our TSC isn't advancing after 100us, we're pretty much
+		 * hosed.
+		 */
+		VERIFY3P(tsc_end, >, tsc_start);
+
+		tsc_sum += tsc_end - tsc_start;
+
+		/*
+		 * We likely did not end exactly HPET_SAMPLE_TICKS after
+		 * we started, so save the actual amount.
+		 */
+		hpet_sum += hpet_num_ticks + hpet_now - hpet_end;
+	}
+
+	uint64_t hpet_avg = hpet_sum / TSC_NUM_SAMPLES;
+	uint64_t tsc_avg = tsc_sum / TSC_NUM_SAMPLES;
+	uint64_t hpet_ns = hpet_avg * hpet_info.period / HPET_FEMTO_TO_NANO;
+
+	PRM_POINT("HPET calibration complete");
+
+	*freqp = tsc_avg * NANOSEC / hpet_ns;
+	PRM_DEBUG(*freqp);
+
+	return (B_TRUE);
+}
+
+static tsc_calibrate_t tsc_calibration_hpet = {
+	.tscc_source = "HPET",
+	.tscc_preference = 50,
+	.tscc_calibrate = tsc_calibrate_hpet,
+};
+TSC_CALIBRATION_SOURCE(tsc_calibration_hpet);
diff --git a/usr/src/uts/i86pc/sys/apic_timer.h b/usr/src/uts/i86pc/sys/apic_timer.h
index 7f7285fb83..fd0e5eb605 100644
--- a/usr/src/uts/i86pc/sys/apic_timer.h
+++ b/usr/src/uts/i86pc/sys/apic_timer.h
@@ -24,6 +24,8 @@
 /*
  * Copyright (c) 2010, Intel Corporation.
  * All rights reserved.
+ *
+ * Copyright 2020 Joyent, Inc.
  */
 
 #ifndef _SYS_APIC_TIMER_H
@@ -71,6 +73,9 @@ extern void	apic_timer_reprogram(hrtime_t);
 extern void	apic_timer_enable(void);
 extern void	apic_timer_disable(void);
 
+extern hrtime_t	apic_timer_stop_count(void);
+extern void	apic_timer_restart(hrtime_t);
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/usr/src/uts/i86pc/sys/hpet.h b/usr/src/uts/i86pc/sys/hpet.h
index 1ee9910441..33906bd3f0 100644
--- a/usr/src/uts/i86pc/sys/hpet.h
+++ b/usr/src/uts/i86pc/sys/hpet.h
@@ -21,6 +21,8 @@
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ *
+ * Copyright 2020 Joyent, Inc.
  */
 
 #ifndef	_HPET_H
@@ -69,7 +71,11 @@ typedef struct hpet {
  */
 extern hpet_t hpet;
 
-int hpet_acpi_init(int *hpet_vect, iflag_t *hpet_flags);
+int hpet_early_init(void);
+boolean_t hpet_timer_is_readable(void);
+uint64_t hpet_read_timer(void);
+int hpet_acpi_init(int *hpet_vect, iflag_t *hpet_flags, hrtime_t (*)(void),
+    void (*)(hrtime_t));
 void hpet_acpi_fini(void);
 uint32_t hpet_proxy_ipl(void);