OS-7662 need a way to disable SMT

Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Approved by: Jerry Jelinek <jerry.jelinek@joyent.com>

15 files changed, 165 insertions, 99 deletions

diff --git a/usr/src/uts/common/cpr/cpr_main.c b/usr/src/uts/common/cpr/cpr_main.c
index 7db797c848..66791cd1f4 100644
--- a/usr/src/uts/common/cpr/cpr_main.c
+++ b/usr/src/uts/common/cpr/cpr_main.c
@@ -24,6 +24,10 @@
  */
 
 /*
+ * Copyright 2019 Joyent, Inc.
+ */
+
+/*
  * This module contains the guts of checkpoint-resume mechanism.
  * All code in this module is platform independent.
  */
@@ -1324,7 +1328,7 @@ cpr_all_online(void)
 	do {
 		cp->cpu_cpr_flags &= ~CPU_CPR_ONLINE;
 		if (!CPU_ACTIVE(cp)) {
-			if ((rc = cpu_online(cp)) != 0)
+			if ((rc = cpu_online(cp, 0)) != 0)
 				break;
 			CPU_SET_CPR_FLAGS(cp, CPU_CPR_ONLINE);
 		}
diff --git a/usr/src/uts/common/cpr/cpr_misc.c b/usr/src/uts/common/cpr/cpr_misc.c
index b1906eb9fa..ee7cd5ddf4 100644
--- a/usr/src/uts/common/cpr/cpr_misc.c
+++ b/usr/src/uts/common/cpr/cpr_misc.c
@@ -22,6 +22,7 @@
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
+ * Copyright 2019 Joyent, Inc.
  */
 
 #include <sys/types.h>
@@ -703,7 +704,7 @@ cpr_statefile_ok(vnode_t *vp, int alloc_retry)
 		 * Estimate space needed for the state file.
 		 *
 		 * State file size in bytes:
-		 * 	kernel size + non-cache pte seg +
+		 *	kernel size + non-cache pte seg +
 		 *	bitmap size + cpr state file headers size
 		 * (round up to fs->fs_bsize)
 		 */
@@ -996,7 +997,7 @@ cpr_p_online(cpu_t *cp, int state)
 
 	switch (state) {
 	case CPU_CPR_ONLINE:
-		rc = cpu_online(cp);
+		rc = cpu_online(cp, 0);
 		break;
 	case CPU_CPR_OFFLINE:
 		rc = cpu_offline(cp, CPU_FORCED);
diff --git a/usr/src/uts/common/disp/disp.c b/usr/src/uts/common/disp/disp.c
index 4898a18bf2..7e933bccc4 100644
--- a/usr/src/uts/common/disp/disp.c
+++ b/usr/src/uts/common/disp/disp.c
@@ -24,7 +24,7 @@
  */
 
 /*
- * Copyright (c) 2018, Joyent, Inc. All rights reserved.
+ * Copyright 2019 Joyent, Inc.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
@@ -60,7 +60,7 @@
 #include <sys/dtrace.h>
 #include <sys/sdt.h>
 #include <sys/archsystm.h>
-#include <sys/ht.h>
+#include <sys/smt.h>
 
 #include <vm/as.h>
 
@@ -1239,13 +1239,13 @@ setbackdq(kthread_t *tp)
 		 * We'll generally let this thread continue to run where
 		 * it last ran...but will consider migration if:
 		 * - The thread probably doesn't have much cache warmth.
-		 * - HT exclusion would prefer us to run elsewhere
+		 * - SMT exclusion would prefer us to run elsewhere
 		 * - The CPU where it last ran is the target of an offline
 		 *   request.
 		 * - The thread last ran outside its home lgroup.
 		 */
 		if ((!THREAD_HAS_CACHE_WARMTH(tp)) ||
-		    !ht_should_run(tp, tp->t_cpu) ||
+		    !smt_should_run(tp, tp->t_cpu) ||
 		    (tp->t_cpu == cpu_inmotion) ||
 		    !LGRP_CONTAINS_CPU(tp->t_lpl->lpl_lgrp, tp->t_cpu)) {
 			cp = disp_lowpri_cpu(tp->t_cpu, tp, tpri);
@@ -1277,7 +1277,7 @@ setbackdq(kthread_t *tp)
 					newcp = cp->cpu_next_part;
 				}
 
-				if (ht_should_run(tp, newcp) &&
+				if (smt_should_run(tp, newcp) &&
 				    RUNQ_LEN(newcp, tpri) < qlen) {
 					DTRACE_PROBE3(runq__balance,
 					    kthread_t *, tp,
@@ -2579,7 +2579,7 @@ disp_cpu_inactive(cpu_t *cp)
  *
  * Otherwise we'll use double the effective dispatcher priority for the CPU.
  *
- * We do this so ht_adjust_cpu_score() can increment the score if needed,
+ * We do this so smt_adjust_cpu_score() can increment the score if needed,
  * without ending up over-riding a dispatcher priority.
  */
 static pri_t
@@ -2599,7 +2599,7 @@ cpu_score(cpu_t *cp, kthread_t *tp)
 	if (2 * cp->cpu_chosen_level > score)
 		score = 2 * cp->cpu_chosen_level;
 
-	return (ht_adjust_cpu_score(tp, cp, score));
+	return (smt_adjust_cpu_score(tp, cp, score));
 }
 
 /*
@@ -2726,7 +2726,7 @@ disp_choose_best_cpu(void)
 	ASSERT(t->t_state == TS_ONPROC);
 	ASSERT(t->t_schedflag & TS_VCPU);
 
-	if (ht_should_run(t, curcpu))
+	if (smt_should_run(t, curcpu))
 		return (curcpu);
 
 	return (disp_lowpri_cpu(curcpu, t, t->t_pri));
diff --git a/usr/src/uts/common/disp/thread.c b/usr/src/uts/common/disp/thread.c
index d576738e75..bf1f121b67 100644
--- a/usr/src/uts/common/disp/thread.c
+++ b/usr/src/uts/common/disp/thread.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2018 Joyent, Inc.
+ * Copyright 2019 Joyent, Inc.
  */
 
 #include <sys/types.h>
@@ -75,7 +75,7 @@
 #include <sys/cpucaps.h>
 #include <sys/kiconv.h>
 #include <sys/ctype.h>
-#include <sys/ht.h>
+#include <sys/smt.h>
 
 #ifndef	STACK_GROWTH_DOWN
 #error Stacks do not grow downward; 3b2 zombie attack detected!
@@ -1421,7 +1421,7 @@ thread_unpin()
 	itp = t->t_intr;		/* interrupted thread */
 	t->t_intr = NULL;		/* clear interrupt ptr */
 
-	ht_end_intr();
+	smt_end_intr();
 
 	/*
 	 * Get state from interrupt thread for the one
diff --git a/usr/src/uts/common/dtrace/dtrace.c b/usr/src/uts/common/dtrace/dtrace.c
index 8d5ccdc64b..769337294b 100644
--- a/usr/src/uts/common/dtrace/dtrace.c
+++ b/usr/src/uts/common/dtrace/dtrace.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2018, Joyent, Inc.
+ * Copyright 2019 Joyent, Inc.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
@@ -6922,6 +6922,16 @@ dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
 		return;
 
 	cookie = dtrace_interrupt_disable();
+
+	/*
+	 * Also refuse to process any probe firings that might happen on a
+	 * disabled CPU.
+	 */
+	if (CPU->cpu_flags & CPU_DISABLED) {
+		dtrace_interrupt_enable(cookie);
+		return;
+	}
+
 	probe = dtrace_probes[id - 1];
 	cpuid = CPU->cpu_id;
 	onintr = CPU_ON_INTR(CPU);
diff --git a/usr/src/uts/common/fs/zfs/zvol.c b/usr/src/uts/common/fs/zfs/zvol.c
index 33bac61d21..af5c9bdda1 100644
--- a/usr/src/uts/common/fs/zfs/zvol.c
+++ b/usr/src/uts/common/fs/zfs/zvol.c
@@ -26,7 +26,7 @@
  * Copyright 2017 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2012, 2017 by Delphix. All rights reserved.
  * Copyright (c) 2014 Integros [integros.com]
- * Copyright (c) 2019, Joyent, Inc.
+ * Copyright 2019 Joyent, Inc.
  */
 
 /*
@@ -90,7 +90,7 @@
 #include <sys/zfeature.h>
 #include <sys/zio_checksum.h>
 #include <sys/zil_impl.h>
-#include <sys/ht.h>
+#include <sys/smt.h>
 #include <sys/dkioc_free_util.h>
 #include <sys/zfs_rlock.h>
 
@@ -1278,7 +1278,7 @@ zvol_strategy(buf_t *bp)
 	    (zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS)) &&
 	    !doread && !is_dumpified;
 
-	ht_begin_unsafe();
+	smt_begin_unsafe();
 
 	/*
 	 * There must be no buffer changes when doing a dmu_sync() because
@@ -1327,7 +1327,7 @@ zvol_strategy(buf_t *bp)
 		zil_commit(zv->zv_zilog, ZVOL_OBJ);
 	biodone(bp);
 
-	ht_end_unsafe();
+	smt_end_unsafe();
 
 	return (0);
 }
@@ -1409,7 +1409,7 @@ zvol_read(dev_t dev, uio_t *uio, cred_t *cr)
 		return (error);
 	}
 
-	ht_begin_unsafe();
+	smt_begin_unsafe();
 
 	DTRACE_PROBE3(zvol__uio__start, dev_t, dev, uio_t *, uio, int, 0);
 
@@ -1471,7 +1471,7 @@ zvol_read(dev_t dev, uio_t *uio, cred_t *cr)
 	DTRACE_PROBE4(zvol__uio__done, dev_t, dev, uio_t *, uio, int, 0, int,
 	    error);
 
-	ht_end_unsafe();
+	smt_end_unsafe();
 
 	return (error);
 }
@@ -1504,7 +1504,7 @@ zvol_write(dev_t dev, uio_t *uio, cred_t *cr)
 		return (error);
 	}
 
-	ht_begin_unsafe();
+	smt_begin_unsafe();
 
 	DTRACE_PROBE3(zvol__uio__start, dev_t, dev, uio_t *, uio, int, 1);
 
@@ -1555,7 +1555,7 @@ zvol_write(dev_t dev, uio_t *uio, cred_t *cr)
 	DTRACE_PROBE4(zvol__uio__done, dev_t, dev, uio_t *, uio, int, 1, int,
 	    error);
 
-	ht_end_unsafe();
+	smt_end_unsafe();
 
 	mutex_enter(&zonep->zone_vfs_lock);
 	zonep->zone_vfs_rwstats.writes++;
@@ -1827,7 +1827,7 @@ zvol_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *cr, int *rvalp)
 		dkc = (struct dk_callback *)arg;
 		mutex_exit(&zfsdev_state_lock);
 
-		ht_begin_unsafe();
+		smt_begin_unsafe();
 
 		zil_commit(zv->zv_zilog, ZVOL_OBJ);
 		if ((flag & FKIOCTL) && dkc != NULL && dkc->dkc_callback) {
@@ -1835,7 +1835,7 @@ zvol_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *cr, int *rvalp)
 			error = 0;
 		}
 
-		ht_end_unsafe();
+		smt_end_unsafe();
 
 		return (error);
 
@@ -1861,9 +1861,9 @@ zvol_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *cr, int *rvalp)
 		} else {
 			zv->zv_flags &= ~ZVOL_WCE;
 			mutex_exit(&zfsdev_state_lock);
-			ht_begin_unsafe();
+			smt_begin_unsafe();
 			zil_commit(zv->zv_zilog, ZVOL_OBJ);
-			ht_end_unsafe();
+			smt_end_unsafe();
 		}
 		return (0);
 	}
@@ -1916,7 +1916,7 @@ zvol_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *cr, int *rvalp)
 
 		mutex_exit(&zfsdev_state_lock);
 
-		ht_begin_unsafe();
+		smt_begin_unsafe();
 
 		for (int i = 0; i < dfl->dfl_num_exts; i++) {
 			uint64_t start = dfl->dfl_exts[i].dfle_start,
@@ -1973,7 +1973,7 @@ zvol_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *cr, int *rvalp)
 		if (!(flag & FKIOCTL))
 			dfl_free(dfl);
 
-		ht_end_unsafe();
+		smt_end_unsafe();
 
 		return (error);
 	}
diff --git a/usr/src/uts/common/io/vnd/vnd.c b/usr/src/uts/common/io/vnd/vnd.c
index d03c7ce4ec..ac60fe678b 100644
--- a/usr/src/uts/common/io/vnd/vnd.c
+++ b/usr/src/uts/common/io/vnd/vnd.c
@@ -10,7 +10,7 @@
  */
 
 /*
- * Copyright (c) 2018, Joyent, Inc.
+ * Copyright 2019 Joyent, Inc.
  */
 
 /*
@@ -830,7 +830,7 @@
 #include <sys/disp.h>
 #include <sys/random.h>
 #include <sys/gsqueue.h>
-#include <sys/ht.h>
+#include <sys/smt.h>
 
 #include <inet/ip.h>
 #include <inet/ip6.h>
@@ -3721,7 +3721,7 @@ vnd_squeue_tx_drain(void *arg, mblk_t *drain_mp, gsqueue_t *gsp, void *dummy)
 	 * We're potentially going deep into the networking layer; make sure the
 	 * guest can't run concurrently.
 	 */
-	ht_begin_unsafe();
+	smt_begin_unsafe();
 
 	nmps = 0;
 	mptot = 0;
@@ -3743,7 +3743,7 @@ vnd_squeue_tx_drain(void *arg, mblk_t *drain_mp, gsqueue_t *gsp, void *dummy)
 		}
 	}
 
-	ht_end_unsafe();
+	smt_end_unsafe();
 
 	empty = vnd_dq_is_empty(&vsp->vns_dq_write);
 
diff --git a/usr/src/uts/common/os/cpu.c b/usr/src/uts/common/os/cpu.c
index 620f26034f..e53c75b64e 100644
--- a/usr/src/uts/common/os/cpu.c
+++ b/usr/src/uts/common/os/cpu.c
@@ -59,6 +59,7 @@
 #include <sys/time.h>
 #include <sys/archsystm.h>
 #include <sys/sdt.h>
+#include <sys/smt.h>
 #if defined(__x86) || defined(__amd64)
 #include <sys/x86_archext.h>
 #endif
@@ -144,6 +145,7 @@ processorid_t max_cpu_seqid_ever = 0;
 
 int ncpus = 1;
 int ncpus_online = 1;
+int ncpus_intr_enabled = 1;
 
 /*
  * CPU that we're trying to offline.  Protected by cpu_lock.
@@ -1209,7 +1211,7 @@ cpu_flagged_active(cpu_flag_t cpu_flags)
  * Bring the indicated CPU online.
  */
 int
-cpu_online(cpu_t *cp)
+cpu_online(cpu_t *cp, int flags)
 {
 	int	error = 0;
 
@@ -1223,6 +1225,9 @@ cpu_online(cpu_t *cp)
 
 	ASSERT(MUTEX_HELD(&cpu_lock));
 
+	if ((cp->cpu_flags & CPU_DISABLED) && !smt_can_enable(cp, flags))
+		return (EINVAL);
+
 	/*
 	 * Put all the cpus into a known safe place.
 	 * No mutexes can be entered while CPUs are paused.
@@ -1236,8 +1241,12 @@ cpu_online(cpu_t *cp)
 			cp->cpu_flags &= ~CPU_FAULTED;
 			mp_cpu_faulted_exit(cp);
 		}
+
+		if (cp->cpu_flags & CPU_DISABLED)
+			smt_force_enabled();
+
 		cp->cpu_flags &= ~(CPU_QUIESCED | CPU_OFFLINE | CPU_FROZEN |
-		    CPU_SPARE);
+		    CPU_SPARE | CPU_DISABLED);
 		CPU_NEW_GENERATION(cp);
 		start_cpus();
 		cpu_stats_kstat_create(cp);
@@ -1278,10 +1287,13 @@ cpu_offline(cpu_t *cp, int flags)
 	lpl_t	*cpu_lpl;
 	proc_t	*p;
 	int	lgrp_diff_lpl;
-	boolean_t unbind_all_threads = (flags & CPU_FORCED) != 0;
+	boolean_t forced = (flags & CPU_FORCED) != 0;
 
 	ASSERT(MUTEX_HELD(&cpu_lock));
 
+	if (cp->cpu_flags & CPU_DISABLED)
+		return (EINVAL);
+
 	/*
 	 * If we're going from faulted or spare to offline, just
 	 * clear these flags and update CPU state.
@@ -1309,7 +1321,7 @@ cpu_offline(cpu_t *cp, int flags)
 	 * Unbind all soft-bound threads bound to our CPU and hard bound threads
 	 * if we were asked to.
 	 */
-	error = cpu_unbind(cp->cpu_id, unbind_all_threads);
+	error = cpu_unbind(cp->cpu_id, forced);
 	if (error != 0)
 		return (error);
 	/*
@@ -1614,6 +1626,9 @@ cpu_faulted(cpu_t *cp, int flags)
 	ASSERT(MUTEX_HELD(&cpu_lock));
 	ASSERT(!cpu_is_poweredoff(cp));
 
+	if (cp->cpu_flags & CPU_DISABLED)
+		return (EINVAL);
+
 	if (cpu_is_offline(cp)) {
 		cp->cpu_flags &= ~CPU_SPARE;
 		cp->cpu_flags |= CPU_FAULTED;
@@ -1642,6 +1657,9 @@ cpu_spare(cpu_t *cp, int flags)
 	ASSERT(MUTEX_HELD(&cpu_lock));
 	ASSERT(!cpu_is_poweredoff(cp));
 
+	if (cp->cpu_flags & CPU_DISABLED)
+		return (EINVAL);
+
 	if (cpu_is_offline(cp)) {
 		if (cp->cpu_flags & CPU_FAULTED) {
 			cp->cpu_flags &= ~CPU_FAULTED;
@@ -2267,28 +2285,9 @@ cpu_info_kstat_update(kstat_t *ksp, int rw)
 	if (cpuid_checkpass(cp, 1) == 0)
 		return (ENXIO);
 #endif
-	switch (cp->cpu_type_info.pi_state) {
-	case P_ONLINE:
-		pi_state = PS_ONLINE;
-		break;
-	case P_POWEROFF:
-		pi_state = PS_POWEROFF;
-		break;
-	case P_NOINTR:
-		pi_state = PS_NOINTR;
-		break;
-	case P_FAULTED:
-		pi_state = PS_FAULTED;
-		break;
-	case P_SPARE:
-		pi_state = PS_SPARE;
-		break;
-	case P_OFFLINE:
-		pi_state = PS_OFFLINE;
-		break;
-	default:
-		pi_state = "unknown";
-	}
+
+	pi_state = cpu_get_state_str(cp->cpu_flags);
+
 	(void) strcpy(cpu_info_template.ci_state.value.c, pi_state);
 	cpu_info_template.ci_state_begin.value.l = cp->cpu_state_begin;
 	(void) strncpy(cpu_info_template.ci_cpu_type.value.c,
@@ -3170,33 +3169,41 @@ cpu_set_state(cpu_t *cpu)
  * communication with user applications; cpu_flags provides the in-kernel
  * interface.
  */
-int
-cpu_get_state(cpu_t *cpu)
+static int
+cpu_flags_to_state(cpu_flag_t flags)
 {
-	ASSERT(MUTEX_HELD(&cpu_lock));
-	if (cpu->cpu_flags & CPU_POWEROFF)
+	if (flags & CPU_DISABLED)
+		return (P_DISABLED);
+	else if (flags & CPU_POWEROFF)
 		return (P_POWEROFF);
-	else if (cpu->cpu_flags & CPU_FAULTED)
+	else if (flags & CPU_FAULTED)
 		return (P_FAULTED);
-	else if (cpu->cpu_flags & CPU_SPARE)
+	else if (flags & CPU_SPARE)
 		return (P_SPARE);
-	else if ((cpu->cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)
+	else if ((flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)
 		return (P_OFFLINE);
-	else if (cpu->cpu_flags & CPU_ENABLE)
+	else if (flags & CPU_ENABLE)
 		return (P_ONLINE);
 	else
 		return (P_NOINTR);
 }
 
+int
+cpu_get_state(cpu_t *cpu)
+{
+	ASSERT(MUTEX_HELD(&cpu_lock));
+	return (cpu_flags_to_state(cpu->cpu_flags));
+}
+
 /*
  * Return processor_info(2) state as a string.
  */
 const char *
-cpu_get_state_str(cpu_t *cpu)
+cpu_get_state_str(cpu_flag_t flags)
 {
 	const char *string;
 
-	switch (cpu_get_state(cpu)) {
+	switch (cpu_flags_to_state(flags)) {
 	case P_ONLINE:
 		string = PS_ONLINE;
 		break;
@@ -3215,6 +3222,9 @@ cpu_get_state_str(cpu_t *cpu)
 	case P_OFFLINE:
 		string = PS_OFFLINE;
 		break;
+	case P_DISABLED:
+		string = PS_DISABLED;
+		break;
 	default:
 		string = "unknown";
 		break;
diff --git a/usr/src/uts/common/os/lgrp.c b/usr/src/uts/common/os/lgrp.c
index 6f6aced619..d8039f1a1f 100644
--- a/usr/src/uts/common/os/lgrp.c
+++ b/usr/src/uts/common/os/lgrp.c
@@ -21,7 +21,7 @@
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
- * Copyright 2018 Joyent, Inc.
+ * Copyright 2019 Joyent, Inc.
  */
 
 /*
@@ -91,7 +91,7 @@
 #include <sys/pg.h>
 #include <sys/promif.h>
 #include <sys/sdt.h>
-#include <sys/ht.h>
+#include <sys/smt.h>
 
 lgrp_gen_t	lgrp_gen = 0;		/* generation of lgroup hierarchy */
 lgrp_t *lgrp_table[NLGRPS_MAX]; /* table of all initialized lgrp_t structs */
@@ -522,7 +522,7 @@ lgrp_main_mp_init(void)
 {
 	klgrpset_t changed;
 
-	ht_init();
+	smt_init();
 
 	/*
 	 * Update lgroup topology (if necessary)
diff --git a/usr/src/uts/common/os/main.c b/usr/src/uts/common/os/main.c
index 3364d1e523..9e2d03e392 100644
--- a/usr/src/uts/common/os/main.c
+++ b/usr/src/uts/common/os/main.c
@@ -27,7 +27,7 @@
 /*	  All Rights Reserved		*/
 
 /*
- * Copyright 2018, Joyent, Inc.
+ * Copyright 2019 Joyent, Inc.
  */
 
 #include <sys/types.h>
@@ -74,6 +74,7 @@
 #include <sys/stack.h>
 #include <sys/brand.h>
 #include <sys/mmapobj.h>
+#include <sys/smt.h>
 
 #include <vm/as.h>
 #include <vm/seg_kmem.h>
@@ -625,6 +626,8 @@ main(void)
 	pm_cfb_setup_intr();
 #if defined(__x86)
 	fastboot_post_startup();
+
+	smt_late_init();
 #endif
 
 	/*
diff --git a/usr/src/uts/common/os/pool_pset.c b/usr/src/uts/common/os/pool_pset.c
index 7f056a6c62..661d4db3ff 100644
--- a/usr/src/uts/common/os/pool_pset.c
+++ b/usr/src/uts/common/os/pool_pset.c
@@ -24,7 +24,9 @@
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
+/*
+ * Copyright 2019 Joyent, Inc.
+ */
 
 #include <sys/pool.h>
 #include <sys/pool_impl.h>
@@ -889,7 +891,7 @@ pool_pset_pack(ea_object_t *eo_system)
 			(void) nvlist_dup(cpu->cpu_props, &nvl, KM_SLEEP);
 			(void) nvlist_add_int64(nvl, "cpu.sys_id", cpu->cpu_id);
 			(void) nvlist_add_string(nvl, "cpu.status",
-			    (char *)cpu_get_state_str(cpu));
+			    (char *)cpu_get_state_str(cpu->cpu_flags));
 			buf = NULL;
 			bufsz = 0;
 			(void) nvlist_pack(nvl, &buf, &bufsz,
@@ -967,7 +969,7 @@ pool_cpu_propget(processorid_t cpuid, char *name, nvlist_t *nvl)
 	}
 	if (strcmp(name, "cpu.status") == 0) {
 		ret = nvlist_add_string(nvl, "cpu.status",
-		    (char *)cpu_get_state_str(cpu));
+		    (char *)cpu_get_state_str(cpu->cpu_flags));
 	} else {
 		ret = EINVAL;
 	}
diff --git a/usr/src/uts/common/sys/cpuvar.h b/usr/src/uts/common/sys/cpuvar.h
index e2ac2ef58d..21bdfbd160 100644
--- a/usr/src/uts/common/sys/cpuvar.h
+++ b/usr/src/uts/common/sys/cpuvar.h
@@ -330,10 +330,13 @@ extern cpu_core_t cpu_core[];
  * suspended (in the suspend path), or have yet to be resumed (in the resume
  * case).
  *
+ * CPU_DISABLED is used for disabling SMT. It is similar to CPU_OFFLINE, but
+ * cannot be onlined without being forced.
+ *
  * On some platforms CPUs can be individually powered off.
  * The following flags are set for powered off CPUs: CPU_QUIESCED,
  * CPU_OFFLINE, and CPU_POWEROFF.  The following flags are cleared:
- * CPU_RUNNING, CPU_READY, CPU_EXISTS, and CPU_ENABLE.
+ * CPU_RUNNING, CPU_READY, CPU_EXISTS, CPU_DISABLED and CPU_ENABLE.
  */
 #define	CPU_RUNNING	0x001		/* CPU running */
 #define	CPU_READY	0x002		/* CPU ready for cross-calls */
@@ -345,10 +348,7 @@ extern cpu_core_t cpu_core[];
 #define	CPU_FROZEN	0x080		/* CPU is frozen via CPR suspend */
 #define	CPU_SPARE	0x100		/* CPU offline available for use */
 #define	CPU_FAULTED	0x200		/* CPU offline diagnosed faulty */
-
-#define	FMT_CPU_FLAGS							\
-	"\20\12fault\11spare\10frozen"					\
-	"\7poweroff\6offline\5enable\4exist\3quiesced\2ready\1run"
+#define	CPU_DISABLED	0x400		/* CPU explicitly disabled (HT) */
 
 #define	CPU_ACTIVE(cpu)	(((cpu)->cpu_flags & CPU_OFFLINE) == 0)
 
@@ -517,6 +517,7 @@ extern cpuset_t		cpu_active_set;	/* cached set of active CPUs */
 extern cpuset_t		cpu_available;	/* cached set of available CPUs */
 extern int		ncpus;		/* number of CPUs present */
 extern int		ncpus_online;	/* number of CPUs not quiesced */
+extern int		ncpus_intr_enabled; /* nr of CPUs taking I/O intrs */
 extern int		max_ncpus;	/* max present before ncpus is known */
 extern int		boot_max_ncpus;	/* like max_ncpus but for real */
 extern int		boot_ncpus;	/* # cpus present @ boot */
@@ -634,12 +635,12 @@ int	cpus_paused(void);
 void	cpu_pause_init(void);
 cpu_t	*cpu_get(processorid_t cpun);	/* get the CPU struct associated */
 
-int	cpu_online(cpu_t *cp);			/* take cpu online */
-int	cpu_offline(cpu_t *cp, int flags);	/* take cpu offline */
-int	cpu_spare(cpu_t *cp, int flags);	/* take cpu to spare */
-int	cpu_faulted(cpu_t *cp, int flags);	/* take cpu to faulted */
-int	cpu_poweron(cpu_t *cp);		/* take powered-off cpu to offline */
-int	cpu_poweroff(cpu_t *cp);	/* take offline cpu to powered-off */
+int	cpu_online(cpu_t *, int);	/* take cpu online */
+int	cpu_offline(cpu_t *, int);	/* take cpu offline */
+int	cpu_spare(cpu_t *, int);	/* take cpu to spare */
+int	cpu_faulted(cpu_t *, int);	/* take cpu to faulted */
+int	cpu_poweron(cpu_t *);		/* take powered-off cpu to offline */
+int	cpu_poweroff(cpu_t *);		/* take offline cpu to powered-off */
 
 cpu_t	*cpu_intr_next(cpu_t *cp);	/* get next online CPU taking intrs */
 int	cpu_intr_count(cpu_t *cp);	/* count # of CPUs handling intrs */
@@ -672,7 +673,7 @@ int	cpu_flagged_poweredoff(cpu_flag_t); /* flags show CPU is powered off */
  */
 void	cpu_set_state(cpu_t *);		/* record/timestamp current state */
 int	cpu_get_state(cpu_t *);		/* get current cpu state */
-const char *cpu_get_state_str(cpu_t *);	/* get current cpu state as string */
+const char *cpu_get_state_str(cpu_flag_t);
 
 
 void	cpu_set_curr_clock(uint64_t);	/* indicate the current CPU's freq */
diff --git a/usr/src/uts/common/sys/processor.h b/usr/src/uts/common/sys/processor.h
index ec4b7471e5..4264a89a2a 100644
--- a/usr/src/uts/common/sys/processor.h
+++ b/usr/src/uts/common/sys/processor.h
@@ -31,6 +31,10 @@
  * Use is subject to license terms.
  */
 
+/*
+ * Copyright 2019 Joyent, Inc.
+ */
+
 #ifndef _SYS_PROCESSOR_H
 #define	_SYS_PROCESSOR_H
 
@@ -68,8 +72,9 @@ typedef int	chipid_t;
 #define	P_POWEROFF	0x0005	/* processor is powered off */
 #define	P_NOINTR	0x0006	/* processor is online, but no I/O interrupts */
 #define	P_SPARE		0x0007	/* processor is offline, can be reactivated */
+#define	P_DISABLED	0x0008	/* processor is explicitly disabled for use */
 #define	P_BAD		P_FAULTED	/* unused but defined by USL */
-#define	P_FORCED 	0x10000000	/* force processor offline */
+#define	P_FORCED	0x10000000	/* force processor offline */
 
 /*
  * String names for processor states defined above.
@@ -80,6 +85,7 @@ typedef int	chipid_t;
 #define	PS_POWEROFF	"powered-off"
 #define	PS_NOINTR	"no-intr"
 #define	PS_SPARE	"spare"
+#define	PS_DISABLED	"disabled"
 
 /*
  * Structure filled in by processor_info(2). This structure
@@ -95,7 +101,7 @@ typedef int	chipid_t;
 #define	PI_FPUTYPE	32	/* max size of FPU types string */
 
 typedef struct {
-	int	pi_state;  			/* processor state, see above */
+	int	pi_state;			/* processor state, see above */
 	char	pi_processor_type[PI_TYPELEN];	/* ASCII CPU type */
 	char	pi_fputypes[PI_FPUTYPE];	/* ASCII FPU types */
 	int	pi_clock;			/* CPU clock freq in MHz */
@@ -111,6 +117,11 @@ typedef struct {
 #define	PBIND_QUERY_TYPE	-5	/* Return binding type */
 
 /*
+ * Sentinel values for p_online(2)
+ */
+#define	P_ALL_SIBLINGS (-1)
+
+/*
  * User-level system call interface prototypes
  */
 #ifndef _KERNEL
diff --git a/usr/src/uts/common/sys/thread.h b/usr/src/uts/common/sys/thread.h
index 6cc474f864..955ba918cd 100644
--- a/usr/src/uts/common/sys/thread.h
+++ b/usr/src/uts/common/sys/thread.h
@@ -25,7 +25,7 @@
  */
 
 /*
- * Copyright 2018 Joyent, Inc.
+ * Copyright 2019 Joyent, Inc.
  */
 
 #ifndef	_SYS_THREAD_H
@@ -355,7 +355,7 @@ typedef struct _kthread {
 
 	char		*t_name;	/* thread name */
 
-	uint64_t	t_unsafe;	/* unsafe to run with HT VCPU thread */
+	uint64_t	t_unsafe;	/* unsafe to run with SMT VCPU thread */
 } kthread_t;
 
 /*
diff --git a/usr/src/uts/common/syscall/p_online.c b/usr/src/uts/common/syscall/p_online.c
index 88a0340bf7..e3156aeab1 100644
--- a/usr/src/uts/common/syscall/p_online.c
+++ b/usr/src/uts/common/syscall/p_online.c
@@ -23,6 +23,10 @@
  * Use is subject to license terms.
  */
 
+/*
+ * Copyright 2019 Joyent, Inc.
+ */
+
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/var.h>
@@ -37,13 +41,15 @@
 #include <sys/processor.h>
 #include <sys/debug.h>
 #include <sys/policy.h>
+#include <sys/smt.h>
 
 /*
  * CPU state diagram
  *
- *                   P_SPARE
+ *                  P_SPARE
  * P_POWEROFF <---> P_OFFLINE <---> P_ONLINE <---> P_NOINTR
  *                  P_FAULTED
+ *                  P_DISABLED
  */
 int
 p_online_internal_locked(processorid_t cpun, int new_status, int *old_status)
@@ -53,10 +59,17 @@ p_online_internal_locked(processorid_t cpun, int new_status, int *old_status)
 	int	error = 0;
 	int	flags = 0;
 
-	/*
-	 * Try to get a pointer to the requested CPU structure.
-	 */
 	ASSERT(MUTEX_HELD(&cpu_lock));
+
+	if (cpun == P_ALL_SIBLINGS) {
+		if (new_status != P_DISABLED) {
+			error = EINVAL;
+			goto out;
+		}
+
+		return (smt_disable());
+	}
+
 	if ((cp = cpu_get(cpun)) == NULL) {
 		error = EINVAL;
 		goto out;
@@ -81,8 +94,10 @@ p_online_internal_locked(processorid_t cpun, int new_status, int *old_status)
 		if (secpolicy_ponline(CRED()) != 0)
 			error = EPERM;
 		break;
+	case P_DISABLED:
 	default:
 		error = EINVAL;
+		break;
 	}
 
 	if (error)
@@ -105,6 +120,7 @@ p_online_internal_locked(processorid_t cpun, int new_status, int *old_status)
 				break;
 			ASSERT(cpu_get_state(cp) == P_OFFLINE);
 			/* FALLTHROUGH */
+		case P_DISABLED:
 		case P_OFFLINE:
 		case P_FAULTED:
 		case P_SPARE:
@@ -112,7 +128,7 @@ p_online_internal_locked(processorid_t cpun, int new_status, int *old_status)
 			 * If CPU is in one of the offline states,
 			 * bring it online.
 			 */
-			error = cpu_online(cp);
+			error = cpu_online(cp, flags);
 			break;
 		case P_NOINTR:
 			cpu_intr_enable(cp);
@@ -130,6 +146,7 @@ p_online_internal_locked(processorid_t cpun, int new_status, int *old_status)
 			cpu_intr_enable(cp);
 			/* FALLTHROUGH */
 		case P_ONLINE:
+		case P_DISABLED:
 		case P_FAULTED:
 		case P_SPARE:
 			/*
@@ -143,6 +160,7 @@ p_online_internal_locked(processorid_t cpun, int new_status, int *old_status)
 			 * If CPU is powered off, power it on.
 			 */
 			error = cpu_poweron(cp);
+			break;
 		}
 		break;
 
@@ -156,13 +174,14 @@ p_online_internal_locked(processorid_t cpun, int new_status, int *old_status)
 				break;
 			ASSERT(cpu_get_state(cp) == P_OFFLINE);
 			/* FALLTHROUGH */
+		case P_DISABLED:
 		case P_OFFLINE:
 		case P_FAULTED:
 		case P_SPARE:
 			/*
 			 * First, bring the CPU online.
 			 */
-			if (error = cpu_online(cp))
+			if (error = cpu_online(cp, flags))
 				break;
 			/* FALLTHROUGH */
 		case P_ONLINE:
@@ -170,6 +189,7 @@ p_online_internal_locked(processorid_t cpun, int new_status, int *old_status)
 			 * CPU is now online.  Try to disable interrupts.
 			 */
 			error = cpu_intr_disable(cp);
+			break;
 		}
 		break;
 
@@ -183,14 +203,16 @@ p_online_internal_locked(processorid_t cpun, int new_status, int *old_status)
 				break;
 			ASSERT(cpu_get_state(cp) == P_OFFLINE);
 			/*FALLTHROUGH*/
+		case P_DISABLED:
 		case P_OFFLINE:
-		case P_SPARE:
 		case P_ONLINE:
 		case P_NOINTR:
+		case P_SPARE:
 			/*
 			 * Mark this CPU as faulted.
 			 */
 			error = cpu_faulted(cp, flags);
+			break;
 		}
 		break;
 
@@ -204,6 +226,7 @@ p_online_internal_locked(processorid_t cpun, int new_status, int *old_status)
 				break;
 			ASSERT(cpu_get_state(cp) == P_OFFLINE);
 			/*FALLTHROUGH*/
+		case P_DISABLED:
 		case P_OFFLINE:
 		case P_FAULTED:
 		case P_ONLINE:
@@ -212,6 +235,7 @@ p_online_internal_locked(processorid_t cpun, int new_status, int *old_status)
 			 * Mark this CPU as a spare.
 			 */
 			error = cpu_spare(cp, flags);
+			break;
 		}
 		break;
 	}