15078 bhyve should enable VM_CAP_HALT_EXIT by default

Reviewed by: Luqman Aden <luqman@oxide.computer>
Reviewed by: Toomas Soome <tsoome@me.com>
Approved by: Dan McDonald <danmcd@mnx.io>

3 files changed, 37 insertions, 26 deletions

diff --git a/usr/src/uts/intel/io/vmm/amd/svm.c b/usr/src/uts/intel/io/vmm/amd/svm.c
index 3492ebd7e1..f4f01ea4b6 100644
--- a/usr/src/uts/intel/io/vmm/amd/svm.c
+++ b/usr/src/uts/intel/io/vmm/amd/svm.c
@@ -355,6 +355,9 @@ vmcb_init(struct svm_softc *sc, int vcpu, uint64_t iopm_base_pa,
 	svm_enable_intercept(sc, vcpu, VMCB_CTRL1_INTCPT,
 	    VMCB_INTCPT_FERR_FREEZE);
 
+	/* Enable exit-on-hlt by default */
+	svm_enable_intercept(sc, vcpu, VMCB_CTRL1_INTCPT, VMCB_INTCPT_HLT);
+
 	svm_enable_intercept(sc, vcpu, VMCB_CTRL2_INTCPT, VMCB_INTCPT_MONITOR);
 	svm_enable_intercept(sc, vcpu, VMCB_CTRL2_INTCPT, VMCB_INTCPT_MWAIT);
 
diff --git a/usr/src/uts/intel/io/vmm/intel/vmx.c b/usr/src/uts/intel/io/vmm/intel/vmx.c
index ccf506d608..c16fe1f1d4 100644
--- a/usr/src/uts/intel/io/vmm/intel/vmx.c
+++ b/usr/src/uts/intel/io/vmm/intel/vmx.c
@@ -98,10 +98,16 @@ __FBSDID("$FreeBSD$");
 	(PROCBASED_INT_WINDOW_EXITING	|				\
 	PROCBASED_NMI_WINDOW_EXITING)
 
-/* We consider TSC offset a necessity for unsynched TSC handling */
+/*
+ * Distinct from FreeBSD bhyve, we consider several additional proc-based
+ * controls necessary:
+ * - TSC offsetting
+ * - HLT exiting
+ */
 #define	PROCBASED_CTLS_ONE_SETTING					\
 	(PROCBASED_SECONDARY_CONTROLS	|				\
 	PROCBASED_TSC_OFFSET		|				\
+	PROCBASED_HLT_EXITING		|				\
 	PROCBASED_MWAIT_EXITING		|				\
 	PROCBASED_MONITOR_EXITING	|				\
 	PROCBASED_IO_EXITING		|				\
@@ -179,9 +185,6 @@ static int vmx_initialized;
  * Optional capabilities
  */
 
-/* HLT triggers a VM-exit */
-static int cap_halt_exit;
-
 /* PAUSE triggers a VM-exit */
 static int cap_pause_exit;
 
@@ -474,7 +477,10 @@ vmx_init(void)
 		return (error);
 	}
 
-	/* Clear the processor-based ctl bits that are set on demand */
+	/*
+	 * Clear interrupt-window/NMI-window exiting from the default proc-based
+	 * controls. They are set and cleared based on runtime vCPU events.
+	 */
 	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
 
 	/* Check support for secondary processor-based VM-execution controls */
@@ -532,11 +538,6 @@ vmx_init(void)
 	 * Check support for optional features by testing them
 	 * as individual bits
 	 */
-	cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
-	    MSR_VMX_TRUE_PROCBASED_CTLS,
-	    PROCBASED_HLT_EXITING, 0,
-	    &tmp) == 0);
-
 	cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 	    MSR_VMX_PROCBASED_CTLS,
 	    PROCBASED_MTF, 0,
@@ -748,6 +749,21 @@ vmx_vminit(struct vm *vm)
 		pin_ctls |= PINBASED_POSTED_INTERRUPT;
 	}
 
+	/* Reflect any enabled defaults in the cap set */
+	int cap_defaults = 0;
+	if ((proc_ctls & PROCBASED_HLT_EXITING) != 0) {
+		cap_defaults |= (1 << VM_CAP_HALT_EXIT);
+	}
+	if ((proc_ctls & PROCBASED_PAUSE_EXITING) != 0) {
+		cap_defaults |= (1 << VM_CAP_PAUSE_EXIT);
+	}
+	if ((proc_ctls & PROCBASED_MTF) != 0) {
+		cap_defaults |= (1 << VM_CAP_MTRAP_EXIT);
+	}
+	if ((proc2_ctls & PROCBASED2_ENABLE_INVPCID) != 0) {
+		cap_defaults |= (1 << VM_CAP_ENABLE_INVPCID);
+	}
+
 	maxcpus = vm_get_maxcpus(vm);
 	datasel = vmm_get_host_datasel();
 	for (i = 0; i < maxcpus; i++) {
@@ -857,7 +873,7 @@ vmx_vminit(struct vm *vm)
 
 		vmcs_clear(vmx->vmcs_pa[i]);
 
-		vmx->cap[i].set = 0;
+		vmx->cap[i].set = cap_defaults;
 		vmx->cap[i].proc_ctls = proc_ctls;
 		vmx->cap[i].proc_ctls2 = proc2_ctls;
 		vmx->cap[i].exc_bitmap = exc_bitmap;
@@ -1019,11 +1035,6 @@ vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu)
 	vmx_invvpid(vmx, vcpu, 1);
 }
 
-/*
- * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
- */
-CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
-
 static __inline void
 vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
 {
@@ -3295,8 +3306,7 @@ vmx_getcap(void *arg, int vcpu, int type, int *retval)
 
 	switch (type) {
 	case VM_CAP_HALT_EXIT:
-		if (cap_halt_exit)
-			ret = 0;
+		ret = 0;
 		break;
 	case VM_CAP_PAUSE_EXIT:
 		if (cap_pause_exit)
@@ -3336,13 +3346,11 @@ vmx_setcap(void *arg, int vcpu, int type, int val)
 
 	switch (type) {
 	case VM_CAP_HALT_EXIT:
-		if (cap_halt_exit) {
-			error = 0;
-			pptr = &vmx->cap[vcpu].proc_ctls;
-			baseval = *pptr;
-			flag = PROCBASED_HLT_EXITING;
-			reg = VMCS_PRI_PROC_BASED_CTLS;
-		}
+		error = 0;
+		pptr = &vmx->cap[vcpu].proc_ctls;
+		baseval = *pptr;
+		flag = PROCBASED_HLT_EXITING;
+		reg = VMCS_PRI_PROC_BASED_CTLS;
 		break;
 	case VM_CAP_MTRAP_EXIT:
 		if (cap_monitor_trap) {
diff --git a/usr/src/uts/intel/sys/vmm_dev.h b/usr/src/uts/intel/sys/vmm_dev.h
index ce5d47fba9..3737686974 100644
--- a/usr/src/uts/intel/sys/vmm_dev.h
+++ b/usr/src/uts/intel/sys/vmm_dev.h
@@ -402,7 +402,7 @@ struct vm_legacy_cpuid {
  * best-effort activity.  Nothing is to be inferred about the magnitude of a
  * change when the version is modified.  It follows no rules like semver.
  */
-#define	VMM_CURRENT_INTERFACE_VERSION	5
+#define	VMM_CURRENT_INTERFACE_VERSION	6
 
 
 #define	VMMCTL_IOC_BASE		(('V' << 16) | ('M' << 8))