13256 bhyve should shadow %cr0 on AMD

13338 bhyve should be able to emulate CLTS
Reviewed by: Toomas Soome <tsoome@me.com>
Reviewed by: Joshua M. Clulow <josh@sysmgr.org>
Reviewed by: Andy Fiddaman <andy@omniosce.org>
Approved by: Richard Lowe <richlowe@richlowe.net>

8 files changed, 441 insertions, 27 deletions

diff --git a/usr/src/uts/i86pc/io/vmm/amd/svm.c b/usr/src/uts/i86pc/io/vmm/amd/svm.c
index 0f81bcc22b..b17515c259 100644
--- a/usr/src/uts/i86pc/io/vmm/amd/svm.c
+++ b/usr/src/uts/i86pc/io/vmm/amd/svm.c
@@ -132,17 +132,19 @@ static VMM_STAT_AMD(VCPU_INTINFO_INJECTED, "Events pending at VM entry");
 static VMM_STAT_AMD(VMEXIT_VINTR, "VM exits due to interrupt window");
 
 static int svm_setreg(void *arg, int vcpu, int ident, uint64_t val);
+static int svm_getreg(void *arg, int vcpu, int ident, uint64_t *val);
+static void flush_asid(struct svm_softc *sc, int vcpuid);
 
-static __inline int
+static __inline bool
 flush_by_asid(void)
 {
-	return (svm_feature & AMD_CPUID_SVM_FLUSH_BY_ASID);
+	return ((svm_feature & AMD_CPUID_SVM_FLUSH_BY_ASID) != 0);
 }
 
-static __inline int
+static __inline bool
 decode_assist(void)
 {
-	return (svm_feature & AMD_CPUID_SVM_DECODE_ASSIST);
+	return ((svm_feature & AMD_CPUID_SVM_DECODE_ASSIST) != 0);
 }
 
 #ifdef __FreeBSD__
@@ -476,6 +478,13 @@ vmcb_init(struct svm_softc *sc, int vcpu, uint64_t iopm_base_pa,
 	}
 
 	/*
+	 * Selectively intercept writes to %cr0.  This triggers on operations
+	 * which would change bits other than TS or MP.
+	 */
+	svm_enable_intercept(sc, vcpu, VMCB_CTRL1_INTCPT,
+	    VMCB_INTCPT_CR0_WRITE);
+
+	/*
 	 * Intercept everything when tracing guest exceptions otherwise
 	 * just intercept machine check exception.
 	 */
@@ -884,6 +893,166 @@ svm_handle_mmio_emul(struct svm_softc *svm_sc, int vcpu, struct vm_exit *vmexit,
 	vie_init_mmio(vie, inst_bytes, inst_len, &paging, gpa);
 }
 
+/*
+ * Do not allow CD, NW, or invalid high bits to be asserted in the value of cr0
+ * which is live in the guest.  They are visible via the shadow instead.
+ */
+#define	SVM_CR0_MASK	~(CR0_CD | CR0_NW | 0xffffffff00000000)
+
+static void
+svm_set_cr0(struct svm_softc *svm_sc, int vcpu, uint64_t val, bool guest_write)
+{
+	struct vmcb_state *state;
+	struct svm_regctx *regctx;
+	uint64_t masked, old, diff;
+
+	state = svm_get_vmcb_state(svm_sc, vcpu);
+	regctx = svm_get_guest_regctx(svm_sc, vcpu);
+
+	old = state->cr0 | (regctx->sctx_cr0_shadow & ~SVM_CR0_MASK);
+	diff = old ^ val;
+
+	/* No further work needed if register contents remain the same */
+	if (diff == 0) {
+		return;
+	}
+
+	/* Flush the TLB if the paging or write-protect bits are changing */
+	if ((diff & CR0_PG) != 0 || (diff & CR0_WP) != 0) {
+		flush_asid(svm_sc, vcpu);
+	}
+
+	/*
+	 * If the change in %cr0 is due to a guest action (via interception)
+	 * then other CPU state updates may be required.
+	 */
+	if (guest_write) {
+		if ((diff & CR0_PG) != 0) {
+			uint64_t efer = state->efer;
+
+			/* Keep the long-mode state in EFER in sync */
+			if ((val & CR0_PG) != 0 && (efer & EFER_LME) != 0) {
+				state->efer |= EFER_LMA;
+			}
+			if ((val & CR0_PG) == 0 && (efer & EFER_LME) != 0) {
+				state->efer &= ~EFER_LMA;
+			}
+		}
+	}
+
+	masked = val & SVM_CR0_MASK;
+	regctx->sctx_cr0_shadow = val;
+	state->cr0 = masked;
+	svm_set_dirty(svm_sc, vcpu, VMCB_CACHE_CR);
+
+	if ((masked ^ val) != 0) {
+		/*
+		 * The guest has set bits in %cr0 which we are masking out and
+		 * exposing via shadow.
+		 *
+		 * We must intercept %cr0 reads in order to make the shadowed
+		 * view available to the guest.
+		 *
+		 * Writes to %cr0 must also be intercepted (unconditionally,
+		 * unlike the VMCB_INTCPT_CR0_WRITE mechanism) so we can catch
+		 * if/when the guest clears those shadowed bits.
+		 */
+		svm_enable_intercept(svm_sc, vcpu, VMCB_CR_INTCPT,
+		    BIT(0) | BIT(16));
+	} else {
+		/*
+		 * When no bits remain in %cr0 which require shadowing, the
+		 * unconditional intercept of reads/writes to %cr0 can be
+		 * disabled.
+		 *
+		 * The selective write intercept (VMCB_INTCPT_CR0_WRITE) remains
+		 * in place so we can be notified of operations which change
+		 * bits other than TS or MP.
+		 */
+		svm_disable_intercept(svm_sc, vcpu, VMCB_CR_INTCPT,
+		    BIT(0) | BIT(16));
+	}
+	svm_set_dirty(svm_sc, vcpu, VMCB_CACHE_I);
+}
+
+static void
+svm_get_cr0(struct svm_softc *svm_sc, int vcpu, uint64_t *val)
+{
+	struct vmcb *vmcb;
+	struct svm_regctx *regctx;
+
+	vmcb = svm_get_vmcb(svm_sc, vcpu);
+	regctx = svm_get_guest_regctx(svm_sc, vcpu);
+
+	/*
+	 * Include the %cr0 bits which exist only in the shadow along with those
+	 * in the running vCPU state.
+	 */
+	*val = vmcb->state.cr0 | (regctx->sctx_cr0_shadow & ~SVM_CR0_MASK);
+}
+
+static void
+svm_handle_cr0_read(struct svm_softc *svm_sc, int vcpu, enum vm_reg_name reg)
+{
+	uint64_t val;
+	int err;
+
+	svm_get_cr0(svm_sc, vcpu, &val);
+	err = svm_setreg(svm_sc, vcpu, reg, val);
+	ASSERT(err == 0);
+}
+
+static void
+svm_handle_cr0_write(struct svm_softc *svm_sc, int vcpu, enum vm_reg_name reg)
+{
+	struct vmcb_state *state;
+	uint64_t val;
+	int err;
+
+	state = svm_get_vmcb_state(svm_sc, vcpu);
+
+	err = svm_getreg(svm_sc, vcpu, reg, &val);
+	ASSERT(err == 0);
+
+	if ((val & CR0_NW) != 0 && (val & CR0_CD) == 0) {
+		/* NW without CD is nonsensical */
+		vm_inject_gp(svm_sc->vm, vcpu);
+		return;
+	}
+	if ((val & CR0_PG) != 0 && (val & CR0_PE) == 0) {
+		/* PG requires PE */
+		vm_inject_gp(svm_sc->vm, vcpu);
+		return;
+	}
+	if ((state->cr0 & CR0_PG) == 0 && (val & CR0_PG) != 0) {
+		/* When enabling paging, PAE must be enabled if LME is. */
+		if ((state->efer & EFER_LME) != 0 &&
+		    (state->cr4 & CR4_PAE) == 0) {
+			vm_inject_gp(svm_sc->vm, vcpu);
+			return;
+		}
+	}
+
+	svm_set_cr0(svm_sc, vcpu, val, true);
+}
+
+static void
+svm_inst_emul_other(struct svm_softc *svm_sc, int vcpu, struct vm_exit *vmexit)
+{
+	struct vie *vie;
+	struct vm_guest_paging paging;
+
+	/* Let the instruction emulation (hopefully in-kernel) handle it */
+	vmexit->exitcode = VM_EXITCODE_INST_EMUL;
+	bzero(&vmexit->u.inst_emul, sizeof (vmexit->u.inst_emul));
+	vie = vm_vie_ctx(svm_sc->vm, vcpu);
+	svm_paging_info(svm_get_vmcb(svm_sc, vcpu), &paging);
+	vie_init_other(vie, &paging);
+
+	/* The instruction emulation will handle advancing %rip */
+	vmexit->inst_length = 0;
+}
+
 static void
 svm_update_virqinfo(struct svm_softc *sc, int vcpu)
 {
@@ -1282,6 +1451,41 @@ svm_vmexit(struct svm_softc *svm_sc, int vcpu, struct vm_exit *vmexit)
 	svm_save_exitintinfo(svm_sc, vcpu);
 
 	switch (code) {
+	case VMCB_EXIT_CR0_READ:
+		if (VMCB_CRx_INFO1_VALID(info1) != 0) {
+			svm_handle_cr0_read(svm_sc, vcpu,
+			    vie_regnum_map(VMCB_CRx_INFO1_GPR(info1)));
+			handled = 1;
+		} else {
+			/*
+			 * If SMSW is used to read the contents of %cr0, then
+			 * the VALID bit will not be set in `info1`, since the
+			 * handling is different from the mov-to-reg case.
+			 *
+			 * Punt to the instruction emulation to handle it.
+			 */
+			svm_inst_emul_other(svm_sc, vcpu, vmexit);
+		}
+		break;
+	case VMCB_EXIT_CR0_WRITE:
+	case VMCB_EXIT_CR0_SEL_WRITE:
+		if (VMCB_CRx_INFO1_VALID(info1) != 0) {
+			svm_handle_cr0_write(svm_sc, vcpu,
+			    vie_regnum_map(VMCB_CRx_INFO1_GPR(info1)));
+			handled = 1;
+		} else {
+			/*
+			 * Writes to %cr0 without VALID being set in `info1` are
+			 * initiated by the LMSW and CLTS instructions.  While
+			 * LMSW (like SMSW) sees little use in modern OSes and
+			 * bootloaders, CLTS is still used for handling FPU
+			 * state transitions.
+			 *
+			 * Punt to the instruction emulation to handle them.
+			 */
+			svm_inst_emul_other(svm_sc, vcpu, vmexit);
+		}
+		break;
 	case VMCB_EXIT_IRET:
 		/*
 		 * Restart execution at "iret" but with the intercept cleared.
@@ -1844,6 +2048,27 @@ check_asid(struct svm_softc *sc, int vcpuid, pmap_t pmap, uint_t thiscpu)
 	ctrl->tlb_ctrl = flush;
 	vcpustate->eptgen = eptgen;
 }
+
+static void
+flush_asid(struct svm_softc *sc, int vcpuid)
+{
+	struct svm_vcpu *vcpustate = svm_get_vcpu(sc, vcpuid);
+	struct vmcb_ctrl *ctrl = svm_get_vmcb_ctrl(sc, vcpuid);
+	uint8_t flush;
+
+	flush = hma_svm_asid_update(&vcpustate->hma_asid, flush_by_asid(),
+	    true);
+
+	ASSERT(flush != VMCB_TLB_FLUSH_NOTHING);
+	ctrl->asid = vcpustate->hma_asid.hsa_asid;
+	ctrl->tlb_ctrl = flush;
+	svm_set_dirty(sc, vcpuid, VMCB_CACHE_ASID);
+	/*
+	 * A potential future optimization: We could choose to update the eptgen
+	 * associated with the vCPU, since any pending eptgen change requiring a
+	 * flush will be satisfied by the one which has just now been queued.
+	 */
+}
 #endif /* __FreeBSD__ */
 
 static __inline void
@@ -2180,6 +2405,8 @@ svm_getreg(void *arg, int vcpu, int ident, uint64_t *val)
 		break;
 
 	case VM_REG_GUEST_CR0:
+		svm_get_cr0(sc, vcpu, val);
+		break;
 	case VM_REG_GUEST_CR2:
 	case VM_REG_GUEST_CR3:
 	case VM_REG_GUEST_CR4:
@@ -2251,6 +2478,8 @@ svm_setreg(void *arg, int vcpu, int ident, uint64_t val)
 		break;
 
 	case VM_REG_GUEST_CR0:
+		svm_set_cr0(sc, vcpu, val, false);
+		break;
 	case VM_REG_GUEST_CR2:
 	case VM_REG_GUEST_CR3:
 	case VM_REG_GUEST_CR4:
diff --git a/usr/src/uts/i86pc/io/vmm/amd/svm.h b/usr/src/uts/i86pc/io/vmm/amd/svm.h
index a3a83dba19..127c04ab6e 100644
--- a/usr/src/uts/i86pc/io/vmm/amd/svm.h
+++ b/usr/src/uts/i86pc/io/vmm/amd/svm.h
@@ -53,6 +53,7 @@ struct svm_regctx {
 	uint64_t	sctx_dr1;
 	uint64_t	sctx_dr2;
 	uint64_t	sctx_dr3;
+	uint64_t	sctx_cr0_shadow;
 
 	uint64_t	host_dr0;
 	uint64_t	host_dr1;
diff --git a/usr/src/uts/i86pc/io/vmm/amd/vmcb.c b/usr/src/uts/i86pc/io/vmm/amd/vmcb.c
index b00f974c23..5be5240129 100644
--- a/usr/src/uts/i86pc/io/vmm/amd/vmcb.c
+++ b/usr/src/uts/i86pc/io/vmm/amd/vmcb.c
@@ -91,11 +91,6 @@ vmcb_regptr(struct vmcb *vmcb, int ident, uint32_t *dirtyp)
 	state = &vmcb->state;
 
 	switch (ident) {
-	case VM_REG_GUEST_CR0:
-		res = &state->cr0;
-		dirty = VMCB_CACHE_CR;
-		break;
-
 	case VM_REG_GUEST_CR2:
 		res = &state->cr2;
 		dirty = VMCB_CACHE_CR2;
diff --git a/usr/src/uts/i86pc/io/vmm/amd/vmcb.h b/usr/src/uts/i86pc/io/vmm/amd/vmcb.h
index 41bbf98097..0685165530 100644
--- a/usr/src/uts/i86pc/io/vmm/amd/vmcb.h
+++ b/usr/src/uts/i86pc/io/vmm/amd/vmcb.h
@@ -141,10 +141,15 @@ struct svm_softc;
 #define	VMCB_EVENTINJ_TYPE_INTn		(4 << 8)
 
 /* VMCB exit code, APM vol2 Appendix C */
+#define	VMCB_EXIT_CR0_READ		0x00
+#define	VMCB_EXIT_CR15_READ		0x0f
+#define	VMCB_EXIT_CR0_WRITE		0x10
+#define	VMCB_EXIT_CR15_WRITE		0x1f
 #define	VMCB_EXIT_MC			0x52
 #define	VMCB_EXIT_INTR			0x60
 #define	VMCB_EXIT_NMI			0x61
 #define	VMCB_EXIT_VINTR			0x64
+#define	VMCB_EXIT_CR0_SEL_WRITE		0x65
 #define	VMCB_EXIT_PUSHF			0x70
 #define	VMCB_EXIT_POPF			0x71
 #define	VMCB_EXIT_CPUID			0x72
@@ -170,6 +175,13 @@ struct svm_softc;
 #define	VMCB_EXIT_INVALID		-1
 
 /*
+ * Move to/from CRx
+ * Bit definitions to decode EXITINFO1
+ */
+#define	VMCB_CRx_INFO1_GPR(x)		((x) & 0xf)
+#define	VMCB_CRx_INFO1_VALID(x)		((x) & (1UL << 63))
+
+/*
  * Nested page fault.
  * Bit definitions to decode EXITINFO1.
  */
diff --git a/usr/src/uts/i86pc/io/vmm/sys/vmm_instruction_emul.h b/usr/src/uts/i86pc/io/vmm/sys/vmm_instruction_emul.h
index 75abfeeaf6..4680c86a56 100644
--- a/usr/src/uts/i86pc/io/vmm/sys/vmm_instruction_emul.h
+++ b/usr/src/uts/i86pc/io/vmm/sys/vmm_instruction_emul.h
@@ -51,10 +51,13 @@ struct vie;
 struct vie *vie_alloc();
 void vie_free(struct vie *);
 
+enum vm_reg_name vie_regnum_map(uint8_t);
+
 void vie_init_mmio(struct vie *vie, const char *inst_bytes, uint8_t inst_length,
     const struct vm_guest_paging *paging, uint64_t gpa);
 void vie_init_inout(struct vie *vie, const struct vm_inout *inout,
     uint8_t inst_len, const struct vm_guest_paging *paging);
+void vie_init_other(struct vie *vie, const struct vm_guest_paging *paging);
 
 int vie_fulfill_mmio(struct vie *vie, const struct vm_mmio *res);
 int vie_fulfill_inout(struct vie *vie, const struct vm_inout *res);
@@ -64,12 +67,15 @@ bool vie_pending(const struct vie *vie);
 uint64_t vie_mmio_gpa(const struct vie *vie);
 void vie_exitinfo(const struct vie *vie, struct vm_exit *vme);
 void vie_fallback_exitinfo(const struct vie *vie, struct vm_exit *vme);
+void vie_cs_info(const struct vie *vie, struct vm *vm, int vcpuid,
+    uint64_t *cs_base, int *cs_d);
 
 void vie_reset(struct vie *vie);
 void vie_advance_pc(struct vie *vie, uint64_t *nextrip);
 
 int vie_emulate_mmio(struct vie *vie, struct vm *vm, int vcpuid);
 int vie_emulate_inout(struct vie *vie, struct vm *vm, int vcpuid);
+int vie_emulate_other(struct vie *vie, struct vm *vm, int vcpuid);
 
 /*
  * APIs to fetch and decode the instruction from nested page fault handler.
diff --git a/usr/src/uts/i86pc/io/vmm/vmm.c b/usr/src/uts/i86pc/io/vmm/vmm.c
index cd235b9e4c..1cd0b23a1c 100644
--- a/usr/src/uts/i86pc/io/vmm/vmm.c
+++ b/usr/src/uts/i86pc/io/vmm/vmm.c
@@ -1615,9 +1615,10 @@ vm_handle_mmio_emul(struct vm *vm, int vcpuid)
 			return (error);
 		} else if (fault) {
 			/*
-			 * If a fault during instruction fetch was encounted, it
-			 * will have asserted that the appropriate exception be
-			 * injected at next entry.  No further work is required.
+			 * If a fault during instruction fetch was encountered,
+			 * it will have asserted that the appropriate exception
+			 * be injected at next entry.
+			 * No further work is required.
 			 */
 			return (0);
 		}
@@ -1724,6 +1725,56 @@ repeat:
 }
 
 static int
+vm_handle_inst_emul(struct vm *vm, int vcpuid)
+{
+	struct vie *vie;
+	struct vcpu *vcpu;
+	struct vm_exit *vme;
+	uint64_t cs_base;
+	int error, fault, cs_d;
+
+	vcpu = &vm->vcpu[vcpuid];
+	vme = &vcpu->exitinfo;
+	vie = vcpu->vie_ctx;
+
+	vie_cs_info(vie, vm, vcpuid, &cs_base, &cs_d);
+
+	/* Fetch the faulting instruction */
+	ASSERT(vie_needs_fetch(vie));
+	error = vie_fetch_instruction(vie, vm, vcpuid, vme->rip + cs_base,
+	    &fault);
+	if (error != 0) {
+		return (error);
+	} else if (fault) {
+		/*
+		 * If a fault during instruction fetch was encounted, it will
+		 * have asserted that the appropriate exception be injected at
+		 * next entry.  No further work is required.
+		 */
+		return (0);
+	}
+
+	if (vie_decode_instruction(vie, vm, vcpuid, cs_d) != 0) {
+		/* Dump (unrecognized) instruction bytes in userspace */
+		vie_fallback_exitinfo(vie, vme);
+		return (-1);
+	}
+
+	error = vie_emulate_other(vie, vm, vcpuid);
+	if (error != 0) {
+		/*
+		 * Instruction emulation was unable to complete successfully, so
+		 * kick it out to userspace for handling.
+		 */
+		vie_fallback_exitinfo(vie, vme);
+	} else {
+		/* Update %rip now that instruction has been emulated */
+		vie_advance_pc(vie, &vcpu->nextrip);
+	}
+	return (error);
+}
+
+static int
 vm_handle_suspend(struct vm *vm, int vcpuid)
 {
 #ifdef __FreeBSD__
@@ -2362,6 +2413,9 @@ restart:
 	case VM_EXITCODE_INOUT:
 		error = vm_handle_inout(vm, vcpuid, vme);
 		break;
+	case VM_EXITCODE_INST_EMUL:
+		error = vm_handle_inst_emul(vm, vcpuid);
+		break;
 	case VM_EXITCODE_MONITOR:
 	case VM_EXITCODE_MWAIT:
 	case VM_EXITCODE_VMINSN:
diff --git a/usr/src/uts/i86pc/io/vmm/vmm_instruction_emul.c b/usr/src/uts/i86pc/io/vmm/vmm_instruction_emul.c
index 02b87a79f6..b0501a60ad 100644
--- a/usr/src/uts/i86pc/io/vmm/vmm_instruction_emul.c
+++ b/usr/src/uts/i86pc/io/vmm/vmm_instruction_emul.c
@@ -69,13 +69,14 @@ enum vie_status {
 	VIES_INIT		= (1U << 0),
 	VIES_MMIO		= (1U << 1),
 	VIES_INOUT		= (1U << 2),
-	VIES_INST_FETCH		= (1U << 3),
-	VIES_INST_DECODE	= (1U << 4),
-	VIES_PENDING_MMIO	= (1U << 5),
-	VIES_PENDING_INOUT	= (1U << 6),
-	VIES_REPEAT		= (1U << 7),
-	VIES_USER_FALLBACK	= (1U << 8),
-	VIES_COMPLETE		= (1U << 9),
+	VIES_OTHER		= (1U << 3),
+	VIES_INST_FETCH		= (1U << 4),
+	VIES_INST_DECODE	= (1U << 5),
+	VIES_PENDING_MMIO	= (1U << 6),
+	VIES_PENDING_INOUT	= (1U << 7),
+	VIES_REPEAT		= (1U << 8),
+	VIES_USER_FALLBACK	= (1U << 9),
+	VIES_COMPLETE		= (1U << 10),
 };
 
 /* State of request to perform emulated access (inout or MMIO) */
@@ -181,6 +182,7 @@ enum {
 	VIE_OP_TYPE_ADD,
 	VIE_OP_TYPE_TEST,
 	VIE_OP_TYPE_BEXTR,
+	VIE_OP_TYPE_CLTS,
 	VIE_OP_TYPE_LAST
 };
 
@@ -199,6 +201,11 @@ static const struct vie_op three_byte_opcodes_0f38[256] = {
 };
 
 static const struct vie_op two_byte_opcodes[256] = {
+	[0x06] = {
+		.op_byte = 0x06,
+		.op_type = VIE_OP_TYPE_CLTS,
+		.op_flags = VIE_OP_F_NO_MODRM | VIE_OP_F_NO_GLA_VERIFICATION
+	},
 	[0xAE] = {
 		.op_byte = 0xAE,
 		.op_type = VIE_OP_TYPE_TWOB_GRP15,
@@ -407,6 +414,13 @@ vie_free(struct vie *vie)
 	kmem_free(vie, sizeof (struct vie));
 }
 
+enum vm_reg_name
+vie_regnum_map(uint8_t regnum)
+{
+	VERIFY3U(regnum, <, 16);
+	return (gpr_map[regnum]);
+}
+
 static void
 vie_calc_bytereg(struct vie *vie, enum vm_reg_name *reg, int *lhbr)
 {
@@ -1876,6 +1890,30 @@ vie_emulate_twob_group15(struct vie *vie, struct vm *vm, int vcpuid,
 }
 
 static int
+vie_emulate_clts(struct vie *vie, struct vm *vm, int vcpuid)
+{
+	uint64_t val;
+	int error;
+
+	if (vie->paging.cpl != 0) {
+		vm_inject_gp(vm, vcpuid);
+		vie->num_processed = 0;
+		return (0);
+	}
+
+	error = vm_get_register(vm, vcpuid, VM_REG_GUEST_CR0, &val);
+	ASSERT(error == 0);
+
+	/* Clear %cr0.TS */
+	val &= ~CR0_TS;
+
+	error = vm_set_register(vm, vcpuid, VM_REG_GUEST_CR0, val);
+	ASSERT(error == 0);
+
+	return (0);
+}
+
+static int
 vie_mmio_read(struct vie *vie, struct vm *vm, int cpuid, uint64_t gpa,
     uint64_t *rval, int bytes)
 {
@@ -2261,6 +2299,28 @@ vie_emulate_inout(struct vie *vie, struct vm *vm, int vcpuid)
 	return (err);
 }
 
+int
+vie_emulate_other(struct vie *vie, struct vm *vm, int vcpuid)
+{
+	int error;
+
+	if ((vie->status & (VIES_INST_DECODE | VIES_OTHER)) !=
+	    (VIES_INST_DECODE | VIES_OTHER)) {
+		return (EINVAL);
+	}
+
+	switch (vie->op.op_type) {
+	case VIE_OP_TYPE_CLTS:
+		error = vie_emulate_clts(vie, vm, vcpuid);
+		break;
+	default:
+		error = EINVAL;
+		break;
+	}
+
+	return (error);
+}
+
 void
 vie_reset(struct vie *vie)
 {
@@ -2338,6 +2398,35 @@ vie_fallback_exitinfo(const struct vie *vie, struct vm_exit *vme)
 	vme->exitcode = VM_EXITCODE_INST_EMUL;
 }
 
+void
+vie_cs_info(const struct vie *vie, struct vm *vm, int vcpuid, uint64_t *cs_base,
+    int *cs_d)
+{
+	struct seg_desc cs_desc;
+	int error;
+
+	error = vm_get_seg_desc(vm, vcpuid, VM_REG_GUEST_CS, &cs_desc);
+	ASSERT(error == 0);
+
+	/* Initialization required for the paging info to be populated */
+	VERIFY(vie->status & VIES_INIT);
+	switch (vie->paging.cpu_mode) {
+	case CPU_MODE_REAL:
+		*cs_base = cs_desc.base;
+		*cs_d = 0;
+		break;
+	case CPU_MODE_PROTECTED:
+	case CPU_MODE_COMPATIBILITY:
+		*cs_base = cs_desc.base;
+		*cs_d = SEG_DESC_DEF32(cs_desc.access) ? 1 : 0;
+		break;
+	default:
+		*cs_base = 0;
+		*cs_d = 0;
+		break;
+	}
+}
+
 bool
 vie_pending(const struct vie *vie)
 {
@@ -2556,6 +2645,19 @@ vie_init_inout(struct vie *vie, const struct vm_inout *inout, uint8_t inst_len,
 	vie->num_processed = inst_len;
 }
 
+void
+vie_init_other(struct vie *vie, const struct vm_guest_paging *paging)
+{
+	bzero(vie, sizeof (struct vie));
+
+	vie->base_register = VM_REG_LAST;
+	vie->index_register = VM_REG_LAST;
+	vie->segment_register = VM_REG_LAST;
+	vie->status = VIES_INIT | VIES_OTHER;
+
+	vie->paging = *paging;
+}
+
 int
 vie_fulfill_mmio(struct vie *vie, const struct vm_mmio *result)
 {
@@ -2873,7 +2975,7 @@ vie_fetch_instruction(struct vie *vie, struct vm *vm, int vcpuid, uint64_t rip,
 	struct vm_copyinfo copyinfo[2];
 	int error, prot;
 
-	if (vie->status != (VIES_INIT|VIES_MMIO)) {
+	if ((vie->status & VIES_INIT) == 0) {
 		return (EINVAL);
 	}
 
diff --git a/usr/src/uts/i86pc/os/hma.c b/usr/src/uts/i86pc/os/hma.c
index a53c797e4b..215243ea98 100644
--- a/usr/src/uts/i86pc/os/hma.c
+++ b/usr/src/uts/i86pc/os/hma.c
@@ -18,6 +18,7 @@
 #include <sys/types.h>
 #include <sys/errno.h>
 #include <sys/machsystm.h>
+#include <sys/archsystm.h>
 #include <sys/controlregs.h>
 #include <sys/x86_archext.h>
 #include <sys/id_space.h>
@@ -522,9 +523,9 @@ uint8_t
 hma_svm_asid_update(hma_svm_asid_t *vcp, boolean_t flush_by_asid,
     boolean_t npt_flush)
 {
-	hma_svm_asid_t *hcp = &hma_svm_cpu_asid[CPU->cpu_seqid];
-
-	ASSERT(curthread->t_preempt != 0);
+	hma_svm_asid_t *hcp;
+	ulong_t iflag;
+	uint8_t res = VMCB_FLUSH_NOTHING;
 
 	/*
 	 * If NPT changes dictate a TLB flush and by-ASID flushing is not
@@ -534,6 +535,17 @@ hma_svm_asid_update(hma_svm_asid_t *vcp, boolean_t flush_by_asid,
 		vcp->hsa_gen = 0;
 	}
 
+	/*
+	 * It is expected that ASID resource updates will commonly be done
+	 * inside a VMM critical section where the GIF is already cleared,
+	 * preventing any possibility of interruption.  Since that cannot be
+	 * checked (there is no easy way to read the GIF), %rflags.IF is also
+	 * cleared for edge cases where an ASID update is performed outside of
+	 * such a GIF-safe critical section.
+	 */
+	iflag = intr_clear();
+
+	hcp = &hma_svm_cpu_asid[CPU->cpu_seqid];
 	if (vcp->hsa_gen != hcp->hsa_gen) {
 		hcp->hsa_asid++;
 
@@ -556,14 +568,17 @@ hma_svm_asid_update(hma_svm_asid_t *vcp, boolean_t flush_by_asid,
 		ASSERT3U(vcp->hsa_asid, <, hma_svm_max_asid);
 
 		if (flush_by_asid) {
-			return (VMCB_FLUSH_ASID);
+			res = VMCB_FLUSH_ASID;
+		} else {
+			res = VMCB_FLUSH_ALL;
 		}
-		return (VMCB_FLUSH_ALL);
 	} else if (npt_flush) {
 		ASSERT(flush_by_asid);
-		return (VMCB_FLUSH_ASID);
+		res = VMCB_FLUSH_ASID;
 	}
-	return (VMCB_FLUSH_NOTHING);
+
+	intr_restore(iflag);
+	return (res);
 }
 
 static int