OS-6759 want CPU localization for vmm callouts

Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: John Levon <john.levon@joyent.com>
Approved by: Mike Gerdts <mike.gerdts@joyent.com>

9 files changed, 123 insertions, 3 deletions

diff --git a/usr/src/compat/freebsd/sys/callout.h b/usr/src/compat/freebsd/sys/callout.h
index 17b6e31507..6087a09f54 100644
--- a/usr/src/compat/freebsd/sys/callout.h
+++ b/usr/src/compat/freebsd/sys/callout.h
@@ -11,6 +11,7 @@
 
 /*
  * Copyright 2014 Pluribus Networks Inc.
+ * Copyright 2018 Joyent, Inc.
  */
 
 #ifndef _COMPAT_FREEBSD_SYS_CALLOUT_H_
@@ -41,6 +42,9 @@ int	vmm_glue_callout_reset_sbt(struct callout *c, sbintime_t sbt,
 int	vmm_glue_callout_stop(struct callout *c);
 int	vmm_glue_callout_drain(struct callout *c);
 
+/* illumos-custom function for resource locality optimization */
+void	vmm_glue_callout_localize(struct callout *c);
+
 static __inline void
 callout_init(struct callout *c, int mpsafe)
 {
diff --git a/usr/src/uts/i86pc/io/vmm/io/vhpet.c b/usr/src/uts/i86pc/io/vmm/io/vhpet.c
index a8711eac6a..b881bd0e03 100644
--- a/usr/src/uts/i86pc/io/vmm/io/vhpet.c
+++ b/usr/src/uts/i86pc/io/vmm/io/vhpet.c
@@ -27,6 +27,10 @@
  * $FreeBSD$
  */
 
+/*
+ * Copyright 2018 Joyent, Inc.
+ */
+
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
@@ -762,3 +766,12 @@ vhpet_getcap(struct vm_hpet_cap *cap)
 	cap->capabilities = vhpet_capabilities();
 	return (0);
 }
+#ifndef __FreeBSD__
+void
+vhpet_localize_resources(struct vhpet *vhpet)
+{
+	for (uint_t i = 0; i < VHPET_NUM_TIMERS; i++) {
+		vmm_glue_callout_localize(&vhpet->timer[i].callout);
+	}
+}
+#endif /* __FreeBSD */
diff --git a/usr/src/uts/i86pc/io/vmm/io/vhpet.h b/usr/src/uts/i86pc/io/vmm/io/vhpet.h
index 330e01739a..3533d163e7 100644
--- a/usr/src/uts/i86pc/io/vmm/io/vhpet.h
+++ b/usr/src/uts/i86pc/io/vmm/io/vhpet.h
@@ -27,6 +27,10 @@
  * $FreeBSD$
  */
 
+/*
+ * Copyright 2018 Joyent, Inc.
+ */
+
 #ifndef _VHPET_H_
 #define	_VHPET_H_
 
@@ -41,4 +45,8 @@ int	vhpet_mmio_read(void *vm, int vcpuid, uint64_t gpa, uint64_t *val,
 	    int size, void *arg);
 int	vhpet_getcap(struct vm_hpet_cap *cap);
 
+#ifndef __FreeBSD__
+void vhpet_localize_resources(struct vhpet *vhpet);
+#endif
+
 #endif	/* _VHPET_H_ */
diff --git a/usr/src/uts/i86pc/io/vmm/io/vlapic.c b/usr/src/uts/i86pc/io/vmm/io/vlapic.c
index 6fed322180..8b07d68571 100644
--- a/usr/src/uts/i86pc/io/vmm/io/vlapic.c
+++ b/usr/src/uts/i86pc/io/vmm/io/vlapic.c
@@ -36,7 +36,7 @@
  * http://www.illumos.org/license/CDDL.
  *
  * Copyright 2014 Pluribus Networks Inc.
- * Copyright 2017 Joyent, Inc.
+ * Copyright 2018 Joyent, Inc.
  */
 
 #include <sys/cdefs.h>
@@ -1687,3 +1687,11 @@ vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys,
 	VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector);
 	vlapic_set_tmr(vlapic, vector, true);
 }
+
+#ifndef __FreeBSD__
+void
+vlapic_localize_resources(struct vlapic *vlapic)
+{
+	vmm_glue_callout_localize(&vlapic->callout);
+}
+#endif /* __FreeBSD */
diff --git a/usr/src/uts/i86pc/io/vmm/io/vlapic.h b/usr/src/uts/i86pc/io/vmm/io/vlapic.h
index 0e68b2fe82..c299a7fda3 100644
--- a/usr/src/uts/i86pc/io/vmm/io/vlapic.h
+++ b/usr/src/uts/i86pc/io/vmm/io/vlapic.h
@@ -26,6 +26,10 @@
  * $FreeBSD$
  */
 
+/*
+ * Copyright 2018 Joyent, Inc.
+ */
+
 #ifndef _VLAPIC_H_
 #define	_VLAPIC_H_
 
@@ -106,4 +110,9 @@ void vlapic_icrtmr_write_handler(struct vlapic *vlapic);
 void vlapic_dcr_write_handler(struct vlapic *vlapic);
 void vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset);
 void vlapic_self_ipi_handler(struct vlapic *vlapic, uint64_t val);
+
+#ifndef __FreeBSD__
+void vlapic_localize_resources(struct vlapic *vlapic);
+#endif
+
 #endif	/* _VLAPIC_H_ */
diff --git a/usr/src/uts/i86pc/io/vmm/io/vrtc.c b/usr/src/uts/i86pc/io/vmm/io/vrtc.c
index 18ebc4b98f..8452d28cd2 100644
--- a/usr/src/uts/i86pc/io/vmm/io/vrtc.c
+++ b/usr/src/uts/i86pc/io/vmm/io/vrtc.c
@@ -24,6 +24,10 @@
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+/*
+ * Copyright 2018 Joyent, Inc.
+ */
+
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
@@ -1017,3 +1021,11 @@ vrtc_cleanup(struct vrtc *vrtc)
 	callout_drain(&vrtc->callout);
 	free(vrtc, M_VRTC);
 }
+
+#ifndef __FreeBSD__
+void
+vrtc_localize_resources(struct vrtc *vrtc)
+{
+	vmm_glue_callout_localize(&vrtc->callout);
+}
+#endif /* __FreeBSD */
diff --git a/usr/src/uts/i86pc/io/vmm/io/vrtc.h b/usr/src/uts/i86pc/io/vmm/io/vrtc.h
index 6fbbc9c810..ffab3a5af0 100644
--- a/usr/src/uts/i86pc/io/vmm/io/vrtc.h
+++ b/usr/src/uts/i86pc/io/vmm/io/vrtc.h
@@ -26,6 +26,10 @@
  * $FreeBSD$
  */
 
+/*
+ * Copyright 2018 Joyent, Inc.
+ */
+
 #ifndef _VRTC_H_
 #define	_VRTC_H_
 
@@ -47,4 +51,8 @@ int vrtc_addr_handler(struct vm *vm, int vcpuid, bool in, int port, int bytes,
 int vrtc_data_handler(struct vm *vm, int vcpuid, bool in, int port, int bytes,
     uint32_t *val);
 
+#ifndef __FreeBSD__
+void vrtc_localize_resources(struct vrtc *);
+#endif
+
 #endif
diff --git a/usr/src/uts/i86pc/io/vmm/vmm.c b/usr/src/uts/i86pc/io/vmm/vmm.c
index bad3ac6317..2e5198310c 100644
--- a/usr/src/uts/i86pc/io/vmm/vmm.c
+++ b/usr/src/uts/i86pc/io/vmm/vmm.c
@@ -36,7 +36,7 @@
  * http://www.illumos.org/license/CDDL.
  *
  * Copyright 2015 Pluribus Networks Inc.
- * Copyright 2017 Joyent, Inc.
+ * Copyright 2018 Joyent, Inc.
  */
 
 #include <sys/cdefs.h>
@@ -1735,6 +1735,39 @@ vm_exit_astpending(struct vm *vm, int vcpuid, uint64_t rip)
 	vmm_stat_incr(vm, vcpuid, VMEXIT_ASTPENDING, 1);
 }
 
+#ifndef __FreeBSD__
+/*
+ * Some vmm resources, such as the lapic, may have CPU-specific resources
+ * allocated to them which would benefit from migration onto the host CPU which
+ * is processing the vcpu state.  When running on a host CPU different from
+ * previous activity, attempt to localize resources when possible.
+ */
+static void
+vm_localize_resources(struct vm *vm, struct vcpu *vcpu)
+{
+	if (vcpu->hostcpu == curcpu)
+		return;
+
+	/*
+	 * The cyclic backing the LAPIC timer is nice to have local as
+	 * reprogramming operations would otherwise require a crosscall.
+	 */
+	vlapic_localize_resources(vcpu->vlapic);
+
+	/*
+	 * Localize system-wide resources to the primary boot vCPU.  While any
+	 * of the other vCPUs may access them, it keeps the potential interrupt
+	 * footprint constrained to CPUs involved with this instance.
+	 */
+	if (vcpu == &vm->vcpu[0]) {
+		vhpet_localize_resources(vm->vhpet);
+		vrtc_localize_resources(vm->vrtc);
+	}
+
+}
+#endif /* __FreeBSD */
+
+
 int
 vm_run(struct vm *vm, struct vm_run *vmrun)
 {
@@ -1783,6 +1816,8 @@ restart:
 #endif
 
 #ifndef	__FreeBSD__
+	vm_localize_resources(vm, vcpu);
+
 	installctx(curthread, vcpu, save_guest_fpustate,
 	    restore_guest_fpustate, NULL, NULL, NULL, NULL);
 #endif
diff --git a/usr/src/uts/i86pc/io/vmm/vmm_sol_glue.c b/usr/src/uts/i86pc/io/vmm/vmm_sol_glue.c
index 2751e72101..32f54e85d8 100644
--- a/usr/src/uts/i86pc/io/vmm/vmm_sol_glue.c
+++ b/usr/src/uts/i86pc/io/vmm/vmm_sol_glue.c
@@ -36,7 +36,7 @@
  * http://www.illumos.org/license/CDDL.
  *
  * Copyright 2014 Pluribus Networks Inc.
- * Copyright 2017 Joyent, Inc.
+ * Copyright 2018 Joyent, Inc.
  */
 
 #include <sys/types.h>
@@ -259,13 +259,28 @@ mtx_destroy(struct mtx *mtx)
 void
 critical_enter(void)
 {
+	kthread_t *tp = curthread;
+
 	kpreempt_disable();
+	if (tp->t_preempt == 1) {
+		/*
+		 * Avoid extra work when nested calls to this are made and only
+		 * set affinity on the top-level entry.  This also means only
+		 * removing the affinity in critical_exit() when at last call.
+		 */
+		thread_affinity_set(tp, CPU_CURRENT);
+	}
 }
 
 void
 critical_exit(void)
 {
+	kthread_t *tp = curthread;
+
 	kpreempt_enable();
+	if (tp->t_preempt == 0) {
+		thread_affinity_clear(tp);
+	}
 }
 
 struct unrhdr;
@@ -410,6 +425,14 @@ vmm_glue_callout_drain(struct callout *c)
 }
 
 void
+vmm_glue_callout_localize(struct callout *c)
+{
+	mutex_enter(&cpu_lock);
+	cyclic_move_here(c->c_cyc_id);
+	mutex_exit(&cpu_lock);
+}
+
+void
 ipi_cpu(int cpu, u_int ipi)
 {
 	/*