14817 bhyve VMs should be capable of auto-destruct

14864 want device for testing vmm_drv interface
Reviewed by: Greg Colombo <greg@oxidecomputer.com>
Reviewed by: Andy Fiddaman <andy@omnios.org>
Approved by: Richard Lowe <richlowe@richlowe.net>

11 files changed, 473 insertions, 12 deletions

diff --git a/usr/src/uts/intel/Makefile.files b/usr/src/uts/intel/Makefile.files
index 8974c80a63..4336f77a23 100644
--- a/usr/src/uts/intel/Makefile.files
+++ b/usr/src/uts/intel/Makefile.files
@@ -384,3 +384,8 @@ PPT_OBJS = ppt.o
 # bhyve VT-d
 #
 VMM_VTD_OBJS = vtd.o
+
+#
+# bhyve drv test interface
+#
+VMM_DRV_TEST_OBJS = vmm_drv_test.o
diff --git a/usr/src/uts/intel/Makefile.intel b/usr/src/uts/intel/Makefile.intel
index f93a9d600f..3c0bce4e3e 100644
--- a/usr/src/uts/intel/Makefile.intel
+++ b/usr/src/uts/intel/Makefile.intel
@@ -753,3 +753,8 @@ DRV_KMODS	+= viona
 #
 DRV_KMODS	+= ppt
 DRV_KMODS	+= vmm_vtd
+
+#
+# bhyve drv test interface
+#
+DRV_KMODS	+= vmm_drv_test
diff --git a/usr/src/uts/intel/Makefile.rules b/usr/src/uts/intel/Makefile.rules
index f7c05bdf22..31b87f55b0 100644
--- a/usr/src/uts/intel/Makefile.rules
+++ b/usr/src/uts/intel/Makefile.rules
@@ -276,6 +276,10 @@ $(OBJS_DIR)/%.o:		$(UTSBASE)/intel/io/viona/%.c
 	$(COMPILE.c) -o $@ $<
 	$(CTFCONVERT_O)
 
+$(OBJS_DIR)/%.o:		$(UTSBASE)/intel/io/vmm/%.c
+	$(COMPILE.c) -o $@ $<
+	$(CTFCONVERT_O)
+
 #
 # krtld compiled into unix
 #
diff --git a/usr/src/uts/intel/io/vmm/sys/vmm_impl.h b/usr/src/uts/intel/io/vmm/sys/vmm_impl.h
index 2b6f41ec54..2638fa06b1 100644
--- a/usr/src/uts/intel/io/vmm/sys/vmm_impl.h
+++ b/usr/src/uts/intel/io/vmm/sys/vmm_impl.h
@@ -13,7 +13,7 @@
 /*
  * Copyright 2014 Pluribus Networks Inc.
  * Copyright 2019 Joyent, Inc.
- * Copyright 2021 Oxide Computer Company
+ * Copyright 2022 Oxide Computer Company
  */
 
 #ifndef _VMM_IMPL_H_
@@ -67,6 +67,7 @@ struct vmm_softc {
 	list_t		vmm_holds;
 	uint_t		vmm_flags;
 	boolean_t	vmm_is_open;
+	boolean_t	vmm_autodestruct;
 
 	kmutex_t	vmm_lease_lock;
 	list_t		vmm_lease_list;
@@ -88,7 +89,7 @@ void vmm_zsd_init(void);
 void vmm_zsd_fini(void);
 int vmm_zsd_add_vm(vmm_softc_t *sc);
 void vmm_zsd_rem_vm(vmm_softc_t *sc);
-int vmm_do_vm_destroy(vmm_softc_t *, boolean_t);
+int vmm_zone_vm_destroy(vmm_softc_t *);
 
 #define	VMM_MODULE_NAME	"vmm"
 
diff --git a/usr/src/uts/intel/io/vmm/vmm_drv_test.c b/usr/src/uts/intel/io/vmm/vmm_drv_test.c
new file mode 100644
index 0000000000..728bc820d0
--- /dev/null
+++ b/usr/src/uts/intel/io/vmm/vmm_drv_test.c
@@ -0,0 +1,293 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+/* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */
+
+/*
+ * Copyright 2022 Oxide Computer Company
+ */
+
+#include <sys/types.h>
+#include <sys/conf.h>
+#include <sys/ddi.h>
+#include <sys/mkdev.h>
+#include <sys/sunddi.h>
+#include <sys/id_space.h>
+#include <sys/stat.h>
+
+#include <sys/vmm_drv.h>
+#include <sys/vmm_drv_test.h>
+
+#define	VDT_CTL_NAME	"vmm_drv_test"
+#define	VDT_CTL_MINOR	0
+
+static dev_info_t	*vdt_dip;
+static void		*vdt_state;
+static id_space_t	*vdt_minors;
+
+typedef struct vdt_soft_state {
+	kmutex_t	vss_lock;
+	vmm_hold_t	*vss_hold;
+} vdt_soft_state_t;
+
+
+static int
+vdt_open(dev_t *devp, int flag, int otype, cred_t *cr)
+{
+	id_t minor;
+
+	if (otype != OTYP_CHR) {
+		return (EINVAL);
+	}
+	if (getminor(*devp) != VDT_CTL_MINOR) {
+		return (ENXIO);
+	}
+
+	minor = id_alloc_nosleep(vdt_minors);
+	if (minor == -1) {
+		return (EBUSY);
+	}
+	if (ddi_soft_state_zalloc(vdt_state, minor) != DDI_SUCCESS) {
+		id_free(vdt_minors, minor);
+		return (ENOMEM);
+	}
+
+	vdt_soft_state_t *ss;
+	ss = ddi_get_soft_state(vdt_state, minor);
+	mutex_init(&ss->vss_lock, NULL, MUTEX_DEFAULT, NULL);
+	*devp = makedevice(getmajor(*devp), minor);
+
+	return (0);
+}
+
+static int
+vdt_close(dev_t dev, int flag, int otype, cred_t *cr)
+{
+	if (otype != OTYP_CHR) {
+		return (EINVAL);
+	}
+
+	id_t minor = getminor(dev);
+	vdt_soft_state_t *ss = ddi_get_soft_state(vdt_state, minor);
+	if (ss == NULL) {
+		return (ENXIO);
+	}
+
+	mutex_destroy(&ss->vss_lock);
+	ddi_soft_state_free(vdt_state, minor);
+	id_free(vdt_minors, minor);
+
+	return (0);
+}
+
+static int
+vdt_ioc_hold(vdt_soft_state_t *ss, cred_t *cr, int vmm_fd)
+{
+	mutex_enter(&ss->vss_lock);
+	if (ss->vss_hold != NULL) {
+		mutex_exit(&ss->vss_lock);
+		return (EEXIST);
+	}
+
+	file_t *fp = getf(vmm_fd);
+	if (fp == NULL) {
+		mutex_exit(&ss->vss_lock);
+		return (EBADF);
+	}
+
+	int err = vmm_drv_hold(fp, cr, &ss->vss_hold);
+	releasef(vmm_fd);
+	mutex_exit(&ss->vss_lock);
+	return (err);
+}
+
+static int
+vdt_ioc_rele(vdt_soft_state_t *ss)
+{
+	mutex_enter(&ss->vss_lock);
+	if (ss->vss_hold == NULL) {
+		mutex_exit(&ss->vss_lock);
+		return (ENODEV);
+	}
+
+	vmm_drv_rele(ss->vss_hold);
+	ss->vss_hold = NULL;
+	mutex_exit(&ss->vss_lock);
+	return (0);
+}
+
+static int
+vdt_ioctl(dev_t dev, int cmd, intptr_t data, int md, cred_t *cr, int *rv)
+{
+	vdt_soft_state_t *ss = ddi_get_soft_state(vdt_state, getminor(dev));
+	if (ss == NULL) {
+		return (ENXIO);
+	}
+
+	int err = 0;
+	*rv = 0;
+	switch (cmd) {
+	case VDT_IOC_HOLD:
+		err = vdt_ioc_hold(ss, cr, (int)data);
+		break;
+	case VDT_IOC_RELE:
+		err = vdt_ioc_rele(ss);
+		break;
+	default:
+		err = ENOTTY;
+		break;
+	}
+
+	return (err);
+}
+
+static int
+vdt_info(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
+{
+	switch (cmd) {
+	case DDI_INFO_DEVT2DEVINFO:
+		*result = (void *)vdt_dip;
+		return (DDI_SUCCESS);
+	case DDI_INFO_DEVT2INSTANCE:
+		*result = (void *)0;
+		return (DDI_SUCCESS);
+	default:
+		return (DDI_FAILURE);
+	}
+}
+
+static int
+vdt_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
+{
+	if (cmd != DDI_ATTACH) {
+		return (DDI_FAILURE);
+	}
+
+	if (vdt_dip != NULL) {
+		return (DDI_FAILURE);
+	}
+
+	/* Create "control" node from which other instances are spawned */
+	if (ddi_create_minor_node(dip, VDT_CTL_NAME, S_IFCHR, VDT_CTL_MINOR,
+	    DDI_PSEUDO, 0) != 0) {
+		return (DDI_FAILURE);
+	}
+
+	ddi_report_dev(dip);
+	vdt_dip = dip;
+	return (DDI_SUCCESS);
+}
+
+static int
+vdt_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
+{
+	if (cmd != DDI_DETACH) {
+		return (DDI_FAILURE);
+	}
+
+	ddi_remove_minor_node(vdt_dip, NULL);
+	vdt_dip = NULL;
+
+	return (DDI_SUCCESS);
+}
+
+static struct cb_ops vdt_cb_ops = {
+	.cb_open	= vdt_open,
+	.cb_close	= vdt_close,
+	.cb_strategy	= nodev,
+	.cb_print	= nodev,
+	.cb_dump	= nodev,
+	.cb_read	= nodev,
+	.cb_write	= nodev,
+	.cb_ioctl	= vdt_ioctl,
+	.cb_devmap	= nodev,
+	.cb_mmap	= nodev,
+	.cb_segmap	= nodev,
+	.cb_chpoll	= nochpoll,
+	.cb_prop_op	= ddi_prop_op,
+
+	.cb_str		= NULL,
+
+	.cb_flag	= D_NEW | D_MP | D_DEVMAP,
+	.cb_rev		= CB_REV,
+	.cb_aread	= nodev,
+	.cb_awrite	= nodev,
+};
+
+static struct dev_ops vdt_ops = {
+	.devo_rev	= DEVO_REV,
+	.devo_refcnt	= 0,
+
+	.devo_getinfo	= vdt_info,
+	.devo_identify	= nulldev,
+	.devo_probe	= nulldev,
+	.devo_attach	= vdt_attach,
+	.devo_detach	= vdt_detach,
+	.devo_reset	= nodev,
+	.devo_cb_ops	= &vdt_cb_ops,
+
+	.devo_bus_ops	= NULL,
+	.devo_power	= ddi_power,
+	.devo_quiesce	= ddi_quiesce_not_needed,
+};
+
+static struct modldrv modldrv = {
+	&mod_driverops,
+	"bhyve vmm drv test",
+	&vdt_ops
+};
+
+static struct modlinkage modlinkage = {
+	MODREV_1,
+	&modldrv,
+	NULL
+};
+
+int
+_init(void)
+{
+	int err;
+
+	vdt_minors = id_space_create("vmm_drv_test_minors",
+	    VDT_CTL_MINOR + 1, MAXMIN32);
+
+	err = ddi_soft_state_init(&vdt_state, sizeof (vdt_soft_state_t), 0);
+	if (err != 0) {
+		return (err);
+	}
+
+	err = mod_install(&modlinkage);
+	if (err != 0) {
+		ddi_soft_state_fini(&vdt_state);
+	}
+
+	return (0);
+}
+
+int
+_fini(void)
+{
+	int err = mod_remove(&modlinkage);
+	if (err != 0) {
+		return (err);
+	}
+
+	ddi_soft_state_fini(&vdt_state);
+
+	id_space_destroy(vdt_minors);
+
+	return (0);
+}
+
+int
+_info(struct modinfo *modinfop)
+{
+	return (mod_info(&modlinkage, modinfop));
+}
diff --git a/usr/src/uts/intel/io/vmm/vmm_drv_test.conf b/usr/src/uts/intel/io/vmm/vmm_drv_test.conf
new file mode 100644
index 0000000000..01ac124f2c
--- /dev/null
+++ b/usr/src/uts/intel/io/vmm/vmm_drv_test.conf
@@ -0,0 +1,14 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# Copyright 2022 Oxide Computer Company
+#
+
+name="vmm_drv_test" parent="pseudo";
diff --git a/usr/src/uts/intel/io/vmm/vmm_sol_dev.c b/usr/src/uts/intel/io/vmm/vmm_sol_dev.c
index da83735e43..e1882d7a73 100644
--- a/usr/src/uts/intel/io/vmm/vmm_sol_dev.c
+++ b/usr/src/uts/intel/io/vmm/vmm_sol_dev.c
@@ -111,6 +111,26 @@ struct vmm_lease {
 	struct vmm_hold		*vml_hold;
 };
 
+/* Options for vmm_destroy_locked */
+typedef enum vmm_destroy_opts {
+	VDO_DEFAULT		= 0,
+	/*
+	 * Request that zone-specific-data associated with this VM not be
+	 * cleaned up as part of the destroy.  Skipping ZSD clean-up is
+	 * necessary when VM is being destroyed as part of zone destruction,
+	 * when said ZSD is already being cleaned up.
+	 */
+	VDO_NO_CLEAN_ZSD	= (1 << 0),
+	/*
+	 * Skip any attempt to wait for vmm_drv consumers when attempting to
+	 * purge them from the instance.  When performing an auto-destruct, it
+	 * is not desirable to wait, since said consumer might exist in a
+	 * "higher" file descriptor which has not yet been closed.
+	 */
+	VDO_NO_PURGE_WAIT	= (1 << 1),
+} vmm_destroy_opts_t;
+
+static int vmm_destroy_locked(vmm_softc_t *, vmm_destroy_opts_t, boolean_t *);
 static int vmm_drv_block_hook(vmm_softc_t *, boolean_t);
 static void vmm_lease_block(vmm_softc_t *);
 static void vmm_lease_unblock(vmm_softc_t *);
@@ -502,6 +522,7 @@ vmmdev_do_ioctl(vmm_softc_t *sc, int cmd, intptr_t arg, int md,
 	case VM_IOAPIC_PINCOUNT:
 	case VM_SUSPEND:
 	case VM_DESC_FPU_AREA:
+	case VM_SET_AUTODESTRUCT:
 	default:
 		break;
 	}
@@ -835,6 +856,17 @@ vmmdev_do_ioctl(vmm_softc_t *sc, int cmd, intptr_t arg, int md,
 		}
 		break;
 	}
+	case VM_SET_AUTODESTRUCT: {
+		/*
+		 * Since this has to do with controlling the lifetime of the
+		 * greater vmm_softc_t, the flag is protected by vmm_mtx, rather
+		 * than the vcpu-centric or rwlock exclusion mechanisms.
+		 */
+		mutex_enter(&vmm_mtx);
+		sc->vmm_autodestruct = (arg != 0);
+		mutex_exit(&vmm_mtx);
+		break;
+	}
 
 	case VM_ISA_ASSERT_IRQ: {
 		struct vm_isa_irq isa_irq;
@@ -1921,6 +1953,7 @@ void
 vmm_drv_rele(vmm_hold_t *hold)
 {
 	vmm_softc_t *sc;
+	boolean_t hma_release = B_FALSE;
 
 	ASSERT(hold != NULL);
 	ASSERT(hold->vmh_sc != NULL);
@@ -1932,9 +1965,25 @@ vmm_drv_rele(vmm_hold_t *hold)
 	if (list_is_empty(&sc->vmm_holds)) {
 		sc->vmm_flags &= ~VMM_HELD;
 		cv_broadcast(&sc->vmm_cv);
+
+		/*
+		 * If pending hold(s) had prevented an auto-destruct of the
+		 * instance when it was closed, finish that clean-up now.
+		 */
+		if (sc->vmm_autodestruct && !sc->vmm_is_open) {
+			int err = vmm_destroy_locked(sc,
+			    VDO_NO_PURGE_WAIT, &hma_release);
+
+			VERIFY0(err);
+			VERIFY(hma_release);
+		}
 	}
 	mutex_exit(&vmm_mtx);
 	kmem_free(hold, sizeof (*hold));
+
+	if (hma_release) {
+		vmm_hma_release();
+	}
 }
 
 boolean_t
@@ -2227,7 +2276,7 @@ vmm_drv_ioport_unhook(vmm_hold_t *hold, void **cookie)
 }
 
 static int
-vmm_drv_purge(vmm_softc_t *sc)
+vmm_drv_purge(vmm_softc_t *sc, boolean_t no_wait)
 {
 	ASSERT(MUTEX_HELD(&vmm_mtx));
 
@@ -2258,7 +2307,13 @@ vmm_drv_purge(vmm_softc_t *sc)
 		 * fashion to waiting for any lingering holds to be dropped.
 		 */
 		while ((sc->vmm_flags & VMM_HELD) != 0) {
-			if (cv_wait_sig(&sc->vmm_cv, &vmm_mtx) <= 0) {
+			/*
+			 * Some holds remain, so wait (if acceptable) for them
+			 * to be cleaned up.
+			 */
+			if (no_wait ||
+			    cv_wait_sig(&sc->vmm_cv, &vmm_mtx) <= 0) {
+				sc->vmm_flags &= ~VMM_CLEANUP;
 				return (EINTR);
 			}
 		}
@@ -2303,7 +2358,7 @@ done:
 }
 
 static int
-vmm_do_vm_destroy_locked(vmm_softc_t *sc, boolean_t clean_zsd,
+vmm_destroy_locked(vmm_softc_t *sc, vmm_destroy_opts_t opts,
     boolean_t *hma_release)
 {
 	dev_info_t	*pdip = ddi_get_parent(vmmdev_dip);
@@ -2313,11 +2368,11 @@ vmm_do_vm_destroy_locked(vmm_softc_t *sc, boolean_t clean_zsd,
 
 	*hma_release = B_FALSE;
 
-	if (vmm_drv_purge(sc) != 0) {
+	if (vmm_drv_purge(sc, (opts & VDO_NO_PURGE_WAIT) != 0) != 0) {
 		return (EINTR);
 	}
 
-	if (clean_zsd) {
+	if ((opts & VDO_NO_CLEAN_ZSD) == 0) {
 		vmm_zsd_rem_vm(sc);
 	}
 
@@ -2344,13 +2399,13 @@ vmm_do_vm_destroy_locked(vmm_softc_t *sc, boolean_t clean_zsd,
 }
 
 int
-vmm_do_vm_destroy(vmm_softc_t *sc, boolean_t clean_zsd)
+vmm_zone_vm_destroy(vmm_softc_t *sc)
 {
 	boolean_t	hma_release = B_FALSE;
 	int		err;
 
 	mutex_enter(&vmm_mtx);
-	err = vmm_do_vm_destroy_locked(sc, clean_zsd, &hma_release);
+	err = vmm_destroy_locked(sc, VDO_NO_CLEAN_ZSD, &hma_release);
 	mutex_exit(&vmm_mtx);
 
 	if (hma_release)
@@ -2384,7 +2439,7 @@ vmmdev_do_vm_destroy(const struct vm_destroy_req *req, cred_t *cr)
 		mutex_exit(&vmm_mtx);
 		return (EPERM);
 	}
-	err = vmm_do_vm_destroy_locked(sc, B_TRUE, &hma_release);
+	err = vmm_destroy_locked(sc, VDO_DEFAULT, &hma_release);
 
 	mutex_exit(&vmm_mtx);
 
@@ -2589,6 +2644,16 @@ vmm_close(dev_t dev, int flag, int otyp, cred_t *credp)
 		ddi_soft_state_free(vmm_statep, minor);
 		id_free(vmm_minors, minor);
 		hma_release = B_TRUE;
+	} else if (sc->vmm_autodestruct) {
+		/*
+		 * Attempt auto-destruct on instance if requested.
+		 *
+		 * Do not wait for existing holds to be purged from the
+		 * instance, since there is no guarantee that will happen in a
+		 * timely manner.  Auto-destruction will resume when the last
+		 * hold is released. (See: vmm_drv_rele)
+		 */
+		(void) vmm_destroy_locked(sc, VDO_NO_PURGE_WAIT, &hma_release);
 	}
 	mutex_exit(&vmm_mtx);
 
diff --git a/usr/src/uts/intel/io/vmm/vmm_zsd.c b/usr/src/uts/intel/io/vmm/vmm_zsd.c
index d441c9dad6..16f05ea173 100644
--- a/usr/src/uts/intel/io/vmm/vmm_zsd.c
+++ b/usr/src/uts/intel/io/vmm/vmm_zsd.c
@@ -12,6 +12,7 @@
 
 /*
  * Copyright (c) 2018, Joyent, Inc.
+ * Copyright 2022 Oxide Computer Company
  */
 
 #include <sys/cpuvar.h>
@@ -188,7 +189,7 @@ vmm_zsd_destroy(zoneid_t zid, void *data)
 		 * This frees all resources associated with the vm, including
 		 * sc.
 		 */
-		err = vmm_do_vm_destroy(sc, B_FALSE);
+		err = vmm_zone_vm_destroy(sc);
 		ASSERT3S(err, ==, 0);
 	}
 
diff --git a/usr/src/uts/intel/sys/vmm_dev.h b/usr/src/uts/intel/sys/vmm_dev.h
index 8d1b2713dd..b8c87217b4 100644
--- a/usr/src/uts/intel/sys/vmm_dev.h
+++ b/usr/src/uts/intel/sys/vmm_dev.h
@@ -385,7 +385,7 @@ struct vm_data_xfer {
  * best-effort activity.  Nothing is to be inferred about the magnitude of a
  * change when the version is modified.  It follows no rules like semver.
  */
-#define	VMM_CURRENT_INTERFACE_VERSION	3
+#define	VMM_CURRENT_INTERFACE_VERSION	4
 
 
 #define	VMMCTL_IOC_BASE		(('V' << 16) | ('M' << 8))
@@ -494,6 +494,8 @@ struct vm_data_xfer {
 #define	VM_DATA_READ			(VMM_IOC_BASE | 0x22)
 #define	VM_DATA_WRITE			(VMM_IOC_BASE | 0x23)
 
+#define	VM_SET_AUTODESTRUCT		(VMM_IOC_BASE | 0x24)
+
 #define	VM_DEVMEM_GETOFFSET		(VMM_IOC_BASE | 0xff)
 
 #define	VMM_CTL_DEV		"/dev/vmmctl"
diff --git a/usr/src/uts/intel/sys/vmm_drv_test.h b/usr/src/uts/intel/sys/vmm_drv_test.h
new file mode 100644
index 0000000000..1cf26dd51a
--- /dev/null
+++ b/usr/src/uts/intel/sys/vmm_drv_test.h
@@ -0,0 +1,25 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+/* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */
+
+/*
+ * Copyright 2022 Oxide Computer Company
+ */
+
+#ifndef _VMM_DRV_TEST_H_
+#define	_VMM_DRV_TEST_H_
+
+#define	VDT_IOC_BASE		(('V' << 16) | ('D' << 8))
+
+#define	VDT_IOC_HOLD		(VDT_IOC_BASE | 0x01)
+#define	VDT_IOC_RELE		(VDT_IOC_BASE | 0x02)
+
+#endif /* _VMM_DRV_TEST_H_ */
diff --git a/usr/src/uts/intel/vmm_drv_test/Makefile b/usr/src/uts/intel/vmm_drv_test/Makefile
new file mode 100644
index 0000000000..4281a66ca4
--- /dev/null
+++ b/usr/src/uts/intel/vmm_drv_test/Makefile
@@ -0,0 +1,46 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright 2022 Oxide Computer Company
+#
+
+UTSBASE	= ../..
+
+MODULE		= vmm_drv_test
+OBJECTS		= $(VMM_DRV_TEST_OBJS:%=$(OBJS_DIR)/%)
+ROOTMODULE	= $(USR_DRV_DIR)/$(MODULE)
+CONF_SRCDIR	= $(UTSBASE)/intel/io/vmm
+
+include $(UTSBASE)/intel/Makefile.intel
+
+ALL_TARGET	= $(BINARY) $(SRC_CONFILE)
+INSTALL_TARGET	= $(BINARY) $(ROOTMODULE) $(ROOT_CONFFILE)
+
+ALL_BUILDS	= $(ALL_BUILDSONLY64)
+DEF_BUILDS	= $(DEF_BUILDSONLY64)
+
+CFLAGS		+= $(CCVERBOSE)
+LDFLAGS		+= -Ndrv/vmm
+
+.KEEP_STATE:
+
+def:		$(DEF_DEPS)
+
+all:		$(ALL_DEPS)
+
+clean:		$(CLEAN_DEPS)
+
+clobber:	$(CLOBBER_DEPS)
+
+install:	$(INSTALL_DEPS)
+
+include $(UTSBASE)/intel/Makefile.targ