13674 bhyve upstream sync 2021 March

Reviewed by: Patrick Mooney <pmooney@pfmooney.com>
Reviewed by: Jorge Schrauwen <sjorge@blackdot.be>
Reviewed by: Mike Zeller <mike.zeller@joyent.com>
Reviewed by: C Fraire <cfraire@me.com>
Approved by: Dan McDonald <danmcd@joyent.com>

86 files changed, 4370 insertions, 2113 deletions

diff --git a/exception_lists/copyright b/exception_lists/copyright
index 0395a6f257..c091d526fa 100644
--- a/exception_lists/copyright
+++ b/exception_lists/copyright
@@ -471,6 +471,7 @@ usr/src/uts/common/sys/scsi/adapters/mpt_sas/mpi/*
 usr/src/uts/sparc/nsmb/ioc_check.ref
 
 # bhyve sources
+usr/src/cmd/bhyve/README.sync
 usr/src/cmd/bhyve/THIRDPARTYLICENSE
 usr/src/cmd/bhyve/THIRDPARTYLICENSE.descrip
 usr/src/cmd/bhyve/acpi.[ch]
@@ -480,21 +481,26 @@ usr/src/cmd/bhyve/bhyvegc.[ch]
 usr/src/cmd/bhyve/bhyverun.[ch]
 usr/src/cmd/bhyve/block_if.[ch]
 usr/src/cmd/bhyve/bootrom.[ch]
+usr/src/cmd/bhyve/config.[ch]
 usr/src/cmd/bhyve/console.[ch]
 usr/src/cmd/bhyve/consport.c
 usr/src/cmd/bhyve/dbgport.[ch]
 usr/src/cmd/bhyve/fwctl.[ch]
 usr/src/cmd/bhyve/gdb.[ch]
+usr/src/cmd/bhyve/hda_codec.c
 usr/src/cmd/bhyve/inout.[ch]
 usr/src/cmd/bhyve/ioapic.[ch]
 usr/src/cmd/bhyve/mem.[ch]
 usr/src/cmd/bhyve/mevent.[ch]
 usr/src/cmd/bhyve/mevent_test.c
 usr/src/cmd/bhyve/mptbl.[ch]
+usr/src/cmd/bhyve/net_backends.[ch]
+usr/src/cmd/bhyve/net_utils.[ch]
 usr/src/cmd/bhyve/pci_ahci.c
 usr/src/cmd/bhyve/pci_e82545.c
 usr/src/cmd/bhyve/pci_emul.[ch]
 usr/src/cmd/bhyve/pci_fbuf.c
+usr/src/cmd/bhyve/pci_hda.[ch]
 usr/src/cmd/bhyve/pci_hostbridge.c
 usr/src/cmd/bhyve/pci_irq.[ch]
 usr/src/cmd/bhyve/pci_lpc.[ch]
@@ -502,10 +508,13 @@ usr/src/cmd/bhyve/pci_nvme.c
 usr/src/cmd/bhyve/pci_passthru.c
 usr/src/cmd/bhyve/pci_uart.c
 usr/src/cmd/bhyve/pci_virtio_block.c
+usr/src/cmd/bhyve/pci_virtio_console.c
 usr/src/cmd/bhyve/pci_virtio_net.c
 usr/src/cmd/bhyve/pci_virtio_rnd.c
 usr/src/cmd/bhyve/pci_virtio_scsi.c
+usr/src/cmd/bhyve/pci_virtio_viona.c
 usr/src/cmd/bhyve/pci_xhci.[ch]
+usr/src/cmd/bhyve/pctestdev.[ch]
 usr/src/cmd/bhyve/pm.c
 usr/src/cmd/bhyve/pmtmr.c
 usr/src/cmd/bhyve/post.c
diff --git a/exception_lists/cstyle b/exception_lists/cstyle
index b92b19cfd1..bf1856d5f0 100644
--- a/exception_lists/cstyle
+++ b/exception_lists/cstyle
@@ -1326,6 +1326,7 @@ usr/src/cmd/bhyve/bhyvegc.[ch]
 usr/src/cmd/bhyve/bhyverun.[ch]
 usr/src/cmd/bhyve/block_if.[ch]
 usr/src/cmd/bhyve/bootrom.[ch]
+usr/src/cmd/bhyve/config.[ch]
 usr/src/cmd/bhyve/console.[ch]
 usr/src/cmd/bhyve/consport.c
 usr/src/cmd/bhyve/dbgport.[ch]
@@ -1362,6 +1363,7 @@ usr/src/cmd/bhyve/pci_virtio_net.c
 usr/src/cmd/bhyve/pci_virtio_rnd.c
 usr/src/cmd/bhyve/pci_virtio_scsi.c
 usr/src/cmd/bhyve/pci_xhci.[ch]
+usr/src/cmd/bhyve/pctestdev.c
 usr/src/cmd/bhyve/pm.c
 usr/src/cmd/bhyve/post.c
 usr/src/cmd/bhyve/ps2kbd.[ch]
diff --git a/exception_lists/hdrchk b/exception_lists/hdrchk
index a8a90a7493..fc022b3782 100644
--- a/exception_lists/hdrchk
+++ b/exception_lists/hdrchk
@@ -404,6 +404,7 @@ usr/src/cmd/bhyve/pci_emul.h
 usr/src/cmd/bhyve/pci_hda.h
 usr/src/cmd/bhyve/pci_irq.h
 usr/src/cmd/bhyve/pci_lpc.h
+usr/src/cmd/bhyve/pctestdev.h
 usr/src/cmd/bhyve/ps2kbd.h
 usr/src/cmd/bhyve/ps2mouse.h
 usr/src/cmd/bhyve/rfb.h
diff --git a/exception_lists/wscheck b/exception_lists/wscheck
index d2a64a3f72..481aa391c9 100644
--- a/exception_lists/wscheck
+++ b/exception_lists/wscheck
@@ -10,6 +10,7 @@
 #
 # Copyright 2019 Joyent, Inc.
 # Copyright 2020 Oxide Computer Company
+# Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
 #
 syntax: glob
 
@@ -39,6 +40,7 @@ usr/src/cmd/bhyve/bhyvegc.[ch]
 usr/src/cmd/bhyve/bhyverun.[ch]
 usr/src/cmd/bhyve/block_if.[ch]
 usr/src/cmd/bhyve/bootrom.[ch]
+usr/src/cmd/bhyve/config.[ch]
 usr/src/cmd/bhyve/console.[ch]
 usr/src/cmd/bhyve/consport.c
 usr/src/cmd/bhyve/dbgport.[ch]
diff --git a/usr/src/cmd/bhyve/Makefile b/usr/src/cmd/bhyve/Makefile
index 1c1b99c52b..662a155ec5 100644
--- a/usr/src/cmd/bhyve/Makefile
+++ b/usr/src/cmd/bhyve/Makefile
@@ -13,6 +13,7 @@
 # Copyright 2014 Pluribus Networks Inc.
 # Copyright 2020 Joyent, Inc.
 # Copyright 2020 Oxide Computer Company
+# Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
 #
 
 PROG =		bhyve
@@ -35,8 +36,7 @@ SRCS =	acpi.c			\
 	block_if.c		\
 	bootrom.c		\
 	console.c		\
-	consport.c		\
-	dbgport.c		\
+	config.c		\
 	fwctl.c			\
 	gdb.c			\
 	inout.c			\
@@ -125,13 +125,17 @@ CPPFLAGS =	-I$(COMPAT)/bhyve -I$(CONTRIB)/bhyve \
 		-DWITHOUT_CAPSICUM
 
 pci_nvme.o := CERRWARN += -_gcc=-Wno-pointer-sign
+pci_nvme.o := CERRWARN += -_gcc10=-Wno-address-of-packed-member
 pci_nvme.o := SMOFF += kmalloc_wrong_size
 
+pci_passthru.o := CERRWARN += -_gcc10=-Wno-address-of-packed-member
+
+pci_xhci.o := CERRWARN += -_gcc10=-Wno-address-of-packed-member
+
 SMOFF += all_func_returns,leaks,no_if_block
 
 # Force c99 for everything
 CSTD=		$(CSTD_GNU99)
-C99MODE=	-xc99=%all
 
 $(PROG) := LDLIBS += \
 	-lsocket \
@@ -139,6 +143,7 @@ $(PROG) := LDLIBS += \
 	-ldlpi \
 	-ldladm \
 	-lmd \
+	-lnvpair \
 	-lcrypto \
 	-luuid \
 	-lvmmapi \
diff --git a/usr/src/cmd/bhyve/README.sync b/usr/src/cmd/bhyve/README.sync
index 2384413853..bed6671c73 100644
--- a/usr/src/cmd/bhyve/README.sync
+++ b/usr/src/cmd/bhyve/README.sync
@@ -4,18 +4,18 @@ public Git repository at https://git.freebsd.org/src.git
 The bhyve kernel module and its associated userland consumers have been updated
 to the latest upstream FreeBSD sources as of:
 
-commit 2bb4be0f86501ec0565dba3d37ce0f7d7fc9c464
-Author: grehan <grehan@FreeBSD.org>
-Date:   Fri Dec 18 00:38:48 2020 +0000
-
-    Fix issues with various VNC clients.
-
-    PR: 250795
-    Submitted by:   Marko Kiiskila <marko@apache.org>
-    Reviewed by:    jhb (bhyve)
-    MFC after:      3 weeks
-    Relnotes:       yes
-    Differential Revision:  https://reviews.freebsd.org/D27605
+commit 9f40a3be3d5dbddf782c3d1eeaadcd022a4dad01
+Author: John Baldwin <jhb@FreeBSD.org>
+Date:   Wed Mar 24 09:29:15 2021 -0700
+
+    bhyve hostbridge: Rename "device" property to "devid".
+
+    "device" is already used as the generic PCI-level name of the device
+    model to use (e.g. "hostbridge").  The result was that parsing
+    "hostbridge" as an integer failed and the host bridge used a device ID
+    of 0.  The EFI ROM asserts that the device ID of the hostbridge is not
+    0, so booting with the current EFI ROM was failing during the ROM
+    boot.
 
 Divergence Notes:
 
diff --git a/usr/src/cmd/bhyve/bhyverun.c b/usr/src/cmd/bhyve/bhyverun.c
index 53a92f6dd7..3e2a72ced8 100644
--- a/usr/src/cmd/bhyve/bhyverun.c
+++ b/usr/src/cmd/bhyve/bhyverun.c
@@ -90,8 +90,8 @@ __FBSDID("$FreeBSD$");
 #include "atkbdc.h"
 #include "console.h"
 #include "bootrom.h"
+#include "config.h"
 #include "inout.h"
-#include "dbgport.h"
 #include "debug.h"
 #include "fwctl.h"
 #include "gdb.h"
@@ -186,26 +186,11 @@ static const char * const vmx_exit_reason_desc[] = {
 typedef int (*vmexit_handler_t)(struct vmctx *, struct vm_exit *, int *vcpu);
 extern int vmexit_task_switch(struct vmctx *, struct vm_exit *, int *vcpu);
 
-char *vmname;
-
 int guest_ncpus;
 uint16_t cores, maxcpus, sockets, threads;
 
-char *guest_uuid_str;
-
 int raw_stdio = 0;
 
-static int gdb_port = 0;
-static int guest_vmexit_on_hlt, guest_vmexit_on_pause;
-static int virtio_msix = 1;
-static int x2apic_mode = 0;	/* default is xAPIC */
-static int destroy_on_poweroff = 0;
-
-static int strictio;
-static int strictmsr = 1;
-
-static int acpi;
-
 static char *progname;
 static const int BSP = 0;
 
@@ -246,16 +231,16 @@ usage(int code)
 
         fprintf(stderr,
 #ifdef	__FreeBSD__
-		"Usage: %s [-abehuwxACDHPSWY]\n"
+		"Usage: %s [-aehuwxACDHPSWY]\n"
 #else
-		"Usage: %s [-abdehuwxACDHPSWY]\n"
+		"Usage: %s [-adehuwxACDHPSWY]\n"
 #endif
 		"       %*s [-c [[cpus=]numcpus][,sockets=n][,cores=n][,threads=n]]\n"
-		"       %*s [-g <gdb port>] [-l <lpc>]\n"
+		"       %*s [-k <file>] [-l <lpc>] [-m mem] [-o <var>=<value>]\n"
 #ifdef	__FreeBSD__
-		"       %*s [-m mem] [-p vcpu:hostcpu] [-s <pci>] [-U uuid] <vm>\n"
+		"       %*s [-p vcpu:hostcpu] [-s <pci>] [-U uuid] [<vm>]\n"
 #else
-		"       %*s [-m mem] [-s <pci>] [-U uuid] <vm>\n"
+		"       %*s [-s <pci>] [-U uuid] [<vm>]\n"
 #endif
 		"       -a: local apic is in xAPIC mode (deprecated)\n"
 		"       -A: create ACPI tables\n"
@@ -266,11 +251,12 @@ usage(int code)
 #endif
 		"       -D: destroy on power-off\n"
 		"       -e: exit on unhandled I/O access\n"
-		"       -g: gdb port\n"
 		"       -h: help\n"
 		"       -H: vmexit from the guest on hlt\n"
+		"       -k: key=value flat config file\n"
 		"       -l: LPC device configuration\n"
 		"       -m: memory size\n"
+		"       -o: set config 'var' to 'value'\n"
 #ifdef	__FreeBSD__
 		"       -p: pin 'vcpu' to 'hostcpu'\n"
 #endif
@@ -291,11 +277,8 @@ usage(int code)
 
 /*
  * XXX This parser is known to have the following issues:
- * 1.  It accepts null key=value tokens ",,".
- * 2.  It accepts whitespace after = and before value.
- * 3.  Values out of range of INT are silently wrapped.
- * 4.  It doesn't check non-final values.
- * 5.  The apparently bogus limits of UINT16_MAX are for future expansion.
+ * 1.  It accepts null key=value tokens ",," as setting "cpus" to an
+ *     empty string.
  *
  * The acceptance of a null specification ('-c ""') is by design to match the
  * manual page syntax specification, this results in a topology of 1 vCPU.
@@ -303,80 +286,116 @@ usage(int code)
 static int
 topology_parse(const char *opt)
 {
-	uint64_t ncpus;
-	int c, chk, n, s, t, tmp;
 	char *cp, *str;
-	bool ns, scts;
 
-	c = 1, n = 1, s = 1, t = 1;
-	ns = false, scts = false;
+	if (*opt == '\0') {
+		set_config_value("sockets", "1");
+		set_config_value("cores", "1");
+		set_config_value("threads", "1");
+		set_config_value("cpus", "1");
+		return (0);
+	}
+
 	str = strdup(opt);
 	if (str == NULL)
-		goto out;
+		errx(4, "Failed to allocate memory");
 
 	while ((cp = strsep(&str, ",")) != NULL) {
-		if (sscanf(cp, "%i%n", &tmp, &chk) == 1) {
-			n = tmp;
-			ns = true;
-		} else if (sscanf(cp, "cpus=%i%n", &tmp, &chk) == 1) {
-			n = tmp;
-			ns = true;
-		} else if (sscanf(cp, "sockets=%i%n", &tmp, &chk) == 1) {
-			s = tmp;
-			scts = true;
-		} else if (sscanf(cp, "cores=%i%n", &tmp, &chk) == 1) {
-			c = tmp;
-			scts = true;
-		} else if (sscanf(cp, "threads=%i%n", &tmp, &chk) == 1) {
-			t = tmp;
-			scts = true;
+		if (strncmp(cp, "cpus=", strlen("cpus=")) == 0)
+			set_config_value("cpus", cp + strlen("cpus="));
+		else if (strncmp(cp, "sockets=", strlen("sockets=")) == 0)
+			set_config_value("sockets", cp + strlen("sockets="));
+		else if (strncmp(cp, "cores=", strlen("cores=")) == 0)
+			set_config_value("cores", cp + strlen("cores="));
+		else if (strncmp(cp, "threads=", strlen("threads=")) == 0)
+			set_config_value("threads", cp + strlen("threads="));
 #ifdef notyet  /* Do not expose this until vmm.ko implements it */
-		} else if (sscanf(cp, "maxcpus=%i%n", &tmp, &chk) == 1) {
-			m = tmp;
+		else if (strncmp(cp, "maxcpus=", strlen("maxcpus=")) == 0)
+			set_config_value("maxcpus", cp + strlen("maxcpus="));
 #endif
-		/* Skip the empty argument case from -c "" */
-		} else if (cp[0] == '\0')
-			continue;
-		else
-			goto out;
-		/* Any trailing garbage causes an error */
-		if (cp[chk] != '\0')
+		else if (strchr(cp, '=') != NULL)
 			goto out;
+		else
+			set_config_value("cpus", cp);
 	}
 	free(str);
-	str = NULL;
-
-	/*
-	 * Range check 1 <= n <= UINT16_MAX all values
-	 */
-	if (n < 1 || s < 1 || c < 1 || t < 1 ||
-	    n > UINT16_MAX || s > UINT16_MAX || c > UINT16_MAX  ||
-	    t > UINT16_MAX)
-		return (-1);
-
-	/* If only the cpus was specified, use that as sockets */
-	if (!scts)
-		s = n;
-	/*
-	 * Compute sockets * cores * threads avoiding overflow
-	 * The range check above insures these are 16 bit values
-	 * If n was specified check it against computed ncpus
-	 */
-	ncpus = (uint64_t)s * c * t;
-	if (ncpus > UINT16_MAX || (ns && n != ncpus))
-		return (-1);
-
-	guest_ncpus = ncpus;
-	sockets = s;
-	cores = c;
-	threads = t;
-	return(0);
+	return (0);
 
 out:
 	free(str);
 	return (-1);
 }
 
+static int
+parse_int_value(const char *key, const char *value, int minval, int maxval)
+{
+	char *cp;
+	long lval;
+
+	errno = 0;
+	lval = strtol(value, &cp, 0);
+	if (errno != 0 || *cp != '\0' || cp == value || lval < minval ||
+	    lval > maxval)
+		errx(4, "Invalid value for %s: '%s'", key, value);
+	return (lval);
+}
+
+/*
+ * Set the sockets, cores, threads, and guest_cpus variables based on
+ * the configured topology.
+ *
+ * The limits of UINT16_MAX are due to the types passed to
+ * vm_set_topology().  vmm.ko may enforce tighter limits.
+ */
+static void
+calc_topolopgy(void)
+{
+	const char *value;
+	bool explicit_cpus;
+	uint64_t ncpus;
+
+	value = get_config_value("cpus");
+	if (value != NULL) {
+		guest_ncpus = parse_int_value("cpus", value, 1, UINT16_MAX);
+		explicit_cpus = true;
+	} else {
+		guest_ncpus = 1;
+		explicit_cpus = false;
+	}
+	value = get_config_value("cores");
+	if (value != NULL)
+		cores = parse_int_value("cores", value, 1, UINT16_MAX);
+	else
+		cores = 1;
+	value = get_config_value("threads");
+	if (value != NULL)
+		threads = parse_int_value("threads", value, 1, UINT16_MAX);
+	else
+		threads = 1;
+	value = get_config_value("sockets");
+	if (value != NULL)
+		sockets = parse_int_value("sockets", value, 1, UINT16_MAX);
+	else
+		sockets = guest_ncpus;
+
+	/*
+	 * Compute sockets * cores * threads avoiding overflow.  The
+	 * range check above insures these are 16 bit values.
+	 */
+	ncpus = (uint64_t)sockets * cores * threads;
+	if (ncpus > UINT16_MAX)
+		errx(4, "Computed number of vCPUs too high: %ju",
+		    (uintmax_t)ncpus);
+
+	if (explicit_cpus) {
+		if (guest_ncpus != ncpus)
+			errx(4, "Topology (%d sockets, %d cores, %d threads) "
+			    "does not match %d vCPUs", sockets, cores, threads,
+			    guest_ncpus);
+	} else
+		guest_ncpus = ncpus;
+}
+
 #ifndef WITHOUT_CAPSICUM
 /*
  * 11-stable capsicum helpers
@@ -437,17 +456,85 @@ pincpu_parse(const char *opt)
 		return (-1);
 	}
 
-	if (vcpumap[vcpu] == NULL) {
-		if ((vcpumap[vcpu] = malloc(sizeof(cpuset_t))) == NULL) {
-			perror("malloc");
-			return (-1);
-		}
-		CPU_ZERO(vcpumap[vcpu]);
+	snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu);
+	value = get_config_value(key);
+
+	if (asprintf(&newval, "%s%s%d", value != NULL ? value : "",
+	    value != NULL ? "," : "", pcpu) == -1) {
+		perror("failed to build new cpuset string");
+		return (-1);
 	}
-	CPU_SET(pcpu, vcpumap[vcpu]);
+
+	set_config_value(key, newval);
+	free(newval);
 	return (0);
 }
 
+static void
+parse_cpuset(int vcpu, const char *list, cpuset_t *set)
+{
+	char *cp, *token;
+	int pcpu, start;
+
+	CPU_ZERO(set);
+	start = -1;
+	token = __DECONST(char *, list);
+	for (;;) {
+		pcpu = strtoul(token, &cp, 0);
+		if (cp == token)
+			errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list);
+		if (pcpu < 0 || pcpu >= CPU_SETSIZE)
+			errx(4, "hostcpu '%d' outside valid range from 0 to %d",
+			    pcpu, CPU_SETSIZE - 1);
+		switch (*cp) {
+		case ',':
+		case '\0':
+			if (start >= 0) {
+				if (start > pcpu)
+					errx(4, "Invalid hostcpu range %d-%d",
+					    start, pcpu);
+				while (start < pcpu) {
+					CPU_SET(start, vcpumap[vcpu]);
+					start++;
+				}
+				start = -1;
+			}
+			CPU_SET(pcpu, vcpumap[vcpu]);
+			break;
+		case '-':
+			if (start >= 0)
+				errx(4, "invalid cpuset for vcpu %d: '%s'",
+				    vcpu, list);
+			start = pcpu;
+			break;
+		default:
+			errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list);
+		}
+		if (*cp == '\0')
+			break;
+		token = cp + 1;
+	}
+}
+
+static void
+build_vcpumaps(void)
+{
+	char key[16];
+	const char *value;
+	int vcpu;
+
+	for (vcpu = 0; vcpu < guest_ncpus; vcpu++) {
+		snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu);
+		value = get_config_value(key);
+		if (value == NULL)
+			continue;
+		vcpumap[vcpu] = malloc(sizeof(cpuset_t));
+		if (vcpumap[vcpu] == NULL)
+			err(4, "Failed to allocate cpuset for vcpu %d", vcpu);
+		parse_cpuset(vcpu, value, vcpumap[vcpu]);
+	}
+}
+
 void
 vm_inject_fault(void *arg, int vcpu, int vector, int errcode_valid,
     int errcode)
@@ -472,24 +559,10 @@ paddr_guest2host(struct vmctx *ctx, uintptr_t gaddr, size_t len)
 }
 
 int
-fbsdrun_vmexit_on_pause(void)
-{
-
-	return (guest_vmexit_on_pause);
-}
-
-int
-fbsdrun_vmexit_on_hlt(void)
-{
-
-	return (guest_vmexit_on_hlt);
-}
-
-int
 fbsdrun_virtio_msix(void)
 {
 
-	return (virtio_msix);
+	return (get_config_bool_default("virtio_msix", true));
 }
 
 static void *
@@ -505,8 +578,7 @@ fbsdrun_start_thread(void *param)
 	snprintf(tname, sizeof(tname), "vcpu %d", vcpu);
 	pthread_set_name_np(mtp->mt_thr, tname);
 
-	if (gdb_port != 0)
-		gdb_cpu_add(vcpu);
+	gdb_cpu_add(vcpu);
 
 	vm_loop(mtp->mt_ctx, vcpu, mtp->mt_startrip);
 
@@ -667,7 +739,7 @@ vmexit_inout(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
 		return (error);
 	}
 
-	error = emulate_inout(ctx, vcpu, &inout, strictio != 0);
+	error = emulate_inout(ctx, vcpu, &inout);
 	if (error) {
 		fprintf(stderr, "Unhandled %s%c 0x%04x at 0x%lx\n",
 		    in ? "in" : "out",
@@ -700,7 +772,7 @@ vmexit_rdmsr(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
 	if (error != 0) {
 		fprintf(stderr, "rdmsr to register %#x on vcpu %d\n",
 		    vme->u.msr.code, *pvcpu);
-		if (strictmsr) {
+		if (get_config_bool("x86.strictmsr")) {
 			vm_inject_gp(ctx, *pvcpu);
 			return (VMEXIT_CONTINUE);
 		}
@@ -726,7 +798,7 @@ vmexit_wrmsr(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
 	if (error != 0) {
 		fprintf(stderr, "wrmsr to register %#x(%#lx) on vcpu %d\n",
 		    vme->u.msr.code, vme->u.msr.wval, *pvcpu);
-		if (strictmsr) {
+		if (get_config_bool("x86.strictmsr")) {
 			vm_inject_gp(ctx, *pvcpu);
 			return (VMEXIT_CONTINUE);
 		}
@@ -880,11 +952,8 @@ vmexit_mtrap(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 
 	stats.vmexit_mtrap++;
 
-	if (gdb_port == 0) {
-		fprintf(stderr, "vm_loop: unexpected VMEXIT_MTRAP\n");
-		exit(4);
-	}
 	gdb_cpu_mtrap(*pvcpu);
+
 	return (VMEXIT_CONTINUE);
 }
 
@@ -986,7 +1055,7 @@ vmexit_suspend(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 	case VM_SUSPEND_RESET:
 		exit(0);
 	case VM_SUSPEND_POWEROFF:
-		if (destroy_on_poweroff)
+		if (get_config_bool_default("destroy_on_poweroff", false))
 			vm_destroy(ctx);
 		exit(1);
 	case VM_SUSPEND_HALT:
@@ -1004,10 +1073,6 @@ static int
 vmexit_debug(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 {
 
-	if (gdb_port == 0) {
-		fprintf(stderr, "vm_loop: unexpected VMEXIT_DEBUG\n");
-		exit(4);
-	}
 	gdb_cpu_suspend(*pvcpu);
 	return (VMEXIT_CONTINUE);
 }
@@ -1016,10 +1081,6 @@ static int
 vmexit_breakpoint(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 {
 
-	if (gdb_port == 0) {
-		fprintf(stderr, "vm_loop: unexpected VMEXIT_DEBUG\n");
-		exit(4);
-	}
 	gdb_cpu_breakpoint(*pvcpu, vmexit);
 	return (VMEXIT_CONTINUE);
 }
@@ -1132,7 +1193,7 @@ fbsdrun_set_capabilities(struct vmctx *ctx, int cpu)
 {
 	int err, tmp;
 
-	if (fbsdrun_vmexit_on_hlt()) {
+	if (get_config_bool_default("x86.vmexit_on_hlt", false)) {
 		err = vm_get_capability(ctx, cpu, VM_CAP_HALT_EXIT, &tmp);
 		if (err < 0) {
 			fprintf(stderr, "VM exit on HLT not supported\n");
@@ -1143,7 +1204,7 @@ fbsdrun_set_capabilities(struct vmctx *ctx, int cpu)
 			handler[VM_EXITCODE_HLT] = vmexit_hlt;
 	}
 
-        if (fbsdrun_vmexit_on_pause()) {
+	if (get_config_bool_default("x86.vmexit_on_pause", false)) {
 		/*
 		 * pause exit support required for this mode
 		 */
@@ -1158,7 +1219,7 @@ fbsdrun_set_capabilities(struct vmctx *ctx, int cpu)
 			handler[VM_EXITCODE_PAUSE] = vmexit_pause;
         }
 
-	if (x2apic_mode)
+	if (get_config_bool_default("x86.x2apic", false))
 		err = vm_set_x2apic_state(ctx, cpu, X2APIC_ENABLED);
 	else
 		err = vm_set_x2apic_state(ctx, cpu, X2APIC_DISABLED);
@@ -1248,63 +1309,103 @@ do_open(const char *vmname)
 	return (ctx);
 }
 
+static bool
+parse_config_option(const char *option)
+{
+	const char *value;
+	char *path;
+
+	value = strchr(option, '=');
+	if (value == NULL || value[1] == '\0')
+		return (false);
+	path = strndup(option, value - option);
+	if (path == NULL)
+		err(4, "Failed to allocate memory");
+	set_config_value(path, value + 1);
+	return (true);
+}
+
+static void
+parse_simple_config_file(const char *path)
+{
+	FILE *fp;
+	char *line, *cp;
+	size_t linecap;
+	unsigned int lineno;
+
+	fp = fopen(path, "r");
+	if (fp == NULL)
+		err(4, "Failed to open configuration file %s", path);
+	line = NULL;
+	linecap = 0;
+	lineno = 1;
+	for (lineno = 1; getline(&line, &linecap, fp) > 0; lineno++) {
+		if (*line == '#' || *line == '\n')
+			continue;
+		cp = strchr(line, '\n');
+		if (cp != NULL)
+			*cp = '\0';
+		if (!parse_config_option(line))
+			errx(4, "%s line %u: invalid config option '%s'", path,
+			    lineno, line);
+	}
+	free(line);
+	fclose(fp);
+}
+
+static void
+set_defaults(void)
+{
+
+	set_config_bool("acpi_tables", false);
+	set_config_value("memory.size", "256M");
+	set_config_bool("x86.strictmsr", true);
+}
+
 int
 main(int argc, char *argv[])
 {
-	int c, error, dbg_port, err, bvmcons;
-	int max_vcpus, mptgen, memflags;
-	int rtc_localtime;
-	bool gdb_stop;
-#ifndef __FreeBSD__
-	bool suspend = false;
-#endif
+	int c, error, err;
+	int max_vcpus, memflags;
 	struct vmctx *ctx;
 	uint64_t rip;
 	size_t memsize;
+	const char *value, *vmname;
 	char *optstr;
 
-	bvmcons = 0;
+	init_config();
+	set_defaults();
 	progname = basename(argv[0]);
-	dbg_port = 0;
-	gdb_stop = false;
-	guest_ncpus = 1;
-	sockets = cores = threads = 1;
-	maxcpus = 0;
-	memsize = 256 * MB;
-	mptgen = 1;
-	rtc_localtime = 1;
-	memflags = 0;
 
 #ifdef	__FreeBSD__
-	optstr = "abehuwxACDHIPSWYp:g:G:c:s:m:l:B:U:";
+	optstr = "aehuwxACDHIPSWYk:o:p:G:c:s:m:l:U:";
 #else
-	optstr = "abdehuwxACDHIPSWYg:G:c:s:m:l:B:U:";
+	/* +d, +B, -p */
+	optstr = "adehuwxACDHIPSWYk:o:G:c:s:m:l:B:U:";
 #endif
 	while ((c = getopt(argc, argv, optstr)) != -1) {
 		switch (c) {
 		case 'a':
-			x2apic_mode = 0;
+			set_config_bool("x86.x2apic", false);
 			break;
 		case 'A':
-			acpi = 1;
-			break;
-		case 'b':
-			bvmcons = 1;
+			set_config_bool("acpi_tables", true);
 			break;
 		case 'D':
-			destroy_on_poweroff = 1;
+			set_config_bool("destroy_on_poweroff", true);
 			break;
+#ifndef	__FreeBSD__
 		case 'B':
 			if (smbios_parse(optarg) != 0) {
 				errx(EX_USAGE, "invalid SMBIOS "
 				    "configuration '%s'", optarg);
 			}
 			break;
-#ifndef	__FreeBSD__
 		case 'd':
-			suspend = true;
+			set_config_bool("suspend_at_boot", true);
 			break;
-#else
+#endif
+#ifdef	__FreeBSD__
 		case 'p':
 			if (pincpu_parse(optarg) != 0) {
 				errx(EX_USAGE, "invalid vcpu pinning "
@@ -1319,17 +1420,17 @@ main(int argc, char *argv[])
 			}
 			break;
 		case 'C':
-			memflags |= VM_MEM_F_INCORE;
-			break;
-		case 'g':
-			dbg_port = atoi(optarg);
+			set_config_bool("memory.guest_in_core", true);
 			break;
 		case 'G':
 			if (optarg[0] == 'w') {
-				gdb_stop = true;
+				set_config_bool("gdb.wait", true);
 				optarg++;
 			}
-			gdb_port = atoi(optarg);
+			set_config_value("gdb.port", optarg);
+			break;
+		case 'k':
+			parse_simple_config_file(optarg);
 			break;
 		case 'l':
 			if (strncmp(optarg, "help", strlen(optarg)) == 0) {
@@ -1349,15 +1450,17 @@ main(int argc, char *argv[])
 			else
 				break;
 		case 'S':
-			memflags |= VM_MEM_F_WIRED;
+			set_config_bool("memory.wired", true);
 			break;
                 case 'm':
-			error = vm_parse_memsize(optarg, &memsize);
-			if (error)
-				errx(EX_USAGE, "invalid memsize '%s'", optarg);
+			set_config_value("memory.size", optarg);
+			break;
+		case 'o':
+			if (!parse_config_option(optarg))
+				errx(EX_USAGE, "invalid configuration option '%s'", optarg);
 			break;
 		case 'H':
-			guest_vmexit_on_hlt = 1;
+			set_config_bool("x86.vmexit_on_hlt", true);
 			break;
 		case 'I':
 			/*
@@ -1369,28 +1472,28 @@ main(int argc, char *argv[])
 			 */
 			break;
 		case 'P':
-			guest_vmexit_on_pause = 1;
+			set_config_bool("x86.vmexit_on_pause", true);
 			break;
 		case 'e':
-			strictio = 1;
+			set_config_bool("x86.strictio", true);
 			break;
 		case 'u':
-			rtc_localtime = 0;
+			set_config_bool("rtc.use_localtime", false);
 			break;
 		case 'U':
-			guest_uuid_str = optarg;
+			set_config_value("uuid", optarg);
 			break;
 		case 'w':
-			strictmsr = 0;
+			set_config_bool("x86.strictmsr", false);
 			break;
 		case 'W':
-			virtio_msix = 0;
+			set_config_bool("virtio_msix", false);
 			break;
 		case 'x':
-			x2apic_mode = 1;
+			set_config_bool("x86.x2apic", true);
 			break;
 		case 'Y':
-			mptgen = 0;
+			set_config_bool("x86.mptable", false);
 			break;
 		case 'h':
 			usage(0);			
@@ -1401,14 +1504,35 @@ main(int argc, char *argv[])
 	argc -= optind;
 	argv += optind;
 
-	if (argc != 1)
+	if (argc > 1)
+		usage(1);
+
+	if (argc == 1)
+		set_config_value("name", argv[0]);
+
+	vmname = get_config_value("name");
+	if (vmname == NULL)
 		usage(1);
 
-	vmname = argv[0];
+	if (get_config_bool_default("config.dump", false)) {
+		dump_config();
+		exit(1);
+	}
+
+	calc_topolopgy();
+#ifdef __FreeBSD__
+	build_vcpumaps();
+#endif
+
+	value = get_config_value("memory.size");
+	error = vm_parse_memsize(value, &memsize);
+	if (error)
+		errx(EX_USAGE, "invalid memsize '%s'", value);
+
 	ctx = do_open(vmname);
 
-	max_vcpus = num_vcpus_allowed(ctx);
-	if (guest_ncpus > max_vcpus) {
+        max_vcpus = num_vcpus_allowed(ctx);
+        if (guest_ncpus > max_vcpus) {
 		fprintf(stderr, "%d vCPUs requested but only %d available\n",
 			guest_ncpus, max_vcpus);
 		exit(4);
@@ -1416,6 +1540,11 @@ main(int argc, char *argv[])
 
 	fbsdrun_set_capabilities(ctx, BSP);
 
+	memflags = 0;
+	if (get_config_bool_default("memory.wired", false))
+		memflags |= VM_MEM_F_WIRED;
+	if (get_config_bool_default("memory.guest_in_core", false))
+		memflags |= VM_MEM_F_INCORE;
 	vm_set_memflags(ctx, memflags);
 #ifdef	__FreeBSD__
 	err = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
@@ -1432,7 +1561,7 @@ main(int argc, char *argv[])
 	} while (error == ENOMEM);
 #endif
 	if (err) {
-		fprintf(stderr, "Unable to setup memory (%d)\n", errno);
+		fprintf(stderr, "Unable to set up memory (%d)\n", errno);
 		exit(4);
 	}
 
@@ -1452,7 +1581,7 @@ main(int argc, char *argv[])
 	pci_irq_init(ctx);
 	ioapic_init(ctx);
 
-	rtc_init(ctx, rtc_localtime);
+	rtc_init(ctx);
 	sci_init(ctx);
 #ifndef	__FreeBSD__
 	pmtmr_init(ctx);
@@ -1470,30 +1599,28 @@ main(int argc, char *argv[])
 	 * Initialize after PCI, to allow a bootrom file to reserve the high
 	 * region.
 	 */
-	if (acpi)
+	if (get_config_bool("acpi_tables"))
 		vmgenc_init(ctx);
 
-	if (dbg_port != 0)
-		init_dbgport(dbg_port);
-
+	value = get_config_value("gdb.port");
 #ifdef __FreeBSD__
-	if (gdb_port != 0)
-		init_gdb(ctx, gdb_port, gdb_stop);
+	if (value != NULL)
+		init_gdb(ctx, atoi(value), get_config_bool_default("gdb.wait",
+		    false));
 #else
-	if (gdb_port < 0) {
-		/*
-		 * Set up the internal gdb state needed for basic debugging, but
-		 * skip the step of listening on a port for the GDB server.
-		 */
-		init_mdb(ctx, gdb_stop);
-	} else if (gdb_port != 0) {
-		init_gdb(ctx, gdb_port, gdb_stop);
+	if (value != NULL) {
+		int port = atoi(value);
+
+		if (port < 0) {
+			init_mdb(ctx,
+			    get_config_bool_default("gdb.wait", false));
+		} else {
+			init_gdb(ctx, port,
+			    get_config_bool_default("gdb.wait", false));
+		}
 	}
 #endif
 
-	if (bvmcons)
-		init_bvmcons();
-
 	vga_init(1);
 
 	if (lpc_bootrom()) {
@@ -1516,7 +1643,7 @@ main(int argc, char *argv[])
 	/*
  	 * build the guest tables, MP etc.
 	 */
-	if (mptgen) {
+	if (get_config_bool_default("x86.mptable", true)) {
 		error = mptable_build(ctx, guest_ncpus);
 		if (error) {
 			perror("error to build the guest tables");
@@ -1524,10 +1651,13 @@ main(int argc, char *argv[])
 		}
 	}
 
+#ifndef __FreeBSD__
+	smbios_apply();
+#endif
 	error = smbios_build(ctx);
 	assert(error == 0);
 
-	if (acpi) {
+	if (get_config_bool("acpi_tables")) {
 		error = acpi_build(ctx, guest_ncpus);
 		assert(error == 0);
 	}
@@ -1559,7 +1689,8 @@ main(int argc, char *argv[])
 	/* Set BSP to run (unlike the APs which wait for INIT) */
 	error = vm_set_run_state(ctx, BSP, VRS_RUN, 0);
 	assert(error == 0);
-	fbsdrun_addcpu(ctx, BSP, rip, suspend);
+	fbsdrun_addcpu(ctx, BSP, rip,
+	    get_config_bool_default("suspend_at_boot", false));
 
 	/* Add subsequent CPUs, which will wait until INIT/SIPI-ed */
 	for (uint_t i = 1; i < guest_ncpus; i++) {
diff --git a/usr/src/cmd/bhyve/bhyverun.h b/usr/src/cmd/bhyve/bhyverun.h
index f2582512ea..24ff115345 100644
--- a/usr/src/cmd/bhyve/bhyverun.h
+++ b/usr/src/cmd/bhyve/bhyverun.h
@@ -49,8 +49,6 @@
 struct vmctx;
 extern int guest_ncpus;
 extern uint16_t cores, sockets, threads;
-extern char *guest_uuid_str;
-extern char *vmname;
 
 void *paddr_guest2host(struct vmctx *ctx, uintptr_t addr, size_t len);
 
@@ -60,9 +58,6 @@ void fbsdrun_addcpu(struct vmctx *ctx, int fromcpu, int newcpu, uint64_t rip);
 #else
 void fbsdrun_addcpu(struct vmctx *ctx, int newcpu, uint64_t rip, bool suspend);
 #endif
-int  fbsdrun_muxed(void);
-int  fbsdrun_vmexit_on_hlt(void);
-int  fbsdrun_vmexit_on_pause(void);
-int  fbsdrun_disable_x2apic(void);
 int  fbsdrun_virtio_msix(void);
+
 #endif
diff --git a/usr/src/cmd/bhyve/block_if.c b/usr/src/cmd/bhyve/block_if.c
index 23a04c0f5b..f15ff1ae7b 100644
--- a/usr/src/cmd/bhyve/block_if.c
+++ b/usr/src/cmd/bhyve/block_if.c
@@ -69,10 +69,12 @@ __FBSDID("$FreeBSD$");
 #include <machine/atomic.h>
 
 #include "bhyverun.h"
+#include "config.h"
 #include "debug.h"
 #ifdef	__FreeBSD__
 #include "mevent.h"
 #endif
+#include "pci_emul.h"
 #include "block_if.h"
 
 #define BLOCKIF_SIG	0xb109b109
@@ -488,16 +490,34 @@ blockif_init(void)
 #endif
 }
 
+int
+blockif_legacy_config(nvlist_t *nvl, const char *opts)
+{
+	char *cp, *path;
+
+	if (opts == NULL)
+		return (0);
+
+	cp = strchr(opts, ',');
+	if (cp == NULL) {
+		set_config_value_node(nvl, "path", opts);
+		return (0);
+	}
+	path = strndup(opts, cp - opts);
+	set_config_value_node(nvl, "path", path);
+	free(path);
+	return (pci_parse_legacy_config(nvl, cp + 1));
+}
+
 struct blockif_ctxt *
-blockif_open(const char *optstr, const char *ident)
+blockif_open(nvlist_t *nvl, const char *ident)
 {
 	char tname[MAXCOMLEN + 1];
 #ifdef	__FreeBSD__
 	char name[MAXPATHLEN];
-	char *nopt, *xopts, *cp;
-#else
-	char *nopt, *xopts, *cp = NULL;
 #endif
+	const char *path, *pssval, *ssval;
+	char *cp;
 	struct blockif_ctxt *bc;
 	struct stat sbuf;
 #ifdef	__FreeBSD__
@@ -507,7 +527,7 @@ blockif_open(const char *optstr, const char *ident)
 #endif
 	off_t size, psectsz, psectoff;
 	int extra, fd, i, sectsz;
-	int nocache, sync, ro, candelete, geom, ssopt, pssopt;
+	int ro, candelete, geom, ssopt, pssopt;
 	int nodelete;
 
 #ifndef WITHOUT_CAPSICUM
@@ -518,59 +538,65 @@ blockif_open(const char *optstr, const char *ident)
 	pthread_once(&blockif_once, blockif_init);
 
 	fd = -1;
+	extra = 0;
 	ssopt = 0;
-	nocache = 0;
-	sync = 0;
+#ifndef __FreeBSD__
+	pssopt = 0;
+#endif
 	ro = 0;
 	nodelete = 0;
 
-	/*
-	 * The first element in the optstring is always a pathname.
-	 * Optional elements follow
-	 */
-	nopt = xopts = strdup(optstr);
-	while (xopts != NULL) {
-		cp = strsep(&xopts, ",");
-		if (cp == nopt)		/* file or device pathname */
-			continue;
-		else if (!strcmp(cp, "nocache"))
-			nocache = 1;
-		else if (!strcmp(cp, "nodelete"))
-			nodelete = 1;
-		else if (!strcmp(cp, "sync") || !strcmp(cp, "direct"))
-			sync = 1;
-		else if (!strcmp(cp, "ro"))
-			ro = 1;
-		else if (sscanf(cp, "sectorsize=%d/%d", &ssopt, &pssopt) == 2)
-			;
-		else if (sscanf(cp, "sectorsize=%d", &ssopt) == 1)
+	if (get_config_bool_node_default(nvl, "nocache", false))
+		extra |= O_DIRECT;
+	if (get_config_bool_node_default(nvl, "nodelete", false))
+		nodelete = 1;
+	if (get_config_bool_node_default(nvl, "sync", false) ||
+	    get_config_bool_node_default(nvl, "direct", false))
+		extra |= O_SYNC;
+	if (get_config_bool_node_default(nvl, "ro", false))
+		ro = 1;
+	ssval = get_config_value_node(nvl, "sectorsize");
+	if (ssval != NULL) {
+		ssopt = strtol(ssval, &cp, 10);
+		if (cp == ssval) {
+			EPRINTLN("Invalid sector size \"%s\"", ssval);
+			goto err;
+		}
+		if (*cp == '\0') {
 			pssopt = ssopt;
-		else {
-			EPRINTLN("Invalid device option \"%s\"", cp);
+		} else if (*cp == '/') {
+			pssval = cp + 1;
+			pssopt = strtol(pssval, &cp, 10);
+			if (cp == pssval || *cp != '\0') {
+				EPRINTLN("Invalid sector size \"%s\"", ssval);
+				goto err;
+			}
+		} else {
+			EPRINTLN("Invalid sector size \"%s\"", ssval);
 			goto err;
 		}
 	}
 
-	extra = 0;
-	if (nocache)
-		extra |= O_DIRECT;
-	if (sync)
-		extra |= O_SYNC;
+	path = get_config_value_node(nvl, "path");
+	if (path == NULL) {
+		EPRINTLN("Missing \"path\" for block device.");
+		goto err;
+	}
 
-	fd = open(nopt, (ro ? O_RDONLY : O_RDWR) | extra);
+	fd = open(path, (ro ? O_RDONLY : O_RDWR) | extra);
 	if (fd < 0 && !ro) {
 		/* Attempt a r/w fail with a r/o open */
-		fd = open(nopt, O_RDONLY | extra);
+		fd = open(path, O_RDONLY | extra);
 		ro = 1;
 	}
 
 	if (fd < 0) {
-		warn("Could not open backing file: %s", nopt);
+		warn("Could not open backing file: %s", path);
 		goto err;
 	}
 
         if (fstat(fd, &sbuf) < 0) {
-		warn("Could not stat backing file %s", nopt);
+		warn("Could not stat backing file %s", path);
 		goto err;
         }
 
@@ -759,7 +785,6 @@ blockif_open(const char *optstr, const char *ident)
 err:
 	if (fd >= 0)
 		close(fd);
-	free(nopt);
 	return (NULL);
 }
 
diff --git a/usr/src/cmd/bhyve/block_if.h b/usr/src/cmd/bhyve/block_if.h
index bff2b42768..d0a7f8812f 100644
--- a/usr/src/cmd/bhyve/block_if.h
+++ b/usr/src/cmd/bhyve/block_if.h
@@ -38,6 +38,7 @@
 #ifndef _BLOCK_IF_H_
 #define _BLOCK_IF_H_
 
+#include <sys/nv.h>
 #include <sys/uio.h>
 #include <sys/unistd.h>
 
@@ -59,7 +60,8 @@ struct blockif_req {
 };
 
 struct blockif_ctxt;
-struct blockif_ctxt *blockif_open(const char *optstr, const char *ident);
+int	blockif_legacy_config(nvlist_t *nvl, const char *opts);
+struct blockif_ctxt *blockif_open(nvlist_t *nvl, const char *ident);
 off_t	blockif_size(struct blockif_ctxt *bc);
 void	blockif_chs(struct blockif_ctxt *bc, uint16_t *c, uint8_t *h,
     uint8_t *s);
diff --git a/usr/src/cmd/bhyve/config.c b/usr/src/cmd/bhyve/config.c
new file mode 100644
index 0000000000..db28cc5516
--- /dev/null
+++ b/usr/src/cmd/bhyve/config.c
@@ -0,0 +1,439 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2021 John H. Baldwin <jhb@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <assert.h>
+#include <err.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "config.h"
+
+static nvlist_t *config_root;
+
+void
+init_config(void)
+{
+	config_root = nvlist_create(0);
+	if (config_root == NULL)
+		err(4, "Failed to create configuration root nvlist");
+}
+
+static nvlist_t *
+_lookup_config_node(nvlist_t *parent, const char *path, bool create)
+{
+	char *copy, *name, *tofree;
+	nvlist_t *nvl, *new_nvl;
+
+	copy = strdup(path);
+	if (copy == NULL)
+		errx(4, "Failed to allocate memory");
+	tofree = copy;
+	nvl = parent;
+	while ((name = strsep(&copy, ".")) != NULL) {
+		if (*name == '\0') {
+			warnx("Invalid configuration node: %s", path);
+			nvl = NULL;
+			break;
+		}
+		if (nvlist_exists_nvlist(nvl, name))
+			nvl = (nvlist_t *)nvlist_get_nvlist(nvl, name);
+		else if (nvlist_exists(nvl, name)) {
+			for (copy = tofree; copy < name; copy++)
+				if (*copy == '\0')
+					*copy = '.';
+			warnx(
+		    "Configuration node %s is a child of existing variable %s",
+			    path, tofree);
+			nvl = NULL;
+			break;
+		} else if (create) {
+			new_nvl = nvlist_create(0);
+			if (new_nvl == NULL)
+				errx(4, "Failed to allocate memory");
+#ifdef __FreeBSD__
+			nvlist_move_nvlist(nvl, name, new_nvl);
+#else
+			if (nvlist_add_nvlist(nvl, name, new_nvl) != 0)
+				errx(4, "Failed to allocate memory");
+			(void) nvlist_free(new_nvl);
+			if (nvlist_lookup_nvlist(nvl, name, &new_nvl) != 0)
+				errx(4, "Failed to retrieve new nvlist");
+#endif
+			nvl = new_nvl;
+		} else {
+			nvl = NULL;
+			break;
+		}
+	}
+	free(tofree);
+	return (nvl);
+}
+
+nvlist_t *
+create_config_node(const char *path)
+{
+
+	return (_lookup_config_node(config_root, path, true));
+}
+
+nvlist_t *
+find_config_node(const char *path)
+{
+
+	return (_lookup_config_node(config_root, path, false));
+}
+
+nvlist_t *
+create_relative_config_node(nvlist_t *parent, const char *path)
+{
+
+	return (_lookup_config_node(parent, path, true));
+}
+
+nvlist_t *
+find_relative_config_node(nvlist_t *parent, const char *path)
+{
+
+	return (_lookup_config_node(parent, path, false));
+}
+
+void
+set_config_value_node(nvlist_t *parent, const char *name, const char *value)
+{
+
+	if (strchr(name, '.') != NULL)
+		errx(4, "Invalid config node name %s", name);
+	if (parent == NULL)
+		parent = config_root;
+	if (nvlist_exists_string(parent, name))
+		nvlist_free_string(parent, name);
+	else if (nvlist_exists(parent, name))
+		errx(4,
+		    "Attemping to add value %s to existing node %s of list %p",
+		    value, name, parent);
+	nvlist_add_string(parent, name, value);
+}
+
+void
+set_config_value(const char *path, const char *value)
+{
+	const char *name;
+	char *node_name;
+	nvlist_t *nvl;
+
+	/* Look for last separator. */
+	name = strrchr(path, '.');
+	if (name == NULL) {
+		nvl = config_root;
+		name = path;
+	} else {
+		node_name = strndup(path, name - path);
+		if (node_name == NULL)
+			errx(4, "Failed to allocate memory");
+		nvl = create_config_node(node_name);
+		if (nvl == NULL)
+			errx(4, "Failed to create configuration node %s",
+			    node_name);
+		free(node_name);
+
+		/* Skip over '.'. */
+		name++;
+	}
+
+	if (nvlist_exists_nvlist(nvl, name))
+		errx(4, "Attempting to add value %s to existing node %s",
+		    value, path);
+	set_config_value_node(nvl, name, value);
+}
+
+static const char *
+get_raw_config_value(const char *path)
+{
+	const char *name;
+	char *node_name;
+	nvlist_t *nvl;
+
+	/* Look for last separator. */
+	name = strrchr(path, '.');
+	if (name == NULL) {
+		nvl = config_root;
+		name = path;
+	} else {
+		node_name = strndup(path, name - path);
+		if (node_name == NULL)
+			errx(4, "Failed to allocate memory");
+		nvl = find_config_node(node_name);
+		free(node_name);
+		if (nvl == NULL)
+			return (NULL);
+
+		/* Skip over '.'. */
+		name++;
+	}
+
+	if (nvlist_exists_string(nvl, name))
+		return (nvlist_get_string(nvl, name));
+	if (nvlist_exists_nvlist(nvl, name))
+		warnx("Attempting to fetch value of node %s", path);
+	return (NULL);
+}
+
+static char *
+_expand_config_value(const char *value, int depth)
+{
+	FILE *valfp;
+	const char *cp, *vp;
+	char *nestedval, *path, *valbuf;
+	size_t valsize;
+
+	valfp = open_memstream(&valbuf, &valsize);
+	if (valfp == NULL)
+		errx(4, "Failed to allocate memory");
+
+	vp = value;
+	while (*vp != '\0') {
+		switch (*vp) {
+		case '%':
+			if (depth > 15) {
+				warnx(
+		    "Too many recursive references in configuration value");
+				fputc('%', valfp);
+				vp++;
+				break;
+			}				
+			if (vp[1] != '(' || vp[2] == '\0')
+				cp = NULL;
+			else
+				cp = strchr(vp + 2, ')');
+			if (cp == NULL) {
+				warnx(
+			    "Invalid reference in configuration value \"%s\"",
+				    value);
+				fputc('%', valfp);
+				vp++;
+				break;
+			}
+			vp += 2;
+
+			if (cp == vp) {
+				warnx(
+			    "Empty reference in configuration value \"%s\"",
+				    value);
+				vp++;
+				break;
+			}
+
+			/* Allocate a C string holding the path. */
+			path = strndup(vp, cp - vp);
+			if (path == NULL)
+				errx(4, "Failed to allocate memory");
+
+			/* Advance 'vp' past the reference. */
+			vp = cp + 1;
+
+			/* Fetch the referenced value. */
+			cp = get_raw_config_value(path);
+			if (cp == NULL)
+				warnx(
+		    "Failed to fetch referenced configuration variable %s",
+				    path);
+			else {
+				nestedval = _expand_config_value(cp, depth + 1);
+				fputs(nestedval, valfp);
+				free(nestedval);
+			}
+			free(path);
+			break;
+		case '\\':
+			vp++;
+			if (*vp == '\0') {
+				warnx(
+			    "Trailing \\ in configuration value \"%s\"",
+				    value);
+				break;
+			}
+			/* FALLTHROUGH */
+		default:
+			fputc(*vp, valfp);
+			vp++;
+			break;
+		}
+	}
+	fclose(valfp);
+	return (valbuf);
+}
+
+const char *
+expand_config_value(const char *value)
+{
+	static char *valbuf;
+
+	if (strchr(value, '%') == NULL)
+		return (value);
+
+	free(valbuf);
+	valbuf = _expand_config_value(value, 0);
+	return (valbuf);
+}
+
+const char *
+get_config_value(const char *path)
+{
+	const char *value;
+
+	value = get_raw_config_value(path);
+	if (value == NULL)
+		return (NULL);
+	return (expand_config_value(value));
+}
+
+const char *
+get_config_value_node(const nvlist_t *parent, const char *name)
+{
+
+	if (strchr(name, '.') != NULL)
+		errx(4, "Invalid config node name %s", name);
+	if (parent == NULL)
+		parent = config_root;
+
+	if (nvlist_exists_nvlist(parent, name))
+		warnx("Attempt to fetch value of node %s of list %p", name,
+		    parent);
+	if (!nvlist_exists_string(parent, name))
+		return (NULL);
+
+	return (expand_config_value(nvlist_get_string(parent, name)));
+}
+
+bool
+_bool_value(const char *name, const char *value)
+{
+
+	if (strcasecmp(value, "true") == 0 ||
+	    strcasecmp(value, "on") == 0 ||
+	    strcasecmp(value, "yes") == 0 ||
+	    strcmp(value, "1") == 0)
+		return (true);
+	if (strcasecmp(value, "false") == 0 ||
+	    strcasecmp(value, "off") == 0 ||
+	    strcasecmp(value, "no") == 0 ||
+	    strcmp(value, "0") == 0)
+		return (false);
+	err(4, "Invalid value %s for boolean variable %s", value, name);
+}
+
+bool
+get_config_bool(const char *path)
+{
+	const char *value;
+
+	value = get_config_value(path);
+	if (value == NULL)
+		err(4, "Failed to fetch boolean variable %s", path);
+	return (_bool_value(path, value));
+}
+
+bool
+get_config_bool_default(const char *path, bool def)
+{
+	const char *value;
+
+	value = get_config_value(path);
+	if (value == NULL)
+		return (def);
+	return (_bool_value(path, value));
+}
+
+bool
+get_config_bool_node(const nvlist_t *parent, const char *name)
+{
+	const char *value;
+
+	value = get_config_value_node(parent, name);
+	if (value == NULL)
+		err(4, "Failed to fetch boolean variable %s", name);
+	return (_bool_value(name, value));
+}
+
+bool
+get_config_bool_node_default(const nvlist_t *parent, const char *name,
+    bool def)
+{
+	const char *value;
+
+	value = get_config_value_node(parent, name);
+	if (value == NULL)
+		return (def);
+	return (_bool_value(name, value));
+}
+
+void
+set_config_bool(const char *path, bool value)
+{
+
+	set_config_value(path, value ? "true" : "false");
+}
+
+void
+set_config_bool_node(nvlist_t *parent, const char *name, bool value)
+{
+
+	set_config_value_node(parent, name, value ? "true" : "false");
+}
+
+static void
+dump_tree(const char *prefix, const nvlist_t *nvl)
+{
+	const char *name;
+	void *cookie;
+	int type;
+
+	cookie = NULL;
+	while ((name = nvlist_next(nvl, &type, &cookie)) != NULL) {
+		if (type == NV_TYPE_NVLIST) {
+			char *new_prefix;
+
+			asprintf(&new_prefix, "%s%s.", prefix, name);
+			dump_tree(new_prefix, nvlist_get_nvlist(nvl, name));
+			free(new_prefix);
+		} else {
+			assert(type == NV_TYPE_STRING);
+			printf("%s%s=%s\n", prefix, name,
+			    nvlist_get_string(nvl, name));
+		}
+	}
+}
+
+void
+dump_config(void)
+{
+	dump_tree("", config_root);
+}
diff --git a/usr/src/cmd/bhyve/config.h b/usr/src/cmd/bhyve/config.h
new file mode 100644
index 0000000000..9f82afb267
--- /dev/null
+++ b/usr/src/cmd/bhyve/config.h
@@ -0,0 +1,119 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2021 John H. Baldwin <jhb@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef __CONFIG_H__
+#define	__CONFIG_H__
+
+#include <sys/nv.h>
+
+/*-
+ * Manages a configuration database backed by an nv(9) list.
+ *
+ * The database only stores string values.  Callers should parse
+ * values into other types if needed.  String values can reference
+ * other configuration variables using a '%(name)' syntax.  In this
+ * case, the name must be the the full path of the configuration
+ * variable.  The % character can be escaped with a preceding \ to
+ * avoid expansion.  Any \ characters must be escaped.
+ *
+ * Configuration variables are stored in a tree.  The full path of a
+ * variable is specified as a dot-separated name similar to sysctl(8)
+ * OIDs.
+ */ 
+
+/*
+ * Fetches the value of a configuration variable.  If the "raw" value
+ * contains references to other configuration variables, this function
+ * expands those references and returns a pointer to the parsed
+ * string.  The string's storage is only stable until the next call to
+ * this function.
+ *
+ * If no node is found, returns NULL.
+ *
+ * If 'parent' is NULL, 'name' is assumed to be a top-level variable.
+ */
+const char *get_config_value_node(const nvlist_t *parent, const char *name);
+
+/*
+ * Similar to get_config_value_node but expects a full path to the
+ * leaf node.
+ */
+const char *get_config_value(const char *path);
+
+/* Initializes the tree to an empty state. */
+void	init_config(void);
+
+/*
+ * Creates an existing configuration node via a dot-separated OID
+ * path.  Will fail if the path names an existing leaf configuration
+ * variable.  If the node already exists, this returns a pointer to
+ * the existing node.
+ */
+nvlist_t *create_config_node(const char *path);
+
+/*
+ * Looks for an existing configuration node via a dot-separated OID
+ * path.  Will fail if the path names an existing leaf configuration
+ * variable.
+ */
+nvlist_t *find_config_node(const char *path);
+
+/*
+ * Similar to the above, but treats the path relative to an existing
+ * 'parent' node rather than as an absolute path.
+ */
+nvlist_t *create_relative_config_node(nvlist_t *parent, const char *path);
+nvlist_t *find_relative_config_node(nvlist_t *parent, const char *path);
+
+/*
+ * Adds or replaces the value of the specified variable.
+ *
+ * If 'parent' is NULL, 'name' is assumed to be a top-level variable.
+ */
+void	set_config_value_node(nvlist_t *parent, const char *name,
+    const char *value);
+
+/*
+ * Similar to set_config_value_node but expects a full path to the
+ * leaf node.
+ */
+void	set_config_value(const char *path, const char *value);
+
+/* Convenience wrappers for boolean variables. */
+bool	get_config_bool(const char *path);
+bool	get_config_bool_node(const nvlist_t *parent, const char *name);
+bool	get_config_bool_default(const char *path, bool def);
+bool	get_config_bool_node_default(const nvlist_t *parent, const char *name,
+    bool def);
+void	set_config_bool(const char *path, bool value);
+void	set_config_bool_node(nvlist_t *parent, const char *name, bool value);
+
+void	dump_config(void);
+
+#endif /* !__CONFIG_H__ */
diff --git a/usr/src/cmd/bhyve/consport.c b/usr/src/cmd/bhyve/consport.c
deleted file mode 100644
index 42ba910f76..0000000000
--- a/usr/src/cmd/bhyve/consport.c
+++ /dev/null
@@ -1,182 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
- *
- * Copyright (c) 2011 NetApp, Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-#ifndef WITHOUT_CAPSICUM
-#include <sys/capsicum.h>
-#endif
-#include <sys/select.h>
-
-#ifndef WITHOUT_CAPSICUM
-#include <capsicum_helpers.h>
-#endif
-#include <err.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <termios.h>
-#include <unistd.h>
-#include <stdbool.h>
-#include <sysexits.h>
-
-#include "inout.h"
-#include "pci_lpc.h"
-#include "debug.h"
-
-#define	BVM_CONSOLE_PORT	0x220
-#define	BVM_CONS_SIG		('b' << 8 | 'v')
-
-#ifdef	__FreeBSD__
-static struct termios tio_orig, tio_new;
-
-static void
-ttyclose(void)
-{
-	tcsetattr(STDIN_FILENO, TCSANOW, &tio_orig);
-}
-#endif
-
-static void
-ttyopen(void)
-{
-#ifdef	__FreeBSD__
-	tcgetattr(STDIN_FILENO, &tio_orig);
-
-	cfmakeraw(&tio_new);
-	tcsetattr(STDIN_FILENO, TCSANOW, &tio_new);	
-	raw_stdio = 1;
-
-	atexit(ttyclose);
-#endif
-}
-
-static bool
-tty_char_available(void)
-{
-	fd_set rfds;
-	struct timeval tv;
-
-	FD_ZERO(&rfds);
-	FD_SET(STDIN_FILENO, &rfds);
-	tv.tv_sec = 0;
-	tv.tv_usec = 0;
-	if (select(STDIN_FILENO + 1, &rfds, NULL, NULL, &tv) > 0) {
-		return (true);
-	} else {
-		return (false);
-	}
-}
-
-static int
-ttyread(void)
-{
-	char rb;
-
-	if (tty_char_available()) {
-		read(STDIN_FILENO, &rb, 1);
-		return (rb & 0xff);
-	} else {
-		return (-1);
-	}
-}
-
-static void
-ttywrite(unsigned char wb)
-{
-	(void) write(STDOUT_FILENO, &wb, 1);
-}
-
-static int
-console_handler(struct vmctx *ctx, int vcpu, int in, int port, int bytes,
-		uint32_t *eax, void *arg)
-{
-	static int opened;
-#ifndef WITHOUT_CAPSICUM
-	cap_rights_t rights;
-	cap_ioctl_t cmds[] = { TIOCGETA, TIOCSETA, TIOCGWINSZ };
-#endif
-
-	if (bytes == 2 && in) {
-		*eax = BVM_CONS_SIG;
-		return (0);
-	}
-
-	/*
-	 * Guests might probe this port to look for old ISA devices
-	 * using single-byte reads.  Return 0xff for those.
-	 */
-	if (bytes == 1 && in) {
-		*eax = 0xff;
-		return (0);
-	}
-
-	if (bytes != 4)
-		return (-1);
-
-	if (!opened) {
-#ifndef WITHOUT_CAPSICUM
-		cap_rights_init(&rights, CAP_EVENT, CAP_IOCTL, CAP_READ,
-		    CAP_WRITE);
-		if (caph_rights_limit(STDIN_FILENO, &rights) == -1)
-			errx(EX_OSERR, "Unable to apply rights for sandbox");
-		if (caph_ioctls_limit(STDIN_FILENO, cmds, nitems(cmds)) == -1)
-			errx(EX_OSERR, "Unable to apply rights for sandbox");
-#endif
-		ttyopen();
-		opened = 1;
-	}
-	
-	if (in)
-		*eax = ttyread();
-	else
-		ttywrite(*eax);
-
-	return (0);
-}
-
-SYSRES_IO(BVM_CONSOLE_PORT, 4);
-
-static struct inout_port consport = {
-	"bvmcons",
-	BVM_CONSOLE_PORT,
-	1,
-	IOPORT_F_INOUT,
-	console_handler
-};
-
-void
-init_bvmcons(void)
-{
-
-	register_inout(&consport);
-}
diff --git a/usr/src/cmd/bhyve/dbgport.c b/usr/src/cmd/bhyve/dbgport.c
deleted file mode 100644
index 88a616b50d..0000000000
--- a/usr/src/cmd/bhyve/dbgport.c
+++ /dev/null
@@ -1,180 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
- *
- * Copyright (c) 2011 NetApp, Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-#ifndef WITHOUT_CAPSICUM
-#include <sys/capsicum.h>
-#endif
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <netinet/tcp.h>
-#include <sys/uio.h>
-
-#ifndef WITHOUT_CAPSICUM
-#include <capsicum_helpers.h>
-#endif
-#include <err.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sysexits.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <errno.h>
-
-#include "inout.h"
-#include "dbgport.h"
-#include "pci_lpc.h"
-
-#define	BVM_DBG_PORT	0x224
-#define	BVM_DBG_SIG	('B' << 8 | 'V')
-
-static int listen_fd, conn_fd;
-
-static struct sockaddr_in sin;
-
-static int
-dbg_handler(struct vmctx *ctx, int vcpu, int in, int port, int bytes,
-	    uint32_t *eax, void *arg)
-{
-	int nwritten, nread, printonce;
-	int on = 1;
-	char ch;
-
-	if (bytes == 2 && in) {
-		*eax = BVM_DBG_SIG;
-		return (0);
-	}
-
-	if (bytes != 4)
-		return (-1);
-
-again:
-	printonce = 0;
-	while (conn_fd < 0) {
-		if (!printonce) {
-			printf("Waiting for connection from gdb\r\n");
-			printonce = 1;
-		}
-		conn_fd = accept4(listen_fd, NULL, NULL, SOCK_NONBLOCK);
-		if (conn_fd >= 0) {
-			/* Avoid EPIPE after the client drops off. */
-			(void)setsockopt(conn_fd, SOL_SOCKET, SO_NOSIGPIPE,
-			    &on, sizeof(on));
-			/* Improve latency for one byte at a time tranfers. */
-			(void)setsockopt(conn_fd, IPPROTO_TCP, TCP_NODELAY,
-			    &on, sizeof(on));
-		} else if (errno != EINTR) {
-			perror("accept");
-		}
-	}
-
-	if (in) {
-		nread = read(conn_fd, &ch, 1);
-		if (nread == -1 && errno == EAGAIN)
-			*eax = -1;
-		else if (nread == 1)
-			*eax = ch;
-		else {
-			close(conn_fd);
-			conn_fd = -1;
-			goto again;
-		}
-	} else {
-		ch = *eax;
-		nwritten = write(conn_fd, &ch, 1);
-		if (nwritten != 1) {
-			close(conn_fd);
-			conn_fd = -1;
-			goto again;
-		}
-	}
-	return (0);
-}
-
-static struct inout_port dbgport = {
-	"bvmdbg",
-	BVM_DBG_PORT,
-	1,
-	IOPORT_F_INOUT,
-	dbg_handler
-};
-
-SYSRES_IO(BVM_DBG_PORT, 4);
-
-void
-init_dbgport(int sport)
-{
-	int reuse;
-#ifndef WITHOUT_CAPSICUM
-	cap_rights_t rights;
-#endif
-
-	conn_fd = -1;
-
-	if ((listen_fd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
-		perror("cannot create socket");
-		exit(4);
-	}
-
-#ifdef	__FreeBSD__
-	sin.sin_len = sizeof(sin);
-#endif
-	sin.sin_family = AF_INET;
-	sin.sin_addr.s_addr = htonl(INADDR_ANY);
-	sin.sin_port = htons(sport);
-
-	reuse = 1;
-	if (setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &reuse,
-	    sizeof(reuse)) < 0) {
-		perror("cannot set socket options");
-		exit(4);
-	}
-
-	if (bind(listen_fd, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
-		perror("cannot bind socket");
-		exit(4);
-	}
-
-	if (listen(listen_fd, 1) < 0) {
-		perror("cannot listen socket");
-		exit(4);
-	}
-
-#ifndef WITHOUT_CAPSICUM
-	cap_rights_init(&rights, CAP_ACCEPT, CAP_READ, CAP_WRITE);
-	if (caph_rights_limit(listen_fd, &rights) == -1)
-		errx(EX_OSERR, "Unable to apply rights for sandbox");
-#endif
-
-	register_inout(&dbgport);
-}
diff --git a/usr/src/cmd/bhyve/dbgport.h b/usr/src/cmd/bhyve/dbgport.h
deleted file mode 100644
index 407ff3ffbf..0000000000
--- a/usr/src/cmd/bhyve/dbgport.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
- *
- * Copyright (c) 2011 NetApp, Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#ifndef _DBGPORT_H_
-#define	_DBGPORT_H_
-
-void	init_dbgport(int port);
-
-#endif
diff --git a/usr/src/cmd/bhyve/gdb.c b/usr/src/cmd/bhyve/gdb.c
index 720a55c58b..60b3b8ab6d 100644
--- a/usr/src/cmd/bhyve/gdb.c
+++ b/usr/src/cmd/bhyve/gdb.c
@@ -135,6 +135,7 @@ static int cur_fd = -1;
 static TAILQ_HEAD(, breakpoint) breakpoints;
 static struct vcpu_state *vcpu_state;
 static int cur_vcpu, stopped_vcpu;
+static bool gdb_active = false;
 
 const int gdb_regset[] = {
 	VM_REG_GUEST_RAX,
@@ -753,6 +754,8 @@ static void
 _gdb_cpu_suspend(int vcpu, bool report_stop)
 {
 
+	if (!gdb_active)
+		return;
 	debug("$vCPU %d suspending\n", vcpu);
 	CPU_SET(vcpu, &vcpus_waiting);
 	if (report_stop && CPU_CMP(&vcpus_waiting, &vcpus_suspended) == 0)
@@ -771,6 +774,8 @@ void
 gdb_cpu_add(int vcpu)
 {
 
+	if (!gdb_active)
+		return;
 	debug("$vCPU %d starting\n", vcpu);
 	pthread_mutex_lock(&gdb_lock);
 	assert(vcpu < guest_ncpus);
@@ -852,6 +857,8 @@ gdb_cpu_mtrap(int vcpu)
 {
 	struct vcpu_state *vs;
 
+	if (!gdb_active)
+		return;
 	debug("$vCPU %d MTRAP\n", vcpu);
 	pthread_mutex_lock(&gdb_lock);
 	vs = &vcpu_state[vcpu];
@@ -892,6 +899,10 @@ gdb_cpu_breakpoint(int vcpu, struct vm_exit *vmexit)
 	uint64_t gpa;
 	int error;
 
+	if (!gdb_active) {
+		fprintf(stderr, "vm_loop: unexpected VMEXIT_DEBUG\n");
+		exit(4);
+	}
 	pthread_mutex_lock(&gdb_lock);
 	error = guest_vaddr2paddr(vcpu, vmexit->rip, &gpa);
 	assert(error == 1);
@@ -1931,4 +1942,5 @@ init_gdb(struct vmctx *_ctx, int sport, bool wait)
 	limit_gdb_socket(s);
 #endif
 	mevent_add(s, EVF_READ, new_connection, NULL);
+	gdb_active = true;
 }
diff --git a/usr/src/cmd/bhyve/hda_codec.c b/usr/src/cmd/bhyve/hda_codec.c
index 41e8121ae2..7a6ba345d8 100644
--- a/usr/src/cmd/bhyve/hda_codec.c
+++ b/usr/src/cmd/bhyve/hda_codec.c
@@ -205,7 +205,7 @@ struct hda_codec_softc {
  * HDA Codec module function declarations
  */
 static int hda_codec_init(struct hda_codec_inst *hci, const char *play,
-    const char *rec, const char *opts);
+    const char *rec);
 static int hda_codec_reset(struct hda_codec_inst *hci);
 static int hda_codec_command(struct hda_codec_inst *hci, uint32_t cmd_data);
 static int hda_codec_notify(struct hda_codec_inst *hci, uint8_t run,
@@ -391,7 +391,7 @@ verb_func_t hda_codec_verb_handlers[HDA_CODEC_NODES_COUNT] = {
 
 static int
 hda_codec_init(struct hda_codec_inst *hci, const char *play,
-    const char *rec, const char *opts)
+    const char *rec)
 {
 	struct hda_codec_softc *sc = NULL;
 	struct hda_codec_stream *st = NULL;
@@ -400,8 +400,6 @@ hda_codec_init(struct hda_codec_inst *hci, const char *play,
 	if (!(play || rec))
 		return (-1);
 
-	DPRINTF("cad: 0x%x opts: %s", hci->cad, opts);
-
 	sc = calloc(1, sizeof(*sc));
 	if (!sc)
 		return (-1);
diff --git a/usr/src/cmd/bhyve/inout.c b/usr/src/cmd/bhyve/inout.c
index 27068023d3..0548807730 100644
--- a/usr/src/cmd/bhyve/inout.c
+++ b/usr/src/cmd/bhyve/inout.c
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
 #include <assert.h>
 
 #include "bhyverun.h"
+#include "config.h"
 #include "inout.h"
 
 SET_DECLARE(inout_port_set, struct inout_port);
@@ -116,7 +117,7 @@ register_default_iohandler(int start, int size)
 }
 
 int
-emulate_inout(struct vmctx *ctx, int vcpu, struct vm_inout *inout, bool strict)
+emulate_inout(struct vmctx *ctx, int vcpu, struct vm_inout *inout)
 {
 	struct inout_handler handler;
 	inout_func_t hfunc;
@@ -134,7 +135,8 @@ emulate_inout(struct vmctx *ctx, int vcpu, struct vm_inout *inout, bool strict)
 	hfunc = handler.handler;
 	harg = handler.arg;
 
-	if (strict && hfunc == default_inout)
+	if (hfunc == default_inout &&
+	    get_config_bool_default("x86.strictio", false))
 		return (-1);
 
 	if (in) {
diff --git a/usr/src/cmd/bhyve/inout.h b/usr/src/cmd/bhyve/inout.h
index b026e18e92..a3b64b9001 100644
--- a/usr/src/cmd/bhyve/inout.h
+++ b/usr/src/cmd/bhyve/inout.h
@@ -47,7 +47,9 @@
 
 struct vmctx;
 struct vm_exit;
+#ifndef __FreeBSD__
 struct vm_inout;
+#endif
 
 /*
  * inout emulation handlers return 0 on success and -1 on failure.
@@ -85,10 +87,12 @@ struct inout_port {
 	DATA_SET(inout_port_set, __CONCAT(__inout_port, __LINE__))
 
 void	init_inout(void);
-int	emulate_inout(struct vmctx *, int vcpu, struct vm_inout *inout,
-    bool strict);
+#ifdef __FreeBSD__
+int	emulate_inout(struct vmctx *, int vcpu, struct vm_exit *vmexit);
+#else
+int	emulate_inout(struct vmctx *, int vcpu, struct vm_inout *inout);
+#endif
 int	register_inout(struct inout_port *iop);
 int	unregister_inout(struct inout_port *iop);
-void	init_bvmcons(void);
 
 #endif	/* _INOUT_H_ */
diff --git a/usr/src/cmd/bhyve/mevent.c b/usr/src/cmd/bhyve/mevent.c
index f3ffc6bcaf..3eb55e3d3a 100644
--- a/usr/src/cmd/bhyve/mevent.c
+++ b/usr/src/cmd/bhyve/mevent.c
@@ -82,8 +82,6 @@ __FBSDID("$FreeBSD$");
 #define	EV_DELETE	0x04
 #endif
 
-extern char *vmname;
-
 static pthread_t mevent_tid;
 static int mevent_timid = 43;
 static int mevent_pipefd[2];
diff --git a/usr/src/cmd/bhyve/mevent_test.c b/usr/src/cmd/bhyve/mevent_test.c
index 4da3adb5ae..dad8b7773e 100644
--- a/usr/src/cmd/bhyve/mevent_test.c
+++ b/usr/src/cmd/bhyve/mevent_test.c
@@ -64,8 +64,6 @@ static pthread_cond_t accept_condvar = PTHREAD_COND_INITIALIZER;
 
 static struct mevent *tevp;
 
-char *vmname = "test vm";
-
 
 #define MEVENT_ECHO
 
diff --git a/usr/src/cmd/bhyve/net_backends.c b/usr/src/cmd/bhyve/net_backends.c
index 884ffb8241..30c26aea45 100644
--- a/usr/src/cmd/bhyve/net_backends.c
+++ b/usr/src/cmd/bhyve/net_backends.c
@@ -75,10 +75,12 @@ __FBSDID("$FreeBSD$");
 #include <netgraph.h>
 #endif
 
+#include "config.h"
 #include "debug.h"
 #include "iov.h"
 #include "mevent.h"
 #include "net_backends.h"
+#include "pci_emul.h"
 
 #include <sys/linker_set.h>
 
@@ -96,7 +98,7 @@ struct net_backend {
 	 * and should not be called by the frontend.
 	 */
 	int (*init)(struct net_backend *be, const char *devname,
-	    const char *opts, net_be_rxeof_t cb, void *param);
+	    nvlist_t *nvl, net_be_rxeof_t cb, void *param);
 	void (*cleanup)(struct net_backend *be);
 
 	/*
@@ -204,7 +206,7 @@ tap_cleanup(struct net_backend *be)
 
 static int
 tap_init(struct net_backend *be, const char *devname,
-	 const char *opts, net_be_rxeof_t cb, void *param)
+	 nvlist_t *nvl, net_be_rxeof_t cb, void *param)
 {
 	struct tap_priv *priv = (struct tap_priv *)be->opaque;
 	char tbuf[80];
@@ -398,18 +400,14 @@ DATA_SET(net_backend_set, vmnet_backend);
 
 static int
 ng_init(struct net_backend *be, const char *devname,
-	 const char *opts, net_be_rxeof_t cb, void *param)
+	 nvlist_t *nvl, net_be_rxeof_t cb, void *param)
 {
 	struct tap_priv *p = (struct tap_priv *)be->opaque;
 	struct ngm_connect ngc;
-	char *ngopts, *tofree;
-	char nodename[NG_NODESIZ];
+	const char *value, *nodename;
 	int sbsz;
 	int ctrl_sock;
 	int flags;
-	int path_provided;
-	int peerhook_provided;
-	int socket_provided;
 	unsigned long maxsbsz;
 	size_t msbsz;
 #ifndef WITHOUT_CAPSICUM
@@ -425,56 +423,27 @@ ng_init(struct net_backend *be, const char *devname,
 
 	memset(&ngc, 0, sizeof(ngc));
 
-	strncpy(ngc.ourhook, "vmlink", NG_HOOKSIZ - 1);
-
-	tofree = ngopts = strdup(opts);
-
-	if (ngopts == NULL) {
-		WPRINTF(("strdup error"));
-		return (-1);
-	}
-
-	socket_provided = 0;
-	path_provided = 0;
-	peerhook_provided = 0;
-
-	while (ngopts != NULL) {
-		char *value = ngopts;
-		char *key;
-
-		key = strsep(&value, "=");
-		if (value == NULL)
-			break;
-		ngopts = value;
-		(void) strsep(&ngopts, ",");
-
-		if (strcmp(key, "socket") == 0) {
-			strncpy(nodename, value, NG_NODESIZ - 1);
-			socket_provided = 1;
-		} else if (strcmp(key, "path") == 0) {
-			strncpy(ngc.path, value, NG_PATHSIZ - 1);
-			path_provided = 1;
-		} else if (strcmp(key, "hook") == 0) {
-			strncpy(ngc.ourhook, value, NG_HOOKSIZ - 1);
-		} else if (strcmp(key, "peerhook") == 0) {
-			strncpy(ngc.peerhook, value, NG_HOOKSIZ - 1);
-			peerhook_provided = 1;
-		}
-	}
-
-	free(tofree);
-
-	if (!path_provided) {
+	value = get_config_value_node(nvl, "path");
+	if (value == NULL) {
 		WPRINTF(("path must be provided"));
 		return (-1);
 	}
+	strncpy(ngc.path, value, NG_PATHSIZ - 1);
 
-	if (!peerhook_provided) {
+	value = get_config_value_node(nvl, "hook");
+	if (value == NULL)
+		value = "vmlink";
+	strncpy(ngc.ourhook, value, NG_HOOKSIZ - 1);
+
+	value = get_config_value_node(nvl, "peerhook");
+	if (value == NULL) {
 		WPRINTF(("peer hook must be provided"));
 		return (-1);
 	}
+	strncpy(ngc.peerhook, value, NG_HOOKSIZ - 1);
 
-	if (NgMkSockNode(socket_provided ? nodename : NULL,
+	nodename = get_config_value_node(nvl, "socket");
+	if (NgMkSockNode(nodename,
 		&ctrl_sock, &be->fd) < 0) {
 		WPRINTF(("can't get Netgraph sockets"));
 		return (-1);
@@ -664,7 +633,7 @@ netmap_set_cap(struct net_backend *be, uint64_t features,
 
 static int
 netmap_init(struct net_backend *be, const char *devname,
-	    const char *opts, net_be_rxeof_t cb, void *param)
+	    nvlist_t *nvl, net_be_rxeof_t cb, void *param)
 {
 	struct netmap_priv *priv = (struct netmap_priv *)be->opaque;
 
@@ -924,10 +893,29 @@ static struct net_backend vale_backend = {
 DATA_SET(net_backend_set, netmap_backend);
 DATA_SET(net_backend_set, vale_backend);
 
+int
+netbe_legacy_config(nvlist_t *nvl, const char *opts)
+{
+	char *backend, *cp;
+
+	if (opts == NULL)
+		return (0);
+
+	cp = strchr(opts, ',');
+	if (cp == NULL) {
+		set_config_value_node(nvl, "backend", opts);
+		return (0);
+	}
+	backend = strndup(opts, cp - opts);
+	set_config_value_node(nvl, "backend", backend);
+	free(backend);
+	return (pci_parse_legacy_config(nvl, cp + 1));
+}
+
 /*
  * Initialize a backend and attach to the frontend.
  * This is called during frontend initialization.
- *  @pbe is a pointer to the backend to be initialized
+ *  @ret is a pointer to the backend to be initialized
  *  @devname is the backend-name as supplied on the command line,
  * 	e.g. -s 2:0,frontend-name,backend-name[,other-args]
  *  @cb is the receive callback supplied by the frontend,
@@ -937,21 +925,19 @@ DATA_SET(net_backend_set, vale_backend);
  *	the argument for the callback.
  */
 int
-netbe_init(struct net_backend **ret, const char *opts, net_be_rxeof_t cb,
+netbe_init(struct net_backend **ret, nvlist_t *nvl, net_be_rxeof_t cb,
     void *param)
 {
 	struct net_backend **pbe, *nbe, *tbe = NULL;
+	const char *value;
 	char *devname;
-	char *options;
 	int err;
 
-	devname = options = strdup(opts);
-
-	if (devname == NULL) {
+	value = get_config_value_node(nvl, "backend");
+	if (value == NULL) {
 		return (-1);
 	}
-
-	devname = strsep(&options, ",");
+	devname = strdup(value);
 
 	/*
 	 * Find the network backend that matches the user-provided
@@ -985,7 +971,7 @@ netbe_init(struct net_backend **ret, const char *opts, net_be_rxeof_t cb,
 	nbe->fe_vnet_hdr_len = 0;
 
 	/* Initialize the backend. */
-	err = nbe->init(nbe, devname, options, cb, param);
+	err = nbe->init(nbe, devname, nvl, cb, param);
 	if (err) {
 		free(devname);
 		free(nbe);
diff --git a/usr/src/cmd/bhyve/net_backends.h b/usr/src/cmd/bhyve/net_backends.h
index b55437fc7b..bc7834546b 100644
--- a/usr/src/cmd/bhyve/net_backends.h
+++ b/usr/src/cmd/bhyve/net_backends.h
@@ -37,8 +37,9 @@ typedef struct net_backend net_backend_t;
 
 /* Interface between network frontends and the network backends. */
 typedef void (*net_be_rxeof_t)(int, enum ev_type, void *param);
-int	netbe_init(net_backend_t **be, const char *opts, net_be_rxeof_t cb,
+int	netbe_init(net_backend_t **be, nvlist_t *nvl, net_be_rxeof_t cb,
             void *param);
+int	netbe_legacy_config(nvlist_t *nvl, const char *opts);
 void	netbe_cleanup(net_backend_t *be);
 uint64_t netbe_get_cap(net_backend_t *be);
 int	 netbe_set_cap(net_backend_t *be, uint64_t cap,
diff --git a/usr/src/cmd/bhyve/net_utils.c b/usr/src/cmd/bhyve/net_utils.c
index d602cac3eb..8088fa89a5 100644
--- a/usr/src/cmd/bhyve/net_utils.c
+++ b/usr/src/cmd/bhyve/net_utils.c
@@ -40,11 +40,12 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 
 #include "bhyverun.h"
+#include "config.h"
 #include "debug.h"
 #include "net_utils.h"
 
 int
-net_parsemac(char *mac_str, uint8_t *mac_addr)
+net_parsemac(const char *mac_str, uint8_t *mac_addr)
 {
         struct ether_addr *ea;
         char zero_addr[ETHER_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 };
@@ -107,7 +108,7 @@ net_genmac(struct pci_devinst *pi, uint8_t *macaddr)
 	char nstr[80];
 
 	snprintf(nstr, sizeof(nstr), "%d-%d-%s", pi->pi_slot,
-	    pi->pi_func, vmname);
+	    pi->pi_func, get_config_value("name"));
 
 	MD5Init(&mdctx);
 	MD5Update(&mdctx, nstr, (unsigned int)strlen(nstr));
diff --git a/usr/src/cmd/bhyve/net_utils.h b/usr/src/cmd/bhyve/net_utils.h
index 1ca20ddb74..3dd4457258 100644
--- a/usr/src/cmd/bhyve/net_utils.h
+++ b/usr/src/cmd/bhyve/net_utils.h
@@ -34,7 +34,7 @@
 #include "pci_emul.h"
 
 void	net_genmac(struct pci_devinst *pi, uint8_t *macaddr);
-int	net_parsemac(char *mac_str, uint8_t *mac_addr);
+int	net_parsemac(const char *mac_str, uint8_t *mac_addr);
 int	net_parsemtu(const char *mtu_str, unsigned long *mtu);
 
 #endif /* _NET_UTILS_H_ */
diff --git a/usr/src/cmd/bhyve/pci_ahci.c b/usr/src/cmd/bhyve/pci_ahci.c
index 0d4951a61e..2416edd166 100644
--- a/usr/src/cmd/bhyve/pci_ahci.c
+++ b/usr/src/cmd/bhyve/pci_ahci.c
@@ -56,6 +56,8 @@ __FBSDID("$FreeBSD$");
 #include <md5.h>
 
 #include "bhyverun.h"
+#include "config.h"
+#include "debug.h"
 #include "pci_emul.h"
 #include "ahci.h"
 #include "block_if.h"
@@ -2308,20 +2310,115 @@ pci_ahci_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx,
 	return (value);
 }
 
+/*
+ * Each AHCI controller has a "port" node which contains nodes for
+ * each port named after the decimal number of the port (no leading
+ * zeroes).  Port nodes contain a "type" ("hd" or "cd"), as well as
+ * options for blockif.  For example:
+ *
+ * pci.0.1.0
+ *          .device="ahci"
+ *          .port
+ *               .0
+ *                 .type="hd"
+ *                 .path="/path/to/image"
+ */
 static int
-pci_ahci_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts, int atapi)
+pci_ahci_legacy_config_port(nvlist_t *nvl, int port, const char *type,
+    const char *opts)
+{
+	char node_name[sizeof("XX")];
+	nvlist_t *port_nvl;
+
+	snprintf(node_name, sizeof(node_name), "%d", port);
+	port_nvl = create_relative_config_node(nvl, node_name);
+	set_config_value_node(port_nvl, "type", type);
+	return (blockif_legacy_config(port_nvl, opts));
+}
+
+static int
+pci_ahci_legacy_config(nvlist_t *nvl, const char *opts)
+{
+	nvlist_t *ports_nvl;
+	const char *type;
+	char *next, *next2, *str, *tofree;
+	int p, ret;
+
+	if (opts == NULL)
+		return (0);
+
+	ports_nvl = create_relative_config_node(nvl, "port");
+	ret = 1;
+	tofree = str = strdup(opts);
+	for (p = 0; p < MAX_PORTS && str != NULL; p++, str = next) {
+		/* Identify and cut off type of present port. */
+		if (strncmp(str, "hd:", 3) == 0) {
+			type = "hd";
+			str += 3;
+		} else if (strncmp(str, "cd:", 3) == 0) {
+			type = "cd";
+			str += 3;
+		} else
+			type = NULL;
+
+		/* Find and cut off the next port options. */
+		next = strstr(str, ",hd:");
+		next2 = strstr(str, ",cd:");
+		if (next == NULL || (next2 != NULL && next2 < next))
+			next = next2;
+		if (next != NULL) {
+			next[0] = 0;
+			next++;
+		}
+
+		if (str[0] == 0)
+			continue;
+
+		if (type == NULL) {
+			EPRINTLN("Missing or invalid type for port %d: \"%s\"",
+			    p, str);
+			goto out;
+		}
+
+		if (pci_ahci_legacy_config_port(ports_nvl, p, type, str) != 0)
+			goto out;
+	}
+	ret = 0;
+out:
+	free(tofree);
+	return (ret);
+}
+
+static int
+pci_ahci_cd_legacy_config(nvlist_t *nvl, const char *opts)
+{
+	nvlist_t *ports_nvl;
+
+	ports_nvl = create_relative_config_node(nvl, "port");
+	return (pci_ahci_legacy_config_port(ports_nvl, 0, "cd", opts));
+}
+
+static int
+pci_ahci_hd_legacy_config(nvlist_t *nvl, const char *opts)
+{
+	nvlist_t *ports_nvl;
+
+	ports_nvl = create_relative_config_node(nvl, "port");
+	return (pci_ahci_legacy_config_port(ports_nvl, 0, "hd", opts));
+}
+
+static int
+pci_ahci_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	char bident[sizeof("XX:XX:XX")];
+	char node_name[sizeof("XX")];
 	struct blockif_ctxt *bctxt;
 	struct pci_ahci_softc *sc;
-	int ret, slots, p;
+	int atapi, ret, slots, p;
 	MD5_CTX mdctx;
 	u_char digest[16];
-	char *next, *next2;
-	char *bopt, *uopt, *xopts, *config;
-	FILE* fp;
-	size_t block_len;
-	int comma, optpos;
+	const char *path, *type, *value;
+	nvlist_t *ports_nvl, *port_nvl;
 
 	ret = 0;
 
@@ -2337,98 +2434,24 @@ pci_ahci_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts, int atapi)
 	sc->pi = 0;
 	slots = 32;
 
-	for (p = 0; p < MAX_PORTS && opts != NULL; p++, opts = next) {
+	ports_nvl = find_relative_config_node(nvl, "port");
+	for (p = 0; p < MAX_PORTS; p++) {
 		struct ata_params *ata_ident = &sc->port[p].ata_ident;
-		memset(ata_ident, 0, sizeof(struct ata_params));
+		char ident[AHCI_PORT_IDENT];
 
-		/* Identify and cut off type of present port. */
-		if (strncmp(opts, "hd:", 3) == 0) {
-			atapi = 0;
-			opts += 3;
-		} else if (strncmp(opts, "cd:", 3) == 0) {
-			atapi = 1;
-			opts += 3;
-		}
-
-		/* Find and cut off the next port options. */
-		next = strstr(opts, ",hd:");
-		next2 = strstr(opts, ",cd:");
-		if (next == NULL || (next2 != NULL && next2 < next))
-			next = next2;
-		if (next != NULL) {
-			next[0] = 0;
-			next++;
-		}
-
-		if (opts[0] == 0)
+		snprintf(node_name, sizeof(node_name), "%d", p);
+		port_nvl = find_relative_config_node(ports_nvl, node_name);
+		if (port_nvl == NULL)
 			continue;
 
-		uopt = strdup(opts);
-		bopt = NULL;
-		fp = open_memstream(&bopt, &block_len);
-		comma = 0;
-		optpos = 0;
-
-		for (xopts = strtok(uopt, ",");
-		     xopts != NULL;
-		     xopts = strtok(NULL, ",")) {
-
-			/* First option assume as block filename. */
-			if (optpos == 0) {
-				/*
-				 * Create an identifier for the backing file.
-				 * Use parts of the md5 sum of the filename
-				 */
-				char ident[AHCI_PORT_IDENT];
-				MD5Init(&mdctx);
-				MD5Update(&mdctx, opts, strlen(opts));
-				MD5Final(digest, &mdctx);
-				snprintf(ident, AHCI_PORT_IDENT,
-					"BHYVE-%02X%02X-%02X%02X-%02X%02X",
-					digest[0], digest[1], digest[2], digest[3], digest[4],
-					digest[5]);
-				ata_string((uint8_t*)&ata_ident->serial, ident, 20);
-				ata_string((uint8_t*)&ata_ident->revision, "001", 8);
-				if (atapi) {
-					ata_string((uint8_t*)&ata_ident->model, "BHYVE SATA DVD ROM", 40);
-				}
-				else {
-					ata_string((uint8_t*)&ata_ident->model, "BHYVE SATA DISK", 40);
-				}
-			}
-
-			if ((config = strchr(xopts, '=')) != NULL) {
-				*config++ = '\0';
-				if (!strcmp("nmrr", xopts)) {
-					ata_ident->media_rotation_rate = atoi(config);
-				}
-				else if (!strcmp("ser", xopts)) {
-					ata_string((uint8_t*)(&ata_ident->serial), config, 20);
-				}
-				else if (!strcmp("rev", xopts)) {
-					ata_string((uint8_t*)(&ata_ident->revision), config, 8);
-				}
-				else if (!strcmp("model", xopts)) {
-					ata_string((uint8_t*)(&ata_ident->model), config, 40);
-				}
-				else {
-					/* Pass all other options to blockif_open. */
-					*--config = '=';
-					fprintf(fp, "%s%s", comma ? "," : "", xopts);
-					comma = 1;
-				}
-			}
-			else {
-				/* Pass all other options to blockif_open. */
-				fprintf(fp, "%s%s", comma ? "," : "", xopts);
-				comma = 1;
-			}
-			optpos++;
-		}
-		free(uopt);
-		fclose(fp);
+		type = get_config_value_node(port_nvl, "type");
+		if (type == NULL)
+			continue;
 
-		DPRINTF("%s\n", bopt);
+		if (strcmp(type, "hd") == 0)
+			atapi = 0;
+		else
+			atapi = 1;
 
 		/*
 		 * Attempt to open the backing image. Use the PCI slot/func
@@ -2436,9 +2459,8 @@ pci_ahci_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts, int atapi)
 		 */
 		snprintf(bident, sizeof(bident), "%d:%d:%d", pi->pi_slot,
 		    pi->pi_func, p);
-		bctxt = blockif_open(bopt, bident);
-		free(bopt);
 
+		bctxt = blockif_open(port_nvl, bident);
 		if (bctxt == NULL) {
 			sc->ports = p;
 			ret = 1;
@@ -2449,6 +2471,40 @@ pci_ahci_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts, int atapi)
 		sc->port[p].port = p;
 		sc->port[p].atapi = atapi;
 
+		/*
+		 * Create an identifier for the backing file.
+		 * Use parts of the md5 sum of the filename
+		 */
+		path = get_config_value_node(port_nvl, "path");
+		MD5Init(&mdctx);
+		MD5Update(&mdctx, path, strlen(path));
+		MD5Final(digest, &mdctx);
+		snprintf(ident, AHCI_PORT_IDENT,
+			"BHYVE-%02X%02X-%02X%02X-%02X%02X",
+			digest[0], digest[1], digest[2], digest[3], digest[4],
+			digest[5]);
+
+		memset(ata_ident, 0, sizeof(struct ata_params));
+		ata_string((uint8_t*)&ata_ident->serial, ident, 20);
+		ata_string((uint8_t*)&ata_ident->revision, "001", 8);
+		if (atapi)
+			ata_string((uint8_t*)&ata_ident->model, "BHYVE SATA DVD ROM", 40);
+		else
+			ata_string((uint8_t*)&ata_ident->model, "BHYVE SATA DISK", 40);
+		value = get_config_value_node(port_nvl, "nmrr");
+		if (value != NULL)
+			ata_ident->media_rotation_rate = atoi(value);
+		value = get_config_value_node(port_nvl, "ser");
+		if (value != NULL)
+			ata_string((uint8_t*)(&ata_ident->serial), value, 20);
+		value = get_config_value_node(port_nvl, "rev");
+		if (value != NULL)
+			ata_string((uint8_t*)(&ata_ident->revision), value, 8);
+		value = get_config_value_node(port_nvl, "model");
+		if (value != NULL)
+			ata_string((uint8_t*)(&ata_ident->model), value, 40);
+		ata_identify_init(&sc->port[p], atapi);
+
 #ifndef __FreeBSD__
 		/*
 		 * Attempt to enable the write cache for this device, as the
@@ -2460,8 +2516,6 @@ pci_ahci_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts, int atapi)
 		(void) blockif_set_wce(bctxt, 1);
 #endif
 
-		ata_identify_init(&sc->port[p], atapi);
-
 		/*
 		 * Allocate blockif request structures and add them
 		 * to the free list
@@ -2513,43 +2567,28 @@ open_fail:
 	return (ret);
 }
 
-static int
-pci_ahci_hd_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
-{
-
-	return (pci_ahci_init(ctx, pi, opts, 0));
-}
-
-static int
-pci_ahci_atapi_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
-{
-
-	return (pci_ahci_init(ctx, pi, opts, 1));
-}
-
 /*
  * Use separate emulation names to distinguish drive and atapi devices
  */
 struct pci_devemu pci_de_ahci = {
 	.pe_emu =	"ahci",
-	.pe_init =	pci_ahci_hd_init,
+	.pe_init =	pci_ahci_init,
+	.pe_legacy_config = pci_ahci_legacy_config,
 	.pe_barwrite =	pci_ahci_write,
-	.pe_barread =	pci_ahci_read
+	.pe_barread =	pci_ahci_read,
 };
 PCI_EMUL_SET(pci_de_ahci);
 
 struct pci_devemu pci_de_ahci_hd = {
 	.pe_emu =	"ahci-hd",
-	.pe_init =	pci_ahci_hd_init,
-	.pe_barwrite =	pci_ahci_write,
-	.pe_barread =	pci_ahci_read
+	.pe_legacy_config = pci_ahci_hd_legacy_config,
+	.pe_alias =	"ahci",
 };
 PCI_EMUL_SET(pci_de_ahci_hd);
 
 struct pci_devemu pci_de_ahci_cd = {
 	.pe_emu =	"ahci-cd",
-	.pe_init =	pci_ahci_atapi_init,
-	.pe_barwrite =	pci_ahci_write,
-	.pe_barread =	pci_ahci_read
+	.pe_legacy_config = pci_ahci_cd_legacy_config,
+	.pe_alias =	"ahci",
 };
 PCI_EMUL_SET(pci_de_ahci_cd);
diff --git a/usr/src/cmd/bhyve/pci_e82545.c b/usr/src/cmd/bhyve/pci_e82545.c
index cb7e074540..598deff980 100644
--- a/usr/src/cmd/bhyve/pci_e82545.c
+++ b/usr/src/cmd/bhyve/pci_e82545.c
@@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$");
 #include "mii.h"
 
 #include "bhyverun.h"
+#include "config.h"
 #include "debug.h"
 #include "pci_emul.h"
 #include "mevent.h"
@@ -360,7 +361,7 @@ static void e82545_reset(struct e82545_softc *sc, int dev);
 static void e82545_rx_enable(struct e82545_softc *sc);
 static void e82545_rx_disable(struct e82545_softc *sc);
 #ifdef	__FreeBSD__
-static void e82545_tap_callback(int fd, enum ev_type type, void *param);
+static void e82545_rx_callback(int fd, enum ev_type type, void *param);
 #endif
 static void e82545_tx_start(struct e82545_softc *sc);
 static void e82545_tx_enable(struct e82545_softc *sc);
@@ -841,7 +842,7 @@ static uint8_t dummybuf[2048];
 
 /* XXX one packet at a time until this is debugged */
 static void
-e82545_tap_callback(int fd, enum ev_type type, void *param)
+e82545_rx_callback(int fd, enum ev_type type, void *param)
 {
 	struct e82545_softc *sc = param;
 	struct e1000_rx_desc *rxd;
@@ -2236,24 +2237,24 @@ e82545_reset(struct e82545_softc *sc, int drvr)
 }
 
 static void
-e82545_open_tap(struct e82545_softc *sc, char *opts)
+e82545_open_tap(struct e82545_softc *sc, const char *path)
 {
 	char tbuf[80];
 #ifndef WITHOUT_CAPSICUM
 	cap_rights_t rights;
 #endif
 	
-	if (opts == NULL) {
+	if (path == NULL) {
 		sc->esc_tapfd = -1;
 		return;
 	}
 
 	strcpy(tbuf, "/dev/");
-	strlcat(tbuf, opts, sizeof(tbuf));
+	strlcat(tbuf, path, sizeof(tbuf));
 
 	sc->esc_tapfd = open(tbuf, O_RDWR);
 	if (sc->esc_tapfd == -1) {
-		DPRINTF("unable to open tap device %s\n", opts);
+		DPRINTF("unable to open tap device %s\n", path);
 		exit(4);
 	}
 
@@ -2277,7 +2278,7 @@ e82545_open_tap(struct e82545_softc *sc, char *opts)
 #ifdef	__FreeBSD__
 	sc->esc_mevp = mevent_add(sc->esc_tapfd,
 				  EVF_READ,
-				  e82545_tap_callback,
+				  e82545_rx_callback,
 				  sc);
 	if (sc->esc_mevp == NULL) {
 		DPRINTF("Could not register mevent %d\n", EVF_READ);
@@ -2288,15 +2289,12 @@ e82545_open_tap(struct e82545_softc *sc, char *opts)
 }
 
 static int
-e82545_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+e82545_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	char nstr[80];
 	struct e82545_softc *sc;
-	char *optscopy;
-	char *vtopts;
-	int mac_provided;
-
-	DPRINTF("Loading with options: %s", opts);
+	const char *mac;
+	int err;
 
 	/* Setup our softc */
 	sc = calloc(1, sizeof(*sc));
@@ -2334,60 +2332,59 @@ e82545_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	pci_emul_alloc_bar(pi, E82545_BAR_IO, PCIBAR_IO,
 		E82545_BAR_IO_LEN);
 
-	/*
-	 * Attempt to open the tap device and read the MAC address
-	 * if specified.  Copied from virtio-net, slightly modified.
-	 */
-	mac_provided = 0;
-	sc->esc_tapfd = -1;
-	if (opts != NULL) {
-		int err = 0;
+	mac = get_config_value_node(nvl, "mac");
+	if (mac != NULL) {
+		err = net_parsemac(mac, sc->esc_mac.octet);
+		if (err) {
+			free(sc);
+			return (err);
+		}
+	} else
+		net_genmac(pi, sc->esc_mac.octet);
 
-		optscopy = vtopts = strdup(opts);
-		(void) strsep(&vtopts, ",");
+	const char *tap = get_config_value_node(nvl, "tap");
+	if (tap != NULL && (strncmp(tap, "tap", 3) == 0 ||
+	    strncmp(tap, "vmnet", 5) == 0))
+		e82545_open_tap(sc, tap);
 
-		/*
-		 * Parse the list of options in the form
-		 *     key1=value1,...,keyN=valueN.
-		 */
-		while (vtopts != NULL) {
-			char *value = vtopts;
-			char *key;
+	/* H/w initiated reset */
+	e82545_reset(sc, 0);
 
-			key = strsep(&value, "=");
-			if (value == NULL)
-				break;
-			vtopts = value;
-			(void) strsep(&vtopts, ",");
-
-			if (strcmp(key, "mac") == 0) {
-				err = net_parsemac(value, sc->esc_mac.octet);
-				if (err)
-					break;
-				mac_provided = 1;
-			}
-		}
+	return (0);
+}
 
-		if (strncmp(optscopy, "tap", 3) == 0 ||
-		    strncmp(optscopy, "vmnet", 5) == 0)
-			e82545_open_tap(sc, optscopy);
+#ifndef __FreeBSD__
+static int
+e82545_legacy_config(nvlist_t *nvl, const char *opt)
+{
+	char *config, *name, *tofree, *value;
 
-		free(optscopy);
-	}
+	if (opt == NULL)
+		return (0);
 
-	if (!mac_provided) {
-		net_genmac(pi, sc->esc_mac.octet);
+	config = tofree = strdup(opt);
+	while ((name = strsep(&config, ",")) != NULL) {
+		value = strchr(name, '=');
+		if (value != NULL) {
+			*value++ = '\0';
+			set_config_value_node(nvl, name, value);
+		} else {
+			set_config_value_node(nvl, "tap", name);
+		}
 	}
-
-	/* H/w initiated reset */
-	e82545_reset(sc, 0);
-
+	free(tofree);
 	return (0);
 }
+#endif
 
 struct pci_devemu pci_de_e82545 = {
 	.pe_emu = 	"e1000",
 	.pe_init =	e82545_init,
+#ifdef __FreeBSD__
+	.pe_legacy_config = netbe_legacy_config,
+#else
+	.pe_legacy_config = e82545_legacy_config,
+#endif
 	.pe_barwrite =	e82545_write,
 	.pe_barread =	e82545_read
 };
diff --git a/usr/src/cmd/bhyve/pci_emul.c b/usr/src/cmd/bhyve/pci_emul.c
index 90602f715b..ce218b8185 100644
--- a/usr/src/cmd/bhyve/pci_emul.c
+++ b/usr/src/cmd/bhyve/pci_emul.c
@@ -62,6 +62,7 @@ __FBSDID("$FreeBSD$");
 
 #include "acpi.h"
 #include "bhyverun.h"
+#include "config.h"
 #include "debug.h"
 #include "inout.h"
 #include "ioapic.h"
@@ -80,8 +81,8 @@ __FBSDID("$FreeBSD$");
 #define	MAXFUNCS	(PCI_FUNCMAX + 1)
 
 struct funcinfo {
-	char	*fi_name;
-	char	*fi_param;
+	nvlist_t *fi_config;
+	struct pci_devemu *fi_pde;
 	struct pci_devinst *fi_devi;
 };
 
@@ -123,7 +124,7 @@ SYSRES_MEM(PCI_EMUL_ECFG_BASE, PCI_EMUL_ECFG_SIZE);
 #define	PCI_EMUL_MEMBASE64	0xD000000000UL
 #define	PCI_EMUL_MEMLIMIT64	0xFD00000000UL
 
-static struct pci_devemu *pci_emul_finddev(char *name);
+static struct pci_devemu *pci_emul_finddev(const char *name);
 static void pci_lintr_route(struct pci_devinst *pi);
 static void pci_lintr_update(struct pci_devinst *pi);
 static void pci_cfgrw(struct vmctx *ctx, int vcpu, int in, int bus, int slot,
@@ -179,13 +180,56 @@ pci_parse_slot_usage(char *aopt)
 	EPRINTLN("Invalid PCI slot info field \"%s\"", aopt);
 }
 
+/*
+ * Helper function to parse a list of comma-separated options where
+ * each option is formatted as "name[=value]".  If no value is
+ * provided, the option is treated as a boolean and is given a value
+ * of true.
+ */
+int
+pci_parse_legacy_config(nvlist_t *nvl, const char *opt)
+{
+	char *config, *name, *tofree, *value;
+
+	if (opt == NULL)
+		return (0);
+
+	config = tofree = strdup(opt);
+	while ((name = strsep(&config, ",")) != NULL) {
+		value = strchr(name, '=');
+		if (value != NULL) {
+			*value = '\0';
+			value++;
+			set_config_value_node(nvl, name, value);
+		} else
+			set_config_bool_node(nvl, name, true);
+	}
+	free(tofree);
+	return (0);
+}
+
+/*
+ * PCI device configuration is stored in MIBs that encode the device's
+ * location:
+ *
+ * pci.<bus>.<slot>.<func>
+ *
+ * Where "bus", "slot", and "func" are all decimal values without
+ * leading zeroes.  Each valid device must have a "device" node which
+ * identifies the driver model of the device.
+ *
+ * Device backends can provide a parser for the "config" string.  If
+ * a custom parser is not provided, pci_parse_legacy_config() is used
+ * to parse the string.
+ */
 int
 pci_parse_slot(char *opt)
 {
-	struct businfo *bi;
-	struct slotinfo *si;
+	char node_name[sizeof("pci.XXX.XX.X")];
+	struct pci_devemu *pde;
 	char *emul, *config, *str, *cp;
 	int error, bnum, snum, fnum;
+	nvlist_t *nvl;
 
 	error = -1;
 	str = strdup(opt);
@@ -222,32 +266,33 @@ pci_parse_slot(char *opt)
 		goto done;
 	}
 
-	if (pci_businfo[bnum] == NULL)
-		pci_businfo[bnum] = calloc(1, sizeof(struct businfo));
-
-	bi = pci_businfo[bnum];
-	si = &bi->slotinfo[snum];
-
-	if (si->si_funcs[fnum].fi_name != NULL) {
-		EPRINTLN("pci slot %d:%d already occupied!",
-			snum, fnum);
+	pde = pci_emul_finddev(emul);
+	if (pde == NULL) {
+		EPRINTLN("pci slot %d:%d:%d: unknown device \"%s\"", bnum, snum,
+		    fnum, emul);
 		goto done;
 	}
 
-	if (pci_emul_finddev(emul) == NULL) {
-		EPRINTLN("pci slot %d:%d: unknown device \"%s\"",
-			snum, fnum, emul);
+	snprintf(node_name, sizeof(node_name), "pci.%d.%d.%d", bnum, snum,
+	    fnum);
+	nvl = find_config_node(node_name);
+	if (nvl != NULL) {
+		EPRINTLN("pci slot %d:%d:%d already occupied!", bnum, snum,
+		    fnum);
 		goto done;
 	}
+	nvl = create_config_node(node_name);
+	if (pde->pe_alias != NULL)
+		set_config_value_node(nvl, "device", pde->pe_alias);
+	else
+		set_config_value_node(nvl, "device", pde->pe_emu);
 
-	error = 0;
-	si->si_funcs[fnum].fi_name = emul;
-	si->si_funcs[fnum].fi_param = config;
-
+	if (pde->pe_legacy_config != NULL)
+		error = pde->pe_legacy_config(nvl, config);
+	else
+		error = pci_parse_legacy_config(nvl, config);
 done:
-	if (error)
-		free(str);
-
+	free(str);
 	return (error);
 }
 
@@ -470,10 +515,12 @@ pci_emul_alloc_resource(uint64_t *baseptr, uint64_t limit, uint64_t size,
 static void
 modify_bar_registration(struct pci_devinst *pi, int idx, int registration)
 {
+	struct pci_devemu *pe;
 	int error;
 	struct inout_port iop;
 	struct mem_range mr;
 
+	pe = pi->pi_d;
 	switch (pi->pi_bar[idx].type) {
 	case PCIBAR_IO:
 		bzero(&iop, sizeof(struct inout_port));
@@ -487,6 +534,9 @@ modify_bar_registration(struct pci_devinst *pi, int idx, int registration)
 			error = register_inout(&iop);
 		} else
 			error = unregister_inout(&iop);
+		if (pe->pe_baraddr != NULL)
+			(*pe->pe_baraddr)(pi->pi_vmctx, pi, idx, registration,
+					  pi->pi_bar[idx].addr);
 		break;
 	case PCIBAR_MEM32:
 	case PCIBAR_MEM64:
@@ -502,6 +552,9 @@ modify_bar_registration(struct pci_devinst *pi, int idx, int registration)
 			error = register_mem(&mr);
 		} else
 			error = unregister_mem(&mr);
+		if (pe->pe_baraddr != NULL)
+			(*pe->pe_baraddr)(pi->pi_vmctx, pi, idx, registration,
+					  pi->pi_bar[idx].addr);
 		break;
 	default:
 		error = EINVAL;
@@ -729,7 +782,7 @@ pci_emul_add_capability(struct pci_devinst *pi, u_char *capdata, int caplen)
 }
 
 static struct pci_devemu *
-pci_emul_finddev(char *name)
+pci_emul_finddev(const char *name)
 {
 	struct pci_devemu **pdpp, *pdp;
 
@@ -770,7 +823,7 @@ pci_emul_init(struct vmctx *ctx, struct pci_devemu *pde, int bus, int slot,
 
 	pci_set_cfgdata8(pdi, PCIR_COMMAND, PCIM_CMD_BUSMASTEREN);
 
-	err = (*pde->pe_init)(ctx, pdi, fi->fi_param);
+	err = (*pde->pe_init)(ctx, pdi, fi->fi_config);
 	if (err == 0)
 		fi->fi_devi = pdi;
 	else
@@ -1099,11 +1152,14 @@ pci_ecfg_base(void)
 int
 init_pci(struct vmctx *ctx)
 {
+	char node_name[sizeof("pci.XXX.XX.X")];
 	struct mem_range mr;
 	struct pci_devemu *pde;
 	struct businfo *bi;
 	struct slotinfo *si;
 	struct funcinfo *fi;
+	nvlist_t *nvl;
+	const char *emul;
 	size_t lowmem;
 	int bus, slot, func;
 	int error;
@@ -1113,8 +1169,13 @@ init_pci(struct vmctx *ctx)
 	pci_emul_membase64 = PCI_EMUL_MEMBASE64;
 
 	for (bus = 0; bus < MAXBUSES; bus++) {
-		if ((bi = pci_businfo[bus]) == NULL)
+		snprintf(node_name, sizeof(node_name), "pci.%d", bus);
+		nvl = find_config_node(node_name);
+		if (nvl == NULL)
 			continue;
+		pci_businfo[bus] = calloc(1, sizeof(struct businfo));
+		bi = pci_businfo[bus];
+
 		/*
 		 * Keep track of the i/o and memory resources allocated to
 		 * this bus.
@@ -1127,10 +1188,34 @@ init_pci(struct vmctx *ctx)
 			si = &bi->slotinfo[slot];
 			for (func = 0; func < MAXFUNCS; func++) {
 				fi = &si->si_funcs[func];
-				if (fi->fi_name == NULL)
+				snprintf(node_name, sizeof(node_name),
+				    "pci.%d.%d.%d", bus, slot, func);
+				nvl = find_config_node(node_name);
+				if (nvl == NULL)
 					continue;
-				pde = pci_emul_finddev(fi->fi_name);
-				assert(pde != NULL);
+
+				fi->fi_config = nvl;
+				emul = get_config_value_node(nvl, "device");
+				if (emul == NULL) {
+					EPRINTLN("pci slot %d:%d:%d: missing "
+					    "\"device\" value", bus, slot, func);
+					return (EINVAL);
+				}
+				pde = pci_emul_finddev(emul);
+				if (pde == NULL) {
+					EPRINTLN("pci slot %d:%d:%d: unknown "
+					    "device \"%s\"", bus, slot, func,
+					    emul);
+					return (EINVAL);
+				}
+				if (pde->pe_alias != NULL) {
+					EPRINTLN("pci slot %d:%d:%d: legacy "
+					    "device \"%s\", use \"%s\" instead",
+					    bus, slot, func, emul,
+					    pde->pe_alias);
+					return (EINVAL);
+				}
+				fi->fi_pde = pde;
 				error = pci_emul_init(ctx, pde, bus, slot,
 				    func, fi);
 				if (error)
@@ -1992,7 +2077,7 @@ struct pci_emul_dsoftc {
 #define	PCI_EMUL_MSIX_MSGS	16
 
 static int
-pci_emul_dinit(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+pci_emul_dinit(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	int error;
 	struct pci_emul_dsoftc *sc;
@@ -2150,7 +2235,7 @@ struct pci_devemu pci_dummy = {
 	.pe_emu = "dummy",
 	.pe_init = pci_emul_dinit,
 	.pe_barwrite = pci_emul_diow,
-	.pe_barread = pci_emul_dior
+	.pe_barread = pci_emul_dior,
 };
 PCI_EMUL_SET(pci_dummy);
 
diff --git a/usr/src/cmd/bhyve/pci_emul.h b/usr/src/cmd/bhyve/pci_emul.h
index 63e3c89a95..cc3a8c048c 100644
--- a/usr/src/cmd/bhyve/pci_emul.h
+++ b/usr/src/cmd/bhyve/pci_emul.h
@@ -37,6 +37,7 @@
 #include <sys/types.h>
 #include <sys/queue.h>
 #include <sys/kernel.h>
+#include <sys/nv.h>
 #include <sys/_pthreadtypes.h>
 
 #include <dev/pci/pcireg.h>
@@ -54,7 +55,9 @@ struct pci_devemu {
 
 	/* instance creation */
 	int       (*pe_init)(struct vmctx *, struct pci_devinst *,
-			     char *opts);
+			     nvlist_t *);
+	int	(*pe_legacy_config)(nvlist_t *, const char *);
+	const char *pe_alias;
 
 	/* ACPI DSDT enumeration */
 	void	(*pe_write_dsdt)(struct pci_devinst *);
@@ -75,10 +78,13 @@ struct pci_devemu {
 				struct pci_devinst *pi, int baridx,
 				uint64_t offset, int size);
 
+	void	(*pe_baraddr)(struct vmctx *ctx, struct pci_devinst *pi,
+			      int baridx, int enabled, uint64_t address);
 #ifndef __FreeBSD__
 	void	(*pe_lintrupdate)(struct pci_devinst *pi);
 #endif /* __FreeBSD__ */
 };
+
 #define PCI_EMUL_SET(x)   DATA_SET(pci_devemu_set, x);
 
 enum pcibar_type {
@@ -237,6 +243,7 @@ int	pci_msix_enabled(struct pci_devinst *pi);
 int	pci_msix_table_bar(struct pci_devinst *pi);
 int	pci_msix_pba_bar(struct pci_devinst *pi);
 int	pci_msi_maxmsgnum(struct pci_devinst *pi);
+int	pci_parse_legacy_config(nvlist_t *nvl, const char *opt);
 int	pci_parse_slot(char *opt);
 void    pci_print_supported_devices();
 void	pci_populate_msicap(struct msicap *cap, int msgs, int nextptr);
diff --git a/usr/src/cmd/bhyve/pci_fbuf.c b/usr/src/cmd/bhyve/pci_fbuf.c
index d945545b9d..44f65f61c7 100644
--- a/usr/src/cmd/bhyve/pci_fbuf.c
+++ b/usr/src/cmd/bhyve/pci_fbuf.c
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 
 #include "bhyvegc.h"
 #include "bhyverun.h"
+#include "config.h"
 #include "debug.h"
 #include "console.h"
 #include "inout.h"
@@ -100,7 +101,7 @@ struct pci_fbuf_softc {
 	char      *rfb_password;
 	int       rfb_port;
 #ifndef __FreeBSD__
-	char	  *rfb_unix;
+	const char *rfb_unix;
 #endif
 	int       rfb_wait;
 	int       vga_enabled;
@@ -119,15 +120,6 @@ static struct pci_fbuf_softc *fbuf_sc;
 #define	PCI_FBUF_MSI_MSGS	 4
 
 static void
-pci_fbuf_usage(char *opt)
-{
-
-	EPRINTLN("Invalid fbuf emulation option \"%s\"", opt);
-	EPRINTLN("fbuf: {wait,}{vga=on|io|off,}rfb=<ip>:port"
-	    "{,w=width}{,h=height}");
-}
-
-static void
 pci_fbuf_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 	       int baridx, uint64_t offset, int size, uint64_t value)
 {
@@ -227,107 +219,137 @@ pci_fbuf_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 	return (value);
 }
 
-static int
-pci_fbuf_parse_opts(struct pci_fbuf_softc *sc, char *opts)
+static void
+pci_fbuf_baraddr(struct vmctx *ctx, struct pci_devinst *pi, int baridx,
+		 int enabled, uint64_t address)
 {
-	char	*uopts, *uoptsbak, *xopts, *config;
-	char	*tmpstr;
-	int	ret;
-
-	ret = 0;
-	uoptsbak = uopts = strdup(opts);
-	while ((xopts = strsep(&uopts, ",")) != NULL) {
-		if (strcmp(xopts, "wait") == 0) {
-			sc->rfb_wait = 1;
-			continue;
-		}
+	struct pci_fbuf_softc *sc;
+	int prot;
 
-		if ((config = strchr(xopts, '=')) == NULL) {
-			pci_fbuf_usage(xopts);
-			ret = -1;
-			goto done;
-		}
+	if (baridx != 1)
+		return;
 
-		*config++ = '\0';
-
-		DPRINTF(DEBUG_VERBOSE, ("pci_fbuf option %s = %s",
-		   xopts, config));
-
-		if (!strcmp(xopts, "tcp") || !strcmp(xopts, "rfb")) {
-			/*
-			 * IPv4 -- host-ip:port
-			 * IPv6 -- [host-ip%zone]:port
-			 * XXX for now port is mandatory.
-			 */
-			tmpstr = strsep(&config, "]");
-			if (config) {
-				if (tmpstr[0] == '[')
-					tmpstr++;
-				sc->rfb_host = strdup(tmpstr);
-				if (config[0] == ':')
-					config++;
-				else {
-					pci_fbuf_usage(xopts);
-					ret = -1;
-					goto done;
+	sc = pi->pi_arg;
+	if (!enabled && sc->fbaddr != 0) {
+		if (vm_munmap_memseg(ctx, sc->fbaddr, FB_SIZE) != 0)
+			EPRINTLN("pci_fbuf: munmap_memseg failed");
+		sc->fbaddr = 0;
+	} else if (sc->fb_base != NULL && sc->fbaddr == 0) {
+		prot = PROT_READ | PROT_WRITE;
+		if (vm_mmap_memseg(ctx, address, VM_FRAMEBUFFER, 0, FB_SIZE, prot) != 0)
+			EPRINTLN("pci_fbuf: mmap_memseg failed");
+		sc->fbaddr = address;
+	}
+}
+
+
+static int
+pci_fbuf_parse_config(struct pci_fbuf_softc *sc, nvlist_t *nvl)
+{
+	const char *value;
+	char *cp;
+
+	sc->rfb_wait = get_config_bool_node_default(nvl, "wait", false);
+
+	/* Prefer "rfb" to "tcp". */
+	value = get_config_value_node(nvl, "rfb");
+	if (value == NULL)
+		value = get_config_value_node(nvl, "tcp");
+	if (value != NULL) {
+		/*
+		 * IPv4 -- host-ip:port
+		 * IPv6 -- [host-ip%zone]:port
+		 * XXX for now port is mandatory for IPv4.
+		 */
+		if (value[0] == '[') {
+			cp = strchr(value + 1, ']');
+			if (cp == NULL || cp == value + 1) {
+				EPRINTLN("fbuf: Invalid IPv6 address: \"%s\"",
+				    value);
+				return (-1);
+			}
+			sc->rfb_host = strndup(value + 1, cp - (value + 1));
+			cp++;
+			if (*cp == ':') {
+				cp++;
+				if (*cp == '\0') {
+					EPRINTLN(
+					    "fbuf: Missing port number: \"%s\"",
+					    value);
+					return (-1);
 				}
-				sc->rfb_port = atoi(config);
+				sc->rfb_port = atoi(cp);
+			} else if (*cp != '\0') {
+				EPRINTLN("fbuf: Invalid IPv6 address: \"%s\"",
+				    value);
+				return (-1);
+			}
+		} else {
+			cp = strchr(value, ':');
+			if (cp == NULL) {
+				sc->rfb_port = atoi(value);
 			} else {
-				config = tmpstr;
-				tmpstr = strsep(&config, ":");
-				if (!config)
-					sc->rfb_port = atoi(tmpstr);
-				else {
-					sc->rfb_port = atoi(config);
-					sc->rfb_host = strdup(tmpstr);
+				sc->rfb_host = strndup(value, cp - value);
+				cp++;
+				if (*cp == '\0') {
+					EPRINTLN(
+					    "fbuf: Missing port number: \"%s\"",
+					    value);
+					return (-1);
 				}
+				sc->rfb_port = atoi(cp);
 			}
+		}
+	}
+
 #ifndef __FreeBSD__
-		} else if (!strcmp(xopts, "unix")) {
-			sc->rfb_unix = strdup(config);
+	sc->rfb_unix = get_config_value_node(nvl, "unix");
 #endif
-	        } else if (!strcmp(xopts, "vga")) {
-			if (!strcmp(config, "off")) {
-				sc->vga_enabled = 0;
-			} else if (!strcmp(config, "io")) {
-				sc->vga_enabled = 1;
-				sc->vga_full = 0;
-			} else if (!strcmp(config, "on")) {
-				sc->vga_enabled = 1;
-				sc->vga_full = 1;
-			} else {
-				pci_fbuf_usage(xopts);
-				ret = -1;
-				goto done;
-			}
-	        } else if (!strcmp(xopts, "w")) {
-		        sc->memregs.width = atoi(config);
-			if (sc->memregs.width > COLS_MAX) {
-				pci_fbuf_usage(xopts);
-				ret = -1;
-				goto done;
-			} else if (sc->memregs.width == 0)
-				sc->memregs.width = 1920;
-		} else if (!strcmp(xopts, "h")) {
-			sc->memregs.height = atoi(config);
-			if (sc->memregs.height > ROWS_MAX) {
-				pci_fbuf_usage(xopts);
-				ret = -1;
-				goto done;
-			} else if (sc->memregs.height == 0)
-				sc->memregs.height = 1080;
-		} else if (!strcmp(xopts, "password")) {
-			sc->rfb_password = strdup(config);
+
+	value = get_config_value_node(nvl, "vga");
+	if (value != NULL) {
+		if (strcmp(value, "off") == 0) {
+			sc->vga_enabled = 0;
+		} else if (strcmp(value, "io") == 0) {
+			sc->vga_enabled = 1;
+			sc->vga_full = 0;
+		} else if (strcmp(value, "on") == 0) {
+			sc->vga_enabled = 1;
+			sc->vga_full = 1;
 		} else {
-			pci_fbuf_usage(xopts);
-			ret = -1;
-			goto done;
+			EPRINTLN("fbuf: Invalid vga setting: \"%s\"", value);
+			return (-1);
 		}
 	}
 
-done:
-	free(uoptsbak);
-	return (ret);
+	value = get_config_value_node(nvl, "w");
+	if (value != NULL) {
+		sc->memregs.width = atoi(value);
+		if (sc->memregs.width > COLS_MAX) {
+			EPRINTLN("fbuf: width %d too large", sc->memregs.width);
+			return (-1);
+		}
+		if (sc->memregs.width == 0)
+			sc->memregs.width = 1920;
+	}
+
+	value = get_config_value_node(nvl, "h");
+	if (value != NULL) {
+		sc->memregs.height = atoi(value);
+		if (sc->memregs.height > ROWS_MAX) {
+			EPRINTLN("fbuf: height %d too large",
+			    sc->memregs.height);
+			return (-1);
+		}
+		if (sc->memregs.height == 0)
+			sc->memregs.height = 1080;
+	}
+
+	value = get_config_value_node(nvl, "password");
+	if (value != NULL)
+		sc->rfb_password = strdup(value);
+
+	return (0);
 }
 
 
@@ -358,7 +380,7 @@ pci_fbuf_render(struct bhyvegc *gc, void *arg)
 }
 
 static int
-pci_fbuf_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+pci_fbuf_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	int error, prot;
 	struct pci_fbuf_softc *sc;
@@ -398,7 +420,7 @@ pci_fbuf_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 
 	sc->fsc_pi = pi;
 
-	error = pci_fbuf_parse_opts(sc, opts);
+	error = pci_fbuf_parse_config(sc, nvl);
 	if (error != 0)
 		goto done;
 
@@ -462,6 +484,7 @@ struct pci_devemu pci_fbuf = {
 	.pe_emu =	"fbuf",
 	.pe_init =	pci_fbuf_init,
 	.pe_barwrite =	pci_fbuf_write,
-	.pe_barread =	pci_fbuf_read
+	.pe_barread =	pci_fbuf_read,
+	.pe_baraddr =	pci_fbuf_baraddr,
 };
 PCI_EMUL_SET(pci_fbuf);
diff --git a/usr/src/cmd/bhyve/pci_hda.c b/usr/src/cmd/bhyve/pci_hda.c
index 86e46a550a..7491944fed 100644
--- a/usr/src/cmd/bhyve/pci_hda.c
+++ b/usr/src/cmd/bhyve/pci_hda.c
@@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");
 
 #include "pci_hda.h"
 #include "bhyverun.h"
+#include "config.h"
 #include "pci_emul.h"
 #include "hdac_reg.h"
 
@@ -147,13 +148,11 @@ static inline uint32_t hda_get_reg_by_offset(struct hda_softc *sc,
 static inline void hda_set_field_by_offset(struct hda_softc *sc,
     uint32_t offset, uint32_t mask, uint32_t value);
 
-static uint8_t hda_parse_config(const char *opts, const char *key, char *val);
-static struct hda_softc *hda_init(const char *opts);
+static struct hda_softc *hda_init(nvlist_t *nvl);
 static void hda_update_intr(struct hda_softc *sc);
 static void hda_response_interrupt(struct hda_softc *sc);
 static int hda_codec_constructor(struct hda_softc *sc,
-    struct hda_codec_class *codec, const char *play, const char *rec,
-    const char *opts);
+    struct hda_codec_class *codec, const char *play, const char *rec);
 static struct hda_codec_class *hda_find_codec_class(const char *name);
 
 static int hda_send_command(struct hda_softc *sc, uint32_t verb);
@@ -210,7 +209,7 @@ static uint64_t hda_get_clock_ns(void);
 /*
  * PCI HDA function declarations
  */
-static int pci_hda_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts);
+static int pci_hda_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl);
 static void pci_hda_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
     int baridx, uint64_t offset, int size, uint64_t value);
 static uint64_t pci_hda_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
@@ -318,67 +317,20 @@ hda_set_field_by_offset(struct hda_softc *sc, uint32_t offset,
 	hda_set_reg_by_offset(sc, offset, reg_value);
 }
 
-static uint8_t
-hda_parse_config(const char *opts, const char *key, char *val)
-{
-	char buf[64];
-	char *s = buf;
-	char *tmp = NULL;
-	size_t len;
-	int i;
-
-	if (!opts)
-		return (0);
-
-	len = strlen(opts);
-	if (len >= sizeof(buf)) {
-		DPRINTF("Opts too big");
-		return (0);
-	}
-
-	DPRINTF("opts: %s", opts);
-
-	strcpy(buf, opts);
-
-	for (i = 0; i < len; i++)
-		if (buf[i] == ',') {
-			buf[i] = 0;
-			tmp = buf + i + 1;
-			break;
-		}
-
-	if (!memcmp(s, key, strlen(key))) {
-		strncpy(val, s + strlen(key), 64);
-		return (1);
-	}
-
-	if (!tmp)
-		return (0);
-
-	s = tmp;
-	if (!memcmp(s, key, strlen(key))) {
-		strncpy(val, s + strlen(key), 64);
-		return (1);
-	}
-
-	return (0);
-}
-
 static struct hda_softc *
-hda_init(const char *opts)
+hda_init(nvlist_t *nvl)
 {
 	struct hda_softc *sc = NULL;
 	struct hda_codec_class *codec = NULL;
-	char play[64];
-	char rec[64];
-	int err, p, r;
+	const char *value;
+	char *play;
+	char *rec;
+	int err;
 
 #if DEBUG_HDA == 1
 	dbg = fopen("/tmp/bhyve_hda.log", "w+");
 #endif
 
-	DPRINTF("opts: %s", opts);
-
 	sc = calloc(1, sizeof(*sc));
 	if (!sc)
 		return (NULL);
@@ -386,19 +338,28 @@ hda_init(const char *opts)
 	hda_reset_regs(sc);
 
 	/*
-	 * TODO search all the codecs declared in opts
+	 * TODO search all configured codecs
 	 * For now we play with one single codec
 	 */
 	codec = hda_find_codec_class("hda_codec");
 	if (codec) {
-		p = hda_parse_config(opts, "play=", play);
-		r = hda_parse_config(opts, "rec=", rec);
+		value = get_config_value_node(nvl, "play");
+		if (value == NULL)
+			play = NULL;
+		else
+			play = strdup(value);
+		value = get_config_value_node(nvl, "rec");
+		if (value == NULL)
+			rec = NULL;
+		else
+			rec = strdup(value);
 		DPRINTF("play: %s rec: %s", play, rec);
-		if (p | r) {
-			err = hda_codec_constructor(sc, codec, p ?	\
-				play : NULL, r ? rec : NULL, NULL);
+		if (play != NULL || rec != NULL) {
+			err = hda_codec_constructor(sc, codec, play, rec);
 			assert(!err);
 		}
+		free(play);
+		free(rec);
 	}
 
 	return (sc);
@@ -470,7 +431,7 @@ hda_response_interrupt(struct hda_softc *sc)
 
 static int
 hda_codec_constructor(struct hda_softc *sc, struct hda_codec_class *codec,
-    const char *play, const char *rec, const char *opts)
+    const char *play, const char *rec)
 {
 	struct hda_codec_inst *hci = NULL;
 
@@ -493,7 +454,7 @@ hda_codec_constructor(struct hda_softc *sc, struct hda_codec_class *codec,
 		return (-1);
 	}
 
-	return (codec->init(hci, play, rec, opts));
+	return (codec->init(hci, play, rec));
 }
 
 static struct hda_codec_class *
@@ -1263,7 +1224,7 @@ static uint64_t hda_get_clock_ns(void)
  * PCI HDA function definitions
  */
 static int
-pci_hda_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+pci_hda_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	struct hda_softc *sc = NULL;
 
@@ -1285,7 +1246,7 @@ pci_hda_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	/* allocate an IRQ pin for our slot */
 	pci_lintr_request(pi);
 
-	sc = hda_init(opts);
+	sc = hda_init(nvl);
 	if (!sc)
 		return (-1);
 
diff --git a/usr/src/cmd/bhyve/pci_hda.h b/usr/src/cmd/bhyve/pci_hda.h
index 7b6bba92e4..65a85f6d60 100644
--- a/usr/src/cmd/bhyve/pci_hda.h
+++ b/usr/src/cmd/bhyve/pci_hda.h
@@ -72,7 +72,7 @@ struct hda_codec_inst {
 struct hda_codec_class {
 	char *name;
 	int (*init)(struct hda_codec_inst *hci, const char *play,
-		const char *rec, const char *opts);
+		const char *rec);
 	int (*reset)(struct hda_codec_inst *hci);
 	int (*command)(struct hda_codec_inst *hci, uint32_t cmd_data);
 	int (*notify)(struct hda_codec_inst *hci, uint8_t run, uint8_t stream,
diff --git a/usr/src/cmd/bhyve/pci_hostbridge.c b/usr/src/cmd/bhyve/pci_hostbridge.c
index b926c7817e..e59affb92e 100644
--- a/usr/src/cmd/bhyve/pci_hostbridge.c
+++ b/usr/src/cmd/bhyve/pci_hostbridge.c
@@ -38,68 +38,12 @@
 #endif
 __FBSDID("$FreeBSD$");
 
-#include "pci_emul.h"
-
-#ifdef __FreeBSD__
-static int
-pci_hostbridge_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
-{
-
-	/* config space */
-	pci_set_cfgdata16(pi, PCIR_VENDOR, 0x1275);	/* NetApp */
-	pci_set_cfgdata16(pi, PCIR_DEVICE, 0x1275);	/* NetApp */
-	pci_set_cfgdata8(pi, PCIR_HDRTYPE, PCIM_HDRTYPE_NORMAL);
-	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_BRIDGE);
-	pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_BRIDGE_HOST);
-
-	pci_emul_add_pciecap(pi, PCIEM_TYPE_ROOT_PORT);
-
-	return (0);
-}
-
-static int
-pci_amd_hostbridge_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
-{
-	(void) pci_hostbridge_init(ctx, pi, opts);
-	pci_set_cfgdata16(pi, PCIR_VENDOR, 0x1022);	/* AMD */
-	pci_set_cfgdata16(pi, PCIR_DEVICE, 0x7432);	/* made up */
-
-	return (0);
-}
-#else
-static void
-pci_hostbridge_setup(struct pci_devinst *pi, uint16_t vendor, uint16_t device)
-{
-	/* config space */
-	pci_set_cfgdata16(pi, PCIR_VENDOR, vendor);
-	pci_set_cfgdata16(pi, PCIR_DEVICE, device);
-	pci_set_cfgdata8(pi, PCIR_HDRTYPE, PCIM_HDRTYPE_NORMAL);
-	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_BRIDGE);
-	pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_BRIDGE_HOST);
-
-	pci_emul_add_pciecap(pi, PCIEM_TYPE_ROOT_PORT);
-}
-
-
-static int
-pci_hostbridge_parse_pci_val(const char *in, uint16_t *val)
-{
-	long num;
-	char *endp = NULL;
+#include <stdlib.h>
 
-	errno = 0;
-	num = strtol(in, &endp, 0);
-	if (errno != 0 || endp == NULL || *endp != '\0') {
-		fprintf(stderr, "pci_hostbridge: invalid num '%s'", in);
-		return (-1);
-	} else if (num < 1 || num > UINT16_MAX) {
-		fprintf(stderr, "pci_hostbridge: 0x%04lx out of range", num);
-		return (-1);
-	}
-	*val = num;
-	return (0);
-}
+#include "config.h"
+#include "pci_emul.h"
 
+#ifndef __FreeBSD__
 static struct pci_hostbridge_model {
 	const char	*phm_model;
 	uint16_t	phm_vendor;
@@ -112,58 +56,30 @@ static struct pci_hostbridge_model {
 };
 
 #define	NUM_HB_MODELS	(sizeof (pci_hb_models) / sizeof (pci_hb_models[0]))
+#endif
 
 static int
-pci_hostbridge_parse_args(char *opts, uint16_t *vendorp, uint16_t *devicep)
+pci_hostbridge_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
-	const char *model = NULL;
-	char *next;
-	uint16_t vendor = 0, device = 0;
-	int err = 0;
-
-	for (; opts != NULL && *opts != '\0'; opts = next) {
-		char *val, *cp;
-
-		if ((cp = strchr(opts, ',')) != NULL) {
-			*cp = '\0';
-			next = cp + 1;
-		} else {
-			next = NULL;
-		}
+	const char *value;
+	u_int vendor, device;
 
-		if ((cp = strchr(opts, '=')) == NULL) {
-			fprintf(stderr,
-			    "pci_hostbridge: expected value for param"
-			    " (%s=VAL)", opts);
-			err = -1;
-			continue;
-		}
+#ifdef __FreeBSD__
+	vendor = 0x1275;	/* NetApp */
+	device = 0x1275;	/* NetApp */
+#else
+	vendor = device = 0;
+#endif
 
-		/* <param>=<value> handling */
-		val = cp + 1;
-		*cp = '\0';
-		if (strcmp(opts, "model") == 0) {
-			model = val;
-		} else if (strcmp(opts, "vendor") == 0) {
-			if (pci_hostbridge_parse_pci_val(val, &vendor) != 0) {
-				err = -1;
-				continue;
-			}
-		} else if (strcmp(opts, "device") == 0) {
-			if (pci_hostbridge_parse_pci_val(val, &device) != 0) {
-				err = -1;
-				continue;
-			}
-		} else {
-			fprintf(stderr,
-			    "pci_hostbridge: unrecognized option '%s'", opts);
-			err = -1;
-			continue;
-		}
-	}
-	if (err != 0) {
-		return (err);
-	}
+	value = get_config_value_node(nvl, "vendor");
+	if (value != NULL)
+		vendor = strtol(value, NULL, 0);
+	value = get_config_value_node(nvl, "devid");
+	if (value != NULL)
+		device = strtol(value, NULL, 0);
+
+#ifndef __FreeBSD__
+	const char *model = get_config_value_node(nvl, "model");
 
 	if (model != NULL && (vendor != 0 || device != 0)) {
 		fprintf(stderr, "pci_hostbridge: cannot specify model "
@@ -175,57 +91,53 @@ pci_hostbridge_parse_args(char *opts, uint16_t *vendorp, uint16_t *devicep)
 		    "device for custom hostbridge");
 		return (-1);
 	}
-	if (model != NULL) {
-		uint_t i;
+	if (model == NULL && vendor == 0 && device == 0)
+		model = "netapp";
 
-		for (i = 0; i < NUM_HB_MODELS; i++) {
+	if (model != NULL) {
+		for (uint_t i = 0; i < NUM_HB_MODELS; i++) {
 			if (strcmp(model, pci_hb_models[i].phm_model) != 0)
 				continue;
 
 			/* found a model match */
-			*vendorp = pci_hb_models[i].phm_vendor;
-			*devicep = pci_hb_models[i].phm_device;
-			return (0);
+			vendor = pci_hb_models[i].phm_vendor;
+			device = pci_hb_models[i].phm_device;
+			break;
+		}
+		if (vendor == 0) {
+			fprintf(stderr, "pci_hostbridge: invalid model '%s'",
+			    model);
+			return (-1);
 		}
-		fprintf(stderr, "pci_hostbridge: invalid model '%s'", model);
-		return (-1);
 	}
+#endif /* !__FreeBSD__ */
 
-	/* custom hostbridge ID was specified */
-	*vendorp = vendor;
-	*devicep = device;
-	return (0);
-}
-
-static int
-pci_hostbridge_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
-{
-	uint16_t vendor, device;
+	/* config space */
+	pci_set_cfgdata16(pi, PCIR_VENDOR, vendor);
+	pci_set_cfgdata16(pi, PCIR_DEVICE, device);
+	pci_set_cfgdata8(pi, PCIR_HDRTYPE, PCIM_HDRTYPE_NORMAL);
+	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_BRIDGE);
+	pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_BRIDGE_HOST);
 
-	if (opts == NULL) {
-		/* Fall back to NetApp default if no options are specified */
-		vendor = 0x1275;
-		device = 0x1275;
-	} else if (pci_hostbridge_parse_args(opts, &vendor, &device) != 0) {
-		return (-1);
-	}
+	pci_emul_add_pciecap(pi, PCIEM_TYPE_ROOT_PORT);
 
-	pci_hostbridge_setup(pi, vendor, device);
 	return (0);
 }
 
 static int
-pci_amd_hostbridge_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+pci_amd_hostbridge_legacy_config(nvlist_t *nvl, const char *opts)
 {
-	pci_hostbridge_setup(pi, 0x1022, 0x7432);
+
+	set_config_value_node(nvl, "vendor", "0x1022");	/* AMD */
+	set_config_value_node(nvl, "devid", "0x7432");	/* made up */
+
 	return (0);
 }
 
-#endif /* __FreeBSD__ */
-
 struct pci_devemu pci_de_amd_hostbridge = {
 	.pe_emu = "amd_hostbridge",
-	.pe_init = pci_amd_hostbridge_init,
+	.pe_legacy_config = pci_amd_hostbridge_legacy_config,
+	.pe_alias = "hostbridge",
 };
 PCI_EMUL_SET(pci_de_amd_hostbridge);
 
diff --git a/usr/src/cmd/bhyve/pci_lpc.c b/usr/src/cmd/bhyve/pci_lpc.c
index 9c8f25e89d..2387f382c5 100644
--- a/usr/src/cmd/bhyve/pci_lpc.c
+++ b/usr/src/cmd/bhyve/pci_lpc.c
@@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$");
 #include "acpi.h"
 #include "debug.h"
 #include "bootrom.h"
+#include "config.h"
 #include "inout.h"
 #include "pci_emul.h"
 #include "pci_irq.h"
@@ -71,20 +72,21 @@ SYSRES_IO(NMISC_PORT, 1);
 
 static struct pci_devinst *lpc_bridge;
 
-static const char *romfile;
-
-#define	LPC_UART_NUM	2
+#define	LPC_UART_NUM	4
 static struct lpc_uart_softc {
 	struct uart_softc *uart_softc;
-	const char *opts;
 	int	iobase;
 	int	irq;
 	int	enabled;
 } lpc_uart_softc[LPC_UART_NUM];
 
-static const char *lpc_uart_names[LPC_UART_NUM] = { "COM1", "COM2" };
+static const char *lpc_uart_names[LPC_UART_NUM] = {
+	"com1", "com2", "com3", "com4"
+};
 
-static bool pctestdev_present;
+static const char *lpc_uart_acpi_names[LPC_UART_NUM] = {
+	"COM1", "COM2", "COM3", "COM4"
+};
 
 /*
  * LPC device configuration is in the following form:
@@ -95,41 +97,38 @@ int
 lpc_device_parse(const char *opts)
 {
 	int unit, error;
-	char *str, *cpy, *lpcdev;
+	char *str, *cpy, *lpcdev, *node_name;
 
 	error = -1;
 	str = cpy = strdup(opts);
 	lpcdev = strsep(&str, ",");
 	if (lpcdev != NULL) {
 		if (strcasecmp(lpcdev, "bootrom") == 0) {
-			romfile = str;
+			set_config_value("lpc.bootrom", str);
 			error = 0;
 			goto done;
 		}
 		for (unit = 0; unit < LPC_UART_NUM; unit++) {
 			if (strcasecmp(lpcdev, lpc_uart_names[unit]) == 0) {
-				lpc_uart_softc[unit].opts = str;
+				asprintf(&node_name, "lpc.%s.path",
+				    lpc_uart_names[unit]);
+				set_config_value(node_name, str);
+				free(node_name);
 				error = 0;
 				goto done;
 			}
 		}
 		if (strcasecmp(lpcdev, pctestdev_getname()) == 0) {
-			if (pctestdev_present) {
-				EPRINTLN("More than one %s device conf is "
-				    "specified; only one is allowed.",
-				    pctestdev_getname());
-			} else if (pctestdev_parse(str) == 0) {
-				pctestdev_present = true;
-				error = 0;
-				free(cpy);
-				goto done;
-			}
+			asprintf(&node_name, "lpc.%s", pctestdev_getname());
+			set_config_bool(node_name, true);
+			free(node_name);
+			error = 0;
+			goto done;
 		}
 	}
 
 done:
-	if (error)
-		free(cpy);
+	free(cpy);
 
 	return (error);
 }
@@ -149,7 +148,7 @@ const char *
 lpc_bootrom(void)
 {
 
-	return (romfile);
+	return (get_config_value("lpc.bootrom"));
 }
 
 static void
@@ -223,9 +222,11 @@ lpc_init(struct vmctx *ctx)
 {
 	struct lpc_uart_softc *sc;
 	struct inout_port iop;
-	const char *name;
+	const char *backend, *name, *romfile;
+	char *node_name;
 	int unit, error;
 
+	romfile = get_config_value("lpc.bootrom");
 	if (romfile != NULL) {
 		error = bootrom_loadrom(ctx, romfile);
 		if (error)
@@ -247,9 +248,12 @@ lpc_init(struct vmctx *ctx)
 		sc->uart_softc = uart_init(lpc_uart_intr_assert,
 				    lpc_uart_intr_deassert, sc);
 
-		if (uart_set_backend(sc->uart_softc, sc->opts) != 0) {
+		asprintf(&node_name, "lpc.%s.path", name);
+		backend = get_config_value(node_name);
+		free(node_name);
+		if (uart_set_backend(sc->uart_softc, backend) != 0) {
 			EPRINTLN("Unable to initialize backend '%s' "
-			    "for LPC device %s", sc->opts, name);
+			    "for LPC device %s", backend, name);
 			return (-1);
 		}
 
@@ -267,11 +271,13 @@ lpc_init(struct vmctx *ctx)
 	}
 
 	/* pc-testdev */
-	if (pctestdev_present) {
+	asprintf(&node_name, "lpc.%s", pctestdev_getname());
+	if (get_config_bool_default(node_name, false)) {
 		error = pctestdev_init(ctx);
 		if (error)
 			return (error);
 	}
+	free(node_name);
 
 	return (0);
 }
@@ -380,7 +386,7 @@ pci_lpc_uart_dsdt(void)
 		if (!sc->enabled)
 			continue;
 		dsdt_line("");
-		dsdt_line("Device (%s)", lpc_uart_names[unit]);
+		dsdt_line("Device (%s)", lpc_uart_acpi_names[unit]);
 		dsdt_line("{");
 		dsdt_line("  Name (_HID, EisaId (\"PNP0501\"))");
 		dsdt_line("  Name (_UID, %d)", unit + 1);
@@ -434,7 +440,7 @@ pci_lpc_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 #define	LPC_VENDOR	0x8086
 
 static int
-pci_lpc_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+pci_lpc_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 
 	/*
diff --git a/usr/src/cmd/bhyve/pci_nvme.c b/usr/src/cmd/bhyve/pci_nvme.c
index a0a8a9571d..c8fd44aa52 100644
--- a/usr/src/cmd/bhyve/pci_nvme.c
+++ b/usr/src/cmd/bhyve/pci_nvme.c
@@ -84,6 +84,7 @@ __FBSDID("$FreeBSD$");
 
 #include "bhyverun.h"
 #include "block_if.h"
+#include "config.h"
 #include "debug.h"
 #include "pci_emul.h"
 
@@ -581,8 +582,9 @@ pci_nvme_init_nsdata(struct pci_nvme_softc *sc,
 		char *data = NULL;
 		uint64_t eui64 = nvstore->eui64;
 
-		asprintf(&data, "%s%u%u%u", vmname, sc->nsc_pi->pi_bus,
-		    sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
+		asprintf(&data, "%s%u%u%u", get_config_value("name"),
+		    sc->nsc_pi->pi_bus, sc->nsc_pi->pi_slot,
+		    sc->nsc_pi->pi_func);
 
 		if (data != NULL) {
 			eui64 = OUI_FREEBSD_NVME_LOW | crc16(0, data, strlen(data));
@@ -1080,9 +1082,13 @@ static int
 nvme_opc_get_log_page(struct pci_nvme_softc* sc, struct nvme_command* command,
 	struct nvme_completion* compl)
 {
-	uint32_t logsize = 0;
+	uint32_t logsize;
 	uint8_t logpage = command->cdw10 & 0xFF;
 
+#ifndef __FreeBSD__
+	logsize = 0;
+#endif
+
 	DPRINTF("%s log page %u len %u", __func__, logpage, logsize);
 
 	pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
@@ -1140,7 +1146,11 @@ nvme_opc_identify(struct pci_nvme_softc* sc, struct nvme_command* command,
 	struct nvme_completion* compl)
 {
 	void *dest;
-	uint16_t status = 0;
+	uint16_t status;
+
+#ifndef __FreeBSD__
+	status = 0;
+#endif
 
 	DPRINTF("%s identify 0x%x nsid 0x%x", __func__,
 	        command->cdw10 & 0xFF, command->nsid);
@@ -1866,7 +1876,11 @@ pci_nvme_io_done(struct blockif_req *br, int err)
 {
 	struct pci_nvme_ioreq *req = br->br_param;
 	struct nvme_submission_queue *sq = req->nvme_sq;
-	uint16_t code, status = 0;
+	uint16_t code, status;
+
+#ifndef __FreeBSD__
+	status = 0;
+#endif
 
 	DPRINTF("%s error %d %s", __func__, err, strerror(err));
 
@@ -1929,7 +1943,11 @@ nvme_write_read_ram(struct pci_nvme_softc *sc,
 {
 	uint8_t *buf = nvstore->ctx;
 	enum nvme_copy_dir dir;
-	uint16_t status = 0;
+	uint16_t status;
+
+#ifndef __FreeBSD__
+	status = 0;
+#endif
 
 	if (is_write)
 		dir = NVME_COPY_TO_PRP;
@@ -2029,11 +2047,15 @@ nvme_opc_write_read(struct pci_nvme_softc *sc,
     struct pci_nvme_ioreq *req,
     uint16_t *status)
 {
-	uint64_t lba, nblocks, bytes = 0;
+	uint64_t lba, nblocks, bytes;
 	size_t offset;
 	bool is_write = cmd->opc == NVME_OPC_WRITE;
 	bool pending = false;
 
+#ifndef __FreeBSD__
+	bytes = 0;
+#endif
+
 	lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10;
 	nblocks = (cmd->cdw12 & 0xFFFF) + 1;
 	if (pci_nvme_out_of_range(nvstore, lba, nblocks)) {
@@ -2081,10 +2103,10 @@ pci_nvme_dealloc_sm(struct blockif_req *br, int err)
 	struct pci_nvme_ioreq *req = br->br_param;
 	struct pci_nvme_softc *sc = req->sc;
 	bool done = true;
-#ifdef __FreeBSD__
 	uint16_t status;
-#else
-	uint16_t status = 0;
+
+#ifndef __FreeBSD__
+	status = 0;
 #endif
 
 	if (err) {
@@ -2121,11 +2143,15 @@ nvme_opc_dataset_mgmt(struct pci_nvme_softc *sc,
     struct pci_nvme_ioreq *req,
     uint16_t *status)
 {
-	struct nvme_dsm_range *range = NULL;
+	struct nvme_dsm_range *range;
 	uint32_t nr, r, non_zero, dr;
 	int err;
 	bool pending = false;
 
+#ifndef __FreeBSD__
+	range = NULL;
+#endif
+
 	if ((sc->ctrldata.oncs & NVME_ONCS_DSM) == 0) {
 		pci_nvme_status_genc(status, NVME_SC_INVALID_OPCODE);
 		goto out;
@@ -2237,9 +2263,13 @@ static void
 pci_nvme_handle_io_cmd(struct pci_nvme_softc* sc, uint16_t idx)
 {
 	struct nvme_submission_queue *sq;
-	uint16_t status = 0;
+	uint16_t status;
 	uint16_t sqhead;
 
+#ifndef __FreeBSD__
+	status = 0;
+#endif
+
 	/* handle all submissions up to sq->tail index */
 	sq = &sc->submit_queues[idx];
 
@@ -2624,14 +2654,12 @@ pci_nvme_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx,
 	return (0);
 }
 
-
 static int
-pci_nvme_parse_opts(struct pci_nvme_softc *sc, char *opts)
+pci_nvme_parse_config(struct pci_nvme_softc *sc, nvlist_t *nvl)
 {
 	char bident[sizeof("XX:X:X")];
-	char	*uopt, *xopts, *config;
+	const char *value;
 	uint32_t sectsz;
-	int optidx;
 
 	sc->max_queues = NVME_QUEUES;
 	sc->max_qentries = NVME_MAX_QENTRIES;
@@ -2640,81 +2668,81 @@ pci_nvme_parse_opts(struct pci_nvme_softc *sc, char *opts)
 	sc->num_cqueues = sc->max_queues;
 	sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO;
 	sectsz = 0;
-
-	uopt = strdup(opts);
-	optidx = 0;
 	snprintf(sc->ctrldata.sn, sizeof(sc->ctrldata.sn),
 	         "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
-	for (xopts = strtok(uopt, ",");
-	     xopts != NULL;
-	     xopts = strtok(NULL, ",")) {
-
-		if ((config = strchr(xopts, '=')) != NULL)
-			*config++ = '\0';
-
-		if (!strcmp("maxq", xopts)) {
-			sc->max_queues = atoi(config);
-		} else if (!strcmp("qsz", xopts)) {
-			sc->max_qentries = atoi(config);
-		} else if (!strcmp("ioslots", xopts)) {
-			sc->ioslots = atoi(config);
-		} else if (!strcmp("sectsz", xopts)) {
-			sectsz = atoi(config);
-		} else if (!strcmp("ser", xopts)) {
-			/*
-			 * This field indicates the Product Serial Number in
-			 * 7-bit ASCII, unused bytes should be space characters.
-			 * Ref: NVMe v1.3c.
-			 */
-			cpywithpad((char *)sc->ctrldata.sn,
-			           sizeof(sc->ctrldata.sn), config, ' ');
-		} else if (!strcmp("ram", xopts)) {
-			uint64_t sz = strtoull(&xopts[4], NULL, 10);
-
-			sc->nvstore.type = NVME_STOR_RAM;
-			sc->nvstore.size = sz * 1024 * 1024;
-			sc->nvstore.ctx = calloc(1, sc->nvstore.size);
-			sc->nvstore.sectsz = 4096;
-			sc->nvstore.sectsz_bits = 12;
-			if (sc->nvstore.ctx == NULL) {
-				perror("Unable to allocate RAM");
-				free(uopt);
-				return (-1);
-			}
-		} else if (!strcmp("eui64", xopts)) {
-			sc->nvstore.eui64 = htobe64(strtoull(config, NULL, 0));
-		} else if (!strcmp("dsm", xopts)) {
-			if (!strcmp("auto", config))
-				sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO;
-			else if (!strcmp("enable", config))
-				sc->dataset_management = NVME_DATASET_MANAGEMENT_ENABLE;
-			else if (!strcmp("disable", config))
-				sc->dataset_management = NVME_DATASET_MANAGEMENT_DISABLE;
-		} else if (optidx == 0) {
-			snprintf(bident, sizeof(bident), "%d:%d",
-			         sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
-			sc->nvstore.ctx = blockif_open(xopts, bident);
-			if (sc->nvstore.ctx == NULL) {
-				perror("Could not open backing file");
-				free(uopt);
-				return (-1);
-			}
-			sc->nvstore.type = NVME_STOR_BLOCKIF;
-			sc->nvstore.size = blockif_size(sc->nvstore.ctx);
-		} else {
-			EPRINTLN("Invalid option %s", xopts);
-			free(uopt);
+
+	value = get_config_value_node(nvl, "maxq");
+	if (value != NULL)
+		sc->max_queues = atoi(value);
+	value = get_config_value_node(nvl, "qsz");
+	if (value != NULL) {
+		sc->max_qentries = atoi(value);
+		if (sc->max_qentries <= 0) {
+			EPRINTLN("nvme: Invalid qsz option %d",
+			    sc->max_qentries);
 			return (-1);
 		}
-
-		optidx++;
 	}
-	free(uopt);
-
-	if (sc->nvstore.ctx == NULL || sc->nvstore.size == 0) {
-		EPRINTLN("backing store not specified");
-		return (-1);
+	value = get_config_value_node(nvl, "ioslots");
+	if (value != NULL) {
+		sc->ioslots = atoi(value);
+		if (sc->ioslots <= 0) {
+			EPRINTLN("Invalid ioslots option %d", sc->ioslots);
+			return (-1);
+		}
 	}
+	value = get_config_value_node(nvl, "sectsz");
+	if (value != NULL)
+		sectsz = atoi(value);
+	value = get_config_value_node(nvl, "ser");
+	if (value != NULL) {
+		/*
+		 * This field indicates the Product Serial Number in
+		 * 7-bit ASCII, unused bytes should be space characters.
+		 * Ref: NVMe v1.3c.
+		 */
+		cpywithpad((char *)sc->ctrldata.sn,
+		    sizeof(sc->ctrldata.sn), value, ' ');
+	}
+	value = get_config_value_node(nvl, "eui64");
+	if (value != NULL)
+		sc->nvstore.eui64 = htobe64(strtoull(value, NULL, 0));
+	value = get_config_value_node(nvl, "dsm");
+	if (value != NULL) {
+		if (strcmp(value, "auto") == 0)
+			sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO;
+		else if (strcmp(value, "enable") == 0)
+			sc->dataset_management = NVME_DATASET_MANAGEMENT_ENABLE;
+		else if (strcmp(value, "disable") == 0)
+			sc->dataset_management = NVME_DATASET_MANAGEMENT_DISABLE;
+	}
+
+	value = get_config_value_node(nvl, "ram");
+	if (value != NULL) {
+		uint64_t sz = strtoull(value, NULL, 10);
+
+		sc->nvstore.type = NVME_STOR_RAM;
+		sc->nvstore.size = sz * 1024 * 1024;
+		sc->nvstore.ctx = calloc(1, sc->nvstore.size);
+		sc->nvstore.sectsz = 4096;
+		sc->nvstore.sectsz_bits = 12;
+		if (sc->nvstore.ctx == NULL) {
+			EPRINTLN("nvme: Unable to allocate RAM");
+			return (-1);
+		}
+	} else {
+		snprintf(bident, sizeof(bident), "%d:%d",
+		    sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
+		sc->nvstore.ctx = blockif_open(nvl, bident);
+		if (sc->nvstore.ctx == NULL) {
+			EPRINTLN("nvme: Could not open backing file: %s",
+			    strerror(errno));
+			return (-1);
+		}
+		sc->nvstore.type = NVME_STOR_BLOCKIF;
+		sc->nvstore.size = blockif_size(sc->nvstore.ctx);
+	}
+
 	if (sectsz == 512 || sectsz == 4096 || sectsz == 8192)
 		sc->nvstore.sectsz = sectsz;
 	else if (sc->nvstore.type != NVME_STOR_RAM)
@@ -2726,20 +2754,11 @@ pci_nvme_parse_opts(struct pci_nvme_softc *sc, char *opts)
 	if (sc->max_queues <= 0 || sc->max_queues > NVME_QUEUES)
 		sc->max_queues = NVME_QUEUES;
 
-	if (sc->max_qentries <= 0) {
-		EPRINTLN("Invalid qsz option");
-		return (-1);
-	}
-	if (sc->ioslots <= 0) {
-		EPRINTLN("Invalid ioslots option");
-		return (-1);
-	}
-
 	return (0);
 }
 
 static int
-pci_nvme_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+pci_nvme_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	struct pci_nvme_softc *sc;
 	uint32_t pci_membar_sz;
@@ -2751,7 +2770,7 @@ pci_nvme_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	pi->pi_arg = sc;
 	sc->nsc_pi = pi;
 
-	error = pci_nvme_parse_opts(sc, opts);
+	error = pci_nvme_parse_config(sc, nvl);
 	if (error < 0)
 		goto done;
 	else
@@ -2828,6 +2847,7 @@ done:
 struct pci_devemu pci_de_nvme = {
 	.pe_emu =	"nvme",
 	.pe_init =	pci_nvme_init,
+	.pe_legacy_config = blockif_legacy_config,
 	.pe_barwrite =	pci_nvme_write,
 	.pe_barread =	pci_nvme_read
 };
diff --git a/usr/src/cmd/bhyve/pci_passthru.c b/usr/src/cmd/bhyve/pci_passthru.c
index c777c56cb1..30f0823726 100644
--- a/usr/src/cmd/bhyve/pci_passthru.c
+++ b/usr/src/cmd/bhyve/pci_passthru.c
@@ -62,6 +62,9 @@ __FBSDID("$FreeBSD$");
 #include <machine/vmm.h>
 #include <vmmapi.h>
 #include <sys/ppt_dev.h>
+
+#include "config.h"
+#include "debug.h"
 #include "pci_emul.h"
 #include "mem.h"
 
@@ -503,11 +506,11 @@ passthru_msix_table_write(struct vmctx *ctx, int vcpu,
 static int
 init_msix_table(struct vmctx *ctx, struct passthru_softc *sc, uint64_t base)
 {
-	int error, idx;
-	size_t len, remaining;
+	int idx;
+	size_t remaining __unused;
 	uint32_t table_size, table_offset;
 	uint32_t pba_size, pba_offset;
-	vm_paddr_t start;
+	vm_paddr_t start __unused;
 	struct pci_devinst *pi = sc->psc_pi;
 
 	assert(pci_msix_table_bar(pi) >= 0 && pci_msix_pba_bar(pi) >= 0);
@@ -561,31 +564,6 @@ init_msix_table(struct vmctx *ctx, struct passthru_softc *sc, uint64_t base)
 		}
 	}
 
-	/* Map everything before the MSI-X table */
-	if (table_offset > 0) {
-		len = table_offset;
-		error = vm_map_pptdev_mmio(ctx, sc->pptfd, start, len, base);
-		if (error)
-			return (error);
-
-		base += len;
-		start += len;
-		remaining -= len;
-	}
-
-	/* Skip the MSI-X table */
-	base += table_size;
-	start += table_size;
-	remaining -= table_size;
-
-	/* Map everything beyond the end of the MSI-X table */
-	if (remaining > 0) {
-		len = remaining;
-		error = vm_map_pptdev_mmio(ctx, sc->pptfd, start, len, base);
-		if (error)
-			return (error);
-	}
-
 	return (0);
 }
 
@@ -631,12 +609,6 @@ cfginitbar(struct vmctx *ctx, struct passthru_softc *sc)
 			error = init_msix_table(ctx, sc, base);
 			if (error)
 				return (-1);
-		} else if (bartype != PCIBAR_IO) {
-			/* Map the physical BAR in the guest MMIO space */
-			error = vm_map_pptdev_mmio(ctx, sc->pptfd,
-			    pi->pi_bar[i].addr, pi->pi_bar[i].size, base);
-			if (error)
-				return (-1);
 		}
 
 		/*
@@ -672,10 +644,23 @@ cfginit(struct vmctx *ctx, struct passthru_softc *sc)
 }
 
 static int
-passthru_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+passthru_legacy_config(nvlist_t *nvl, const char *opts)
+{
+	if (opts == NULL)
+		return (0);
+
+	if (strncmp(opts, "/dev/ppt", 8) == 0)
+		set_config_value_node(nvl, "path", opts);
+
+	return (0);
+}
+
+static int
+passthru_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	int error, memflags, pptfd;
 	struct passthru_softc *sc;
+	const char *path;
 
 	sc = NULL;
 	error = 1;
@@ -686,7 +671,8 @@ passthru_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 		goto done;
 	}
 
-	if (opts == NULL || passthru_dev_open(opts, &pptfd) != 0) {
+	path = get_config_value_node(nvl, "path");
+	if (path == NULL || passthru_dev_open(path, &pptfd) != 0) {
 		warnx("invalid passthru options");
 		goto done;
 	}
@@ -930,12 +916,85 @@ passthru_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx,
 	return (val);
 }
 
+static void
+passthru_msix_addr(struct vmctx *ctx, struct pci_devinst *pi, int baridx,
+    int enabled, uint64_t address)
+{
+	struct passthru_softc *sc;
+	size_t remaining;
+	uint32_t table_size, table_offset;
+
+	sc = pi->pi_arg;
+	table_offset = rounddown2(pi->pi_msix.table_offset, 4096);
+	if (table_offset > 0) {
+		if (!enabled) {
+			if (vm_unmap_pptdev_mmio(ctx, sc->pptfd, address,
+			    table_offset) != 0)
+				warnx("pci_passthru: unmap_pptdev_mmio failed");
+		} else {
+			if (vm_map_pptdev_mmio(ctx, sc->pptfd, address,
+			    table_offset, sc->psc_bar[baridx].addr) != 0)
+				warnx("pci_passthru: map_pptdev_mmio failed");
+		}
+	}
+	table_size = pi->pi_msix.table_offset - table_offset;
+	table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE;
+	table_size = roundup2(table_size, 4096);
+	remaining = pi->pi_bar[baridx].size - table_offset - table_size;
+	if (remaining > 0) {
+		address += table_offset + table_size;
+		if (!enabled) {
+			if (vm_unmap_pptdev_mmio(ctx, sc->pptfd, address,
+			    remaining) != 0)
+				warnx("pci_passthru: unmap_pptdev_mmio failed");
+		} else {
+			if (vm_map_pptdev_mmio(ctx, sc->pptfd, address,
+			    remaining, sc->psc_bar[baridx].addr +
+			    table_offset + table_size) != 0)
+				warnx("pci_passthru: map_pptdev_mmio failed");
+		}
+	}
+}
+
+static void
+passthru_mmio_addr(struct vmctx *ctx, struct pci_devinst *pi, int baridx,
+    int enabled, uint64_t address)
+{
+	struct passthru_softc *sc;
+
+	sc = pi->pi_arg;
+	if (!enabled) {
+		if (vm_unmap_pptdev_mmio(ctx, sc->pptfd, address,
+		    sc->psc_bar[baridx].size) != 0)
+			warnx("pci_passthru: unmap_pptdev_mmio failed");
+	} else {
+		if (vm_map_pptdev_mmio(ctx, sc->pptfd, address,
+		    sc->psc_bar[baridx].size, sc->psc_bar[baridx].addr) != 0)
+			warnx("pci_passthru: map_pptdev_mmio failed");
+	}
+}
+
+static void
+passthru_addr(struct vmctx *ctx, struct pci_devinst *pi, int baridx,
+	      int enabled, uint64_t address)
+{
+
+	if (pi->pi_bar[baridx].type == PCIBAR_IO)
+		return;
+	if (baridx == pci_msix_table_bar(pi))
+		passthru_msix_addr(ctx, pi, baridx, enabled, address);
+	else
+		passthru_mmio_addr(ctx, pi, baridx, enabled, address);
+}
+
 struct pci_devemu passthru = {
 	.pe_emu		= "passthru",
 	.pe_init	= passthru_init,
+	.pe_legacy_config = passthru_legacy_config,
 	.pe_cfgwrite	= passthru_cfgwrite,
 	.pe_cfgread	= passthru_cfgread,
 	.pe_barwrite 	= passthru_write,
 	.pe_barread    	= passthru_read,
+	.pe_baraddr	= passthru_addr,
 };
 PCI_EMUL_SET(passthru);
diff --git a/usr/src/cmd/bhyve/pci_uart.c b/usr/src/cmd/bhyve/pci_uart.c
index 2e8177bafb..25ef1ed662 100644
--- a/usr/src/cmd/bhyve/pci_uart.c
+++ b/usr/src/cmd/bhyve/pci_uart.c
@@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$");
 #include <stdio.h>
 
 #include "bhyverun.h"
+#include "config.h"
 #include "debug.h"
 #include "pci_emul.h"
 #include "uart_emul.h"
@@ -89,9 +90,19 @@ pci_uart_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 }
 
 static int
-pci_uart_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+pci_uart_legacy_config(nvlist_t *nvl, const char *opts)
+{
+
+	if (opts != NULL)
+		set_config_value_node(nvl, "path", opts);
+	return (0);
+}
+
+static int
+pci_uart_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	struct uart_softc *sc;
+	const char *device;
 
 	pci_emul_alloc_bar(pi, 0, PCIBAR_IO, UART_IO_BAR_SIZE);
 	pci_lintr_request(pi);
@@ -104,9 +115,10 @@ pci_uart_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	sc = uart_init(pci_uart_intr_assert, pci_uart_intr_deassert, pi);
 	pi->pi_arg = sc;
 
-	if (uart_set_backend(sc, opts) != 0) {
+	device = get_config_value_node(nvl, "path");
+	if (uart_set_backend(sc, device) != 0) {
 		EPRINTLN("Unable to initialize backend '%s' for "
-		    "pci uart at %d:%d", opts, pi->pi_slot, pi->pi_func);
+		    "pci uart at %d:%d", device, pi->pi_slot, pi->pi_func);
 		return (-1);
 	}
 
@@ -116,6 +128,7 @@ pci_uart_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 struct pci_devemu pci_de_com = {
 	.pe_emu =	"uart",
 	.pe_init =	pci_uart_init,
+	.pe_legacy_config = pci_uart_legacy_config,
 	.pe_barwrite =	pci_uart_write,
 	.pe_barread =	pci_uart_read
 };
diff --git a/usr/src/cmd/bhyve/pci_virtio_block.c b/usr/src/cmd/bhyve/pci_virtio_block.c
index 27d743a770..85c3f07b31 100644
--- a/usr/src/cmd/bhyve/pci_virtio_block.c
+++ b/usr/src/cmd/bhyve/pci_virtio_block.c
@@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$");
 #include <md5.h>
 
 #include "bhyverun.h"
+#include "config.h"
 #include "debug.h"
 #include "pci_emul.h"
 #include "virtio.h"
@@ -416,26 +417,22 @@ pci_vtblk_notify(void *vsc, struct vqueue_info *vq)
 }
 
 static int
-pci_vtblk_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+pci_vtblk_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	char bident[sizeof("XX:X:X")];
 	struct blockif_ctxt *bctxt;
+	const char *path;
 	MD5_CTX mdctx;
 	u_char digest[16];
 	struct pci_vtblk_softc *sc;
 	off_t size;
 	int i, sectsz, sts, sto;
 
-	if (opts == NULL) {
-		WPRINTF(("virtio-block: backing device required"));
-		return (1);
-	}
-
 	/*
 	 * The supplied backing file has to exist
 	 */
 	snprintf(bident, sizeof(bident), "%d:%d", pi->pi_slot, pi->pi_func);
-	bctxt = blockif_open(opts, bident);
+	bctxt = blockif_open(nvl, bident);
 	if (bctxt == NULL) {
 		perror("Could not open backing file");
 		return (1);
@@ -478,8 +475,9 @@ pci_vtblk_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	 * Create an identifier for the backing file. Use parts of the
 	 * md5 sum of the filename
 	 */
+	path = get_config_value_node(nvl, "path");
 	MD5Init(&mdctx);
-	MD5Update(&mdctx, opts, strlen(opts));
+	MD5Update(&mdctx, path, strlen(path));
 	MD5Final(digest, &mdctx);
 	snprintf(sc->vbsc_ident, VTBLK_BLK_ID_BYTES,
 	    "BHYVE-%02X%02X-%02X%02X-%02X%02X",
@@ -520,7 +518,7 @@ pci_vtblk_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_DEV_BLOCK);
 	pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR);
 	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
-	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_TYPE_BLOCK);
+	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_ID_BLOCK);
 	pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR);
 
 	if (vi_intr_init(&sc->vbsc_vs, 1, fbsdrun_virtio_msix())) {
@@ -569,6 +567,7 @@ pci_vtblk_apply_feats(void *vsc, uint64_t caps)
 struct pci_devemu pci_de_vblk = {
 	.pe_emu =	"virtio-blk",
 	.pe_init =	pci_vtblk_init,
+	.pe_legacy_config = blockif_legacy_config,
 	.pe_barwrite =	vi_pci_write,
 	.pe_barread =	vi_pci_read
 };
diff --git a/usr/src/cmd/bhyve/pci_virtio_console.c b/usr/src/cmd/bhyve/pci_virtio_console.c
index 5799b20f6a..f4aa292536 100644
--- a/usr/src/cmd/bhyve/pci_virtio_console.c
+++ b/usr/src/cmd/bhyve/pci_virtio_console.c
@@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$");
 #include <sysexits.h>
 
 #include "bhyverun.h"
+#include "config.h"
 #include "debug.h"
 #include "pci_emul.h"
 #include "virtio.h"
@@ -132,7 +133,6 @@ struct pci_vtcon_softc {
 	uint64_t                 vsc_features;
 	char *                   vsc_rootdir;
 	int                      vsc_kq;
-	int                      vsc_nports;
 	bool                     vsc_ready;
 	struct pci_vtcon_port    vsc_control_port;
  	struct pci_vtcon_port    vsc_ports[VTCON_MAXPORTS];
@@ -246,18 +246,17 @@ pci_vtcon_port_to_vq(struct pci_vtcon_port *port, bool tx_queue)
 }
 
 static struct pci_vtcon_port *
-pci_vtcon_port_add(struct pci_vtcon_softc *sc, const char *name,
+pci_vtcon_port_add(struct pci_vtcon_softc *sc, int port_id, const char *name,
     pci_vtcon_cb_t *cb, void *arg)
 {
 	struct pci_vtcon_port *port;
 
-	if (sc->vsc_nports == VTCON_MAXPORTS) {
+	port = &sc->vsc_ports[port_id];
+	if (port->vsp_enabled) {
 		errno = EBUSY;
 		return (NULL);
 	}
-
-	port = &sc->vsc_ports[sc->vsc_nports++];
-	port->vsp_id = sc->vsc_nports - 1;
+	port->vsp_id = port_id;
 	port->vsp_sc = sc;
 	port->vsp_name = name;
 	port->vsp_cb = cb;
@@ -268,7 +267,7 @@ pci_vtcon_port_add(struct pci_vtcon_softc *sc, const char *name,
 		port->vsp_txq = 0;
 		port->vsp_rxq = 1;
 	} else {
-		port->vsp_txq = sc->vsc_nports * 2;
+		port->vsp_txq = (port_id + 1) * 2;
 		port->vsp_rxq = port->vsp_txq + 1;
 	}
 
@@ -277,22 +276,39 @@ pci_vtcon_port_add(struct pci_vtcon_softc *sc, const char *name,
 }
 
 static int
-pci_vtcon_sock_add(struct pci_vtcon_softc *sc, const char *name,
-    const char *path)
+pci_vtcon_sock_add(struct pci_vtcon_softc *sc, const char *port_name,
+    const nvlist_t *nvl)
 {
-	struct pci_vtcon_sock *sock;
 #ifdef __FreeBSD__
+	struct pci_vtcon_sock *sock;
 	struct sockaddr_un sun;
-	char *pathcopy;
 #else
+	struct pci_vtcon_sock *sock = NULL;
 	/* Our compiler #defines 'sun' as '1'.  Awesome. */
 	struct sockaddr_un addr;
 #endif
+	const char *name, *path;
+	char *cp, *pathcopy;
+	long port;
 	int s = -1, fd = -1, error = 0;
 #ifndef WITHOUT_CAPSICUM
 	cap_rights_t rights;
 #endif
 
+	port = strtol(port_name, &cp, 0);
+	if (*cp != '\0' || port < 0 || port >= VTCON_MAXPORTS) {
+		EPRINTLN("vtcon: Invalid port %s", port_name);
+		error = -1;
+		goto out;
+	}
+
+	path = get_config_value_node(nvl, "path");
+	if (path == NULL) {
+		EPRINTLN("vtcon: required path missing for port %ld", port);
+		error = -1;
+		goto out;
+	}
+
 	sock = calloc(1, sizeof(struct pci_vtcon_sock));
 	if (sock == NULL) {
 		error = -1;
@@ -331,8 +347,9 @@ pci_vtcon_sock_add(struct pci_vtcon_softc *sc, const char *name,
 	}
 #else /* __FreeBSD__ */
 	/* Do a simple bind rather than the FreeBSD bindat() */
+	pathcopy = (char *)path;
 	addr.sun_family = AF_UNIX;
-	(void) strlcpy(addr.sun_path, path, sizeof (addr.sun_path));
+	(void) strlcpy(addr.sun_path, pathcopy, sizeof (addr.sun_path));
 	if (bind(fd, (struct sockaddr *)&addr, sizeof (addr)) < 0) {
 		error = -1;
 		goto out;
@@ -355,7 +372,13 @@ pci_vtcon_sock_add(struct pci_vtcon_softc *sc, const char *name,
 		errx(EX_OSERR, "Unable to apply rights for sandbox");
 #endif
 
-	sock->vss_port = pci_vtcon_port_add(sc, name, pci_vtcon_sock_tx, sock);
+	name = get_config_value_node(nvl, "name");
+	if (name == NULL) {
+		EPRINTLN("vtcon: required name missing for port %ld", port);
+		error = -1;
+		goto out;
+	}
+	sock->vss_port = pci_vtcon_port_add(sc, port, name, pci_vtcon_sock_tx, sock);
 	if (sock->vss_port == NULL) {
 		error = -1;
 		goto out;
@@ -467,10 +490,10 @@ pci_vtcon_sock_tx(struct pci_vtcon_port *port, void *arg, struct iovec *iov,
     int niov)
 {
 	struct pci_vtcon_sock *sock;
-#ifdef __FreeBSD__
 	int i, ret;
-#else
-	int i, ret = 0;
+
+#ifndef __FreeBSD__
+	ret = 0;
 #endif
 
 	sock = (struct pci_vtcon_sock *)arg;
@@ -521,13 +544,13 @@ pci_vtcon_control_tx(struct pci_vtcon_port *port, void *arg, struct iovec *iov,
 		break;
 
 	case VTCON_PORT_READY:
-		if (ctrl->id >= sc->vsc_nports) {
+		tmp = &sc->vsc_ports[ctrl->id];
+		if (ctrl->id >= VTCON_MAXPORTS || !tmp->vsp_enabled) {
 			WPRINTF(("VTCON_PORT_READY event for unknown port %d",
 			    ctrl->id));
 			return;
 		}
 
-		tmp = &sc->vsc_ports[ctrl->id];
 		if (tmp->vsp_console) {
 			resp.event = VTCON_CONSOLE_PORT;
 			resp.id = ctrl->id;
@@ -638,14 +661,63 @@ pci_vtcon_notify_rx(void *vsc, struct vqueue_info *vq)
 	}
 }
 
+#ifdef __FreeBSD__
+/*
+ * Each console device has a "port" node which contains nodes for
+ * each port.  Ports are numbered starting at 0.
+ */
+static int
+pci_vtcon_legacy_config_port(nvlist_t *nvl, int port, char *opt)
+{
+	char *name, *path;
+	char node_name[sizeof("XX")];
+	nvlist_t *port_nvl;
+
+	name = strsep(&opt, "=");
+	path = opt;
+	if (path == NULL) {
+		EPRINTLN("vtcon: port %s requires a path", name);
+		return (-1);
+	}
+	if (port >= VTCON_MAXPORTS) {
+		EPRINTLN("vtcon: too many ports");
+		return (-1);
+	}
+	snprintf(node_name, sizeof(node_name), "%d", port);
+	port_nvl = create_relative_config_node(nvl, node_name);
+	set_config_value_node(port_nvl, "name", name);
+	set_config_value_node(port_nvl, "path", path);
+	return (0);
+}
+
 static int
-pci_vtcon_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+pci_vtcon_legacy_config(nvlist_t *nvl, const char *opts)
+{
+	char *opt, *str, *tofree;
+	nvlist_t *ports_nvl;
+	int error, port;
+
+	ports_nvl = create_relative_config_node(nvl, "port");
+	tofree = str = strdup(opts);
+	error = 0;
+	port = 0;
+	while ((opt = strsep(&str, ",")) != NULL) {
+		error = pci_vtcon_legacy_config_port(ports_nvl, port, opt);
+		if (error)
+			break;
+		port++;
+	}
+	free(tofree);
+	return (error);
+}
+#endif
+
+static int
+pci_vtcon_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	struct pci_vtcon_softc *sc;
-	char *portname = NULL;
-	char *portpath = NULL;
-	char *opt;
-	int i;	
+	nvlist_t *ports_nvl;
+	int i;
 
 	sc = calloc(1, sizeof(struct pci_vtcon_softc));
 	sc->vsc_config = calloc(1, sizeof(struct pci_vtcon_config));
@@ -667,7 +739,7 @@ pci_vtcon_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_DEV_CONSOLE);
 	pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR);
 	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_SIMPLECOMM);
-	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_TYPE_CONSOLE);
+	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_ID_CONSOLE);
 	pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR);
 
 	if (vi_intr_init(&sc->vsc_vs, 1, fbsdrun_virtio_msix()))
@@ -681,15 +753,24 @@ pci_vtcon_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	sc->vsc_control_port.vsp_cb = pci_vtcon_control_tx;
 	sc->vsc_control_port.vsp_enabled = true;
 
-	while ((opt = strsep(&opts, ",")) != NULL) {
-		portname = strsep(&opt, "=");
-		portpath = opt;
-
-		/* create port */
-		if (pci_vtcon_sock_add(sc, portname, portpath) < 0) {
-			EPRINTLN("cannot create port %s: %s",
-			    portname, strerror(errno));
-			return (1);
+	ports_nvl = find_relative_config_node(nvl, "port");
+	if (ports_nvl != NULL) {
+		const char *name;
+		void *cookie;
+		int type;
+
+		cookie = NULL;
+		while ((name = nvlist_next(ports_nvl, &type, &cookie)) !=
+		    NULL) {
+			if (type != NV_TYPE_NVLIST)
+				continue;
+
+			if (pci_vtcon_sock_add(sc, name,
+			    nvlist_get_nvlist(ports_nvl, name)) < 0) {
+				EPRINTLN("cannot create port %s: %s",
+				    name, strerror(errno));
+				return (1);
+			}
 		}
 	}
 
diff --git a/usr/src/cmd/bhyve/pci_virtio_net.c b/usr/src/cmd/bhyve/pci_virtio_net.c
index ded9ca90ea..6736603fb8 100644
--- a/usr/src/cmd/bhyve/pci_virtio_net.c
+++ b/usr/src/cmd/bhyve/pci_virtio_net.c
@@ -83,6 +83,7 @@ __FBSDID("$FreeBSD$");
 #endif
 
 #include "bhyverun.h"
+#include "config.h"
 #include "debug.h"
 #include "pci_emul.h"
 #ifdef __FreeBSD__
@@ -813,7 +814,7 @@ pci_vtnet_ping_ctlq(void *vsc, struct vqueue_info *vq)
 #endif /* __FreeBSD__ */
 
 static void
-pci_vtnet_tap_setup(struct pci_vtnet_softc *sc, char *devname)
+pci_vtnet_tap_setup(struct pci_vtnet_softc *sc, const char *devname)
 {
 	char tbuf[80];
 #ifndef WITHOUT_CAPSICUM
@@ -932,17 +933,17 @@ pci_vtnet_netmap_setup(struct pci_vtnet_softc *sc, char *ifname)
 #endif /* __FreeBSD__ */
 
 static int
-pci_vtnet_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+pci_vtnet_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	struct pci_vtnet_softc *sc;
+	const char *value;
 	char tname[MAXCOMLEN + 1];
 #ifdef __FreeBSD__
-	int mac_provided;
-	int mtu_provided;
 	unsigned long mtu = ETHERMTU;
 #else
 	int use_msix = 1;
 #endif
+	int err;
 
 	/*
 	 * Allocate data structures for further virtio initializations.
@@ -962,96 +963,52 @@ pci_vtnet_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	sc->vsc_queues[VTNET_CTLQ].vq_qsize = VTNET_RINGSZ;
         sc->vsc_queues[VTNET_CTLQ].vq_notify = pci_vtnet_ping_ctlq;
 #endif
- 
-	/*
-	 * Attempt to open the backend device and read the MAC address
-	 * if specified.
-	 */
-#ifdef __FreeBSD__
-	mac_provided = 0;
-	mtu_provided = 0;
-#endif
-	if (opts != NULL) {
-		char *optscopy;
-		char *vtopts;
-		int err = 0;
 
-		/* Get the device name. */
-		optscopy = vtopts = strdup(opts);
-		(void) strsep(&vtopts, ",");
-
-#ifdef __FreeBSD__
-		/*
-		 * Parse the list of options in the form
-		 *     key1=value1,...,keyN=valueN.
-		 */
-		while (vtopts != NULL) {
-			char *value = vtopts;
-			char *key;
-
-			key = strsep(&value, "=");
-			if (value == NULL)
-				break;
-			vtopts = value;
-			(void) strsep(&vtopts, ",");
-
-			if (strcmp(key, "mac") == 0) {
-				err = net_parsemac(value, sc->vsc_config.mac);
-				if (err)
-					break;
-				mac_provided = 1;
-			} else if (strcmp(key, "mtu") == 0) {
-				err = net_parsemtu(value, &mtu);
-				if (err)
-					break;
-
-				if (mtu < VTNET_MIN_MTU || mtu > VTNET_MAX_MTU) {
-					err = EINVAL;
-					errno = EINVAL;
-					break;
-				}
-				mtu_provided = 1;
-			}
+	value = get_config_value_node(nvl, "mac");
+	if (value != NULL) {
+		err = net_parsemac(value, sc->vsc_config.mac);
+		if (err) {
+			free(sc);
+			return (err);
 		}
-#endif
-
-#ifndef __FreeBSD__
-		/* Use the already strsep(",")-ed optscopy */
-		if (strncmp(optscopy, "tap", 3) == 0 ||
-		    strncmp(optscopy, "vmnet", 5) == 0)
-			pci_vtnet_tap_setup(sc, optscopy);
-#endif
-
-		free(optscopy);
+	} else
+		net_genmac(pi, sc->vsc_config.mac);
 
+#ifdef __FreeBSD__
+	value = get_config_value_node(nvl, "mtu");
+	if (value != NULL) {
+		err = net_parsemtu(value, &mtu);
 		if (err) {
 			free(sc);
 			return (err);
 		}
+		if (mtu < VTNET_MIN_MTU || mtu > VTNET_MAX_MTU) {
+			err = EINVAL;
+			errno = EINVAL;
+			free(sc);
+			return (err);
+		}
+		sc->vsc_consts.vc_hv_caps |= VIRTIO_NET_F_MTU;
+	}
+#endif
 
+	/* Permit interfaces without a configured backend. */
+	if (get_config_value_node(nvl, "backend") != NULL) {
 #ifdef __FreeBSD__
-		err = netbe_init(&sc->vsc_be, opts, pci_vtnet_rx_callback,
-		          sc);
+		err = netbe_init(&sc->vsc_be, nvl, pci_vtnet_rx_callback, sc);
 		if (err) {
 			free(sc);
 			return (err);
 		}
-
-		sc->vsc_consts.vc_hv_caps |= VIRTIO_NET_F_MRG_RXBUF |
-		    netbe_get_cap(sc->vsc_be);
+#else
+		pci_vtnet_tap_setup(sc, get_config_value_node(nvl, "backend"));
 #endif
-
 	}
 
-#ifdef __FreeBSD__
-	if (!mac_provided) {
-		net_genmac(pi, sc->vsc_config.mac);
-	}
 
-	sc->vsc_config.mtu = mtu;
-	if (mtu_provided) {
-		sc->vsc_consts.vc_hv_caps |= VIRTIO_NET_F_MTU;
-	}
+#ifdef __FreeBSD__
+	sc->vsc_consts.vc_hv_caps |= VIRTIO_NET_F_MRG_RXBUF |
+	    netbe_get_cap(sc->vsc_be);
 #endif
 
 	/* 
@@ -1064,16 +1021,11 @@ pci_vtnet_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_DEV_NET);
 	pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR);
 	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_NETWORK);
-	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_TYPE_NET);
+	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_ID_NETWORK);
 	pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR);
 
-	/* Link is up if we managed to open tap device or vale port. */
-#ifdef	__FreeBSD__
-	sc->vsc_config.status = (opts == NULL || sc->vsc_tapfd >= 0 ||
-#else
-	sc->vsc_config.status = (opts == NULL || sc->vsc_dlpifd >= 0 ||
-	    sc->vsc_nmd != NULL);
-#endif
+	/* Link is always up. */
+	sc->vsc_config.status = 1;
 
 	/* use BAR 1 to map MSI-X table and PBA, if we're using MSI-X */
 	if (vi_intr_init(&sc->vsc_vs, 1, use_msix))
@@ -1154,9 +1106,38 @@ pci_vtnet_neg_features(void *vsc, uint64_t negotiated_features)
 	pthread_mutex_unlock(&sc->rx_mtx);
 }
 
+#ifndef __FreeBSD__
+static int
+pci_vtnet_legacy_config(nvlist_t *nvl, const char *opt)
+{
+	char *config, *name, *tofree, *value;
+
+	if (opt == NULL)
+		return (0);
+
+	config = tofree = strdup(opt);
+	while ((name = strsep(&config, ",")) != NULL) {
+		value = strchr(name, '=');
+		if (value != NULL) {
+			*value++ = '\0';
+			set_config_value_node(nvl, name, value);
+		} else {
+			set_config_value_node(nvl, "backend", name);
+		}
+	}
+	free(tofree);
+	return (0);
+}
+#endif
+
 struct pci_devemu pci_de_vnet = {
 	.pe_emu = 	"virtio-net",
 	.pe_init =	pci_vtnet_init,
+#ifdef __FreeBSD__
+	.pe_legacy_config = netbe_legacy_config,
+#else
+	.pe_legacy_config = pci_vtnet_legacy_config,
+#endif
 	.pe_barwrite =	vi_pci_write,
 	.pe_barread =	vi_pci_read
 };
diff --git a/usr/src/cmd/bhyve/pci_virtio_rnd.c b/usr/src/cmd/bhyve/pci_virtio_rnd.c
index 4f908324cf..53c3c9e2bc 100644
--- a/usr/src/cmd/bhyve/pci_virtio_rnd.c
+++ b/usr/src/cmd/bhyve/pci_virtio_rnd.c
@@ -143,7 +143,7 @@ pci_vtrnd_notify(void *vsc, struct vqueue_info *vq)
 
 
 static int
-pci_vtrnd_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+pci_vtrnd_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	struct pci_vtrnd_softc *sc;
 	int fd;
@@ -190,7 +190,7 @@ pci_vtrnd_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_DEV_RANDOM);
 	pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR);
 	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_CRYPTO);
-	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_TYPE_ENTROPY);
+	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_ID_ENTROPY);
 	pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR);
 
 	if (vi_intr_init(&sc->vrsc_vs, 1, fbsdrun_virtio_msix()))
@@ -205,6 +205,6 @@ struct pci_devemu pci_de_vrnd = {
 	.pe_emu =	"virtio-rnd",
 	.pe_init =	pci_vtrnd_init,
 	.pe_barwrite =	vi_pci_write,
-	.pe_barread =	vi_pci_read
+	.pe_barread =	vi_pci_read,
 };
 PCI_EMUL_SET(pci_de_vrnd);
diff --git a/usr/src/cmd/bhyve/pci_virtio_scsi.c b/usr/src/cmd/bhyve/pci_virtio_scsi.c
index 92a3311b69..8edf64a113 100644
--- a/usr/src/cmd/bhyve/pci_virtio_scsi.c
+++ b/usr/src/cmd/bhyve/pci_virtio_scsi.c
@@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$");
 #include <camlib.h>
 
 #include "bhyverun.h"
+#include "config.h"
 #include "debug.h"
 #include "pci_emul.h"
 #include "virtio.h"
@@ -244,7 +245,7 @@ static void pci_vtscsi_eventq_notify(void *, struct vqueue_info *);
 static void pci_vtscsi_requestq_notify(void *, struct vqueue_info *);
 static int  pci_vtscsi_init_queue(struct pci_vtscsi_softc *,
     struct pci_vtscsi_queue *, int);
-static int pci_vtscsi_init(struct vmctx *, struct pci_devinst *, char *);
+static int pci_vtscsi_init(struct vmctx *, struct pci_devinst *, nvlist_t *);
 
 static struct virtio_consts vtscsi_vi_consts = {
 	"vtscsi",				/* our name */
@@ -665,32 +666,36 @@ pci_vtscsi_init_queue(struct pci_vtscsi_softc *sc,
 }
 
 static int
-pci_vtscsi_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+pci_vtscsi_legacy_config(nvlist_t *nvl, const char *opts)
+{
+	char *cp, *devname;
+
+	cp = strchr(opts, ',');
+	if (cp == NULL) {
+		set_config_value_node(nvl, "dev", opts);
+		return (0);
+	}
+	devname = strndup(opts, cp - opts);
+	set_config_value_node(nvl, "dev", devname);
+	free(devname);
+	return (pci_parse_legacy_config(nvl, cp + 1));
+}
+
+static int
+pci_vtscsi_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	struct pci_vtscsi_softc *sc;
-	char *opt, *optname;
-	const char *devname;
-	int i, optidx = 0;
+	const char *devname, *value;;
+	int i;
 
 	sc = calloc(1, sizeof(struct pci_vtscsi_softc));
-	devname = "/dev/cam/ctl";
-	while ((opt = strsep(&opts, ",")) != NULL) {
-		optname = strsep(&opt, "=");
-		if (opt == NULL && optidx == 0) {
-			if (optname[0] != 0)
-				devname = optname;
-		} else if (strcmp(optname, "dev") == 0 && opt != NULL) {
-			devname = opt;
-		} else if (strcmp(optname, "iid") == 0 && opt != NULL) {
-			sc->vss_iid = strtoul(opt, NULL, 10);
-		} else {
-			EPRINTLN("Invalid option %s", optname);
-			free(sc);
-			return (1);
-		}
-		optidx++;
-	}
+	value = get_config_value_node(nvl, "iid");
+	if (value != NULL)
+		sc->vss_iid = strtoul(value, NULL, 10);
 
+	devname = get_config_value_node(nvl, "dev");
+	if (devname == NULL)
+		devname = "/dev/cam/ctl";
 	sc->vss_ctl_fd = open(devname, O_RDWR);
 	if (sc->vss_ctl_fd < 0) {
 		WPRINTF(("cannot open %s: %s", devname, strerror(errno)));
@@ -720,7 +725,7 @@ pci_vtscsi_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_DEV_SCSI);
 	pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR);
 	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
-	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_TYPE_SCSI);
+	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_ID_SCSI);
 	pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR);
 
 	if (vi_intr_init(&sc->vss_vs, 1, fbsdrun_virtio_msix()))
diff --git a/usr/src/cmd/bhyve/pci_virtio_viona.c b/usr/src/cmd/bhyve/pci_virtio_viona.c
index 9cafa7b111..07abf20512 100644
--- a/usr/src/cmd/bhyve/pci_virtio_viona.c
+++ b/usr/src/cmd/bhyve/pci_virtio_viona.c
@@ -64,6 +64,7 @@
 #include <vmmapi.h>
 
 #include "bhyverun.h"
+#include "config.h"
 #include "pci_emul.h"
 #include "virtio.h"
 
@@ -135,10 +136,12 @@ struct pci_viona_softc {
 static uint64_t
 pci_viona_iosize(struct pci_devinst *pi)
 {
-	if (pci_msix_enabled(pi))
+	if (pci_msix_enabled(pi)) {
 		return (VIONA_REGSZ);
-	else
-		return (VIONA_REGSZ - (VTCFG_R_CFG1 - VTCFG_R_MSIX));
+	} else {
+		return (VIONA_REGSZ -
+		    (VIRTIO_PCI_CONFIG_OFF(1) - VIRTIO_PCI_CONFIG_OFF(0)));
+	}
 }
 
 static uint16_t
@@ -301,82 +304,80 @@ pci_viona_viona_init(struct vmctx *ctx, struct pci_viona_softc *sc)
 }
 
 static int
-pci_viona_parse_opts(struct pci_viona_softc *sc, char *opts)
+pci_viona_legacy_config(nvlist_t *nvl, const char *opt)
 {
-	char *next, *cp, *vnic = NULL;
+	char *config, *name, *tofree, *value;
+
+	if (opt == NULL)
+		return (0);
+
+	config = tofree = strdup(opt);
+	while ((name = strsep(&config, ",")) != NULL) {
+		value = strchr(name, '=');
+		if (value != NULL) {
+			*value++ = '\0';
+			set_config_value_node(nvl, name, value);
+		} else {
+			set_config_value_node(nvl, "vnic", name);
+		}
+	}
+	free(tofree);
+	return (0);
+}
+
+static int
+pci_viona_parse_opts(struct pci_viona_softc *sc, nvlist_t *nvl)
+{
+	const char *value;
 	int err = 0;
 
 	sc->vsc_vq_size = VIONA_RINGSZ;
 	sc->vsc_feature_mask = 0;
+	sc->vsc_linkname[0] = '\0';
 
-	for (; opts != NULL && *opts != '\0'; opts = next) {
-		char *val;
+	value = get_config_value_node(nvl, "feature_mask");
+	if (value != NULL) {
+		long num;
 
-		if ((cp = strchr(opts, ',')) != NULL) {
-			*cp = '\0';
-			next = cp + 1;
+		errno = 0;
+		num = strtol(value, NULL, 0);
+		if (errno != 0 || num < 0) {
+			fprintf(stderr,
+			    "viona: invalid mask '%s'", value);
 		} else {
-			next = NULL;
+			sc->vsc_feature_mask = num;
 		}
+	}
 
-		if ((cp = strchr(opts, '=')) == NULL) {
-			/* vnic chosen with bare name */
-			if (vnic != NULL) {
-				fprintf(stderr,
-				    "viona: unexpected vnic name '%s'", opts);
-				err = -1;
-			} else {
-				vnic = opts;
-			}
-			continue;
-		}
+	value = get_config_value_node(nvl, "vqsize");
+	if (value != NULL) {
+		long num;
 
-		/* <param>=<value> handling */
-		val = cp + 1;
-		*cp = '\0';
-		if (strcmp(opts, "feature_mask") == 0) {
-			long num;
-
-			errno = 0;
-			num = strtol(val, NULL, 0);
-			if (errno != 0 || num < 0) {
-				fprintf(stderr,
-				    "viona: invalid mask '%s'", val);
-			} else {
-				sc->vsc_feature_mask = num;
-			}
-		} else if (strcmp(opts, "vqsize") == 0) {
-			long num;
-
-			errno = 0;
-			num = strtol(val, NULL, 0);
-			if (errno != 0) {
-				fprintf(stderr,
-				    "viona: invalid vsqize '%s'", val);
-				err = -1;
-			} else if (num <= 2 || num > 32768) {
-				fprintf(stderr,
-				    "viona: vqsize out of range", num);
-				err = -1;
-			} else if ((1 << (ffs(num) - 1)) != num) {
-				fprintf(stderr,
-				    "viona: vqsize must be power of 2", num);
-				err = -1;
-			} else {
-				sc->vsc_vq_size = num;
-			}
-		} else {
+		errno = 0;
+		num = strtol(value, NULL, 0);
+		if (errno != 0) {
+			fprintf(stderr,
+			    "viona: invalid vsqize '%s'", value);
+			err = -1;
+		} else if (num <= 2 || num > 32768) {
+			fprintf(stderr,
+			    "viona: vqsize out of range", num);
+			err = -1;
+		} else if ((1 << (ffs(num) - 1)) != num) {
 			fprintf(stderr,
-			    "viona: unrecognized option '%s'", opts);
+			    "viona: vqsize must be power of 2", num);
 			err = -1;
+		} else {
+			sc->vsc_vq_size = num;
 		}
 	}
-	if (vnic == NULL) {
+
+	value = get_config_value_node(nvl, "vnic");
+	if (value == NULL) {
 		fprintf(stderr, "viona: vnic name required");
-		sc->vsc_linkname[0] = '\0';
 		err = -1;
 	} else {
-		(void) strlcpy(sc->vsc_linkname, vnic, MAXLINKNAMELEN);
+		(void) strlcpy(sc->vsc_linkname, value, MAXLINKNAMELEN);
 	}
 
 	DPRINTF(("viona=%p dev=%s vqsize=%x feature_mask=%x\n", sc,
@@ -385,7 +386,7 @@ pci_viona_parse_opts(struct pci_viona_softc *sc, char *opts)
 }
 
 static int
-pci_viona_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+pci_viona_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	dladm_handle_t		handle;
 	dladm_status_t		status;
@@ -394,8 +395,10 @@ pci_viona_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	int error, i;
 	struct pci_viona_softc *sc;
 	uint64_t ioport;
+	const char *vnic;
 
-	if (opts == NULL) {
+	vnic = get_config_value_node(nvl, "vnic");
+	if (vnic == NULL) {
 		printf("virtio-viona: vnic required\n");
 		return (1);
 	}
@@ -408,7 +411,7 @@ pci_viona_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 
 	pthread_mutex_init(&sc->vsc_mtx, NULL);
 
-	if (pci_viona_parse_opts(sc, opts) != 0) {
+	if (pci_viona_parse_opts(sc, nvl) != 0) {
 		free(sc);
 		return (1);
 	}
@@ -419,18 +422,18 @@ pci_viona_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 		return (1);
 	}
 
-	if (dladm_name2info(handle, sc->vsc_linkname, &sc->vsc_linkid,
-	    NULL, NULL, NULL) != DLADM_STATUS_OK) {
-		WPRINTF(("dladm_name2info() for %s failed: %s\n", opts,
+	if ((status = dladm_name2info(handle, sc->vsc_linkname, &sc->vsc_linkid,
+	    NULL, NULL, NULL)) != DLADM_STATUS_OK) {
+		WPRINTF(("dladm_name2info() for %s failed: %s\n", vnic,
 		    dladm_status2str(status, errmsg)));
 		dladm_close(handle);
 		free(sc);
 		return (1);
 	}
 
-	if (dladm_vnic_info(handle, sc->vsc_linkid, &attr,
-	    DLADM_OPT_ACTIVE) != DLADM_STATUS_OK) {
-		WPRINTF(("dladm_vnic_info() for %s failed: %s\n", opts,
+	if ((status = dladm_vnic_info(handle, sc->vsc_linkid, &attr,
+	    DLADM_OPT_ACTIVE)) != DLADM_STATUS_OK) {
+		WPRINTF(("dladm_vnic_info() for %s failed: %s\n", vnic,
 		    dladm_status2str(status, errmsg)));
 		dladm_close(handle);
 		free(sc);
@@ -454,7 +457,7 @@ pci_viona_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_DEV_NET);
 	pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR);
 	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_NETWORK);
-	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_TYPE_NET);
+	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_ID_NETWORK);
 	pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR);
 
 	/* MSI-X support */
@@ -476,7 +479,7 @@ pci_viona_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	}
 
 	/* Install ioport hook for virtqueue notification */
-	ioport = pi->pi_bar[0].addr + VTCFG_R_QNOTIFY;
+	ioport = pi->pi_bar[0].addr + VIRTIO_PCI_QUEUE_NOTIFY;
 	error = ioctl(sc->vsc_vnafd, VNA_IOC_SET_NOTIFY_IOP, ioport);
 	if (error != 0) {
 		WPRINTF(("could not install ioport hook at %x\n", ioport));
@@ -501,8 +504,10 @@ viona_adjust_offset(struct pci_devinst *pi, uint64_t offset)
 	 * whether MSI-X capability is enabled or not
 	 */
 	if (!pci_msix_enabled(pi)) {
-		if (offset >= VTCFG_R_MSIX)
-			return (offset + (VTCFG_R_CFG1 - VTCFG_R_MSIX));
+		if (offset >= VIRTIO_PCI_CONFIG_OFF(0)) {
+			return (offset + (VIRTIO_PCI_CONFIG_OFF(1) -
+			    VIRTIO_PCI_CONFIG_OFF(0)));
+		}
 	}
 
 	return (offset);
@@ -640,7 +645,7 @@ pci_viona_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 	offset = viona_adjust_offset(pi, offset);
 
 	switch (offset) {
-	case VTCFG_R_GUESTCAP:
+	case VIRTIO_PCI_GUEST_FEATURES:
 		assert(size == 4);
 		value &= ~(sc->vsc_feature_mask);
 		err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_FEATURES, &value);
@@ -651,29 +656,29 @@ pci_viona_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 			sc->vsc_features = value;
 		}
 		break;
-	case VTCFG_R_PFN:
+	case VIRTIO_PCI_QUEUE_PFN:
 		assert(size == 4);
 		pci_viona_ring_init(sc, value);
 		break;
-	case VTCFG_R_QSEL:
+	case VIRTIO_PCI_QUEUE_SEL:
 		assert(size == 2);
 		assert(value < VIONA_MAXQ);
 		sc->vsc_curq = value;
 		break;
-	case VTCFG_R_QNOTIFY:
+	case VIRTIO_PCI_QUEUE_NOTIFY:
 		assert(size == 2);
 		assert(value < VIONA_MAXQ);
 		pci_viona_qnotify(sc, value);
 		break;
-	case VTCFG_R_STATUS:
+	case VIRTIO_PCI_STATUS:
 		assert(size == 1);
 		pci_viona_update_status(sc, value);
 		break;
-	case VTCFG_R_CFGVEC:
+	case VIRTIO_MSI_CONFIG_VECTOR:
 		assert(size == 2);
 		sc->vsc_msix_table_idx[VIONA_CTLQ] = value;
 		break;
-	case VTCFG_R_QVEC:
+	case VIRTIO_MSI_QUEUE_VECTOR:
 		assert(size == 2);
 		assert(sc->vsc_curq != VIONA_CTLQ);
 		sc->vsc_msix_table_idx[sc->vsc_curq] = value;
@@ -699,9 +704,9 @@ pci_viona_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 			*(uint32_t *)ptr = value;
 		}
 		break;
-	case VTCFG_R_HOSTCAP:
-	case VTCFG_R_QNUM:
-	case VTCFG_R_ISR:
+	case VIRTIO_PCI_HOST_FEATURES:
+	case VIRTIO_PCI_QUEUE_NUM:
+	case VIRTIO_PCI_ISR:
 	case VIONA_R_CFG6:
 	case VIONA_R_CFG7:
 		DPRINTF(("viona: write to readonly reg %ld\n\r", offset));
@@ -742,7 +747,7 @@ pci_viona_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 	offset = viona_adjust_offset(pi, offset);
 
 	switch (offset) {
-	case VTCFG_R_HOSTCAP:
+	case VIRTIO_PCI_HOST_FEATURES:
 		assert(size == 4);
 		err = ioctl(sc->vsc_vnafd, VNA_IOC_GET_FEATURES, &value);
 		if (err != 0) {
@@ -751,31 +756,31 @@ pci_viona_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 		}
 		value &= ~sc->vsc_feature_mask;
 		break;
-	case VTCFG_R_GUESTCAP:
+	case VIRTIO_PCI_GUEST_FEATURES:
 		assert(size == 4);
 		value = sc->vsc_features; /* XXX never read ? */
 		break;
-	case VTCFG_R_PFN:
+	case VIRTIO_PCI_QUEUE_PFN:
 		assert(size == 4);
 		value = sc->vsc_pfn[sc->vsc_curq] >> VRING_PFN;
 		break;
-	case VTCFG_R_QNUM:
+	case VIRTIO_PCI_QUEUE_NUM:
 		assert(size == 2);
 		value = pci_viona_qsize(sc, sc->vsc_curq);
 		break;
-	case VTCFG_R_QSEL:
+	case VIRTIO_PCI_QUEUE_SEL:
 		assert(size == 2);
 		value = sc->vsc_curq;  /* XXX never read ? */
 		break;
-	case VTCFG_R_QNOTIFY:
+	case VIRTIO_PCI_QUEUE_NOTIFY:
 		assert(size == 2);
 		value = sc->vsc_curq;  /* XXX never read ? */
 		break;
-	case VTCFG_R_STATUS:
+	case VIRTIO_PCI_STATUS:
 		assert(size == 1);
 		value = sc->vsc_status;
 		break;
-	case VTCFG_R_ISR:
+	case VIRTIO_PCI_ISR:
 		assert(size == 1);
 		value = sc->vsc_isr;
 		sc->vsc_isr = 0;	/* a read clears this flag */
@@ -783,11 +788,11 @@ pci_viona_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 			pci_lintr_deassert(pi);
 		}
 		break;
-	case VTCFG_R_CFGVEC:
+	case VIRTIO_MSI_CONFIG_VECTOR:
 		assert(size == 2);
 		value = sc->vsc_msix_table_idx[VIONA_CTLQ];
 		break;
-	case VTCFG_R_QVEC:
+	case VIRTIO_MSI_QUEUE_VECTOR:
 		assert(size == 2);
 		assert(sc->vsc_curq != VIONA_CTLQ);
 		value = sc->vsc_msix_table_idx[sc->vsc_curq];
@@ -830,6 +835,7 @@ pci_viona_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 struct pci_devemu pci_de_viona = {
 	.pe_emu =	"virtio-net-viona",
 	.pe_init =	pci_viona_init,
+	.pe_legacy_config = pci_viona_legacy_config,
 	.pe_barwrite =	pci_viona_write,
 	.pe_barread =	pci_viona_read,
 	.pe_lintrupdate = pci_viona_lintrupdate
diff --git a/usr/src/cmd/bhyve/pci_xhci.c b/usr/src/cmd/bhyve/pci_xhci.c
index 587e80a91c..bbccc7f0bf 100644
--- a/usr/src/cmd/bhyve/pci_xhci.c
+++ b/usr/src/cmd/bhyve/pci_xhci.c
@@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$");
 #include <xhcireg.h>
 
 #include "bhyverun.h"
+#include "config.h"
 #include "debug.h"
 #include "pci_emul.h"
 #include "pci_xhci.h"
@@ -273,7 +274,6 @@ struct pci_xhci_softc {
 	struct pci_xhci_portregs *portregs;
 	struct pci_xhci_dev_emu  **devices; /* XHCI[port] = device */
 	struct pci_xhci_dev_emu  **slots;   /* slots assigned from 1 */
-	int		ndevices;
 
 	int		usb2_port_start;
 	int		usb3_port_start;
@@ -566,7 +566,8 @@ pci_xhci_get_dev_ctx(struct pci_xhci_softc *sc, uint32_t slot)
 	uint64_t devctx_addr;
 	struct xhci_dev_ctx *devctx;
 
-	assert(slot > 0 && slot <= sc->ndevices);
+	assert(slot > 0 && slot <= XHCI_MAX_DEVS);
+	assert(XHCI_SLOTDEV_PTR(sc, slot) != NULL);
 	assert(sc->opregs.dcbaa_p != NULL);
 
 	devctx_addr = sc->opregs.dcbaa_p->dcba[slot];
@@ -839,7 +840,7 @@ pci_xhci_cmd_disable_slot(struct pci_xhci_softc *sc, uint32_t slot)
 	if (sc->portregs == NULL)
 		goto done;
 
-	if (slot > sc->ndevices) {
+	if (slot > XHCI_MAX_SLOTS) {
 		cmderr = XHCI_TRB_ERROR_SLOT_NOT_ON;
 		goto done;
 	}
@@ -853,7 +854,8 @@ pci_xhci_cmd_disable_slot(struct pci_xhci_softc *sc, uint32_t slot)
 			cmderr = XHCI_TRB_ERROR_SUCCESS;
 			/* TODO: reset events and endpoints */
 		}
-	}
+	} else
+		cmderr = XHCI_TRB_ERROR_SLOT_NOT_ON;
 
 done:
 	return (cmderr);
@@ -1912,7 +1914,7 @@ pci_xhci_device_doorbell(struct pci_xhci_softc *sc, uint32_t slot,
 	DPRINTF(("pci_xhci doorbell slot %u epid %u stream %u",
 	    slot, epid, streamid));
 
-	if (slot == 0 || slot > sc->ndevices) {
+	if (slot == 0 || slot > XHCI_MAX_SLOTS) {
 		DPRINTF(("pci_xhci: invalid doorbell slot %u", slot));
 		return;
 	}
@@ -2656,67 +2658,129 @@ pci_xhci_dev_event(struct usb_hci *hci, enum hci_usbev evid, void *param)
 	return (0);
 }
 
+/*
+ * Each controller contains a "slot" node which contains a list of
+ * child nodes each of which is a device.  Each slot node's name
+ * corresponds to a specific controller slot.  These nodes
+ * contain a "device" variable identifying the device model of the
+ * USB device.  For example:
+ *
+ * pci.0.1.0
+ *          .device="xhci"
+ *          .slot
+ *               .1
+ *                 .device="tablet"
+ */
+static int
+pci_xhci_legacy_config(nvlist_t *nvl, const char *opts)
+{
+	char node_name[16];
+	nvlist_t *slots_nvl, *slot_nvl;
+	char *cp, *opt, *str, *tofree;
+	int slot;
 
+	if (opts == NULL)
+		return (0);
 
-static void
-pci_xhci_device_usage(char *opt)
-{
+	slots_nvl = create_relative_config_node(nvl, "slot");
+	slot = 1;
+	tofree = str = strdup(opts);
+	while ((opt = strsep(&str, ",")) != NULL) {
+		/* device[=<config>] */
+		cp = strchr(opt, '=');
+		if (cp != NULL) {
+			*cp = '\0';
+			cp++;
+		}
+
+		snprintf(node_name, sizeof(node_name), "%d", slot);
+		slot++;
+		slot_nvl = create_relative_config_node(slots_nvl, node_name);
+		set_config_value_node(slot_nvl, "device", opt);
 
-	EPRINTLN("Invalid USB emulation \"%s\"", opt);
+		/*
+		 * NB: Given that we split on commas above, the legacy
+		 * format only supports a single option.
+		 */
+		if (cp != NULL && *cp != '\0')
+			pci_parse_legacy_config(slot_nvl, cp);
+	}
+	free(tofree);
+	return (0);
 }
 
 static int
-pci_xhci_parse_opts(struct pci_xhci_softc *sc, char *opts)
+pci_xhci_parse_devices(struct pci_xhci_softc *sc, nvlist_t *nvl)
 {
-	struct pci_xhci_dev_emu	**devices;
 	struct pci_xhci_dev_emu	*dev;
 	struct usb_devemu	*ue;
-	void	*devsc;
-	char	*uopt, *xopts, *config;
-	int	usb3_port, usb2_port, i;
+	const nvlist_t *slots_nvl, *slot_nvl;
+	const char *name, *device;
+	char	*cp;
+	void	*devsc, *cookie;
+	long	slot;
+	int	type, usb3_port, usb2_port, i, ndevices;
 
-	uopt = NULL;
-	usb3_port = sc->usb3_port_start - 1;
-	usb2_port = sc->usb2_port_start - 1;
-	devices = NULL;
+	usb3_port = sc->usb3_port_start;
+	usb2_port = sc->usb2_port_start;
 
-	if (opts == NULL)
-		goto portsfinal;
+	sc->devices = calloc(XHCI_MAX_DEVS, sizeof(struct pci_xhci_dev_emu *));
+	sc->slots = calloc(XHCI_MAX_SLOTS, sizeof(struct pci_xhci_dev_emu *));
 
-	devices = calloc(XHCI_MAX_DEVS, sizeof(struct pci_xhci_dev_emu *));
+	/* port and slot numbering start from 1 */
+	sc->devices--;
+	sc->slots--;
 
-	sc->slots = calloc(XHCI_MAX_SLOTS, sizeof(struct pci_xhci_dev_emu *));
-	sc->devices = devices;
-	sc->ndevices = 0;
-
-	uopt = strdup(opts);
-	for (xopts = strtok(uopt, ",");
-	     xopts != NULL;
-	     xopts = strtok(NULL, ",")) {
-		if (usb2_port == ((sc->usb2_port_start-1) + XHCI_MAX_DEVS/2) ||
-		    usb3_port == ((sc->usb3_port_start-1) + XHCI_MAX_DEVS/2)) {
+	ndevices = 0;
+
+	slots_nvl = find_relative_config_node(nvl, "slot");
+	if (slots_nvl == NULL)
+		goto portsfinal;
+
+	cookie = NULL;
+	while ((name = nvlist_next(slots_nvl, &type, &cookie)) != NULL) {
+		if (usb2_port == ((sc->usb2_port_start) + XHCI_MAX_DEVS/2) ||
+		    usb3_port == ((sc->usb3_port_start) + XHCI_MAX_DEVS/2)) {
 			WPRINTF(("pci_xhci max number of USB 2 or 3 "
 			     "devices reached, max %d", XHCI_MAX_DEVS/2));
-			usb2_port = usb3_port = -1;
-			goto done;
+			goto bad;
 		}
 
-		/* device[=<config>] */
-		if ((config = strchr(xopts, '=')) == NULL)
-			config = "";		/* no config */
-		else
-			*config++ = '\0';
+		if (type != NV_TYPE_NVLIST) {
+			EPRINTLN(
+			    "pci_xhci: config variable '%s' under slot node",
+			     name);
+			goto bad;
+		}
+
+		slot = strtol(name, &cp, 0);
+		if (*cp != '\0' || slot <= 0 || slot > XHCI_MAX_SLOTS) {
+			EPRINTLN("pci_xhci: invalid slot '%s'", name);
+			goto bad;
+		}
+
+		if (XHCI_SLOTDEV_PTR(sc, slot) != NULL) {
+			EPRINTLN("pci_xhci: duplicate slot '%s'", name);
+			goto bad;
+		}
 
-		ue = usb_emu_finddev(xopts);
+		slot_nvl = nvlist_get_nvlist(slots_nvl, name);
+		device = get_config_value_node(slot_nvl, "device");
+		if (device == NULL) {
+			EPRINTLN(
+			    "pci_xhci: missing \"device\" value for slot '%s'",
+				name);
+			goto bad;
+		}
+
+		ue = usb_emu_finddev(device);
 		if (ue == NULL) {
-			pci_xhci_device_usage(xopts);
-			DPRINTF(("pci_xhci device not found %s", xopts));
-			usb2_port = usb3_port = -1;
-			goto done;
+			EPRINTLN("pci_xhci: unknown device model \"%s\"",
+			    device);
+			goto bad;
 		}
 
-		DPRINTF(("pci_xhci adding device %s, opts \"%s\"",
-		        xopts, config));
+		DPRINTF(("pci_xhci adding device %s", device));
 
 		dev = calloc(1, sizeof(struct pci_xhci_dev_emu));
 		dev->xsc = sc;
@@ -2725,66 +2789,65 @@ pci_xhci_parse_opts(struct pci_xhci_softc *sc, char *opts)
 		dev->hci.hci_event = pci_xhci_dev_event;
 
 		if (ue->ue_usbver == 2) {
-			dev->hci.hci_port = usb2_port + 1;
-			devices[usb2_port] = dev;
+			if (usb2_port == sc->usb2_port_start +
+			    XHCI_MAX_DEVS / 2) {
+				WPRINTF(("pci_xhci max number of USB 2 devices "
+				     "reached, max %d", XHCI_MAX_DEVS / 2));
+				goto bad;
+			}
+			dev->hci.hci_port = usb2_port;
 			usb2_port++;
 		} else {
-			dev->hci.hci_port = usb3_port + 1;
-			devices[usb3_port] = dev;
+			if (usb3_port == sc->usb3_port_start +
+			    XHCI_MAX_DEVS / 2) {
+				WPRINTF(("pci_xhci max number of USB 3 devices "
+				     "reached, max %d", XHCI_MAX_DEVS / 2));
+				goto bad;
+			}
+			dev->hci.hci_port = usb3_port;
 			usb3_port++;
 		}
+		XHCI_DEVINST_PTR(sc, dev->hci.hci_port) = dev;
 
 		dev->hci.hci_address = 0;
-		devsc = ue->ue_init(&dev->hci, config);
+		devsc = ue->ue_init(&dev->hci, nvl);
 		if (devsc == NULL) {
-			pci_xhci_device_usage(xopts);
-			usb2_port = usb3_port = -1;
-			goto done;
+			goto bad;
 		}
 
 		dev->dev_ue = ue;
 		dev->dev_sc = devsc;
 
-		/* assign slot number to device */
-		sc->slots[sc->ndevices] = dev;
-
-		sc->ndevices++;
+		XHCI_SLOTDEV_PTR(sc, slot) = dev;
+		ndevices++;
 	}
 
 portsfinal:
 	sc->portregs = calloc(XHCI_MAX_DEVS, sizeof(struct pci_xhci_portregs));
+	sc->portregs--;
 
-	if (sc->ndevices > 0) {
-		/* port and slot numbering start from 1 */
-		sc->devices--;
-		sc->portregs--;
-		sc->slots--;
-
+	if (ndevices > 0) {
 		for (i = 1; i <= XHCI_MAX_DEVS; i++) {
 			pci_xhci_init_port(sc, i);
 		}
 	} else {
 		WPRINTF(("pci_xhci no USB devices configured"));
-		sc->ndevices = 1;
 	}
+	return (0);
 
-done:
-	if (devices != NULL) {
-		if (usb2_port <= 0 && usb3_port <= 0) {
-			sc->devices = NULL;
-			for (i = 0; devices[i] != NULL; i++)
-				free(devices[i]);
-			sc->ndevices = -1;
-
-			free(devices);
-		}
+bad:
+	for (i = 1; i <= XHCI_MAX_DEVS; i++) {
+		free(XHCI_DEVINST_PTR(sc, i));
 	}
-	free(uopt);
-	return (sc->ndevices);
+
+	free(sc->devices + 1);
+	free(sc->slots + 1);
+
+	return (-1);
 }
 
 static int
-pci_xhci_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
+pci_xhci_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 {
 	struct pci_xhci_softc *sc;
 	int	error;
@@ -2803,7 +2866,7 @@ pci_xhci_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	sc->usb3_port_start = 1;
 
 	/* discover devices */
-	error = pci_xhci_parse_opts(sc, opts);
+	error = pci_xhci_parse_devices(sc, nvl);
 	if (error < 0)
 		goto done;
 	else
@@ -2877,11 +2940,10 @@ done:
 	return (error);
 }
 
-
-
 struct pci_devemu pci_de_xhci = {
 	.pe_emu =	"xhci",
 	.pe_init =	pci_xhci_init,
+	.pe_legacy_config = pci_xhci_legacy_config,
 	.pe_barwrite =	pci_xhci_write,
 	.pe_barread =	pci_xhci_read
 };
diff --git a/usr/src/cmd/bhyve/pctestdev.c b/usr/src/cmd/bhyve/pctestdev.c
index be445e5c75..49e03b2a43 100644
--- a/usr/src/cmd/bhyve/pctestdev.c
+++ b/usr/src/cmd/bhyve/pctestdev.c
@@ -89,15 +89,6 @@ pctestdev_getname(void)
 }
 
 int
-pctestdev_parse(const char *opts)
-{
-	if (opts != NULL && *opts != '\0')
-		return (-1);
-
-	return (0);
-}
-
-int
 pctestdev_init(struct vmctx *ctx)
 {
 	struct mem_range iomem;
diff --git a/usr/src/cmd/bhyve/pctestdev.h b/usr/src/cmd/bhyve/pctestdev.h
index c1c940146e..5808abe6e3 100644
--- a/usr/src/cmd/bhyve/pctestdev.h
+++ b/usr/src/cmd/bhyve/pctestdev.h
@@ -38,6 +38,5 @@ struct vmctx;
 
 const char	*pctestdev_getname(void);
 int		 pctestdev_init(struct vmctx *ctx);
-int		 pctestdev_parse(const char *opts);
 
 #endif
diff --git a/usr/src/cmd/bhyve/rfb.c b/usr/src/cmd/bhyve/rfb.c
index dbd96e67cc..a8e45d03a6 100644
--- a/usr/src/cmd/bhyve/rfb.c
+++ b/usr/src/cmd/bhyve/rfb.c
@@ -437,7 +437,7 @@ static int
 rfb_send_all(struct rfb_softc *rc, int cfd, struct bhyvegc_image *gc)
 {
 	struct rfb_srvr_updt_msg supdt_msg;
-	struct rfb_srvr_rect_hdr srect_hdr;
+        struct rfb_srvr_rect_hdr srect_hdr;
 	ssize_t nwrite;
 	unsigned long zlen;
 	int err;
@@ -631,11 +631,11 @@ rfb_send_screen(struct rfb_softc *rc, int cfd)
 		}
 	}
 
-	/*
-	 * We only send the update if there are changes.
-	 * Restore the pending flag since it was unconditionally cleared
-	 * above.
-	 */
+       /*
+	* We only send the update if there are changes.
+	* Restore the pending flag since it was unconditionally cleared
+	* above.
+	*/
 	if (!changes) {
 		rc->pending = true;
 		goto done;
@@ -1005,7 +1005,7 @@ report_and_done:
 	if (perror == 0)
 		pthread_set_name_np(tid, "rfbout");
 
-	/* Now read in client requests. 1st byte identifies type */
+        /* Now read in client requests. 1st byte identifies type */
 	for (;;) {
 		len = read(cfd, buf, 1);
 		if (len <= 0) {
@@ -1198,7 +1198,7 @@ rfb_init(char *hostname, int port, int wait, char *password)
 
 #ifndef __FreeBSD__
 int
-rfb_init_unix(char *path, int wait, char *password)
+rfb_init_unix(const char *path, int wait, char *password)
 {
 	struct rfb_softc *rc;
 	struct sockaddr_un sock;
diff --git a/usr/src/cmd/bhyve/rfb.h b/usr/src/cmd/bhyve/rfb.h
index 990e2075ac..71227c59ac 100644
--- a/usr/src/cmd/bhyve/rfb.h
+++ b/usr/src/cmd/bhyve/rfb.h
@@ -36,7 +36,7 @@
 
 int	rfb_init(char *hostname, int port, int wait, char *password);
 #ifndef __FreeBSD__
-int	rfb_init_unix(char *path, int wait, char *password);
+int	rfb_init_unix(const char *path, int wait, char *password);
 #endif
 
 #endif /* _RFB_H_ */
diff --git a/usr/src/cmd/bhyve/rtc.c b/usr/src/cmd/bhyve/rtc.c
index 09ca3f61ae..0f63156adb 100644
--- a/usr/src/cmd/bhyve/rtc.c
+++ b/usr/src/cmd/bhyve/rtc.c
@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
 #include <vmmapi.h>
 
 #include "acpi.h"
+#include "config.h"
 #include "pci_lpc.h"
 #include "rtc.h"
 
@@ -59,13 +60,13 @@ __FBSDID("$FreeBSD$");
  * Returns the current RTC time as number of seconds since 00:00:00 Jan 1, 1970
  */
 static time_t
-rtc_time(struct vmctx *ctx, int use_localtime)
+rtc_time(struct vmctx *ctx)
 {
 	struct tm tm;
 	time_t t;
 
 	time(&t);
-	if (use_localtime) {
+	if (get_config_bool_default("rtc.use_localtime", true)) {
 		localtime_r(&t, &tm);
 		t = timegm(&tm);
 	}
@@ -73,7 +74,7 @@ rtc_time(struct vmctx *ctx, int use_localtime)
 }
 
 void
-rtc_init(struct vmctx *ctx, int use_localtime)
+rtc_init(struct vmctx *ctx)
 {	
 	size_t himem;
 	size_t lomem;
@@ -101,7 +102,7 @@ rtc_init(struct vmctx *ctx, int use_localtime)
 	err = vm_rtc_write(ctx, RTC_HMEM_MSB, himem >> 16);
 	assert(err == 0);
 
-	err = vm_rtc_settime(ctx, rtc_time(ctx, use_localtime));
+	err = vm_rtc_settime(ctx, rtc_time(ctx));
 	assert(err == 0);
 }
 
diff --git a/usr/src/cmd/bhyve/rtc.h b/usr/src/cmd/bhyve/rtc.h
index 1c108eed99..c8b3572e93 100644
--- a/usr/src/cmd/bhyve/rtc.h
+++ b/usr/src/cmd/bhyve/rtc.h
@@ -31,6 +31,6 @@
 #ifndef _RTC_H_
 #define _RTC_H_
 
-void	rtc_init(struct vmctx *ctx, int use_localtime);
+void	rtc_init(struct vmctx *ctx);
 
 #endif /* _RTC_H_ */
diff --git a/usr/src/cmd/bhyve/smbiostbl.c b/usr/src/cmd/bhyve/smbiostbl.c
index 3df2012f10..21185292a2 100644
--- a/usr/src/cmd/bhyve/smbiostbl.c
+++ b/usr/src/cmd/bhyve/smbiostbl.c
@@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$");
 #include <vmmapi.h>
 
 #include "bhyverun.h"
+#include "config.h"
 #include "debug.h"
 #include "smbiostbl.h"
 
@@ -589,11 +590,13 @@ smbios_type1_initializer(struct smbios_structure *template_entry,
     uint16_t *n, uint16_t *size)
 {
 	struct smbios_table_type1 *type1;
+	const char *guest_uuid_str;
 
 	smbios_generic_initializer(template_entry, template_strings,
 	    curaddr, endaddr, n, size);
 	type1 = (struct smbios_table_type1 *)curaddr;
 
+	guest_uuid_str = get_config_value("uuid");
 	if (guest_uuid_str != NULL) {
 		uuid_t		uuid;
 		uint32_t	status;
@@ -607,6 +610,7 @@ smbios_type1_initializer(struct smbios_structure *template_entry,
 		MD5_CTX		mdctx;
 		u_char		digest[16];
 		char		hostname[MAXHOSTNAMELEN];
+		const char	*vmname;
 
 		/*
 		 * Universally unique and yet reproducible are an
@@ -617,6 +621,7 @@ smbios_type1_initializer(struct smbios_structure *template_entry,
 			return (-1);
 
 		MD5Init(&mdctx);
+		vmname = get_config_value("name");
 		MD5Update(&mdctx, vmname, strlen(vmname));
 		MD5Update(&mdctx, hostname, sizeof(hostname));
 		MD5Final(digest, &mdctx);
@@ -863,27 +868,44 @@ smbios_build(struct vmctx *ctx)
 	return (0);
 }
 
+#ifndef __FreeBSD__
+struct {
+	const char *key;
+	const char **targetp;
+} type1_map[] = {
+	{ "manufacturer", &smbios_type1_strings[0] },
+	{ "product", &smbios_type1_strings[1] },
+	{ "version", &smbios_type1_strings[2] },
+	{ "serial", &smbios_type1_strings[3] },
+	{ "sku", &smbios_type1_strings[4] },
+	{ "family", &smbios_type1_strings[5] },
+	{ 0 }
+};
+
+void
+smbios_apply(void)
+{
+	nvlist_t *nvl;
+
+	nvl = find_config_node("smbios");
+	if (nvl == NULL)
+		return;
+
+	for (uint_t i = 0; type1_map[i].key != NULL; i++) {
+		const char *value;
+
+		value = get_config_value_node(nvl, type1_map[i].key);
+		if (value != NULL)
+			*type1_map[i].targetp = value;
+	}
+}
+
 int
 smbios_parse(const char *opts)
 {
-	char *buf;
-	char *lasts;
-	char *token;
-	char *end;
+	char *buf, *lasts, *token, *end;
+	nvlist_t *nvl;
 	long type;
-	struct {
-		const char *key;
-		const char **targetp;
-	} type1_map[] = {
-		{ "manufacturer", &smbios_type1_strings[0] },
-		{ "product", &smbios_type1_strings[1] },
-		{ "version", &smbios_type1_strings[2] },
-		{ "serial", &smbios_type1_strings[3] },
-		{ "sku", &smbios_type1_strings[4] },
-		{ "family", &smbios_type1_strings[5] },
-		{ "uuid", (const char **)&guest_uuid_str },
-		{ 0 }
-	};
 
 	if ((buf = strdup(opts)) == NULL) {
 		(void) fprintf(stderr, "out of memory\n");
@@ -909,9 +931,15 @@ smbios_parse(const char *opts)
 		goto fail;
 	}
 
+	nvl = create_config_node("smbios");
+	if (nvl == NULL) {
+		(void) fprintf(stderr, "out of memory\n");
+		return (-1);
+	}
+
 	while ((token = strtok_r(NULL, ",", &lasts)) != NULL) {
 		char *val;
-		int i;
+		uint_t i;
 
 		if ((val = strchr(token, '=')) == NULL) {
 			(void) fprintf(stderr, "invalid key=value: '%s'\n",
@@ -921,6 +949,11 @@ smbios_parse(const char *opts)
 		*val = '\0';
 		val++;
 
+		if (strcmp(token, "uuid") == 0) {
+			set_config_value_node(nvl, token, val);
+			continue;
+		}
+
 		for (i = 0; type1_map[i].key != NULL; i++) {
 			if (strcmp(token, type1_map[i].key) == 0) {
 				break;
@@ -930,7 +963,7 @@ smbios_parse(const char *opts)
 			(void) fprintf(stderr, "invalid key '%s'\n", token);
 			goto fail;
 		}
-		*type1_map[i].targetp = val;
+		set_config_value_node(nvl, token, val);
 	}
 
 	return (0);
@@ -939,3 +972,4 @@ fail:
 	free(buf);
 	return (-1);
 }
+#endif
diff --git a/usr/src/cmd/bhyve/smbiostbl.h b/usr/src/cmd/bhyve/smbiostbl.h
index 81e26309e5..2438fc9083 100644
--- a/usr/src/cmd/bhyve/smbiostbl.h
+++ b/usr/src/cmd/bhyve/smbiostbl.h
@@ -38,6 +38,9 @@
 struct vmctx;
 
 int	smbios_build(struct vmctx *ctx);
+#ifndef __FreeBSD__
 int	smbios_parse(const char *opts);
+void	smbios_apply(void);
+#endif
 
 #endif /* _SMBIOSTBL_H_ */
diff --git a/usr/src/cmd/bhyve/test/Makefile.com b/usr/src/cmd/bhyve/test/Makefile.com
index a2e5bce08f..64f1819b76 100644
--- a/usr/src/cmd/bhyve/test/Makefile.com
+++ b/usr/src/cmd/bhyve/test/Makefile.com
@@ -21,7 +21,6 @@ include $(SRC)/cmd/Makefile.cmd.64
 # Force c99 for everything
 #
 CSTD=		$(CSTD_GNU99)
-C99MODE=	-xc99=%all
 
 CFLAGS +=	$(CCVERBOSE) -_gcc=-Wimplicit-function-declaration \
 		-_gcc=-Wno-parentheses
diff --git a/usr/src/cmd/bhyve/uart_emul.c b/usr/src/cmd/bhyve/uart_emul.c
index 077380a422..a04229b288 100644
--- a/usr/src/cmd/bhyve/uart_emul.c
+++ b/usr/src/cmd/bhyve/uart_emul.c
@@ -77,6 +77,10 @@ __FBSDID("$FreeBSD$");
 #define	COM1_IRQ	4
 #define	COM2_BASE	0x2F8
 #define	COM2_IRQ	3
+#define	COM3_BASE	0x3E8
+#define	COM3_IRQ	4
+#define	COM4_BASE	0x2E8
+#define	COM4_IRQ	3
 
 #define	DEFAULT_RCLK	1843200
 #define	DEFAULT_BAUD	9600
@@ -104,6 +108,8 @@ static struct {
 } uart_lres[] = {
 	{ COM1_BASE, COM1_IRQ, false},
 	{ COM2_BASE, COM2_IRQ, false},
+	{ COM3_BASE, COM3_IRQ, false},
+	{ COM4_BASE, COM4_IRQ, false},
 };
 
 #define	UART_NLDEVS	(sizeof(uart_lres) / sizeof(uart_lres[0]))
@@ -916,7 +922,7 @@ uart_stdio_backend(struct uart_softc *sc)
 }
 
 static int
-uart_tty_backend(struct uart_softc *sc, const char *opts)
+uart_tty_backend(struct uart_softc *sc, const char *path)
 {
 #ifndef WITHOUT_CAPSICUM
 	cap_rights_t rights;
@@ -924,7 +930,7 @@ uart_tty_backend(struct uart_softc *sc, const char *opts)
 #endif
 	int fd;
 
-	fd = open(opts, O_RDWR | O_NONBLOCK);
+	fd = open(path, O_RDWR | O_NONBLOCK);
 	if (fd < 0)
 		return (-1);
 
@@ -948,21 +954,21 @@ uart_tty_backend(struct uart_softc *sc, const char *opts)
 }
 
 int
-uart_set_backend(struct uart_softc *sc, const char *opts)
+uart_set_backend(struct uart_softc *sc, const char *device)
 {
 	int retval;
 
-	if (opts == NULL)
+	if (device == NULL)
 		return (0);
 
 #ifndef __FreeBSD__
-	if (strncmp("socket,", opts, 7) == 0)
-		return (uart_sock_backend(sc, opts));
+	if (strncmp("socket,", device, 7) == 0)
+		return (uart_sock_backend(sc, device));
 #endif
-	if (strcmp("stdio", opts) == 0)
+	if (strcmp("stdio", device) == 0)
 		retval = uart_stdio_backend(sc);
 	else
-		retval = uart_tty_backend(sc, opts);
+		retval = uart_tty_backend(sc, device);
 	if (retval == 0)
 		uart_opentty(sc);
 
diff --git a/usr/src/cmd/bhyve/uart_emul.h b/usr/src/cmd/bhyve/uart_emul.h
index a87202df1f..8c5b983a08 100644
--- a/usr/src/cmd/bhyve/uart_emul.h
+++ b/usr/src/cmd/bhyve/uart_emul.h
@@ -43,5 +43,5 @@ struct uart_softc *uart_init(uart_intr_func_t intr_assert,
 int	uart_legacy_alloc(int unit, int *ioaddr, int *irq);
 uint8_t	uart_read(struct uart_softc *sc, int offset);
 void	uart_write(struct uart_softc *sc, int offset, uint8_t value);
-int	uart_set_backend(struct uart_softc *sc, const char *opt);
+int	uart_set_backend(struct uart_softc *sc, const char *device);
 #endif
diff --git a/usr/src/cmd/bhyve/usb_emul.c b/usr/src/cmd/bhyve/usb_emul.c
index 6ecdd9530e..d97d7b38d2 100644
--- a/usr/src/cmd/bhyve/usb_emul.c
+++ b/usr/src/cmd/bhyve/usb_emul.c
@@ -43,7 +43,7 @@ __FBSDID("$FreeBSD$");
 SET_DECLARE(usb_emu_set, struct usb_devemu);
 
 struct usb_devemu *
-usb_emu_finddev(char *name)
+usb_emu_finddev(const char *name)
 {
 	struct usb_devemu **udpp, *udp;
 
diff --git a/usr/src/cmd/bhyve/usb_emul.h b/usr/src/cmd/bhyve/usb_emul.h
index e55a421b6f..ede0fe0154 100644
--- a/usr/src/cmd/bhyve/usb_emul.h
+++ b/usr/src/cmd/bhyve/usb_emul.h
@@ -32,6 +32,7 @@
 #ifndef _USB_EMUL_H_
 #define _USB_EMUL_H_
 
+#include <sys/nv.h>
 #include <stdlib.h>
 #include <sys/linker_set.h>
 #include <pthread.h>
@@ -57,7 +58,7 @@ struct usb_devemu {
 	int	ue_usbspeed;	/* usb device speed */
 
 	/* instance creation */
-	void	*(*ue_init)(struct usb_hci *hci, char *opt);
+	void	*(*ue_init)(struct usb_hci *hci, nvlist_t *nvl);
 
 	/* handlers */
 	int	(*ue_request)(void *sc, struct usb_data_xfer *xfer);
@@ -155,7 +156,7 @@ enum USB_ERRCODE {
 #define	USB_DATA_XFER_LOCK_HELD(x) MUTEX_HELD(&((x)->mtx))
 #endif
 
-struct usb_devemu *usb_emu_finddev(char *name);
+struct usb_devemu *usb_emu_finddev(const char *name);
 
 struct usb_data_xfer_block *usb_data_xfer_append(struct usb_data_xfer *xfer,
                           void *buf, int blen, void *hci_data, int ccs);
diff --git a/usr/src/cmd/bhyve/usb_mouse.c b/usr/src/cmd/bhyve/usb_mouse.c
index 7790fe0ec9..340fdc0cb0 100644
--- a/usr/src/cmd/bhyve/usb_mouse.c
+++ b/usr/src/cmd/bhyve/usb_mouse.c
@@ -239,8 +239,6 @@ struct umouse_bos_desc umouse_bosd = {
 struct umouse_softc {
 	struct usb_hci *hci;
 
-	char	*opt;
-
 	struct umouse_report um_report;
 	int	newdata;
 	struct {
@@ -297,7 +295,7 @@ umouse_event(uint8_t button, int x, int y, void *arg)
 }
 
 static void *
-umouse_init(struct usb_hci *hci, char *opt)
+umouse_init(struct usb_hci *hci, nvlist_t *nvl)
 {
 	struct umouse_softc *sc;
 
@@ -305,7 +303,6 @@ umouse_init(struct usb_hci *hci, char *opt)
 	sc->hci = hci;
 
 	sc->hid.protocol = 1;	/* REPORT protocol */
-	sc->opt = strdup(opt);
 	pthread_mutex_init(&sc->mtx, NULL);
 	pthread_mutex_init(&sc->ev_mtx, NULL);
 
diff --git a/usr/src/cmd/bhyve/virtio.c b/usr/src/cmd/bhyve/virtio.c
index d899a57795..b7e858694f 100644
--- a/usr/src/cmd/bhyve/virtio.c
+++ b/usr/src/cmd/bhyve/virtio.c
@@ -35,6 +35,10 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/atomic.h>
 
+#ifdef __FreeBSD__
+#include <dev/virtio/pci/virtio_pci_legacy_var.h>
+#endif
+
 #include <stdio.h>
 #include <stdint.h>
 #include <pthread.h>
@@ -126,10 +130,10 @@ vi_set_io_bar(struct virtio_softc *vs, int barnum)
 	size_t size;
 
 	/*
-	 * ??? should we use CFG0 if MSI-X is disabled?
+	 * ??? should we use VIRTIO_PCI_CONFIG_OFF(0) if MSI-X is disabled?
 	 * Existing code did not...
 	 */
-	size = VTCFG_R_CFG1 + vs->vs_vc->vc_cfgsize;
+	size = VIRTIO_PCI_CONFIG_OFF(1) + vs->vs_vc->vc_cfgsize;
 	pci_emul_alloc_bar(vs->vs_pi, barnum, PCIBAR_IO, size);
 }
 
@@ -181,12 +185,12 @@ vi_vq_init(struct virtio_softc *vs, uint32_t pfn)
 	vq = &vs->vs_queues[vs->vs_curq];
 	vq->vq_pfn = pfn;
 	phys = (uint64_t)pfn << VRING_PFN;
-	size = vring_size(vq->vq_qsize);
+	size = vring_size_aligned(vq->vq_qsize);
 	base = paddr_guest2host(vs->vs_pi->pi_vmctx, phys, size);
 
 	/* First page(s) are descriptors... */
-	vq->vq_desc = (struct virtio_desc *)base;
-	base += vq->vq_qsize * sizeof(struct virtio_desc);
+	vq->vq_desc = (struct vring_desc *)base;
+	base += vq->vq_qsize * sizeof(struct vring_desc);
 
 	/* ... immediately followed by "avail" ring (entirely uint16_t's) */
 	vq->vq_avail = (struct vring_avail *)base;
@@ -210,15 +214,15 @@ vi_vq_init(struct virtio_softc *vs, uint32_t pfn)
  * descriptor.
  */
 static inline void
-_vq_record(int i, volatile struct virtio_desc *vd, struct vmctx *ctx,
+_vq_record(int i, volatile struct vring_desc *vd, struct vmctx *ctx,
 	   struct iovec *iov, int n_iov, uint16_t *flags) {
 
 	if (i >= n_iov)
 		return;
-	iov[i].iov_base = paddr_guest2host(ctx, vd->vd_addr, vd->vd_len);
-	iov[i].iov_len = vd->vd_len;
+	iov[i].iov_base = paddr_guest2host(ctx, vd->addr, vd->len);
+	iov[i].iov_len = vd->len;
 	if (flags != NULL)
-		flags[i] = vd->vd_flags;
+		flags[i] = vd->flags;
 }
 #define	VQ_MAX_DESCRIPTORS	512	/* see below */
 
@@ -235,7 +239,7 @@ _vq_record(int i, volatile struct virtio_desc *vd, struct vmctx *ctx,
  * i.e., we do not count the indirect descriptors, only the "real"
  * ones.
  *
- * Basically, this vets the vd_flags and vd_next field of each
+ * Basically, this vets the "flags" and "next" field of each
  * descriptor and tells you how many are involved.  Since some may
  * be indirect, this also needs the vmctx (in the pci_devinst
  * at vs->vs_pi) so that it can find indirect descriptors.
@@ -252,7 +256,7 @@ _vq_record(int i, volatile struct virtio_desc *vd, struct vmctx *ctx,
  *
  * If you want to verify the WRITE flag on each descriptor, pass a
  * non-NULL "flags" pointer to an array of "uint16_t" of the same size
- * as n_iov and we'll copy each vd_flags field after unwinding any
+ * as n_iov and we'll copy each "flags" field after unwinding any
  * indirects.
  *
  * If some descriptor(s) are invalid, this prints a diagnostic message
@@ -268,7 +272,7 @@ vq_getchain(struct vqueue_info *vq, uint16_t *pidx,
 	int i;
 	u_int ndesc, n_indir;
 	u_int idx, next;
-	volatile struct virtio_desc *vdir, *vindir, *vp;
+	volatile struct vring_desc *vdir, *vindir, *vp;
 	struct vmctx *ctx;
 	struct virtio_softc *vs;
 	const char *name;
@@ -278,11 +282,11 @@ vq_getchain(struct vqueue_info *vq, uint16_t *pidx,
 
 	/*
 	 * Note: it's the responsibility of the guest not to
-	 * update vq->vq_avail->va_idx until all of the descriptors
+	 * update vq->vq_avail->idx until all of the descriptors
          * the guest has written are valid (including all their
-         * vd_next fields and vd_flags).
+         * "next" fields and "flags").
 	 *
-	 * Compute (va_idx - last_avail) in integers mod 2**16.  This is
+	 * Compute (vq_avail->idx - last_avail) in integers mod 2**16.  This is
 	 * the number of descriptors the device has made available
 	 * since the last time we updated vq->vq_last_avail.
 	 *
@@ -290,7 +294,7 @@ vq_getchain(struct vqueue_info *vq, uint16_t *pidx,
 	 * then trim off excess bits.
 	 */
 	idx = vq->vq_last_avail;
-	ndesc = (uint16_t)((u_int)vq->vq_avail->va_idx - idx);
+	ndesc = (uint16_t)((u_int)vq->vq_avail->idx - idx);
 	if (ndesc == 0)
 		return (0);
 	if (ndesc > vq->vq_qsize) {
@@ -310,9 +314,9 @@ vq_getchain(struct vqueue_info *vq, uint16_t *pidx,
 	 * index, but we just abort if the count gets excessive.
 	 */
 	ctx = vs->vs_pi->pi_vmctx;
-	*pidx = next = vq->vq_avail->va_ring[idx & (vq->vq_qsize - 1)];
+	*pidx = next = vq->vq_avail->ring[idx & (vq->vq_qsize - 1)];
 	vq->vq_last_avail++;
-	for (i = 0; i < VQ_MAX_DESCRIPTORS; next = vdir->vd_next) {
+	for (i = 0; i < VQ_MAX_DESCRIPTORS; next = vdir->next) {
 		if (next >= vq->vq_qsize) {
 			EPRINTLN(
 			    "%s: descriptor index %u out of range, "
@@ -321,7 +325,7 @@ vq_getchain(struct vqueue_info *vq, uint16_t *pidx,
 			return (-1);
 		}
 		vdir = &vq->vq_desc[next];
-		if ((vdir->vd_flags & VRING_DESC_F_INDIRECT) == 0) {
+		if ((vdir->flags & VRING_DESC_F_INDIRECT) == 0) {
 			_vq_record(i, vdir, ctx, iov, n_iov, flags);
 			i++;
 		} else if ((vs->vs_vc->vc_hv_caps &
@@ -332,16 +336,16 @@ vq_getchain(struct vqueue_info *vq, uint16_t *pidx,
 			    name);
 			return (-1);
 		} else {
-			n_indir = vdir->vd_len / 16;
-			if ((vdir->vd_len & 0xf) || n_indir == 0) {
+			n_indir = vdir->len / 16;
+			if ((vdir->len & 0xf) || n_indir == 0) {
 				EPRINTLN(
 				    "%s: invalid indir len 0x%x, "
 				    "driver confused?",
-				    name, (u_int)vdir->vd_len);
+				    name, (u_int)vdir->len);
 				return (-1);
 			}
 			vindir = paddr_guest2host(ctx,
-			    vdir->vd_addr, vdir->vd_len);
+			    vdir->addr, vdir->len);
 			/*
 			 * Indirects start at the 0th, then follow
 			 * their own embedded "next"s until those run
@@ -352,7 +356,7 @@ vq_getchain(struct vqueue_info *vq, uint16_t *pidx,
 			next = 0;
 			for (;;) {
 				vp = &vindir[next];
-				if (vp->vd_flags & VRING_DESC_F_INDIRECT) {
+				if (vp->flags & VRING_DESC_F_INDIRECT) {
 					EPRINTLN(
 					    "%s: indirect desc has INDIR flag,"
 					    " driver confused?",
@@ -362,9 +366,9 @@ vq_getchain(struct vqueue_info *vq, uint16_t *pidx,
 				_vq_record(i, vp, ctx, iov, n_iov, flags);
 				if (++i > VQ_MAX_DESCRIPTORS)
 					goto loopy;
-				if ((vp->vd_flags & VRING_DESC_F_NEXT) == 0)
+				if ((vp->flags & VRING_DESC_F_NEXT) == 0)
 					break;
-				next = vp->vd_next;
+				next = vp->next;
 				if (next >= n_indir) {
 					EPRINTLN(
 					    "%s: invalid next %u > %u, "
@@ -374,7 +378,7 @@ vq_getchain(struct vqueue_info *vq, uint16_t *pidx,
 				}
 			}
 		}
-		if ((vdir->vd_flags & VRING_DESC_F_NEXT) == 0)
+		if ((vdir->flags & VRING_DESC_F_NEXT) == 0)
 			return (i);
 	}
 loopy:
@@ -401,7 +405,7 @@ void
 vq_relchain_prepare(struct vqueue_info *vq, uint16_t idx, uint32_t iolen)
 {
 	volatile struct vring_used *vuh;
-	volatile struct virtio_used *vue;
+	volatile struct vring_used_elem *vue;
 	uint16_t mask;
 
 	/*
@@ -409,16 +413,13 @@ vq_relchain_prepare(struct vqueue_info *vq, uint16_t idx, uint32_t iolen)
 	 *  - mask is N-1 where N is a power of 2 so computes x % N
 	 *  - vuh points to the "used" data shared with guest
 	 *  - vue points to the "used" ring entry we want to update
-	 *
-	 * (I apologize for the two fields named vu_idx; the
-	 * virtio spec calls the one that vue points to, "id"...)
 	 */
 	mask = vq->vq_qsize - 1;
 	vuh = vq->vq_used;
 
-	vue = &vuh->vu_ring[vq->vq_next_used++ & mask];
-	vue->vu_idx = idx;
-	vue->vu_tlen = iolen;
+	vue = &vuh->ring[vq->vq_next_used++ & mask];
+	vue->id = idx;
+	vue->len = iolen;
 }
 
 void
@@ -430,7 +431,7 @@ vq_relchain_publish(struct vqueue_info *vq)
 	 * (and even on x86 to act as a compiler barrier).
 	 */
 	atomic_thread_fence_rel();
-	vq->vq_used->vu_idx = vq->vq_next_used;
+	vq->vq_used->idx = vq->vq_next_used;
 }
 
 /*
@@ -480,12 +481,12 @@ vq_endchains(struct vqueue_info *vq, int used_all_avail)
 	 */
 	vs = vq->vq_vs;
 	old_idx = vq->vq_save_used;
-	vq->vq_save_used = new_idx = vq->vq_used->vu_idx;
+	vq->vq_save_used = new_idx = vq->vq_used->idx;
 
 	/*
-	 * Use full memory barrier between vu_idx store from preceding
+	 * Use full memory barrier between "idx" store from preceding
 	 * vq_relchain() call and the loads from VQ_USED_EVENT_IDX() or
-	 * va_flags below.
+	 * "flags" field below.
 	 */
 	atomic_thread_fence_seq_cst();
 	if (used_all_avail &&
@@ -501,7 +502,7 @@ vq_endchains(struct vqueue_info *vq, int used_all_avail)
 			(uint16_t)(new_idx - old_idx);
 	} else {
 		intr = new_idx != old_idx &&
-		    !(vq->vq_avail->va_flags & VRING_AVAIL_F_NO_INTERRUPT);
+		    !(vq->vq_avail->flags & VRING_AVAIL_F_NO_INTERRUPT);
 	}
 	if (intr)
 		vq_interrupt(vs, vq);
@@ -514,16 +515,16 @@ static struct config_reg {
 	uint8_t		cr_ro;		/* true => reg is read only */
 	const char	*cr_name;	/* name of reg */
 } config_regs[] = {
-	{ VTCFG_R_HOSTCAP,	4, 1, "HOSTCAP" },
-	{ VTCFG_R_GUESTCAP,	4, 0, "GUESTCAP" },
-	{ VTCFG_R_PFN,		4, 0, "PFN" },
-	{ VTCFG_R_QNUM,		2, 1, "QNUM" },
-	{ VTCFG_R_QSEL,		2, 0, "QSEL" },
-	{ VTCFG_R_QNOTIFY,	2, 0, "QNOTIFY" },
-	{ VTCFG_R_STATUS,	1, 0, "STATUS" },
-	{ VTCFG_R_ISR,		1, 0, "ISR" },
-	{ VTCFG_R_CFGVEC,	2, 0, "CFGVEC" },
-	{ VTCFG_R_QVEC,		2, 0, "QVEC" },
+	{ VIRTIO_PCI_HOST_FEATURES,	4, 1, "HOST_FEATURES" },
+	{ VIRTIO_PCI_GUEST_FEATURES,	4, 0, "GUEST_FEATURES" },
+	{ VIRTIO_PCI_QUEUE_PFN,		4, 0, "QUEUE_PFN" },
+	{ VIRTIO_PCI_QUEUE_NUM,		2, 1, "QUEUE_NUM" },
+	{ VIRTIO_PCI_QUEUE_SEL,		2, 0, "QUEUE_SEL" },
+	{ VIRTIO_PCI_QUEUE_NOTIFY,	2, 0, "QUEUE_NOTIFY" },
+	{ VIRTIO_PCI_STATUS,		1, 0, "STATUS" },
+	{ VIRTIO_PCI_ISR,		1, 0, "ISR" },
+	{ VIRTIO_MSI_CONFIG_VECTOR,	2, 0, "CONFIG_VECTOR" },
+	{ VIRTIO_MSI_QUEUE_VECTOR,	2, 0, "QUEUE_VECTOR" },
 };
 
 static inline struct config_reg *
@@ -585,10 +586,7 @@ vi_pci_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 	if (size != 1 && size != 2 && size != 4)
 		goto bad;
 
-	if (pci_msix_enabled(pi))
-		virtio_config_size = VTCFG_R_CFG1;
-	else
-		virtio_config_size = VTCFG_R_CFG0;
+	virtio_config_size = VIRTIO_PCI_CONFIG_OFF(pci_msix_enabled(pi));
 
 	if (offset >= virtio_config_size) {
 		/*
@@ -622,39 +620,39 @@ bad:
 	}
 
 	switch (offset) {
-	case VTCFG_R_HOSTCAP:
+	case VIRTIO_PCI_HOST_FEATURES:
 		value = vc->vc_hv_caps;
 		break;
-	case VTCFG_R_GUESTCAP:
+	case VIRTIO_PCI_GUEST_FEATURES:
 		value = vs->vs_negotiated_caps;
 		break;
-	case VTCFG_R_PFN:
+	case VIRTIO_PCI_QUEUE_PFN:
 		if (vs->vs_curq < vc->vc_nvq)
 			value = vs->vs_queues[vs->vs_curq].vq_pfn;
 		break;
-	case VTCFG_R_QNUM:
+	case VIRTIO_PCI_QUEUE_NUM:
 		value = vs->vs_curq < vc->vc_nvq ?
 		    vs->vs_queues[vs->vs_curq].vq_qsize : 0;
 		break;
-	case VTCFG_R_QSEL:
+	case VIRTIO_PCI_QUEUE_SEL:
 		value = vs->vs_curq;
 		break;
-	case VTCFG_R_QNOTIFY:
+	case VIRTIO_PCI_QUEUE_NOTIFY:
 		value = 0;	/* XXX */
 		break;
-	case VTCFG_R_STATUS:
+	case VIRTIO_PCI_STATUS:
 		value = vs->vs_status;
 		break;
-	case VTCFG_R_ISR:
+	case VIRTIO_PCI_ISR:
 		value = vs->vs_isr;
 		vs->vs_isr = 0;		/* a read clears this flag */
 		if (value)
 			pci_lintr_deassert(pi);
 		break;
-	case VTCFG_R_CFGVEC:
+	case VIRTIO_MSI_CONFIG_VECTOR:
 		value = vs->vs_msix_cfg_idx;
 		break;
-	case VTCFG_R_QVEC:
+	case VIRTIO_MSI_QUEUE_VECTOR:
 		value = vs->vs_curq < vc->vc_nvq ?
 		    vs->vs_queues[vs->vs_curq].vq_msix_idx :
 		    VIRTIO_MSI_NO_VECTOR;
@@ -705,10 +703,7 @@ vi_pci_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 	if (size != 1 && size != 2 && size != 4)
 		goto bad;
 
-	if (pci_msix_enabled(pi))
-		virtio_config_size = VTCFG_R_CFG1;
-	else
-		virtio_config_size = VTCFG_R_CFG0;
+	virtio_config_size = VIRTIO_PCI_CONFIG_OFF(pci_msix_enabled(pi));
 
 	if (offset >= virtio_config_size) {
 		/*
@@ -746,18 +741,18 @@ bad:
 	}
 
 	switch (offset) {
-	case VTCFG_R_GUESTCAP:
+	case VIRTIO_PCI_GUEST_FEATURES:
 		vs->vs_negotiated_caps = value & vc->vc_hv_caps;
 		if (vc->vc_apply_features)
 			(*vc->vc_apply_features)(DEV_SOFTC(vs),
 			    vs->vs_negotiated_caps);
 		break;
-	case VTCFG_R_PFN:
+	case VIRTIO_PCI_QUEUE_PFN:
 		if (vs->vs_curq >= vc->vc_nvq)
 			goto bad_qindex;
 		vi_vq_init(vs, value);
 		break;
-	case VTCFG_R_QSEL:
+	case VIRTIO_PCI_QUEUE_SEL:
 		/*
 		 * Note that the guest is allowed to select an
 		 * invalid queue; we just need to return a QNUM
@@ -765,7 +760,7 @@ bad:
 		 */
 		vs->vs_curq = value;
 		break;
-	case VTCFG_R_QNOTIFY:
+	case VIRTIO_PCI_QUEUE_NOTIFY:
 		if (value >= vc->vc_nvq) {
 			EPRINTLN("%s: queue %d notify out of range",
 				name, (int)value);
@@ -781,15 +776,15 @@ bad:
 			    "%s: qnotify queue %d: missing vq/vc notify",
 				name, (int)value);
 		break;
-	case VTCFG_R_STATUS:
+	case VIRTIO_PCI_STATUS:
 		vs->vs_status = value;
 		if (value == 0)
 			(*vc->vc_reset)(DEV_SOFTC(vs));
 		break;
-	case VTCFG_R_CFGVEC:
+	case VIRTIO_MSI_CONFIG_VECTOR:
 		vs->vs_msix_cfg_idx = value;
 		break;
-	case VTCFG_R_QVEC:
+	case VIRTIO_MSI_QUEUE_VECTOR:
 		if (vs->vs_curq >= vc->vc_nvq)
 			goto bad_qindex;
 		vq = &vs->vs_queues[vs->vs_curq];
diff --git a/usr/src/cmd/bhyve/virtio.h b/usr/src/cmd/bhyve/virtio.h
index b28e61d353..16d7c5c25e 100644
--- a/usr/src/cmd/bhyve/virtio.h
+++ b/usr/src/cmd/bhyve/virtio.h
@@ -28,12 +28,18 @@
  * $FreeBSD$
  */
 
-#ifndef	_VIRTIO_H_
-#define	_VIRTIO_H_
+#ifndef	_BHYVE_VIRTIO_H_
+#define	_BHYVE_VIRTIO_H_
 
 #include <pthread_np.h>
 #include <machine/atomic.h>
 
+#include <dev/virtio/virtio.h>
+#ifdef __FreeBSD__
+#include <dev/virtio/virtio_ring.h>
+#include <dev/virtio/pci/virtio_pci_var.h>
+#endif
+
 /*
  * These are derived from several virtio specifications.
  *
@@ -126,39 +132,6 @@
  */
 #define VRING_ALIGN	4096
 
-#define VRING_DESC_F_NEXT	(1 << 0)
-#define VRING_DESC_F_WRITE	(1 << 1)
-#define VRING_DESC_F_INDIRECT	(1 << 2)
-
-struct virtio_desc {			/* AKA vring_desc */
-	uint64_t	vd_addr;	/* guest physical address */
-	uint32_t	vd_len;		/* length of scatter/gather seg */
-	uint16_t	vd_flags;	/* VRING_F_DESC_* */
-	uint16_t	vd_next;	/* next desc if F_NEXT */
-} __packed;
-
-struct virtio_used {			/* AKA vring_used_elem */
-	uint32_t	vu_idx;		/* head of used descriptor chain */
-	uint32_t	vu_tlen;	/* length written-to */
-} __packed;
-
-#define VRING_AVAIL_F_NO_INTERRUPT   1
-
-struct vring_avail {
-	uint16_t	va_flags;	/* VRING_AVAIL_F_* */
-	uint16_t	va_idx;		/* counts to 65535, then cycles */
-	uint16_t	va_ring[];	/* size N, reported in QNUM value */
-/*	uint16_t	va_used_event;	-- after N ring entries */
-} __packed;
-
-#define	VRING_USED_F_NO_NOTIFY		1
-struct vring_used {
-	uint16_t	vu_flags;	/* VRING_USED_F_* */
-	uint16_t	vu_idx;		/* counts to 65535, then cycles */
-	struct virtio_used vu_ring[];	/* size N */
-/*	uint16_t	vu_avail_event;	-- after N ring entries */
-} __packed;
-
 /*
  * The address of any given virtual queue is determined by a single
  * Page Frame Number register.  The guest writes the PFN into the
@@ -192,23 +165,6 @@ struct vring_used {
 #define	VRING_PFN		12
 
 /*
- * Virtio device types
- *
- * XXX Should really be merged with <dev/virtio/virtio.h> defines
- */
-#define	VIRTIO_TYPE_NET		1
-#define	VIRTIO_TYPE_BLOCK	2
-#define	VIRTIO_TYPE_CONSOLE	3
-#define	VIRTIO_TYPE_ENTROPY	4
-#define	VIRTIO_TYPE_BALLOON	5
-#define	VIRTIO_TYPE_IOMEMORY	6
-#define	VIRTIO_TYPE_RPMSG	7
-#define	VIRTIO_TYPE_SCSI	8
-#define	VIRTIO_TYPE_9P		9
-
-/* experimental IDs start at 65535 and work down */
-
-/*
  * PCI vendor/device IDs
  */
 #define	VIRTIO_VENDOR		0x1AF4
@@ -217,72 +173,13 @@ struct vring_used {
 #define	VIRTIO_DEV_CONSOLE	0x1003
 #define	VIRTIO_DEV_RANDOM	0x1005
 #define	VIRTIO_DEV_SCSI		0x1008
-
-/*
- * PCI config space constants.
- *
- * If MSI-X is enabled, the ISR register is generally not used,
- * and the configuration vector and queue vector appear at offsets
- * 20 and 22 with the remaining configuration registers at 24.
- * If MSI-X is not enabled, those two registers disappear and
- * the remaining configuration registers start at offset 20.
- */
-#define	VTCFG_R_HOSTCAP		0
-#define	VTCFG_R_GUESTCAP	4
-#define	VTCFG_R_PFN		8
-#define	VTCFG_R_QNUM		12
-#define	VTCFG_R_QSEL		14
-#define	VTCFG_R_QNOTIFY		16
-#define	VTCFG_R_STATUS		18
-#define	VTCFG_R_ISR		19
-#define	VTCFG_R_CFGVEC		20
-#define	VTCFG_R_QVEC		22
-#define	VTCFG_R_CFG0		20	/* No MSI-X */
-#define	VTCFG_R_CFG1		24	/* With MSI-X */
-#define	VTCFG_R_MSIX		20
-
-/*
- * Bits in VTCFG_R_STATUS.  Guests need not actually set any of these,
- * but a guest writing 0 to this register means "please reset".
- */
-#define	VTCFG_STATUS_ACK	0x01	/* guest OS has acknowledged dev */
-#define	VTCFG_STATUS_DRIVER	0x02	/* guest OS driver is loaded */
-#define	VTCFG_STATUS_DRIVER_OK	0x04	/* guest OS driver ready */
-#define	VTCFG_STATUS_FAILED	0x80	/* guest has given up on this dev */
-
-/*
- * Bits in VTCFG_R_ISR.  These apply only if not using MSI-X.
- *
- * (We don't [yet?] ever use CONF_CHANGED.)
- */
-#define	VTCFG_ISR_QUEUES	0x01	/* re-scan queues */
-#define	VTCFG_ISR_CONF_CHANGED	0x80	/* configuration changed */
-
-#define	VIRTIO_MSI_NO_VECTOR	0xFFFF
-
-/*
- * Feature flags.
- * Note: bits 0 through 23 are reserved to each device type.
- */
-#define	VIRTIO_F_NOTIFY_ON_EMPTY	(1 << 24)
-#define	VIRTIO_RING_F_INDIRECT_DESC	(1 << 28)
-#define	VIRTIO_RING_F_EVENT_IDX		(1 << 29)
+#define	VIRTIO_DEV_9P		0x1009
 
 /* From section 2.3, "Virtqueue Configuration", of the virtio specification */
-static inline size_t
-vring_size(u_int qsz)
+static inline int
+vring_size_aligned(u_int qsz)
 {
-	size_t size;
-
-	/* constant 3 below = va_flags, va_idx, va_used_event */
-	size = sizeof(struct virtio_desc) * qsz + sizeof(uint16_t) * (3 + qsz);
-	size = roundup2(size, VRING_ALIGN);
-
-	/* constant 3 below = vu_flags, vu_idx, vu_avail_event */
-	size += sizeof(uint16_t) * 3 + sizeof(struct virtio_used) * qsz;
-	size = roundup2(size, VRING_ALIGN);
-
-	return (size);
+	return (roundup2(vring_size(qsz, VRING_ALIGN), VRING_ALIGN));
 }
 
 struct vmctx;
@@ -392,23 +289,23 @@ struct vqueue_info {
 	uint16_t vq_num;	/* we're the num'th queue in the softc */
 
 	uint16_t vq_flags;	/* flags (see above) */
-	uint16_t vq_last_avail;	/* a recent value of vq_avail->va_idx */
+	uint16_t vq_last_avail;	/* a recent value of vq_avail->idx */
 	uint16_t vq_next_used;	/* index of the next used slot to be filled */
-	uint16_t vq_save_used;	/* saved vq_used->vu_idx; see vq_endchains */
+	uint16_t vq_save_used;	/* saved vq_used->idx; see vq_endchains */
 	uint16_t vq_msix_idx;	/* MSI-X index, or VIRTIO_MSI_NO_VECTOR */
 
 	uint32_t vq_pfn;	/* PFN of virt queue (not shifted!) */
 
-	volatile struct virtio_desc *vq_desc;	/* descriptor array */
+	volatile struct vring_desc *vq_desc;	/* descriptor array */
 	volatile struct vring_avail *vq_avail;	/* the "avail" ring */
 	volatile struct vring_used *vq_used;	/* the "used" ring */
 
 };
 /* as noted above, these are sort of backwards, name-wise */
 #define VQ_AVAIL_EVENT_IDX(vq) \
-	(*(volatile uint16_t *)&(vq)->vq_used->vu_ring[(vq)->vq_qsize])
+	(*(volatile uint16_t *)&(vq)->vq_used->ring[(vq)->vq_qsize])
 #define VQ_USED_EVENT_IDX(vq) \
-	((vq)->vq_avail->va_ring[(vq)->vq_qsize])
+	((vq)->vq_avail->ring[(vq)->vq_qsize])
 
 /*
  * Is this ring ready for I/O?
@@ -429,7 +326,7 @@ vq_has_descs(struct vqueue_info *vq)
 {
 
 	return (vq_ring_ready(vq) && vq->vq_last_avail !=
-	    vq->vq_avail->va_idx);
+	    vq->vq_avail->idx);
 }
 
 /*
@@ -469,11 +366,11 @@ static inline void
 vq_kick_enable(struct vqueue_info *vq)
 {
 
-	vq->vq_used->vu_flags &= ~VRING_USED_F_NO_NOTIFY;
+	vq->vq_used->flags &= ~VRING_USED_F_NO_NOTIFY;
 	/*
-	 * Full memory barrier to make sure the store to vu_flags
-	 * happens before the load from va_idx, which results from
-	 * a subsequent call to vq_has_descs().
+	 * Full memory barrier to make sure the store to vq_used->flags
+	 * happens before the load from vq_avail->idx, which results from a
+	 * subsequent call to vq_has_descs().
 	 */
 	atomic_thread_fence_seq_cst();
 }
@@ -482,7 +379,7 @@ static inline void
 vq_kick_disable(struct vqueue_info *vq)
 {
 
-	vq->vq_used->vu_flags |= VRING_USED_F_NO_NOTIFY;
+	vq->vq_used->flags |= VRING_USED_F_NO_NOTIFY;
 }
 
 struct iovec;
@@ -506,4 +403,4 @@ uint64_t vi_pci_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 		     int baridx, uint64_t offset, int size);
 void	vi_pci_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
 		     int baridx, uint64_t offset, int size, uint64_t value);
-#endif	/* _VIRTIO_H_ */
+#endif	/* _BHYVE_VIRTIO_H_ */
diff --git a/usr/src/cmd/bhyvectl/bhyvectl.c b/usr/src/cmd/bhyvectl/bhyvectl.c
index 2948c14ebb..313a1a37f4 100644
--- a/usr/src/cmd/bhyvectl/bhyvectl.c
+++ b/usr/src/cmd/bhyvectl/bhyvectl.c
@@ -1959,7 +1959,9 @@ main(int argc, char *argv[])
 	if (!error) {
 		ctx = vm_open(vmname);
 		if (ctx == NULL) {
-			printf("VM:%s is not created.\n", vmname);
+			fprintf(stderr,
+			    "vm_open: %s could not be opened: %s\n",
+			    vmname, strerror(errno));
 			exit (1);
 		}
 	}
diff --git a/usr/src/compat/bhyve/dev/virtio/virtio.h b/usr/src/compat/bhyve/dev/virtio/virtio.h
new file mode 100644
index 0000000000..5e996730d0
--- /dev/null
+++ b/usr/src/compat/bhyve/dev/virtio/virtio.h
@@ -0,0 +1,144 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2013  Chris Torek <torek @ torek net>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_BHYVE_COMPAT_VIRTIO_H_
+#define	_BHYVE_COMPAT_VIRTIO_H_
+
+#define VRING_DESC_F_NEXT	(1 << 0)
+#define VRING_DESC_F_WRITE	(1 << 1)
+#define VRING_DESC_F_INDIRECT	(1 << 2)
+
+struct vring_desc {
+	uint64_t	addr;		/* guest physical address */
+	uint32_t	len;		/* length of scatter/gather seg */
+	uint16_t	flags;		/* VRING_F_DESC_* */
+	uint16_t	next;		/* next desc if F_NEXT */
+} __packed;
+
+struct vring_used_elem {
+	uint32_t	id;		/* head of used descriptor chain */
+	uint32_t	len;		/* length written-to */
+} __packed;
+
+#define VRING_AVAIL_F_NO_INTERRUPT   1
+
+struct vring_avail {
+	uint16_t	flags;		/* VRING_AVAIL_F_* */
+	uint16_t	idx;		/* counts to 65535, then cycles */
+	uint16_t	ring[];		/* size N, reported in QNUM value */
+/*	uint16_t	used_event;	-- after N ring entries */
+} __packed;
+
+#define	VRING_USED_F_NO_NOTIFY		1
+struct vring_used {
+	uint16_t	flags;		/* VRING_USED_F_* */
+	uint16_t	idx;		/* counts to 65535, then cycles */
+	struct vring_used_elem ring[];	/* size N */
+/*	uint16_t	avail_event;	-- after N ring entries */
+} __packed;
+
+/*
+ * Virtio device types
+ */
+#define	VIRTIO_ID_NETWORK	1
+#define	VIRTIO_ID_BLOCK		2
+#define	VIRTIO_ID_CONSOLE	3
+#define	VIRTIO_ID_ENTROPY	4
+#define	VIRTIO_ID_BALLOON	5
+#define	VIRTIO_ID_IOMEMORY	6
+#define	VIRTIO_ID_RPMSG		7
+#define	VIRTIO_ID_SCSI		8
+#define	VIRTIO_ID_9P		9
+
+/* experimental IDs start at 65535 and work down */
+
+/*
+ * PCI config space constants.
+ *
+ * If MSI-X is enabled, the ISR register is generally not used,
+ * and the configuration vector and queue vector appear at offsets
+ * 20 and 22 with the remaining configuration registers at 24.
+ * If MSI-X is not enabled, those two registers disappear and
+ * the remaining configuration registers start at offset 20.
+ */
+#define	VIRTIO_PCI_HOST_FEATURES		0
+#define	VIRTIO_PCI_GUEST_FEATURES		4
+#define	VIRTIO_PCI_QUEUE_PFN			8
+#define	VIRTIO_PCI_QUEUE_NUM			12
+#define	VIRTIO_PCI_QUEUE_SEL			14
+#define	VIRTIO_PCI_QUEUE_NOTIFY			16
+#define	VIRTIO_PCI_STATUS			18
+#define	VIRTIO_PCI_ISR				19
+#define	VIRTIO_MSI_CONFIG_VECTOR		20
+#define	VIRTIO_MSI_QUEUE_VECTOR			22
+#define	VIRTIO_PCI_CONFIG_OFF(msix_enabled)	((msix_enabled) ? 24 : 20)
+
+/*
+ * Bits in VTCFG_R_STATUS.  Guests need not actually set any of these,
+ * but a guest writing 0 to this register means "please reset".
+ */
+#define	VTCFG_STATUS_ACK	0x01	/* guest OS has acknowledged dev */
+#define	VTCFG_STATUS_DRIVER	0x02	/* guest OS driver is loaded */
+#define	VTCFG_STATUS_DRIVER_OK	0x04	/* guest OS driver ready */
+#define	VTCFG_STATUS_FAILED	0x80	/* guest has given up on this dev */
+
+/*
+ * Bits in VTCFG_R_ISR.  These apply only if not using MSI-X.
+ *
+ * (We don't [yet?] ever use CONF_CHANGED.)
+ */
+#define	VTCFG_ISR_QUEUES	0x01	/* re-scan queues */
+#define	VTCFG_ISR_CONF_CHANGED	0x80	/* configuration changed */
+
+#define	VIRTIO_MSI_NO_VECTOR	0xFFFF
+
+/*
+ * Feature flags.
+ * Note: bits 0 through 23 are reserved to each device type.
+ */
+#define	VIRTIO_F_NOTIFY_ON_EMPTY	(1 << 24)
+#define	VIRTIO_RING_F_INDIRECT_DESC	(1 << 28)
+#define	VIRTIO_RING_F_EVENT_IDX		(1 << 29)
+
+static inline int
+vring_size(unsigned int num, unsigned long align)
+{
+        int size;
+
+        size = num * sizeof(struct vring_desc);
+        size += sizeof(struct vring_avail) + (num * sizeof(uint16_t)) +
+            sizeof(uint16_t);
+        size = (size + align - 1) & ~(align - 1);
+        size += sizeof(struct vring_used) +
+            (num * sizeof(struct vring_used_elem)) + sizeof(uint16_t);
+        return (size);
+}
+
+#endif	/* _BHYVE_COMPAT_VIRTIO_H_ */
diff --git a/usr/src/compat/bhyve/sys/nv.h b/usr/src/compat/bhyve/sys/nv.h
new file mode 100644
index 0000000000..c446cf4059
--- /dev/null
+++ b/usr/src/compat/bhyve/sys/nv.h
@@ -0,0 +1,93 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
+ */
+
+#ifndef _COMPAT_FREEBSD_SYS_NV_H_
+#define	_COMPAT_FREEBSD_SYS_NV_H_
+
+#include <assert.h>
+#include <libnvpair.h>
+
+#define	NV_TYPE_NVLIST	DATA_TYPE_NVLIST
+#define	NV_TYPE_STRING	DATA_TYPE_STRING
+
+static inline const char *
+nvlist_next(const nvlist_t *nvl, int *type, void **cookie)
+{
+	nvpair_t *nvp = *cookie;
+
+	nvp = nvlist_next_nvpair((nvlist_t *)nvl, nvp);
+	if (nvp == NULL)
+		return (NULL);
+
+	*cookie = nvp;
+	*type = nvpair_type(nvp);
+	return (nvpair_name(nvp));
+}
+
+static inline nvlist_t *
+nvlist_create(int flag)
+{
+	nvlist_t *nvl;
+
+	/*
+	 * We only emulate this with flag == 0, which is equivalent to the
+	 * illumos NV_UNIQUE_NAME.
+	 */
+	assert(flag == 0);
+
+	if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0)
+		return (NULL);
+	return (nvl);
+}
+
+static inline bool
+nvlist_exists_nvlist(const nvlist_t *nvl, const char *name)
+{
+	nvlist_t *snvl;
+
+	return (nvlist_lookup_nvlist((nvlist_t *)nvl, name, &snvl) == 0);
+}
+
+static inline nvlist_t *
+nvlist_get_nvlist(const nvlist_t *nvl, const char *name)
+{
+	nvlist_t *snvl;
+
+	if (nvlist_lookup_nvlist((nvlist_t *)nvl, name, &snvl) == 0)
+		return (snvl);
+	return (NULL);
+}
+
+static inline bool
+nvlist_exists_string(const nvlist_t *nvl, const char *name)
+{
+	char *str;
+
+	return (nvlist_lookup_string((nvlist_t *)nvl, name, &str) == 0);
+}
+
+static inline char *
+nvlist_get_string(const nvlist_t *nvl, const char *name)
+{
+	char *str;
+
+	if (nvlist_lookup_string((nvlist_t *)nvl, name, &str) == 0)
+		return (str);
+	return (NULL);
+}
+
+#define nvlist_free_string(nvl, name) nvlist_remove_all((nvl), (name))
+
+#endif /* _COMPAT_FREEBSD_SYS_NV_H_ */
diff --git a/usr/src/compat/bhyve/uuid.h b/usr/src/compat/bhyve/uuid.h
index 72ef2c7787..ed0ddee1e3 100644
--- a/usr/src/compat/bhyve/uuid.h
+++ b/usr/src/compat/bhyve/uuid.h
@@ -25,9 +25,9 @@
 #define	uuid_s_invalid_string_uuid	2
 
 static __inline void
-uuid_from_string(char *str, uuid_t *uuidp, uint32_t *status)
+uuid_from_string(const char *str, uuid_t *uuidp, uint32_t *status)
 {
-	if (uuid_parse(str, *uuidp) == 0) {
+	if (uuid_parse((char *)str, *uuidp) == 0) {
 		*status = uuid_s_ok;
 	} else {
 		*status = uuid_s_invalid_string_uuid;
diff --git a/usr/src/lib/libvmmapi/common/mapfile-vers b/usr/src/lib/libvmmapi/common/mapfile-vers
index be0a055490..2489376b62 100644
--- a/usr/src/lib/libvmmapi/common/mapfile-vers
+++ b/usr/src/lib/libvmmapi/common/mapfile-vers
@@ -13,6 +13,7 @@
 # Copyright 2013 Pluribus Networks Inc.
 # Copyright 2019 Joyent, Inc.
 # Copyright 2020 Oxide Computer Company
+# Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
 #
 
 #
@@ -65,6 +66,7 @@ SYMBOL_VERSION ILLUMOSprivate {
 		vm_get_pptdev_limits;
 		vm_get_register;
 		vm_get_register_set;
+		vm_get_run_state;
 		vm_get_seg_desc;
 		vm_get_stat_desc;
 		vm_get_stats;
@@ -90,6 +92,7 @@ SYMBOL_VERSION ILLUMOSprivate {
 		vm_map_pptdev_mmio;
 		vm_mmap_getnext;
 		vm_mmap_memseg;
+		vm_munmap_memseg;
 		vm_open;
 		vm_parse_memsize;
 		vm_pmtmr_set_location;
@@ -107,6 +110,7 @@ SYMBOL_VERSION ILLUMOSprivate {
 		vm_set_memflags;
 		vm_set_register;
 		vm_set_register_set;
+		vm_set_run_state;
 		vm_set_topology;
 		vm_set_x2apic_state;
 		vm_setup_memory;
@@ -116,9 +120,8 @@ SYMBOL_VERSION ILLUMOSprivate {
 		vm_suspend_cpu;
 		vm_suspended_cpus;
 		vm_unassign_pptdev;
+		vm_unmap_pptdev_mmio;
 		vm_wrlock_cycle;
-		vm_get_run_state;
-		vm_set_run_state;
 
 	local:
 		*;
diff --git a/usr/src/lib/libvmmapi/common/vmmapi.c b/usr/src/lib/libvmmapi/common/vmmapi.c
index fcb098a74f..ba3fb7f8dd 100644
--- a/usr/src/lib/libvmmapi/common/vmmapi.c
+++ b/usr/src/lib/libvmmapi/common/vmmapi.c
@@ -185,16 +185,7 @@ vm_open(const char *name)
 
 	return (vm);
 err:
-#ifdef __FreeBSD__
-	vm_destroy(vm);
-#else
-	/*
-	 * As libvmmapi is used by other programs to query and control bhyve
-	 * VMs, destroying a VM just because the open failed isn't useful. We
-	 * have to free what we have allocated, though.
-	 */
 	free(vm);
-#endif
 	return (NULL);
 }
 
@@ -315,6 +306,19 @@ vm_mmap_memseg(struct vmctx *ctx, vm_paddr_t gpa, int segid, vm_ooffset_t off,
 }
 
 int
+vm_munmap_memseg(struct vmctx *ctx, vm_paddr_t gpa, size_t len)
+{
+	struct vm_munmap munmap;
+	int error;
+
+	munmap.gpa = gpa;
+	munmap.len = len;
+
+	error = ioctl(ctx->fd, VM_MUNMAP_MEMSEG, &munmap);
+	return (error);
+}
+
+int
 vm_mmap_getnext(struct vmctx *ctx, vm_paddr_t *gpa, int *segid,
     vm_ooffset_t *segoff, size_t *len, int *prot, int *flags)
 {
@@ -1107,6 +1111,22 @@ vm_map_pptdev_mmio(struct vmctx *ctx, int bus, int slot, int func,
 }
 
 int
+vm_unmap_pptdev_mmio(struct vmctx *ctx, int bus, int slot, int func,
+		     vm_paddr_t gpa, size_t len)
+{
+	struct vm_pptdev_mmio pptmmio;
+
+	bzero(&pptmmio, sizeof(pptmmio));
+	pptmmio.bus = bus;
+	pptmmio.slot = slot;
+	pptmmio.func = func;
+	pptmmio.gpa = gpa;
+	pptmmio.len = len;
+
+	return (ioctl(ctx->fd, VM_UNMAP_PPTDEV_MMIO, &pptmmio));
+}
+
+int
 vm_setup_pptdev_msi(struct vmctx *ctx, int vcpu, int bus, int slot, int func,
     uint64_t addr, uint64_t msg, int numvec)
 {
@@ -1210,6 +1230,19 @@ vm_map_pptdev_mmio(struct vmctx *ctx, int pptfd, vm_paddr_t gpa, size_t len,
 }
 
 int
+vm_unmap_pptdev_mmio(struct vmctx *ctx, int pptfd, vm_paddr_t gpa, size_t len)
+{
+	struct vm_pptdev_mmio pptmmio;
+
+	bzero(&pptmmio, sizeof(pptmmio));
+	pptmmio.pptfd = pptfd;
+	pptmmio.gpa = gpa;
+	pptmmio.len = len;
+
+	return (ioctl(ctx->fd, VM_UNMAP_PPTDEV_MMIO, &pptmmio));
+}
+
+int
 vm_setup_pptdev_msi(struct vmctx *ctx, int vcpu, int pptfd, uint64_t addr,
     uint64_t msg, int numvec)
 {
@@ -1919,7 +1952,7 @@ vm_get_ioctls(size_t *len)
 	/* keep in sync with machine/vmm_dev.h */
 	static const cap_ioctl_t vm_ioctl_cmds[] = { VM_RUN, VM_SUSPEND, VM_REINIT,
 	    VM_ALLOC_MEMSEG, VM_GET_MEMSEG, VM_MMAP_MEMSEG, VM_MMAP_MEMSEG,
-	    VM_MMAP_GETNEXT, VM_SET_REGISTER, VM_GET_REGISTER,
+	    VM_MMAP_GETNEXT, VM_MUNMAP_MEMSEG, VM_SET_REGISTER, VM_GET_REGISTER,
 	    VM_SET_SEGMENT_DESCRIPTOR, VM_GET_SEGMENT_DESCRIPTOR,
 	    VM_SET_REGISTER_SET, VM_GET_REGISTER_SET,
 	    VM_SET_KERNEMU_DEV, VM_GET_KERNEMU_DEV,
@@ -1929,7 +1962,7 @@ vm_get_ioctls(size_t *len)
 	    VM_ISA_DEASSERT_IRQ, VM_ISA_PULSE_IRQ, VM_ISA_SET_IRQ_TRIGGER,
 	    VM_SET_CAPABILITY, VM_GET_CAPABILITY, VM_BIND_PPTDEV,
 	    VM_UNBIND_PPTDEV, VM_MAP_PPTDEV_MMIO, VM_PPTDEV_MSI,
-	    VM_PPTDEV_MSIX, VM_PPTDEV_DISABLE_MSIX,
+	    VM_PPTDEV_MSIX, VM_UNMAP_PPTDEV_MMIO, VM_PPTDEV_DISABLE_MSIX,
 	    VM_INJECT_NMI, VM_STATS, VM_STAT_DESC,
 	    VM_SET_X2APIC_STATE, VM_GET_X2APIC_STATE,
 	    VM_GET_HPET_CAPABILITIES, VM_GET_GPA_PMAP, VM_GLA2GPA,
diff --git a/usr/src/lib/libvmmapi/common/vmmapi.h b/usr/src/lib/libvmmapi/common/vmmapi.h
index 72e43a4e3d..79c7dc02ee 100644
--- a/usr/src/lib/libvmmapi/common/vmmapi.h
+++ b/usr/src/lib/libvmmapi/common/vmmapi.h
@@ -132,6 +132,8 @@ int	vm_get_devmem_offset(struct vmctx *ctx, int segid, off_t *mapoff);
 int	vm_mmap_memseg(struct vmctx *ctx, vm_paddr_t gpa, int segid,
 	    vm_ooffset_t segoff, size_t len, int prot);
 
+int	vm_munmap_memseg(struct vmctx *ctx, vm_paddr_t gpa, size_t len);
+
 int	vm_create(const char *name);
 int	vm_get_device_fd(struct vmctx *ctx);
 struct vmctx *vm_open(const char *name);
@@ -220,6 +222,8 @@ int	vm_assign_pptdev(struct vmctx *ctx, int bus, int slot, int func);
 int	vm_unassign_pptdev(struct vmctx *ctx, int bus, int slot, int func);
 int	vm_map_pptdev_mmio(struct vmctx *ctx, int bus, int slot, int func,
 			   vm_paddr_t gpa, size_t len, vm_paddr_t hpa);
+int	vm_unmap_pptdev_mmio(struct vmctx *ctx, int bus, int slot, int func,
+			     vm_paddr_t gpa, size_t len);
 int	vm_setup_pptdev_msi(struct vmctx *ctx, int vcpu, int bus, int slot,
 	    int func, uint64_t addr, uint64_t msg, int numvec);
 int	vm_setup_pptdev_msix(struct vmctx *ctx, int vcpu, int bus, int slot,
@@ -233,6 +237,8 @@ int	vm_assign_pptdev(struct vmctx *ctx, int pptfd);
 int	vm_unassign_pptdev(struct vmctx *ctx, int pptfd);
 int	vm_map_pptdev_mmio(struct vmctx *ctx, int pptfd, vm_paddr_t gpa,
     size_t len, vm_paddr_t hpa);
+int	vm_unmap_pptdev_mmio(struct vmctx *ctx, int pptfd, vm_paddr_t gpa,
+    size_t len);
 int	vm_setup_pptdev_msi(struct vmctx *ctx, int vcpu, int pptfd,
     uint64_t addr, uint64_t msg, int numvec);
 int	vm_setup_pptdev_msix(struct vmctx *ctx, int vcpu, int pptfd,
diff --git a/usr/src/man/man1m/Makefile b/usr/src/man/man1m/Makefile
index fc43842db2..ee30a593f4 100644
--- a/usr/src/man/man1m/Makefile
+++ b/usr/src/man/man1m/Makefile
@@ -18,7 +18,7 @@
 # Copyright 2018 Nexenta Systems, Inc.
 # Copyright (c) 2017, Chris Fraire <cfraire@me.com>.
 # Copyright 2020 Peter Tribble
-# Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
+# Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
 #
 
 include		$(SRC)/Makefile.master
@@ -546,6 +546,8 @@ _MANFILES=	6to4relay.1m		\
 i386_MANFILES=				\
 		acpidump.1m		\
 		acpixtract.1m		\
+		bhyve.1m		\
+		bhyvectl.1m		\
 		nvmeadm.1m		\
 		pptadm.1m		\
 		rdmsr.1m
diff --git a/usr/src/man/man1m/bhyve.1m b/usr/src/man/man1m/bhyve.1m
new file mode 100644
index 0000000000..cd5a002b61
--- /dev/null
+++ b/usr/src/man/man1m/bhyve.1m
@@ -0,0 +1,673 @@
+.\" Copyright (c) 2013 Peter Grehan
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" Portions Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
+.\"
+.Dd March 18, 2021
+.Dt BHYVE 1M
+.Os
+.Sh NAME
+.Nm bhyve
+.Nd "run a guest operating system inside a virtual machine"
+.Sh SYNOPSIS
+.Nm
+.Op Fl aCDdeHhPSuWwxY
+.Oo
+.Sm off
+.Fl c\~
+.Oo
+.Op Cm cpus=
+.Ar numcpus
+.Oc
+.Op Cm ,sockets= Ar n
+.Op Cm ,cores= Ar n
+.Op Cm ,threads= Ar n
+.Oc
+.Sm on
+.Op Fl G Ar port
+.Oo Fl B
+.Sm off
+.Ar type Ns \&,
+.Op Ar key Ns = Ns Ar value
+.Oo \&, Ns Ar key Ns = Ns Ar value Ns Oc \&...
+.Oc
+.Sm on
+.Op Fl k Ar file
+.Oo Fl l
+.Sm off
+.Cm help \&| Ar lpcdev Op Cm \&, Ar conf
+.Sm on
+.Oc
+.Oo Fl m
+.Sm off
+.Ar memsize
+.Oo
+.Cm K No \&| Cm k No \&| Cm M No \&| Cm m No \&| Cm G No \&| Cm g No \&| Cm T No \&| Cm t
+.Oc
+.Sm on
+.Oc
+.Op Fl o Ar var Ns Cm = Ns Ar value
+.Op Fl r Ar file
+.Oo Fl s
+.Sm off
+.Cm help \&| Ar slot Cm \&, Ar emulation Op Cm \&, Ar conf
+.Sm on
+.Oc
+.Op Fl U Ar uuid
+.Ar vmname
+.Sh DESCRIPTION
+.Nm
+is a hypervisor that runs guest operating systems inside a
+virtual machine.
+.Pp
+Parameters such as the number of virtual CPUs, amount of guest memory, and
+I/O connectivity can be specified with command-line parameters.
+.Pp
+.Nm
+runs until the guest operating system reboots or an unhandled hypervisor
+exit is detected.
+.Sh OPTIONS
+.Bl -tag -width 10n
+.It Fl a
+The guest's local APIC is configured in xAPIC mode.
+The xAPIC mode is the default setting so this option is redundant.
+It will be deprecated in a future version.
+.It Xo
+.Fl B
+.Sm off
+.Ar type Ns \&,
+.Op Ar key Ns = Ns Ar value
+.Oo \&, Ns Ar key Ns = Ns Ar value Ns Oc \&...
+.Sm on
+.Xc
+Configure smbios data.
+.Ar type
+must be set to
+.Sy 1 .
+Supported keys are:
+.Sy manufacturer ,
+.Sy product ,
+.Sy version ,
+.Sy serial ,
+.Sy sku ,
+.Sy family
+and
+.Sy uuid .
+.It Xo Fl c
+.Sm off
+.Op Oo Cm cpus= Oc Ar numcpus
+.Op Cm ,sockets= Ar n
+.Op Cm ,cores= Ar n
+.Op Cm ,threads= Ar n
+.Xc
+.Sm on
+Number of guest virtual CPUs
+and/or the CPU topology.
+The default value for each of
+.Ar numcpus ,
+.Ar sockets ,
+.Ar cores ,
+and
+.Ar threads
+is 1.
+The current maximum number of guest virtual CPUs is 32.
+If
+.Ar numcpus
+is not specified then it will be calculated from the other arguments.
+The topology must be consistent in that the
+.Ar numcpus
+must equal the product of
+.Ar sockets ,
+.Ar cores ,
+and
+.Ar threads .
+If a
+.Ar setting
+is specified more than once the last one has precedence.
+.It Fl C
+Include guest memory in core file.
+.It Fl d
+Suspend CPUs at boot.
+.It Fl D
+Destroy the VM on guest initiated power-off.
+.It Fl e
+Force
+.Nm
+to exit when a guest issues an access to an I/O port that is not emulated.
+This is intended for debug purposes.
+.It Fl G Ar port
+Start a debug server that uses the GDB protocol to export guest state to a
+debugger.
+An IPv4 TCP socket will be bound to the supplied
+.Ar port
+to listen for debugger connections.
+Only a single debugger may be attached to the debug server at a time.
+If
+.Ar port
+begins with
+.Sq w ,
+.Nm
+will pause execution at the first instruction waiting for a debugger to attach.
+.It Fl h
+Print help message and exit.
+.It Fl H
+Yield the virtual CPU thread when a HLT instruction is detected.
+If this option is not specified, virtual CPUs will use 100% of a host CPU.
+.It Fl k Ar file
+Set configuration variables from a simple, key-value config file.
+Each line of the config file is expected to consist of a config variable
+name, an equals sign
+.Pq Sq = ,
+and a value.
+No spaces are permitted between the variable name, equals sign, or
+value.
+Blank lines and lines starting with
+.Sq #
+are ignored.
+.It Fl l Cm help Ns \&| Ns Ar lpcdev Ns Op , Ns Ar conf
+Allow devices behind the LPC PCI-ISA bridge to be configured.
+The only supported devices are the TTY-class devices
+.Ar com1
+through
+.Ar com4 ,
+the boot ROM device
+.Ar bootrom ,
+and the debug/test device
+.Ar pc-testdev .
+.Pp
+.Cm help
+print a list of supported LPC devices.
+.It Fl m Ar memsize Ns Op Ar K|k|M|m|G|g|T|t
+Guest physical memory size.
+The size argument may be suffixed with one of K, M, G or T (either upper
+or lower case) to indicate a multiple of kibibytes, mebibytes, gibibytes,
+or tebibytes.
+If no suffix is given, the value is assumed to be in mebibytes.
+.Pp
+.Ar memsize
+defaults to 256MiB.
+.It Fl o Ar var Ns Cm = Ns Ar value
+Set the configuration variable
+.Ar var
+to
+.Ar value .
+.It Fl P
+Force the guest virtual CPU to exit when a PAUSE instruction is detected.
+.It Fl s Cm help Ns \&| Ns Ar slot Ns \&, Ns Ar emulation Ns Op , Ns Ar conf
+Configure a virtual PCI slot and function.
+.Pp
+.Nm
+provides PCI bus emulation and virtual devices that can be attached to
+slots on the bus.
+There are 32 available slots, with the option of providing up to 8 functions
+per slot.
+.Bl -tag -width 10n
+.It Cm help
+print a list of supported PCI devices.
+.It Ar slot
+.Ar pcislot[:function]
+.Ar bus:pcislot:function
+.Pp
+The
+.Ar pcislot
+value is 0 to 31.
+The optional
+.Ar functiwidth on
+value is 0 to 7.
+The optional
+.Ar bus
+value is 0 to 255.
+If not specified, the
+.Ar function
+value defaults to 0.
+If not specified, the
+.Ar bus
+value defaults to 0.
+.It Ar emulation
+.Bl -tag -width 10n
+.It Li hostbridge \&| Li amd_hostbridge
+.Pp
+Provide a simple host bridge.
+This is usually configured at slot 0, and is required by most guest
+operating systems.
+The
+.Li amd_hostbridge
+emulation is identical but uses a PCI vendor ID of
+.Li AMD .
+.It Li passthru
+PCI pass-through device.
+.It Li virtio-net-viona
+Virtio network interface.
+.It Li virtio-blk
+Virtio block storage interface.
+.It Li virtio-rnd
+Virtio random number generator interface.
+.It Li virtio-console
+Virtio console interface, which exposes multiple ports
+to the guest in the form of simple char devices for simple IO
+between the guest and host userspaces.
+.It Li ahci
+AHCI controller attached to arbitrary devices.
+.It Li ahci-cd
+AHCI controller attached to an ATAPI CD/DVD.
+.It Li ahci-hd
+AHCI controller attached to a SATA hard-drive.
+.It Li e1000
+Intel e82545 network interface.
+.It Li uart
+PCI 16550 serial device.
+.It Li lpc
+LPC PCI-ISA bridge with COM1 to COM4 16550 serial ports, a boot ROM, and,
+optionally, the debug/test device.
+The LPC bridge emulation can only be configured on bus 0.
+.It Li fbuf
+Raw framebuffer device attached to VNC server.
+.It Li xhci
+eXtensible Host Controller Interface (xHCI) USB controller.
+.It Li nvme
+NVM Express (NVMe) controller.
+.El
+.It Op Ar conf
+This optional parameter describes the backend for device emulations.
+If
+.Ar conf
+is not specified, the device emulation has no backend and can be
+considered unconnected.
+.Pp
+Host Bridge Devices
+.Bl -tag -width 10n
+.It Cm model Ns = Ns Ar model
+Specify a hostbridge model to emulate.
+Valid model strings, and their associated vendor and device IDs are:
+.Sy amd Pq 0x1022/0x7432 ,
+.Sy netapp Pq 0x1275/0x1275 ,
+.Sy i440fx Pq 0x8086/0x1237
+and
+.Sy q35 Pq 0x8086/0x29b0 .
+The default value of
+.Ar model
+is
+.Cm netapp .
+.It Cm vendor Ns = Ns Ar vendor
+PCI vendor ID.
+.It Cm devid Ns = Ns Ar devid
+PCI device ID.
+.El
+.Pp
+Providing extra configuration parameters for a host bridge is optional, but if
+parameters are provided then either
+.Va model
+by itself, or both of
+.Va vendor
+and
+.Va devid
+must be specified.
+.Pp
+Network backends:
+.Bl -tag -width 10n
+.It Oo Cm vnic Ns = Oc Ns Ar vnic Ns Oo , Ns Cm feature_mask Ns = Ns Ar mask Oc
+.Pp
+.Ar vnic
+is the name of a configured virtual NIC on the system.
+.Ar mask
+is applied to the virtio feature flags which are advertised to the guest.
+Bits set in the
+.Ar mask
+value are removed from the advertised features.
+.El
+.Pp
+Block storage devices:
+.Bl -tag -width 10n
+.It Pa /filename Ns Oo , Ns Ar block-device-options Oc
+.It Pa /dev/xxx Ns Oo , Ns Ar block-device-options Oc
+.El
+.Pp
+The
+.Ar block-device-options
+are:
+.Bl -tag -width 8n
+.It Cm nocache
+Open the file with
+.Dv O_DIRECT .
+.It Cm direct
+Open the file using
+.Dv O_SYNC .
+.It Cm ro
+Force the file to be opened read-only.
+.It Cm sectorsize Ns = Ns Ar logical Ns Oo / Ns Ar physical Oc
+Specify the logical and physical sector sizes of the emulated disk.
+The physical sector size is optional and is equal to the logical sector size
+if not explicitly specified.
+.It Cm nodelete
+Disable emulation of guest trim requests via
+.Dv DIOCGDELETE
+requests.
+.El
+.Pp
+TTY devices:
+.Bl -tag -width 10n
+.It Cm stdio
+Connect the serial port to the standard input and output of
+the
+.Nm
+process.
+.It Pa /dev/xxx
+Use the host TTY device for serial port I/O.
+.El
+.Pp
+Boot ROM device:
+.Bl -tag -width 10n
+.It Pa romfile
+Map
+.Ar romfile
+in the guest address space reserved for boot firmware.
+.El
+.Pp
+Pass-through devices:
+.Bl -tag -width 10n
+.It Pa /dev/ppt Ns Ar N
+Connect to a PCI device on the host identified by the specificed path.
+.El
+.Pp
+Guest memory must be wired using the
+.Fl S
+option when a pass-through device is configured.
+.Pp
+The host device must have been previously attached to the
+.Sy ppt
+driver.
+.Pp
+Virtio console devices:
+.Bl -tag -width 10n
+.It Li port1= Ns Pa /path/to/port1.sock Ns ,anotherport= Ns Pa ...
+A maximum of 16 ports per device can be created.
+Every port is named and corresponds to a UNIX domain socket created by
+.Nm .
+.Nm
+accepts at most one connection per port at a time.
+.Pp
+Limitations:
+.Bl -bullet -offset 2n
+.It
+Due to lack of destructors in
+.Nm ,
+sockets on the filesystem must be cleaned up manually after
+.Nm
+exits.
+.It
+There is no way to use the "console port" feature, nor the console port
+resize at present.
+.It
+Emergency write is advertised, but no-op at present.
+.El
+.El
+.Pp
+Framebuffer devices:
+.Bl -tag -width 10n
+.It Xo
+.Sm off
+.Oo
+.Cm rfb No = Oo Ar IP \&: Oc Ar port
+|
+.Cm unix No = Ar path
+.Oc
+.Op \&, Cm w No = Ar width
+.Op \&, Cm h No = Ar height
+.Op \&, Cm vga No = Ar vgaconf
+.Op \&, Cm wait
+.Op \&, Cm password No = Ar password
+.Sm on
+.Xc
+.Bl -tag -width 8n
+.It Cm unix Ns = Ns Ar path
+The path to a UNIX socket which will be created and where
+.Nm
+will accept VNC connections.
+.It Cm rfb Ns = Ns Ar IPv4:port No or Ar [IPv6%zone]:port
+An
+.Ar IP
+address and a
+.Ar port
+VNC should listen on.
+The default is to listen on localhost IPv4 address and default VNC port 5900.
+An IPv6 address must be enclosed in square brackets and may contain an
+optional zone identifier.
+.It Cm w Ns = Ns Ar width No and Cm h Ns = Ns Ar height
+A display resolution, width and height, respectively.
+If not specified, a default resolution of 1024x768 pixels will be used.
+Minimal supported resolution is 640x480 pixels,
+and maximum is 1920x1200 pixels.
+.It Cm vga Ns = Ns Ar vgaconf
+Possible values for this option are
+.Dq io
+(default),
+.Dq on
+, and
+.Dq off .
+PCI graphics cards have a dual personality in that they are
+standard PCI devices with BAR addressing, but may also
+implicitly decode legacy VGA I/O space
+.Pq Ad 0x3c0-3df
+and memory space
+.Pq 64KB at Ad 0xA0000 .
+The default
+.Dq io
+option should be used for guests that attempt to issue BIOS calls which result
+in I/O port queries, and fail to boot if I/O decode is disabled.
+.Pp
+The
+.Dq on
+option should be used along with the CSM BIOS capability in UEFI
+to boot traditional BIOS guests that require the legacy VGA I/O and
+memory regions to be available.
+.Pp
+The
+.Dq off
+option should be used for the UEFI guests that assume that
+VGA adapter is present if they detect the I/O ports.
+An example of such a guest is
+.Ox
+in UEFI mode.
+.It Cm wait
+Instruct
+.Nm
+to only boot upon the initiation of a VNC connection, simplifying the
+installation of operating systems that require immediate keyboard input.
+This can be removed for post-installation use.
+.It Cm password Ns = Ns Ar password
+This type of authentication is known to be cryptographically weak and is not
+intended for use on untrusted networks.
+Many implementations will want to use stronger security, such as running
+the session over an encrypted channel provided by IPsec or SSH.
+.El
+.El
+.Pp
+xHCI USB devices:
+.Bl -tag -width 10n
+.It Li tablet
+A USB tablet device which provides precise cursor synchronization
+when using VNC.
+.El
+.Pp
+NVMe devices:
+.Bl -tag -width 10n
+.It Li path
+Accepted device paths are:
+.Ar /dev/blockdev
+or
+.Ar /path/to/image
+or
+.Ar ram=size_in_MiB .
+.It Li maxq
+Max number of queues.
+.It Li qsz
+Max elements in each queue.
+.It Li ioslots
+Max number of concurrent I/O requests.
+.It Li sectsz
+Sector size (defaults to blockif sector size).
+.It Li ser
+Serial number with maximum 20 characters.
+.El
+.Pp
+AHCI devices:
+.Bl -tag -width 10n
+.It Li nmrr
+Nominal Media Rotation Rate, known as RPM. value 1 will indicate device as Solid State Disk. default value is 0, not report.
+.It Li ser
+Serial Number with maximum 20 characters.
+.It Li rev
+Revision Number with maximum 8 characters.
+.It Li model
+Model Number with maximum 40 characters.
+.El
+.El
+.It Fl S
+Wire guest memory.
+.It Fl u
+RTC keeps UTC time.
+.It Fl U Ar uuid
+Set the universally unique identifier
+.Pq UUID
+in the guest's System Management BIOS System Information structure.
+By default a UUID is generated from the host's hostname and
+.Ar vmname .
+.It Fl w
+Ignore accesses to unimplemented Model Specific Registers (MSRs).
+This is intended for debug purposes.
+.It Fl W
+Force virtio PCI device emulations to use MSI interrupts instead of MSI-X
+interrupts.
+.It Fl x
+The guest's local APIC is configured in x2APIC mode.
+.It Fl Y
+Disable MPtable generation.
+.It Ar vmname
+Alphanumeric name of the guest.
+.El
+.Sh CONFIGURATION VARIABLES
+.Nm
+uses an internal tree of configuration variables to describe global and
+per-device settings.
+When
+.Nm
+starts,
+it parses command line options (including config files) in the order given
+on the command line.
+Each command line option sets one or more configuration variables.
+For example,
+the
+.Fl s
+option creates a new tree node for a PCI device and sets one or more variables
+under that node including the device model and device model-specific variables.
+Variables may be set multiple times during this parsing stage with the final
+value overriding previous values.
+.Pp
+Once all of the command line options have been processed,
+the configuration values are frozen.
+.Nm
+then uses the value of configuration values to initialize device models
+and global settings.
+.Pp
+More details on configuration variables can be found in
+.Xr bhyve_config 4 .
+.Sh SIGNAL HANDLING
+.Nm
+deals with the following signals:
+.Pp
+.Bl -tag -width indent -compact
+.It SIGTERM
+Trigger ACPI poweroff for a VM
+.El
+.Sh EXIT STATUS
+Exit status indicates how the VM was terminated:
+.Pp
+.Bl -tag -width indent -compact
+.It 0
+rebooted
+.It 1
+powered off
+.It 2
+halted
+.It 3
+triple fault
+.It 4
+exited due to an error
+.El
+.Sh EXAMPLES
+To run a virtual machine with 1GB of memory, two virtual CPUs, a virtio
+block device backed by the
+.Pa /my/image
+filesystem image, and a serial port for the console:
+.Bd -literal -offset indent
+bhyve -c 2 -s 0,hostbridge -s 1,lpc -s 2,virtio-blk,/my/image \e
+  -l com1,stdio -A -H -P -m 1G vm1
+.Ed
+.Pp
+Run a 24GB single-CPU virtual machine with three network ports.
+.Bd -literal -offset indent
+bhyve -s 0,hostbridge -s 1,lpc -s 2:0,virtio-net-viona,vmvnic0 \e
+  -s 2:1,virtio-net-viona,vmvnic1 -s 2:2,virtio-net-viona,vmvnic2 \e
+  -s 3,virtio-blk,/my/image -l com1,stdio \e
+  -A -H -P -m 24G bigvm
+.Ed
+.Pp
+Run an 8GB virtual machine with 2 quad core CPUs, 2 NVMe disks and one other
+disk attached as a Virtio block device, an AHCI ATAPI CD-ROM, a single viona
+network port, an i440fx hostbridge, and the console port connected to a socket.
+.Bd -literal -offset indent
+bhyve -c sockets=2,cores=4,threads=2 \e
+  -s 0,hostbridge,model=i440fx -s 1,lpc \e
+  -s 1:0,nvme,/dev/zvol/rdsk/tank/hdd0 \e
+  -s 1:1,nvme,/dev/zvol/rdsk/tank/hdd1 \e
+  -s 1:2,virtio-blk,/dev/zvol/rdsk/tank/hdd1 \e
+  -s 2:0,ahci,cd:/images/install.iso \e
+  -s 3,virtio-net-viona,vnic=vmvnic0 \e
+  -l com1,socket,/tmp/vm.com1,wait \e
+  -A -H -P -m 8G
+.Ed
+.Pp
+Run a UEFI virtual machine with a display resolution of 800 by 600 pixels
+that can be accessed via VNC at: 0.0.0.0:5900.
+.Bd -literal -offset indent
+bhyve -c 2 -m 4G -w -H \e
+  -s 0,hostbridge \e
+  -s 3,ahci-cd,/path/to/uefi-OS-install.iso \e
+  -s 4,nvme,/dev/zvol/rdsk/tank/hdd0 \e
+  -s 5,virtio-net-viona,vnic=vnmic0 \e
+  -s 29,fbuf,rfb=0.0.0.0:5900,w=800,h=600,wait \e
+  -s 30,xhci,tablet \e
+  -s 31,lpc -l com1,stdio \e
+  -l bootrom,/usr/share/bhyve/firmware/BHYVE_UEFI.fd \e
+   uefivm
+.Ed
+.Sh SEE ALSO
+.Xr bhyvectl 1M ,
+.Xr bhyve_config 4 ,
+.Pp
+.Rs
+.%A Intel
+.%B 64 and IA-32 Architectures Software Developer’s Manual
+.%V Volume 3
+.Re
diff --git a/usr/src/man/man1m/bhyvectl.1m b/usr/src/man/man1m/bhyvectl.1m
new file mode 100644
index 0000000000..baa3667320
--- /dev/null
+++ b/usr/src/man/man1m/bhyvectl.1m
@@ -0,0 +1,85 @@
+.\" Copyright (c) 2015 Christian Brueffer
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" Portions Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
+.\"
+.Dd May 4, 2020
+.Dt BHYVECTL 1M
+.Os
+.Sh NAME
+.Nm bhyvectl
+.Nd "control utility for bhyve instances"
+.Sh SYNOPSIS
+.Nm
+.Fl -vm= Ns Ar <vmname>
+.Op Fl -create
+.Op Fl -destroy
+.Op Fl -get-stats
+.Op Fl -inject-nmi
+.Op Fl -force-reset
+.Op Fl -force-poweroff
+.Op Fl -checkpoint= Ns Ar <filename>
+.Op Fl -suspend= Ns Ar <filename>
+.Sh DESCRIPTION
+The
+.Nm
+command is a control utility for active
+.Xr bhyve 1M
+virtual machine instances.
+.Pp
+.Em Note :
+Most
+.Nm
+flags are intended for querying and setting the state of an active instance.
+These commands are intended for development purposes, and are not documented here.
+A complete list can be obtained by executing
+.Nm
+without any arguments.
+.Pp
+The user-facing options are as follows:
+.Bl -tag -width ".Fl d Ar argument"
+.It Fl -vm= Ns Ar <vmname>
+Operate on the virtual machine
+.Ar <vmname> .
+.It Fl -create
+Create the specified VM.
+.It Fl -destroy
+Destroy the specified VM.
+.It Fl -get-stats
+Retrieve statistics for the specified VM.
+.It Fl -inject-nmi
+Inject a non-maskable interrupt (NMI) into the VM.
+.It Fl -force-reset
+Force the VM to reset.
+.It Fl -force-poweroff
+Force the VM to power off.
+.El
+.Sh EXIT STATUS
+.Ex -std
+.Sh EXAMPLES
+Destroy the VM called fbsd10:
+.Pp
+.Dl "bhyvectl --vm=fbsd10 --destroy"
+.Sh SEE ALSO
+.Xr bhyve 1M
diff --git a/usr/src/man/man4/Makefile b/usr/src/man/man4/Makefile
index 2008cfee3b..642829e62a 100644
--- a/usr/src/man/man4/Makefile
+++ b/usr/src/man/man4/Makefile
@@ -14,11 +14,12 @@
 # Copyright 2015, Joyent, Inc.
 # Copyright 2018 Nexenta Systems, Inc.
 # Copyright 2018 Gary Mills
+# Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
 #
 
 include		$(SRC)/Makefile.master
 
-MANSECT= 	4
+MANSECT=	4
 
 _MANFILES=	Intro.4				\
 		NISLDAPmapping.4		\
@@ -202,7 +203,8 @@ _MANFILES=	Intro.4				\
 
 sparc_MANFILES=	sbus.4
 
-i386_MANFILES=	loader.conf.4			\
+i386_MANFILES=	bhyve_config.4			\
+		loader.conf.4			\
 		sysbus.4
 
 _MANLINKS=	addresses.4			\
diff --git a/usr/src/man/man4/bhyve_config.4 b/usr/src/man/man4/bhyve_config.4
new file mode 100644
index 0000000000..b3d4252a8c
--- /dev/null
+++ b/usr/src/man/man4/bhyve_config.4
@@ -0,0 +1,474 @@
+.\" SPDX-License-Identifier: BSD-2-Clause
+.\"
+.\" Copyright (c) 2021 John H. Baldwin <jhb@FreeBSD.org>
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" Portions Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
+.\"
+.Dd March 24, 2021
+.Dt BHYVE_CONFIG 4
+.Os
+.Sh NAME
+.Nm bhyve_config
+.Nd "bhyve configuration variables"
+.Sh DESCRIPTION
+.Xr bhyve 1M
+uses a hierarchical tree of configuration variables to describe global and
+per-device settings.
+Internal nodes in this tree do not have a value,
+only leaf nodes have values.
+This manual describes the configuration variables understood by
+.Xr bhyve 1M .
+If additional variables are defined,
+.Xr bhyve 1M
+will ignore them and will not emit errors for unknown variables.
+However, these additional variables can be referenced by other
+variables as described below.
+.Sh VARIABLE VALUES
+Configuration variable values are stored as strings.
+A configuration variable value may refer to one or more other
+configuration values by name.
+Instances of the pattern
+.Sq % Ns Pq Ar var
+are replaced by the value of the configuration variable
+.Va var .
+To avoid unwanted expansion,
+.Sq %
+characters can be escaped by a leading
+.Sq % .
+For example,
+if a configuration variable
+.Va disk
+uses the value
+.Pa /dev/zvol/bhyve/%(name) ,
+then the final value of the
+.Va disk
+variable will be set to the path of a ZFS volume whose name matches
+the name of the virtual machine on the pool
+.Pa bhyve .
+.Pp
+Some configuration variables may be interpreted as a boolean value.
+For those variables the following case-insensitive values may be used to
+indicate true:
+.Pp
+.Bl -bullet -offset indent -compact
+.It
+true
+.It
+on
+.It
+yes
+.It
+1
+.El
+.Pp
+The following values may be used to indicate false:
+.Pp
+.Bl -bullet -offset indent -compact
+.It
+false
+.It
+off
+.It
+no
+.It
+0
+.El
+.Pp
+Some configuration variables may be interpreted as an integer.
+For those variables,
+any syntax supported by
+.Xr strtoul 3C
+may be used.
+.Sh GLOBAL SETTINGS
+.Ss Architecture Neutral Settings
+.Bl -column "memory.guest_in_core" "integer" "Default"
+.It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description
+.It Va name Ta string Ta Ta
+The name of the VM.
+.It Va cpus Ta integer Ta 1 Ta
+The total number of virtual CPUs.
+.It Va cores Ta integer Ta 1 Ta
+The number of virtual cores in each virtual socket.
+.It Va threads Ta integer Ta 1 Ta
+The number of virtual CPUs in each virtual core.
+.It Va sockets Ta integer Ta 1 Ta
+The number of virtual sockets.
+.It Va memory.size Ta string Ta 256M Ta
+Guest physical memory size.
+The size argument may be suffixed with one of K, M, G or T (either upper
+or lower case) to indicate a multiple of kibibytes, mebibytes, gibibytes,
+or tebibytes.
+If no suffix is given, the value is assumed to be in mebibytes.
+.It Va memory.wired Ta bool Ta false Ta
+Wire guest memory.
+.It Va acpi_tables Ta bool Ta false Ta
+Generate ACPI tables.
+.It Va destroy_on_poweroff Ta bool Ta false Ta
+Destroy the VM on guest-initiated power-off.
+.It Va gdb.port Ta integer Ta 0 Ta
+TCP port number for the debug server.
+If this is set to a non-zero value, a debug server
+will listen for connections on this port.
+.It Va gdb.wait Ta bool Ta false Ta
+If the debug server is enabled, wait for a debugger to connect
+before starting the guest.
+.It Va rtc.use_localtime Ta bool Ta true Ta
+The real time clock uses the local time of the host.
+If this is set to false, the real time clock uses UTC.
+.It Va uuid Ta string Ta Ta
+The universally unique identifier (UUID) to use in the guest's
+System Management BIOS System Information structure.
+If an explicit value is not set, a valid UUID is generated from
+the host's hostname and the VM name.
+.It Va virtio_msix Ta bool Ta true Ta
+Use MSI-X interrupts for PCI VirtIO devices.
+If set to false, MSI interrupts are used instead.
+.It Va config.dump Ta bool Ta false Ta
+If this value is set to true, then
+.Xr bhyve 1M
+will write all of its configuration variables to stdout and exit
+after it has finished parsing command line options.
+.El
+.Ss x86-Specific Settings
+.Bl -column "x86.vmexit_on_pause" "integer" "Default"
+.It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description
+.It Va x86.mptable Ta bool Ta true Ta
+Generate an MPTable.
+.It Va x86.x2apic Ta bool Ta false Ta
+Configure guest's local APICs in x2APIC mode.
+.It Va x86.strictio Ta bool Ta false Ta
+Exit if a guest accesses an I/O port that is not emulated.
+By default, writes are ignored and reads return all bits set.
+.It Va x86.strictmsr Ta bool Ta true Ta
+Inject a general protection fault if a guest accesses a Model Specific
+Register (MSR) that is not emulated.
+If this is false, writes are ignored and reads return zero.
+.It Va x86.vmexit_on_hlt Ta bool Ta false Ta
+Force a VM exit when a guest CPU executes the
+.Dv HLT
+instruction.
+This allows idle guest CPUs to yield the host CPU.
+.It Va x86.vmexit_on_pause Ta bool Ta false Ta
+Force a VM exit when a guest CPU executes the
+.Dv PAUSE
+instruction.
+.El
+.Sh DEVICE SETTINGS
+Device settings are stored under a device node.
+The device node's name is set by the parent bus of the device.
+.Ss PCI Device Settings
+PCI devices are described by a device node named
+.Dq pci Ns Ar bus . Ns Ar slot . Ns Ar function
+where each of
+.Ar bus ,
+.Ar slot ,
+and
+.Ar function
+are formatted as decimal values with no padding.
+All PCI device nodes must contain a configuration variable named
+.Dq device
+which specifies the device model to use.
+The following PCI device models are supported:
+.Bl -tag -width indent
+.It Li hostbridge
+Provide a simple PCI-Host bridge device.
+This is usually configured at pci0:0:0 and is required by most guest
+operating systems.
+.It Li ahci
+AHCI storage controller.
+.It Li e1000
+Intel e82545 network interface.
+.It Li fbuf
+VGA framebuffer device attached to VNC server.
+.It Li lpc
+LPC PCI-ISA bridge with COM1-COM4 16550 serial ports,
+a boot ROM,
+and an optional debug/test device.
+This device must be configured on bus 0.
+.It Li nvme
+NVM Express (NVMe) controller.
+.It Li passthru
+PCI pass-through device.
+.It Li uart
+PCI 16550 serial device.
+.It Li virtio-blk
+VirtIO block storage interface.
+.It Li virtio-console
+VirtIO console interface.
+.It Li virtio-net-viona
+VirtIO network interface.
+.It Li virtio-rnd
+VirtIO random number generator interface.
+.It Li xhci
+Extensible Host Controller Interface (XHCI) USB controller.
+.El
+.Ss USB Device Settings
+USB controller devices contain zero or more child USB devices
+attached to slots.
+Each USB device stores its settings in a node named
+.Dq slot. Ns Va N
+under the controller's device node.
+.Va N
+is the number of the slot to which the USB device is attached.
+Note that USB slot numbers begin at 1.
+All USB device nodes must contain a configuration variable named
+.Dq device
+which specifies the device model to use.
+The following USB device models are supported:
+.Bl -tag -width indent
+.It Li tablet
+A USB tablet device which provides precise cursor synchronization
+when using VNC.
+.El
+.Ss Block Device Settings
+Block devices use the following settings to configure their backing store.
+These settings are stored in the configuration node of the respective device.
+.Bl -column "sectorsize" "logical[/physical]" "Default"
+.It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description
+.It path Ta string Ta Ta
+The path of the file or disk device to use as the backing store.
+.It nocache Ta bool Ta false Ta
+Disable caching on the backing file by opening the backing file with
+.Dv O_DIRECT .
+.It nodelete Ta bool Ta false Ta
+Disable emulation of guest trim requests via
+.Dv DIOCGDELETE
+requests.
+.It sync Ta bool Ta false Ta
+Write changes to the backing file with synchronous writes.
+.It direct Ta bool Ta false Ta
+An alias for
+.Va sync .
+.It ro Ta bool Ta false Ta
+Disable writes to the backing file.
+.It sectorsize Ta Va logical Ns Op / Ns Va physical Ta Ta
+Specify the logical and physical sector size of the emulated disk.
+If the physical size is not specified, it is set to be equal to the logical
+size.
+.El
+.Ss virtio-net-viona Settings
+Viona network devices use the following settings to configure their backend.
+.Bl -column "feature_flags" "string" "Default"
+.It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description
+.It vnic Ta string Ta Ta
+The VNIC to use for the network connection.
+.It feature_mask Ta integer Ta 0 Ta
+Specify a mask to apply to the virtio features advertised to the guest.
+.El
+.Ss UART Device Settings
+.Bl -column "Name" "Format" "Default"
+.It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description
+.It Va path Ta path Ta Ta
+Backend device for the serial port.
+Either the pathname of a character device or
+.Dq stdio
+to use standard input and output of the
+.Xr bhyve 1M
+process.
+.El
+.Ss Host Bridge Settings
+Host Bridge devices use the following settings.
+When configuring parameters, either the
+.Va model
+by itself, or both of
+.Va vendor
+and
+.Va devid
+must be specified.
+.Bl -column "vendor" "integer" "Default"
+.It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description
+.It Va model Ta string Ta netapp Ta
+Specify a hostbridge model to emulate.
+Valid model strings, and their associated vendor and device IDs are:
+.Sy amd Pq 0x1022/0x7432 ,
+.Sy netapp Pq 0x1275/0x1275 ,
+.Sy i440fx Pq 0x8086/0x1237
+and
+.Sy q35 Pq 0x8086/0x29b0 .
+.It Va vendor Ta integer Ta 0x1275 Ta
+PCI vendor ID.
+.It Va devid Ta integer Ta 0x1275 Ta
+PCI device ID.
+.El
+.Ss AHCI Controller Settings
+AHCI controller devices contain zero or more ports each of which
+provides a storage device.
+Each port stores its settings in a node named
+.Dq port. Ns Va N
+under the controller's device node.
+The
+.Va N
+values are formatted as successive decimal values starting with 0.
+In addition to the block device settings described above, each
+port supports the following settings:
+.Bl -column "model" "integer" "generated"
+.It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description
+.It Va type Ta string Ta Ta
+The type of storage device to emulate.
+Must be set to either
+.Dq cd
+or
+.Dq hd .
+.It Va nmrr Ta integer Ta 0 Ta
+Nominal Media Rotation Rate, also known as RPM.
+A value 1 of indicates a device with no rate such as a Solid State Disk.
+.It Va ser Ta string Ta generated Ta
+Serial number of up to twenty characters.
+A default serial number is generated using a hash of the backing
+store's pathname.
+.It Va rev Ta string Ta 001 Ta
+Revision number of up to eight characters.
+.It Va model Ta string Ta Ta
+Model number of up to forty characters.
+Separate default model strings are used for
+.Dq cd
+and
+.Dq hd
+device types.
+.El
+.Ss Frame Buffer Settings
+.Bl -column "password" "[IP:]port" "127.0.0.1:5900"
+.It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description
+.It Va wait Ta bool Ta false Ta
+Wait for a remote connection before starting the VM.
+.It Va rfb Ta Oo Ar IP Ns \&: Oc Ns Ar port Ta 127.0.0.1:5900 Ta
+TCP address to listen on for remote connections.
+The IP address must be given as a numeric address.
+IPv6 addresses must be enclosed in square brackets and
+support scoped identifiers as described in
+.Xr getaddrinfo 3socket .
+A bare port number may be given in which case the IPv4
+localhost address is used.
+.It Va unix Ta string Ta Ta
+UNIX socket to listen on for VNC connections.
+.It Va vga Ta string Ta io Ta
+VGA configuration.
+More details are provided in
+.Xr bhyve 1M .
+.It Va w Ta integer Ta 1024 Ta
+Frame buffer width in pixels.
+.It Va h Ta integer Ta 768 Ta
+Frame buffer height in pixels.
+.It Va password Ta string Ta Ta
+Password to use for VNC authentication.
+This type of authentication is known to be cryptographically weak and is not
+intended for use on untrusted networks.
+.El
+.Ss LPC Device Settings
+The LPC bridge stores its configuration under a top-level
+.Va lpc
+node rather than under the PCI LPC device's node.
+The following nodes are available under
+.Va lpc :
+.Bl -column "pc-testdev" "Format" "Default"
+.It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description
+.It Va bootrom Ta path Ta Ta
+Path to a boot ROM.
+The contents of this file are copied into the guest's
+memory ending just before the 4GB physical address.
+If a boot ROM is present, a firmware interface device is
+also enabled for use by the boot ROM.
+.It Va com1 Ta node Ta Ta
+Settings for the COM1 serial port device.
+.It Va com2 Ta node Ta Ta
+Settings for the COM2 serial port device.
+.It Va com3 Ta node Ta Ta
+Settings for the COM3 serial port device.
+.It Va com4 Ta node Ta Ta
+Settings for the COM4 serial port device.
+.It Va pc-testdev Ta bool Ta false Ta
+Enable the PC debug/test device.
+.El
+.Ss NVMe Controller Settings
+Each NVMe controller supports a single storage device.
+The device can be backed either by a memory disk described by the
+.Va ram
+variable, or a block device using the block device settings described above.
+In addition, each controller supports the following settings:
+.Bl -column "ioslots" "Format" "Default"
+.It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description
+.It Va maxq Ta integer Ta 16 Ta
+Maximum number of I/O submission and completion queue pairs.
+.It Va qsz Ta integer Ta 2058 Ta
+Number of elements in each I/O queue.
+.It Va ioslots Ta integer Ta 8 Ta
+Maximum number of concurrent I/O requests.
+.It Va sectsz Ta integer Ta Ta
+Sector size.
+Can be one of 512, 4096, or 8192.
+Devices backed by a memory disk use 4096 as the default.
+Devices backed by a block device use the block device's sector size
+as the default.
+.It Va ser Ta string Ta Ta
+Serial number of up to twenty characters.
+A default serial number is generated using a hash of the device's PCI address.
+.It Va eui64 Ta integer Ta Ta
+IEEE Extended Unique Identifier.
+If an EUI is not provided, a default is generated using a checksum of the
+device's PCI address.
+.It Va dsm Ta string Ta auto Ta
+Whether or not to advertise Dataset Management (DSM) support.
+One of
+.Dq auto ,
+.Dq enable ,
+or
+.Dq disable .
+The
+.Dq auto
+setting only advertises support if the backing store supports
+resource freeing, for example via TRIM.
+.It Va ram Ta integer Ta Ta
+If set, allocate a memory disk as the backing store.
+The value of this variable is the size of the memory disk in megabytes.
+.El
+.Ss PCI Passthrough Settings
+.Bl -column "Name" "integer" "Default"
+.It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description
+.It Va path Ta string Ta Ta
+Path to a PCI passthrough device in the form
+.Pa /dev/ppt Ns Ar N
+where
+.Ar N
+is the device number.
+.El
+.Ss VirtIO Console Device Settings
+Each VirtIO Console device contains one or more console ports.
+Each port stores its settings in a node named
+.Dq port. Ns Va N
+under the controller's device node.
+The
+.Va N
+values are formatted as successive decimal values starting with 0.
+Each port supports the following settings:
+.Bl -column "Name" "Format" "Default"
+.It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description
+.It Va name Ta string Ta Ta
+The name of the port exposed to the guest.
+.It Va path Ta path Ta Ta
+The path of a UNIX domain socket providing the host connection for the port.
+.El
+.Sh SEE ALSO
+.Xr bhyve 1M ,
+.Xr strtoul 3c ,
+.Xr getaddrinfo 3socket
diff --git a/usr/src/pkg/manifests/system-bhyve.mf b/usr/src/pkg/manifests/system-bhyve.mf
index 450f83954a..0495d9f649 100644
--- a/usr/src/pkg/manifests/system-bhyve.mf
+++ b/usr/src/pkg/manifests/system-bhyve.mf
@@ -15,7 +15,7 @@
 
 #
 # Copyright 2018 Joyent, Inc.
-# Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
+# Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
 #
 
 #
@@ -38,6 +38,7 @@ dir path=usr/sbin
 dir path=usr/share
 dir path=usr/share/man
 dir path=usr/share/man/man1m
+dir path=usr/share/man/man4
 driver name=ppt
 driver name=viona
 driver name=vmm
@@ -50,7 +51,10 @@ file path=usr/kernel/drv/vmm.conf
 file path=usr/sbin/bhyve mode=0555
 file path=usr/sbin/bhyvectl mode=0555
 file path=usr/sbin/pptadm mode=0555
+file path=usr/share/man/man1m/bhyve.1m
+file path=usr/share/man/man1m/bhyvectl.1m
 file path=usr/share/man/man1m/pptadm.1m
+file path=usr/share/man/man4/bhyve_config.4
 license lic_CDDL license=lic_CDDL
 license usr/src/uts/i86pc/io/vmm/THIRDPARTYLICENSE \
     license=usr/src/uts/i86pc/io/vmm/THIRDPARTYLICENSE
diff --git a/usr/src/uts/i86pc/io/vmm/amd/amdiommu.c b/usr/src/uts/i86pc/io/vmm/amd/amdiommu.c
new file mode 100644
index 0000000000..8634f88e77
--- /dev/null
+++ b/usr/src/uts/i86pc/io/vmm/amd/amdiommu.c
@@ -0,0 +1,185 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2021 The FreeBSD Foundation
+ *
+ * Portions of this software were developed by Ka Ho Ng
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/rman.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include "amdvi_priv.h"
+#include "ivhd_if.h"
+
+struct amdiommu_softc {
+	struct resource *event_res;	/* Event interrupt resource. */
+	void		*event_tag;	/* Event interrupt tag. */
+	int		event_rid;
+};
+
+static int	amdiommu_probe(device_t);
+static int	amdiommu_attach(device_t);
+static int	amdiommu_detach(device_t);
+static int	ivhd_setup_intr(device_t, driver_intr_t, void *,
+		    const char *);
+static int	ivhd_teardown_intr(device_t);
+
+static device_method_t amdiommu_methods[] = {
+	/* device interface */
+	DEVMETHOD(device_probe,			amdiommu_probe),
+	DEVMETHOD(device_attach,		amdiommu_attach),
+	DEVMETHOD(device_detach,		amdiommu_detach),
+	DEVMETHOD(ivhd_setup_intr,		ivhd_setup_intr),
+	DEVMETHOD(ivhd_teardown_intr,		ivhd_teardown_intr),
+	DEVMETHOD_END
+};
+static driver_t amdiommu_driver = {
+	"amdiommu",
+	amdiommu_methods,
+	sizeof (struct amdiommu_softc),
+};
+
+static int
+amdiommu_probe(device_t dev)
+{
+	int error;
+	int capoff;
+
+	/*
+	 * Check base class and sub-class
+	 */
+	if (pci_get_class(dev) != PCIC_BASEPERIPH ||
+	    pci_get_subclass(dev) != PCIS_BASEPERIPH_IOMMU)
+		return (ENXIO);
+
+	/*
+	 * A IOMMU capability block carries a 0Fh capid.
+	 */
+	error = pci_find_cap(dev, PCIY_SECDEV, &capoff);
+	if (error)
+		return (ENXIO);
+
+	/*
+	 * bit [18:16] == 011b indicates the capability block is IOMMU
+	 * capability block. If the field is not set to 011b, bail out.
+	 */
+	if ((pci_read_config(dev, capoff + 2, 2) & 0x7) != 0x3)
+		return (ENXIO);
+
+	return (BUS_PROBE_SPECIFIC);
+}
+
+static int
+amdiommu_attach(device_t dev)
+{
+
+	device_set_desc(dev, "AMD-Vi/IOMMU PCI function");
+	return (0);
+}
+
+static int
+amdiommu_detach(device_t dev)
+{
+
+	return (0);
+}
+
+static int
+ivhd_setup_intr(device_t dev, driver_intr_t handler, void *arg,
+    const char *desc)
+{
+	struct amdiommu_softc *sc;
+	int error, msicnt;
+
+	sc = device_get_softc(dev);
+	msicnt = 1;
+	if (sc->event_res != NULL)
+		panic("%s is called without intr teardown", __func__);
+	sc->event_rid = 1;
+
+	error = pci_alloc_msi(dev, &msicnt);
+	if (error) {
+		device_printf(dev, "Couldn't find event MSI IRQ resource.\n");
+		return (ENOENT);
+	}
+
+	sc->event_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+	    &sc->event_rid, RF_ACTIVE);
+	if (sc->event_res == NULL) {
+		device_printf(dev, "Unable to allocate event INTR resource.\n");
+		error = ENOMEM;
+		goto fail;
+	}
+
+	error = bus_setup_intr(dev, sc->event_res, INTR_TYPE_MISC | INTR_MPSAFE,
+	    NULL, handler, arg, &sc->event_tag);
+	if (error) {
+		device_printf(dev, "Fail to setup event intr\n");
+		goto fail;
+	}
+
+	bus_describe_intr(dev, sc->event_res, sc->event_tag, "%s", desc);
+	return (0);
+
+fail:
+	ivhd_teardown_intr(dev);
+	return (error);
+}
+
+static int
+ivhd_teardown_intr(device_t dev)
+{
+	struct amdiommu_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	if (sc->event_res != NULL) {
+		bus_teardown_intr(dev, sc->event_res, sc->event_tag);
+		sc->event_tag = NULL;
+	}
+	if (sc->event_res != NULL) {
+		bus_release_resource(dev, SYS_RES_IRQ, sc->event_rid,
+		    sc->event_res);
+		sc->event_res = NULL;
+	}
+	pci_release_msi(dev);
+	return (0);
+}
+
+static devclass_t amdiommu_devclass;
+
+/* This driver has to be loaded before ivhd */
+DRIVER_MODULE(amdiommu, pci, amdiommu_driver, amdiommu_devclass, 0, 0);
+MODULE_DEPEND(amdiommu, pci, 1, 1, 1);
diff --git a/usr/src/uts/i86pc/io/vmm/amd/amdvi_hw.c b/usr/src/uts/i86pc/io/vmm/amd/amdvi_hw.c
index e2f298ae09..869b692100 100644
--- a/usr/src/uts/i86pc/io/vmm/amd/amdvi_hw.c
+++ b/usr/src/uts/i86pc/io/vmm/amd/amdvi_hw.c
@@ -52,6 +52,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/vmparam.h>
 #include <machine/pci_cfgreg.h>
 
+#include "ivhd_if.h"
 #include "pcib_if.h"
 
 #include "io/iommu.h"
@@ -514,8 +515,7 @@ amdvi_dump_cmds(struct amdvi_softc *softc, int count)
 		printf("  [CMD%d, off:0x%x] opcode= 0x%x 0x%x"
 		    " 0x%x 0x%lx\n", i, off, cmd->opcode,
 		    cmd->word0, cmd->word1, cmd->addr);
-		off = (off + sizeof(struct amdvi_cmd)) %
-		    (softc->cmd_max * sizeof(struct amdvi_cmd));
+		off = MOD_INC(off, sizeof(struct amdvi_cmd), softc->cmd_max);
 	}
 }
 
@@ -772,99 +772,33 @@ amdvi_free_evt_intr_res(device_t dev)
 {
 
 	struct amdvi_softc *softc;
+	device_t mmio_dev;
 
 	softc = device_get_softc(dev);
-	if (softc->event_tag != NULL) {
-		bus_teardown_intr(dev, softc->event_res, softc->event_tag);
-	}
-	if (softc->event_res != NULL) {
-		bus_release_resource(dev, SYS_RES_IRQ, softc->event_rid,
-		    softc->event_res);
-	}
-	bus_delete_resource(dev, SYS_RES_IRQ, softc->event_rid);
-	PCIB_RELEASE_MSI(device_get_parent(device_get_parent(dev)),
-	    dev, 1, &softc->event_irq);
+	mmio_dev = softc->pci_dev;
+
+	IVHD_TEARDOWN_INTR(mmio_dev);
 }
 
 static bool
 amdvi_alloc_intr_resources(struct amdvi_softc *softc)
 {
 	struct amdvi_ctrl *ctrl;
-	device_t dev, pcib;
-	device_t mmio_dev;
-	uint64_t msi_addr;
-	uint32_t msi_data;
+	device_t dev, mmio_dev;
 	int err;
 
 	dev = softc->dev;
-	pcib = device_get_parent(device_get_parent(dev));
-	mmio_dev = pci_find_bsf(PCI_RID2BUS(softc->pci_rid),
-            PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid));
-	if (device_is_attached(mmio_dev)) {
-		device_printf(dev,
-		    "warning: IOMMU device is claimed by another driver %s\n",
-		    device_get_driver(mmio_dev)->name);
-	}
-
-	softc->event_irq = -1;
-	softc->event_rid = 0;
-
-	/*
-	 * Section 3.7.1 of IOMMU rev 2.0. With MSI, there is only one
-	 * interrupt. XXX: Enable MSI/X support.
-	 */
-	err = PCIB_ALLOC_MSI(pcib, dev, 1, 1, &softc->event_irq);
-	if (err) {
-		device_printf(dev,
-		    "Couldn't find event MSI IRQ resource.\n");
-		return (ENOENT);
-	}
-
-	err = bus_set_resource(dev, SYS_RES_IRQ, softc->event_rid,
-	    softc->event_irq, 1);
-	if (err) {
-		device_printf(dev, "Couldn't set event MSI resource.\n");
-		return (ENXIO);
-	}
-
-	softc->event_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
-	    &softc->event_rid, RF_ACTIVE);
-	if (!softc->event_res) {
-		device_printf(dev,
-		    "Unable to allocate event INTR resource.\n");
-		return (ENOMEM);
-	}
-
-	if (bus_setup_intr(dev, softc->event_res,
-	    INTR_TYPE_MISC | INTR_MPSAFE, NULL, amdvi_event_intr,
-	    softc, &softc->event_tag)) {
-		device_printf(dev, "Fail to setup event intr\n");
-		bus_release_resource(softc->dev, SYS_RES_IRQ,
-		    softc->event_rid, softc->event_res);
-		softc->event_res = NULL;
-		return (ENXIO);
-	}
-
-	bus_describe_intr(dev, softc->event_res, softc->event_tag,
-	    "fault");
-
-	err = PCIB_MAP_MSI(pcib, dev, softc->event_irq, &msi_addr,
-	    &msi_data);
-	if (err) {
-		device_printf(dev,
-		    "Event interrupt config failed, err=%d.\n",
-		    err);
-		amdvi_free_evt_intr_res(softc->dev);
-		return (err);
-	}
+	mmio_dev = softc->pci_dev;
 
 	/* Clear interrupt status bits. */
 	ctrl = softc->ctrl;
 	ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
 
-	/* Now enable MSI interrupt. */
-	pci_enable_msi(mmio_dev, msi_addr, msi_data);
-	return (0);
+	err = IVHD_SETUP_INTR(mmio_dev, amdvi_event_intr, softc, "fault");
+	if (err)
+		device_printf(dev, "Interrupt setup failed on %s\n",
+		    device_get_nameunit(mmio_dev));
+	return (err);
 }
 
 static void
diff --git a/usr/src/uts/i86pc/io/vmm/amd/amdvi_priv.h b/usr/src/uts/i86pc/io/vmm/amd/amdvi_priv.h
index 5d47142a72..8d54d8ab9a 100644
--- a/usr/src/uts/i86pc/io/vmm/amd/amdvi_priv.h
+++ b/usr/src/uts/i86pc/io/vmm/amd/amdvi_priv.h
@@ -375,17 +375,14 @@ enum IvrsType
 struct amdvi_softc {
 	struct amdvi_ctrl *ctrl;	/* Control area. */
 	device_t	dev;		/* IOMMU device. */
+	device_t	pci_dev;	/* IOMMU PCI function device. */
 	enum IvrsType   ivhd_type;	/* IOMMU IVHD type. */
 	bool		iotlb;		/* IOTLB supported by IOMMU */
 	struct amdvi_cmd *cmd;		/* Command descriptor area. */
 	int		cmd_max;	/* Max number of commands. */
 	uint64_t	cmp_data;	/* Command completion write back. */
 	struct amdvi_event *event;	/* Event descriptor area. */
-	struct resource *event_res;	/* Event interrupt resource. */
-	void		*event_tag;	/* Event interrupt tag. */
 	int		event_max;	/* Max number of events. */
-	int		event_irq;
-	int		event_rid;
 	/* ACPI various flags. */
 	uint32_t	ivhd_flag;	/* ACPI IVHD flag. */
 	uint32_t	ivhd_feature;	/* ACPI v1 Reserved or v2 attribute. */
diff --git a/usr/src/uts/i86pc/io/vmm/amd/ivrs_drv.c b/usr/src/uts/i86pc/io/vmm/amd/ivrs_drv.c
index 96241be8f4..3eb9fda2e3 100644
--- a/usr/src/uts/i86pc/io/vmm/amd/ivrs_drv.c
+++ b/usr/src/uts/i86pc/io/vmm/amd/ivrs_drv.c
@@ -2,6 +2,7 @@
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2016, Anish Gupta (anish@freebsd.org)
+ * Copyright (c) 2021 The FreeBSD Foundation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -44,6 +45,8 @@ __FBSDID("$FreeBSD$");
 #include <contrib/dev/acpica/include/acpi.h>
 #include <contrib/dev/acpica/include/accommon.h>
 #include <dev/acpica/acpivar.h>
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
 
 #include "io/iommu.h"
 #include "amdvi_priv.h"
@@ -366,10 +369,11 @@ ivhd_identify(driver_t *driver, device_t parent)
 	for (i = ivhd_count - 1 ; i > 0 ; i--){
 		if (ivhd_is_newer(&ivhd_hdrs[i-1]->Header,
 			&ivhd_hdrs[i]->Header)) {
-			ivhd_hdrs[i-1] = ivhd_hdrs[i];
+			memmove(&ivhd_hdrs[i-1], &ivhd_hdrs[i],
+			    sizeof(void *) * (ivhd_count - i));
 			ivhd_count--;
 		}
-       }
+	}
 
 	ivhd_devs = malloc(sizeof(device_t) * ivhd_count, M_DEVBUF,
 		M_WAITOK | M_ZERO);
@@ -627,6 +631,9 @@ ivhd_attach(device_t dev)
 	softc->dev = dev;
 	ivhd = ivhd_hdrs[unit];
 	KASSERT(ivhd, ("ivhd is NULL"));
+	softc->pci_dev = pci_find_bsf(PCI_RID2BUS(ivhd->Header.DeviceId),
+	    PCI_RID2SLOT(ivhd->Header.DeviceId),
+	    PCI_RID2FUNC(ivhd->Header.DeviceId));
 
 	softc->ivhd_type = ivhd->Header.Type;
 	softc->pci_seg = ivhd->PciSegmentGroup;
diff --git a/usr/src/uts/i86pc/io/vmm/io/ppt.c b/usr/src/uts/i86pc/io/vmm/io/ppt.c
index 02446862ea..8f3a276a93 100644
--- a/usr/src/uts/i86pc/io/vmm/io/ppt.c
+++ b/usr/src/uts/i86pc/io/vmm/io/ppt.c
@@ -866,7 +866,7 @@ fail:
 }
 
 static void
-ppt_unmap_mmio(struct vm *vm, struct pptdev *ppt)
+ppt_unmap_all_mmio(struct vm *vm, struct pptdev *ppt)
 {
 	int i;
 	struct pptseg *seg;
@@ -1084,7 +1084,7 @@ ppt_do_unassign(struct pptdev *ppt)
 
 	pf_set_passthru(ppt->pptd_dip, B_FALSE);
 
-	ppt_unmap_mmio(vm, ppt);
+	ppt_unmap_all_mmio(vm, ppt);
 	ppt_teardown_msi(ppt);
 	ppt_teardown_msix(ppt);
 	iommu_remove_device(vm_iommu_domain(vm), pci_get_bdf(ppt->pptd_dip));
@@ -1172,6 +1172,39 @@ done:
 	return (err);
 }
 
+int
+ppt_unmap_mmio(struct vm *vm, int pptfd, vm_paddr_t gpa, size_t len)
+{
+	struct pptdev *ppt;
+	int err = 0;
+	uint_t i;
+
+	mutex_enter(&pptdev_mtx);
+	err = ppt_findf(vm, pptfd, &ppt);
+	if (err != 0) {
+		mutex_exit(&pptdev_mtx);
+		return (err);
+	}
+
+	for (i = 0; i < MAX_MMIOSEGS; i++) {
+		struct pptseg *seg = &ppt->mmio[i];
+
+		if (seg->gpa == gpa && seg->len == len) {
+			err = vm_unmap_mmio(vm, seg->gpa, seg->len);
+			if (err == 0) {
+				seg->gpa = 0;
+				seg->len = 0;
+			}
+			goto out;
+		}
+	}
+	err = ENOENT;
+out:
+	releasef(pptfd);
+	mutex_exit(&pptdev_mtx);
+	return (err);
+}
+
 static uint_t
 pptintr(caddr_t arg, caddr_t unused)
 {
diff --git a/usr/src/uts/i86pc/io/vmm/io/ppt.h b/usr/src/uts/i86pc/io/vmm/io/ppt.h
index 72a768c085..f69a352fe0 100644
--- a/usr/src/uts/i86pc/io/vmm/io/ppt.h
+++ b/usr/src/uts/i86pc/io/vmm/io/ppt.h
@@ -34,6 +34,7 @@
 int ppt_unassign_all(struct vm *vm);
 int ppt_map_mmio(struct vm *vm, int pptfd, vm_paddr_t gpa, size_t len,
     vm_paddr_t hpa);
+int ppt_unmap_mmio(struct vm *vm, int pptfd, vm_paddr_t gpa, size_t len);
 int ppt_setup_msi(struct vm *vm, int vcpu, int pptfd, uint64_t addr,
     uint64_t msg, int numvec);
 int ppt_setup_msix(struct vm *vm, int vcpu, int pptfd, int idx, uint64_t addr,
diff --git a/usr/src/uts/i86pc/io/vmm/io/ppt.mapfile b/usr/src/uts/i86pc/io/vmm/io/ppt.mapfile
index 1b08b06b58..42d92f0066 100644
--- a/usr/src/uts/i86pc/io/vmm/io/ppt.mapfile
+++ b/usr/src/uts/i86pc/io/vmm/io/ppt.mapfile
@@ -12,6 +12,7 @@
 
 #
 # Copyright 2019 Joyent, Inc.
+# Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
 #
 
 #
@@ -45,6 +46,7 @@ SYMBOL_VERSION ILLUMOSprivate {
 	ppt_unassign_device;
 	ppt_unassign_all;
 	ppt_map_mmio;
+	ppt_unmap_mmio;
 	ppt_setup_msi;
 	ppt_setup_msix;
 	ppt_get_limits;
diff --git a/usr/src/uts/i86pc/io/vmm/io/vioapic.c b/usr/src/uts/i86pc/io/vmm/io/vioapic.c
index 8b259000c6..ee974858c5 100644
--- a/usr/src/uts/i86pc/io/vmm/io/vioapic.c
+++ b/usr/src/uts/i86pc/io/vmm/io/vioapic.c
@@ -133,8 +133,14 @@ vioapic_send_intr(struct vioapic *vioapic, int pin)
 	phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
 	delmode = low & IOART_DELMOD;
 	level = low & IOART_TRGRLVL ? true : false;
-	if (level)
+	if (level) {
+		if ((low & IOART_REM_IRR) != 0) {
+			VIOAPIC_CTR1(vioapic, "ioapic pin%d: irr pending",
+			    pin);
+			return;
+		}
 		vioapic->rtbl[pin].reg |= IOART_REM_IRR;
+	}
 
 	vector = low & IOART_INTVEC;
 	dest = high >> APIC_ID_SHIFT;
@@ -305,17 +311,25 @@ vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data)
 		vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS;
 		vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS;
 
+		/*
+		 * Switching from level to edge triggering will clear the IRR
+		 * bit. This is what FreeBSD will do in order to EOI an
+		 * interrupt when the IO-APIC doesn't support targeted EOI (see
+		 * _ioapic_eoi_source).
+		 */
+		if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG &&
+		    (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0)
+			vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
+
 		VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx",
 		    pin, vioapic->rtbl[pin].reg);
 
 		/*
 		 * Generate an interrupt if the following conditions are met:
-		 * - pin is not masked
-		 * - previous interrupt has been EOIed
+		 * - pin trigger mode is level
 		 * - pin level is asserted
 		 */
-		if ((vioapic->rtbl[pin].reg & IOART_INTMASK) == IOART_INTMCLR &&
-		    (vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0 &&
+		if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL &&
 		    (vioapic->rtbl[pin].acnt > 0)) {
 			VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl "
 			    "write, acnt %d", pin, vioapic->rtbl[pin].acnt);
diff --git a/usr/src/uts/i86pc/io/vmm/sys/vmm_kernel.h b/usr/src/uts/i86pc/io/vmm/sys/vmm_kernel.h
index 09e9afd8a8..7abadbfacf 100644
--- a/usr/src/uts/i86pc/io/vmm/sys/vmm_kernel.h
+++ b/usr/src/uts/i86pc/io/vmm/sys/vmm_kernel.h
@@ -40,6 +40,7 @@
  * Copyright 2015 Pluribus Networks Inc.
  * Copyright 2019 Joyent, Inc.
  * Copyright 2020 Oxide Computer Company
+ * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #ifndef _VMM_KERNEL_H_
@@ -133,6 +134,7 @@ int vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores,
  */
 int vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t off,
     size_t len, int prot, int flags);
+int vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len);
 int vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem);
 void vm_free_memseg(struct vm *vm, int ident);
 int vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa);
diff --git a/usr/src/uts/i86pc/io/vmm/vmm.c b/usr/src/uts/i86pc/io/vmm/vmm.c
index dc7c50c574..68f8651a3a 100644
--- a/usr/src/uts/i86pc/io/vmm/vmm.c
+++ b/usr/src/uts/i86pc/io/vmm/vmm.c
@@ -895,6 +895,24 @@ vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first,
 }
 
 int
+vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len)
+{
+	struct mem_map *m;
+	int i;
+
+	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
+		m = &vm->mem_maps[i];
+		if (m->gpa == gpa && m->len == len &&
+		    (m->flags & VM_MEMMAP_F_IOMMU) == 0) {
+			vm_free_memmap(vm, i);
+			return (0);
+		}
+	}
+
+	return (EINVAL);
+}
+
+int
 vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid,
     vm_ooffset_t *segoff, size_t *len, int *prot, int *flags)
 {
diff --git a/usr/src/uts/i86pc/io/vmm/vmm_sol_dev.c b/usr/src/uts/i86pc/io/vmm/vmm_sol_dev.c
index ebec9bef99..ce7dbaa77f 100644
--- a/usr/src/uts/i86pc/io/vmm/vmm_sol_dev.c
+++ b/usr/src/uts/i86pc/io/vmm/vmm_sol_dev.c
@@ -467,8 +467,10 @@ vmmdev_do_ioctl(vmm_softc_t *sc, int cmd, intptr_t arg, int md,
 	case VM_BIND_PPTDEV:
 	case VM_UNBIND_PPTDEV:
 	case VM_MAP_PPTDEV_MMIO:
+	case VM_UNMAP_PPTDEV_MMIO:
 	case VM_ALLOC_MEMSEG:
 	case VM_MMAP_MEMSEG:
+	case VM_MUNMAP_MEMSEG:
 	case VM_WRLOCK_CYCLE:
 	case VM_PMTMR_LOCATE:
 		vmm_write_lock(sc);
@@ -638,6 +640,17 @@ vmmdev_do_ioctl(vmm_softc_t *sc, int cmd, intptr_t arg, int md,
 		    pptmmio.len, pptmmio.hpa);
 		break;
 	}
+	case VM_UNMAP_PPTDEV_MMIO: {
+		struct vm_pptdev_mmio pptmmio;
+
+		if (ddi_copyin(datap, &pptmmio, sizeof (pptmmio), md)) {
+			error = EFAULT;
+			break;
+		}
+		error = ppt_unmap_mmio(sc->vmm_vm, pptmmio.pptfd, pptmmio.gpa,
+		    pptmmio.len);
+		break;
+	}
 	case VM_BIND_PPTDEV: {
 		struct vm_pptdev pptdev;
 
@@ -849,6 +862,16 @@ vmmdev_do_ioctl(vmm_softc_t *sc, int cmd, intptr_t arg, int md,
 		    mm.len, mm.prot, mm.flags);
 		break;
 	}
+	case VM_MUNMAP_MEMSEG: {
+		struct vm_munmap mu;
+
+		if (ddi_copyin(datap, &mu, sizeof (mu), md)) {
+			error = EFAULT;
+			break;
+		}
+		error = vm_munmap_memseg(sc->vmm_vm, mu.gpa, mu.len);
+		break;
+	}
 	case VM_ALLOC_MEMSEG: {
 		struct vm_memseg vmseg;
 
diff --git a/usr/src/uts/i86pc/sys/vmm_dev.h b/usr/src/uts/i86pc/sys/vmm_dev.h
index f4a68636b3..d5f4845e50 100644
--- a/usr/src/uts/i86pc/sys/vmm_dev.h
+++ b/usr/src/uts/i86pc/sys/vmm_dev.h
@@ -57,6 +57,11 @@ struct vm_memmap {
 #define	VM_MEMMAP_F_WIRED	0x01
 #define	VM_MEMMAP_F_IOMMU	0x02
 
+struct vm_munmap {
+	vm_paddr_t	gpa;
+	size_t		len;
+};
+
 #define	VM_MEMSEG_NAME(m)	((m)->name[0] != '\0' ? (m)->name : NULL)
 struct vm_memseg {
 	int		segid;
@@ -325,6 +330,8 @@ struct vm_run_state {
 #define	VM_ALLOC_MEMSEG		(VMM_LOCK_IOC_BASE | 0x05)
 #define	VM_MMAP_MEMSEG		(VMM_LOCK_IOC_BASE | 0x06)
 #define	VM_PMTMR_LOCATE		(VMM_LOCK_IOC_BASE | 0x07)
+#define	VM_MUNMAP_MEMSEG	(VMM_LOCK_IOC_BASE | 0x08)
+#define	VM_UNMAP_PPTDEV_MMIO	(VMM_LOCK_IOC_BASE | 0x09)
 
 #define	VM_WRLOCK_CYCLE		(VMM_LOCK_IOC_BASE | 0xff)
 
diff --git a/usr/src/uts/intel/ia32/ml/modstubs.s b/usr/src/uts/intel/ia32/ml/modstubs.s
index 070083d8f1..4143c181a3 100644
--- a/usr/src/uts/intel/ia32/ml/modstubs.s
+++ b/usr/src/uts/intel/ia32/ml/modstubs.s
@@ -22,6 +22,7 @@
 /*
  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2019 Joyent, Inc.
+ * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #include <sys/asm_linkage.h>
@@ -1299,6 +1300,7 @@ fcnname/**/_info:							\
 	MODULE(ppt,drv);
 	WSTUB(ppt, ppt_unassign_all,		nomod_zero);
 	WSTUB(ppt, ppt_map_mmio,		nomod_einval);
+	WSTUB(ppt, ppt_unmap_mmio,		nomod_einval);
 	WSTUB(ppt, ppt_setup_msi,		nomod_einval);
 	WSTUB(ppt, ppt_setup_msix,		nomod_einval);
 	WSTUB(ppt, ppt_disable_msix,		nomod_einval);