Re-add the NetBSD/amd64 Virtual Memory Monitor (NVMM) support to QEMU 5.2.0

thanks go to Ryo ONODERA <ryo@tetera.org> for his help with the build logic

15 files changed, 1686 insertions, 5 deletions

diff --git a/emulators/qemu/MESSAGE b/emulators/qemu/MESSAGE
new file mode 100644
index 00000000000..5884b6e4a2e
--- /dev/null
+++ b/emulators/qemu/MESSAGE
@@ -0,0 +1,25 @@
+===========================================================================
+$NetBSD: MESSAGE,v 1.1 2021/03/06 11:19:34 reinoud Exp $
+
+To use the NetBSD/amd64 Virtual Memory Monitor (NVMM) for QEMU :
+
+Load the NVMM kernel driver if not already loaded
+# modload nvmm
+
+You may want to run the NVMM tests to confirm that your setup is correct:
+# cd /usr/tests/lib/libnvmm/
+# atf-run | atf-report
+
+Configure the permissions once
+# chown root:nvmm ${PREFIX}/bin/qemu-system-x86_64
+# chmod g+s ${PREFIX}/bin/qemu-system-x86_64
+# chown root:nvmm ${PREFIX}/bin/qemu-system-i386
+# chmod g+s ${PREFIX}/bin/qemu-system-i386
+
+Append "-accel nvmm" to your usual Qemu command line
+$ qemu-system-x86_64 -smp 4 -m 8G -cdrom Win10_2004_English_x64.iso -accel nvmm
+
+See nvmm(4) and nvmmctl(8) or visit
+   https://m00nbsd.net/4e0798b7f2620c965d0dd9d6a7a2f296.html for more info.
+
+===========================================================================
diff --git a/emulators/qemu/Makefile b/emulators/qemu/Makefile
index a776992f1f6..ca6ed34f873 100644
--- a/emulators/qemu/Makefile
+++ b/emulators/qemu/Makefile
@@ -1,6 +1,7 @@
-# $NetBSD: Makefile,v 1.264 2021/02/23 09:28:33 wiz Exp $
+# $NetBSD: Makefile,v 1.265 2021/03/06 11:19:34 reinoud Exp $
 
 DISTNAME=	qemu-5.2.0
+PKGREVISION=	1
 CATEGORIES=	emulators
 MASTER_SITES=	https://download.qemu.org/
 EXTRACT_SUFX=	.tar.xz
diff --git a/emulators/qemu/distinfo b/emulators/qemu/distinfo
index 5d1e0ca7679..8b2ee52413b 100644
--- a/emulators/qemu/distinfo
+++ b/emulators/qemu/distinfo
@@ -1,4 +1,4 @@
-$NetBSD: distinfo,v 1.171 2021/02/23 09:28:33 wiz Exp $
+$NetBSD: distinfo,v 1.172 2021/03/06 11:19:34 reinoud Exp $
 
 SHA1 (palcode-clipper) = e25ae10a10e0801e47b62b9ee2d10c8ccb4ee940
 RMD160 (palcode-clipper) = a637f1cc38dabfdff36e3f02b6dd02d7c63cb8db
@@ -8,7 +8,9 @@ SHA1 (qemu-5.2.0.tar.xz) = 146578267387e301423502d19024f8ffe35ab332
 RMD160 (qemu-5.2.0.tar.xz) = 2c33e773f012e333f99237e3d4ff1653ea0bc88f
 SHA512 (qemu-5.2.0.tar.xz) = bddd633ce111471ebc651e03080251515178808556b49a308a724909e55dac0be0cc0c79c536ac12d239678ae94c60100dc124be9b9d9538340c03a2f27177f3
 Size (qemu-5.2.0.tar.xz) = 106902800 bytes
+SHA1 (patch-accel_stubs_nvmm-stub.c) = d66d47eabb8bb6728e777da7589b43d491adbcc8
 SHA1 (patch-backends_tpm_tpm__ioctl.h) = fbd6c877ad605f7120290efbb0ac653c69f351de
+SHA1 (patch-configure) = 8b392c5633c70d65f2f27af3b617a53af9772899
 SHA1 (patch-contrib_ivshmem-client_ivshmem-client.c) = 40c8751607cbf66a37e4c4e08f2664b864e2e984
 SHA1 (patch-contrib_ivshmem-server_ivshmem-server.c) = d8f53432b5752f4263dc4ef96108a976a05147a3
 SHA1 (patch-hw-mips-Kconfig) = c7199ad26ac45116ab4d38252db4234ae93bdf9a
@@ -24,9 +26,13 @@ SHA1 (patch-hw_net_xilinx__axienet.c) = ebcd2676d64ce6f31e4a8c976d4fdf530ad5e8b7
 SHA1 (patch-hw_rtc_mc146818rtc.c) = cc7a3b28010966b65b7a16db756226ac2669f310
 SHA1 (patch-hw_scsi_scsi-disk.c) = fdbf2f962a6dcb1a115a7f8a5b8790ff9295fb33
 SHA1 (patch-hw_usb_dev-mtp.c) = 94ddf53a41cc75810cfece1b8aef1831fab4ce43
+SHA1 (patch-include_sysemu_hw_accel.h) = d083cd51434e28eb0d647b5107d34018b0ef63dc
 SHA1 (patch-include_sysemu_kvm.h) = 9847abe3be70bd708a521310f5d5515e45a1a5a0
-SHA1 (patch-meson.build) = 5f9e00312b2dc6a6b1c43b1f28d092e69b09091d
+SHA1 (patch-include_sysemu_nvmm.h) = 1fe49c4f11910d6faf683ae3233f783a0b03ce5a
+SHA1 (patch-meson.build) = 65f1f51403b6def24640396ad26bb7c25a16dae0
+SHA1 (patch-meson__options.txt) = 286d097f596baa5af244a990d2874f1a7ee65198
 SHA1 (patch-net_tap-solaris.c) = cc953c9a624dd55ace4e130d0b31bbfb956c17d5
+SHA1 (patch-qemu-options.hx) = e2f264117f703aa4ccf56219f370c3b1303e8b07
 SHA1 (patch-roms_qemu-palcode_hwrpb.h) = ae7b4c0680367af6f740d62a54dc86352128d76f
 SHA1 (patch-roms_qemu-palcode_init.c) = 7a0ebcd86f4106318791e7d90273fb55a424f1b8
 SHA1 (patch-roms_qemu-palcode_memcpy.c) = 7761774ae9092d0f494deaf302d663ba479a09cf
@@ -40,5 +46,10 @@ SHA1 (patch-roms_qemu-palcode_sys-clipper.h) = 8983d7072b1c1e66bf0a18d2e49e50374
 SHA1 (patch-roms_qemu-palcode_vgaio.c) = c8d7adc053cd6655f005527d16647611040c09d2
 SHA1 (patch-roms_u-boot-sam460ex_Makefile) = 3a1bbf19b1422c10ebdd819eb0b711fafc78e2f2
 SHA1 (patch-roms_u-boot_tools_imx8m__image.sh) = e4c452062f40569e33aa93eec4a65bd3af2e74fc
+SHA1 (patch-target_i386_helper.c) = 3314e65df11492438af2ec2c53ed3082a0b62b09
 SHA1 (patch-target_i386_kvm-stub.c) = 4cd2b7a8d8d8a317829f982b5acff7fdf2479d9f
+SHA1 (patch-target_i386_meson.build) = d0e0d7d4dd96ea43fc386e7166bbabbd71b0f4fc
+SHA1 (patch-target_i386_nvmm_all.c) = dd22747aa0b94bc371303fc865d523d3623ec043
+SHA1 (patch-target_i386_nvmm_cpus.c) = 7f028bf2637fe31d8524f710a9e508c8ce65c822
+SHA1 (patch-target_i386_nvmm_cpus.h) = 0a25e49929cb772fc46a4ace91127ccf3605521d
 SHA1 (patch-target_sparc_translate.c) = 7ec2add2fd808facb48b9a66ccc345599251bf76
diff --git a/emulators/qemu/patches/patch-accel_stubs_nvmm-stub.c b/emulators/qemu/patches/patch-accel_stubs_nvmm-stub.c
new file mode 100644
index 00000000000..a79aeb113df
--- /dev/null
+++ b/emulators/qemu/patches/patch-accel_stubs_nvmm-stub.c
@@ -0,0 +1,50 @@
+$NetBSD: patch-accel_stubs_nvmm-stub.c,v 1.3 2021/03/06 11:19:34 reinoud Exp $
+
+Add NVMM support.
+
+--- accel/stubs/nvmm-stub.c.orig	2020-02-06 16:25:13.966864001 +0000
++++ accel/stubs/nvmm-stub.c
+@@ -0,0 +1,43 @@
++/*
++ * Copyright (c) 2018-2019 Maxime Villard, All rights reserved.
++ *
++ * NetBSD Virtual Machine Monitor (NVMM) accelerator stub.
++ *
++ * This work is licensed under the terms of the GNU GPL, version 2 or later.
++ * See the COPYING file in the top-level directory.
++ */
++
++#include "qemu/osdep.h"
++#include "qemu-common.h"
++#include "cpu.h"
++#include "sysemu/nvmm.h"
++
++int nvmm_init_vcpu(CPUState *cpu)
++{
++    return -1;
++}
++
++int nvmm_vcpu_exec(CPUState *cpu)
++{
++    return -1;
++}
++
++void nvmm_destroy_vcpu(CPUState *cpu)
++{
++}
++
++void nvmm_cpu_synchronize_state(CPUState *cpu)
++{
++}
++
++void nvmm_cpu_synchronize_post_reset(CPUState *cpu)
++{
++}
++
++void nvmm_cpu_synchronize_post_init(CPUState *cpu)
++{
++}
++
++void nvmm_cpu_synchronize_pre_loadvm(CPUState *cpu)
++{
++}
diff --git a/emulators/qemu/patches/patch-configure b/emulators/qemu/patches/patch-configure
new file mode 100644
index 00000000000..5933bfeaa16
--- /dev/null
+++ b/emulators/qemu/patches/patch-configure
@@ -0,0 +1,43 @@
+$NetBSD: patch-configure,v 1.31 2021/03/06 11:19:34 reinoud Exp $
+
+Add NVMM support.
+Fix jemalloc detection.
+
+--- configure.orig	2020-12-08 16:59:44.000000000 +0000
++++ configure
+@@ -334,6 +334,7 @@ vhost_user_fs=""
+ kvm="auto"
+ hax="auto"
+ hvf="auto"
++nvmm="auto"
+ whpx="auto"
+ rdma=""
+ pvrdma=""
+@@ -1102,6 +1103,10 @@ for opt do
+   ;;
+   --enable-hvf) hvf="enabled"
+   ;;
++  --disable-nvmm) nvmm="disabled"
++  ;;
++  --enable-nvmm) nvmm="enabled"
++  ;;
+   --disable-whpx) whpx="disabled"
+   ;;
+   --enable-whpx) whpx="enabled"
+@@ -1783,6 +1788,7 @@ disabled with --disable-FEATURE, default
+   kvm             KVM acceleration support
+   hax             HAX acceleration support
+   hvf             Hypervisor.framework acceleration support
++  nvmm            NVMM acceleration support
+   whpx            Windows Hypervisor Platform acceleration support
+   rdma            Enable RDMA-based migration
+   pvrdma          Enable PVRDMA support
+@@ -7005,7 +7011,7 @@ NINJA=$ninja $meson setup \
+         ${staticpic:+-Db_staticpic=$staticpic} \
+         -Db_coverage=$(if test "$gcov" = yes; then echo true; else echo false; fi) \
+         -Dmalloc=$malloc -Dmalloc_trim=$malloc_trim -Dsparse=$sparse \
+-        -Dkvm=$kvm -Dhax=$hax -Dwhpx=$whpx -Dhvf=$hvf \
++        -Dkvm=$kvm -Dhax=$hax -Dwhpx=$whpx -Dhvf=$hvf -Dnvmm=$nvmm \
+         -Dxen=$xen -Dxen_pci_passthrough=$xen_pci_passthrough -Dtcg=$tcg \
+         -Dcocoa=$cocoa -Dmpath=$mpath -Dsdl=$sdl -Dsdl_image=$sdl_image \
+         -Dvnc=$vnc -Dvnc_sasl=$vnc_sasl -Dvnc_jpeg=$vnc_jpeg -Dvnc_png=$vnc_png \
diff --git a/emulators/qemu/patches/patch-include_sysemu_hw_accel.h b/emulators/qemu/patches/patch-include_sysemu_hw_accel.h
new file mode 100644
index 00000000000..793405df5ac
--- /dev/null
+++ b/emulators/qemu/patches/patch-include_sysemu_hw_accel.h
@@ -0,0 +1,12 @@
+$NetBSD: patch-include_sysemu_hw_accel.h,v 1.1 2021/03/06 11:19:34 reinoud Exp $
+
+--- include/sysemu/hw_accel.h.orig	2020-12-08 16:59:44.000000000 +0000
++++ include/sysemu/hw_accel.h
+@@ -16,6 +16,7 @@
+ #include "sysemu/kvm.h"
+ #include "sysemu/hvf.h"
+ #include "sysemu/whpx.h"
++#include "sysemu/nvmm.h"
+ 
+ void cpu_synchronize_state(CPUState *cpu);
+ void cpu_synchronize_post_reset(CPUState *cpu);
diff --git a/emulators/qemu/patches/patch-include_sysemu_nvmm.h b/emulators/qemu/patches/patch-include_sysemu_nvmm.h
new file mode 100644
index 00000000000..8c1e874796e
--- /dev/null
+++ b/emulators/qemu/patches/patch-include_sysemu_nvmm.h
@@ -0,0 +1,31 @@
+$NetBSD: patch-include_sysemu_nvmm.h,v 1.3 2021/03/06 11:19:34 reinoud Exp $
+
+--- include/sysemu/nvmm.h.orig	2021-03-05 22:29:22.991663471 +0000
++++ include/sysemu/nvmm.h
+@@ -0,0 +1,26 @@
++/*
++ * Copyright (c) 2018-2019 Maxime Villard, All rights reserved.
++ *
++ * NetBSD Virtual Machine Monitor (NVMM) accelerator support.
++ *
++ * This work is licensed under the terms of the GNU GPL, version 2 or later.
++ * See the COPYING file in the top-level directory.
++ */
++
++#ifndef QEMU_NVMM_H
++#define QEMU_NVMM_H
++
++#include "config-host.h"
++#include "qemu-common.h"
++
++#ifdef CONFIG_NVMM
++
++int nvmm_enabled(void);
++
++#else /* CONFIG_NVMM */
++
++#define nvmm_enabled() (0)
++
++#endif /* CONFIG_NVMM */
++
++#endif /* CONFIG_NVMM */
diff --git a/emulators/qemu/patches/patch-meson.build b/emulators/qemu/patches/patch-meson.build
index 2bb2636d334..ee9e2060221 100644
--- a/emulators/qemu/patches/patch-meson.build
+++ b/emulators/qemu/patches/patch-meson.build
@@ -1,11 +1,61 @@
-$NetBSD: patch-meson.build,v 1.1 2021/02/20 22:59:29 ryoon Exp $
+$NetBSD: patch-meson.build,v 1.2 2021/03/06 11:19:34 reinoud Exp $
 
+* Add NetBSD support.
 * Detect iconv in libc properly for pkgsrc (pkgsrc removes -liconv)
   to fix qemu-system-aarch64 link.
 
 --- meson.build.orig	2020-12-08 16:59:44.000000000 +0000
 +++ meson.build
-@@ -535,7 +535,7 @@ if have_system and not get_option('curse
+@@ -84,6 +84,7 @@ if cpu in ['x86', 'x86_64']
+   accelerator_targets += {
+     'CONFIG_HAX': ['i386-softmmu', 'x86_64-softmmu'],
+     'CONFIG_HVF': ['x86_64-softmmu'],
++    'CONFIG_NVMM': ['x86_64-softmmu'],
+     'CONFIG_WHPX': ['i386-softmmu', 'x86_64-softmmu'],
+   }
+ endif
+@@ -169,6 +170,7 @@ version_res = []
+ coref = []
+ iokit = []
+ emulator_link_args = []
++nvmm = []
+ cocoa = not_found
+ hvf = not_found
+ if targetos == 'windows'
+@@ -196,6 +198,10 @@ elif targetos == 'openbsd'
+     # Disable OpenBSD W^X if available
+     emulator_link_args = cc.get_supported_link_arguments('-Wl,-z,wxneeded')
+   endif
++elif targetos == 'netbsd'
++  if not get_option('nvmm').disabled()
++    nvmm = cc.find_library('nvmm')
++  endif
+ endif
+ 
+ accelerators = []
+@@ -228,6 +234,11 @@ if not get_option('hax').disabled()
+     accelerators += 'CONFIG_HAX'
+   endif
+ endif
++if not get_option('nvmm').disabled()
++  if cc.has_header('nvmm.h', required: get_option('nvmm'))
++    accelerators += 'CONFIG_NVMM'
++  endif
++endif
+ if not get_option('tcg').disabled()
+   if cpu not in supported_cpus
+     if 'CONFIG_TCG_INTERPRETER' in config_host
+@@ -246,6 +257,9 @@ endif
+ if 'CONFIG_HVF' not in accelerators and get_option('hvf').enabled()
+   error('HVF not available on this platform')
+ endif
++if 'CONFIG_NVMM' not in accelerators and get_option('nvmm').enabled()
++  error('NVMM not available on this platform')
++endif
+ if 'CONFIG_WHPX' not in accelerators and get_option('whpx').enabled()
+   error('WHPX not available on this platform')
+ endif
+@@ -535,7 +549,7 @@ if have_system and not get_option('curse
      endif
    endif
    if not get_option('iconv').disabled()
@@ -14,3 +64,20 @@ $NetBSD: patch-meson.build,v 1.1 2021/02/20 22:59:29 ryoon Exp $
        # Programs will be linked with glib and this will bring in libiconv on FreeBSD.
        # We need to use libiconv if available because mixing libiconv's headers with
        # the system libc does not work.
+@@ -1815,7 +1829,7 @@ foreach target : target_dirs
+       'name': 'qemu-system-' + target_name,
+       'gui': false,
+       'sources': files('softmmu/main.c'),
+-      'dependencies': []
++      'dependencies': [nvmm]
+     }]
+     if targetos == 'windows' and (sdl.found() or gtk.found())
+       execs += [{
+@@ -2106,6 +2120,7 @@ summary_info += {'Install blobs':     ge
+ summary_info += {'KVM support':       config_all.has_key('CONFIG_KVM')}
+ summary_info += {'HAX support':       config_all.has_key('CONFIG_HAX')}
+ summary_info += {'HVF support':       config_all.has_key('CONFIG_HVF')}
++summary_info += {'NVMM support':      config_all.has_key('CONFIG_NVMM')}
+ summary_info += {'WHPX support':      config_all.has_key('CONFIG_WHPX')}
+ summary_info += {'TCG support':       config_all.has_key('CONFIG_TCG')}
+ if config_all.has_key('CONFIG_TCG')
diff --git a/emulators/qemu/patches/patch-meson__options.txt b/emulators/qemu/patches/patch-meson__options.txt
new file mode 100644
index 00000000000..e53388b4a14
--- /dev/null
+++ b/emulators/qemu/patches/patch-meson__options.txt
@@ -0,0 +1,13 @@
+$NetBSD: patch-meson__options.txt,v 1.1 2021/03/06 11:19:34 reinoud Exp $
+
+--- meson_options.txt.orig	2020-12-08 16:59:44.000000000 +0000
++++ meson_options.txt
+@@ -29,6 +29,8 @@ option('whpx', type: 'feature', value: '
+        description: 'WHPX acceleration support')
+ option('hvf', type: 'feature', value: 'auto',
+        description: 'HVF acceleration support')
++option('nvmm', type: 'feature', value: 'auto',
++       description: 'NVMM acceleration support')
+ option('xen', type: 'feature', value: 'auto',
+        description: 'Xen backend support')
+ option('xen_pci_passthrough', type: 'feature', value: 'auto',
diff --git a/emulators/qemu/patches/patch-qemu-options.hx b/emulators/qemu/patches/patch-qemu-options.hx
new file mode 100644
index 00000000000..f7b0c0df8ef
--- /dev/null
+++ b/emulators/qemu/patches/patch-qemu-options.hx
@@ -0,0 +1,42 @@
+$NetBSD: patch-qemu-options.hx,v 1.4 2021/03/06 11:19:34 reinoud Exp $
+
+Add NVMM support.
+
+--- qemu-options.hx.orig	2020-04-28 16:49:25.000000000 +0000
++++ qemu-options.hx
+@@ -26,7 +26,7 @@ DEF("machine", HAS_ARG, QEMU_OPTION_mach
+     "-machine [type=]name[,prop[=value][,...]]\n"
+     "                selects emulated machine ('-machine help' for list)\n"
+     "                property accel=accel1[:accel2[:...]] selects accelerator\n"
+-    "                supported accelerators are kvm, xen, hax, hvf, whpx or tcg (default: tcg)\n"
++    "                supported accelerators are kvm, xen, hax, hvf, nvmm, whpx or tcg (default: tcg)\n"
+     "                vmport=on|off|auto controls emulation of vmport (default: auto)\n"
+     "                dump-guest-core=on|off include guest memory in a core dump (default=on)\n"
+     "                mem-merge=on|off controls memory merge support (default: on)\n"
+@@ -58,7 +58,7 @@ SRST
+ 
+     ``accel=accels1[:accels2[:...]]``
+         This is used to enable an accelerator. Depending on the target
+-        architecture, kvm, xen, hax, hvf, whpx or tcg can be available.
++        architecture, kvm, xen, hax, hvf, nvmm, whpx or tcg can be available.
+         By default, tcg is used. If there is more than one accelerator
+         specified, the next one is used if the previous one fails to
+         initialize.
+@@ -119,7 +119,7 @@ ERST
+ 
+ DEF("accel", HAS_ARG, QEMU_OPTION_accel,
+     "-accel [accel=]accelerator[,prop[=value][,...]]\n"
+-    "                select accelerator (kvm, xen, hax, hvf, whpx or tcg; use 'help' for a list)\n"
++    "                select accelerator (kvm, xen, hax, hvf, nvmm, whpx or tcg; use 'help' for a list)\n"
+     "                igd-passthru=on|off (enable Xen integrated Intel graphics passthrough, default=off)\n"
+     "                kernel-irqchip=on|off|split controls accelerated irqchip support (default=on)\n"
+     "                kvm-shadow-mem=size of KVM shadow MMU in bytes\n"
+@@ -128,7 +128,7 @@ DEF("accel", HAS_ARG, QEMU_OPTION_accel,
+ SRST
+ ``-accel name[,prop=value[,...]]``
+     This is used to enable an accelerator. Depending on the target
+-    architecture, kvm, xen, hax, hvf, whpx or tcg can be available. By
++    architecture, kvm, xen, hax, hvf, nvmm, whpx or tcg can be available. By
+     default, tcg is used. If there is more than one accelerator
+     specified, the next one is used if the previous one fails to
+     initialize.
diff --git a/emulators/qemu/patches/patch-target_i386_helper.c b/emulators/qemu/patches/patch-target_i386_helper.c
new file mode 100644
index 00000000000..4647e96d5c8
--- /dev/null
+++ b/emulators/qemu/patches/patch-target_i386_helper.c
@@ -0,0 +1,13 @@
+$NetBSD: patch-target_i386_helper.c,v 1.3 2021/03/06 11:19:34 reinoud Exp $
+
+--- target/i386/helper.c.orig	2020-12-08 16:59:44.000000000 +0000
++++ target/i386/helper.c
+@@ -1008,7 +1008,7 @@ void cpu_report_tpr_access(CPUX86State *
+     X86CPU *cpu = env_archcpu(env);
+     CPUState *cs = env_cpu(env);
+ 
+-    if (kvm_enabled() || whpx_enabled()) {
++    if (kvm_enabled() || whpx_enabled() || nvmm_enabled()) {
+         env->tpr_access_type = access;
+ 
+         cpu_interrupt(cs, CPU_INTERRUPT_TPR);
diff --git a/emulators/qemu/patches/patch-target_i386_meson.build b/emulators/qemu/patches/patch-target_i386_meson.build
new file mode 100644
index 00000000000..f0e137da0c4
--- /dev/null
+++ b/emulators/qemu/patches/patch-target_i386_meson.build
@@ -0,0 +1,15 @@
+$NetBSD: patch-target_i386_meson.build,v 1.1 2021/03/06 11:19:34 reinoud Exp $
+
+--- target/i386/meson.build.orig	2020-12-08 16:59:44.000000000 +0000
++++ target/i386/meson.build
+@@ -34,6 +34,10 @@ i386_softmmu_ss.add(when: 'CONFIG_WHPX',
+   'whpx-all.c',
+   'whpx-cpus.c',
+ ))
++i386_softmmu_ss.add(when: 'CONFIG_NVMM', if_true: files(
++  'nvmm-all.c',
++  'nvmm-cpus.c',
++))
+ i386_softmmu_ss.add(when: 'CONFIG_HAX', if_true: files(
+   'hax-all.c',
+   'hax-mem.c',
diff --git a/emulators/qemu/patches/patch-target_i386_nvmm_all.c b/emulators/qemu/patches/patch-target_i386_nvmm_all.c
new file mode 100644
index 00000000000..e00ff0ff5d2
--- /dev/null
+++ b/emulators/qemu/patches/patch-target_i386_nvmm_all.c
@@ -0,0 +1,1221 @@
+$NetBSD: patch-target_i386_nvmm_all.c,v 1.1 2021/03/06 11:19:34 reinoud Exp $
+
+--- target/i386/nvmm-all.c.orig	2021-03-05 20:20:34.189784289 +0000
++++ target/i386/nvmm-all.c
+@@ -0,0 +1,1216 @@
++/*
++ * Copyright (c) 2018-2019 Maxime Villard, All rights reserved.
++ *
++ * NetBSD Virtual Machine Monitor (NVMM) accelerator for QEMU.
++ *
++ * This work is licensed under the terms of the GNU GPL, version 2 or later.
++ * See the COPYING file in the top-level directory.
++ */
++
++#include "qemu/osdep.h"
++#include "cpu.h"
++#include "exec/address-spaces.h"
++#include "exec/ioport.h"
++#include "qemu-common.h"
++#include "strings.h"
++#include "sysemu/accel.h"
++#include "sysemu/nvmm.h"
++#include "sysemu/runstate.h"
++#include "sysemu/sysemu.h"
++#include "sysemu/cpus.h"
++#include "qemu/main-loop.h"
++#include "qemu/error-report.h"
++#include "qemu/queue.h"
++#include "qapi/error.h"
++#include "migration/blocker.h"
++
++#include "nvmm-cpus.h"
++
++#include <nvmm.h>
++
++struct qemu_vcpu {
++    struct nvmm_vcpu vcpu;
++    uint8_t tpr;
++    bool stop;
++
++    /* Window-exiting for INTs/NMIs. */
++    bool int_window_exit;
++    bool nmi_window_exit;
++
++    /* The guest is in an interrupt shadow (POP SS, etc). */
++    bool int_shadow;
++};
++
++struct qemu_machine {
++    struct nvmm_capability cap;
++    struct nvmm_machine mach;
++};
++
++/* -------------------------------------------------------------------------- */
++
++static bool nvmm_allowed;
++static struct qemu_machine qemu_mach;
++
++static struct qemu_vcpu *
++get_qemu_vcpu(CPUState *cpu)
++{
++    return (struct qemu_vcpu *)cpu->hax_vcpu;
++}
++
++static struct nvmm_machine *
++get_nvmm_mach(void)
++{
++    return &qemu_mach.mach;
++}
++
++/* -------------------------------------------------------------------------- */
++
++static void
++nvmm_set_segment(struct nvmm_x64_state_seg *nseg, const SegmentCache *qseg)
++{
++    uint32_t attrib = qseg->flags;
++
++    nseg->selector = qseg->selector;
++    nseg->limit = qseg->limit;
++    nseg->base = qseg->base;
++    nseg->attrib.type = __SHIFTOUT(attrib, DESC_TYPE_MASK);
++    nseg->attrib.s = __SHIFTOUT(attrib, DESC_S_MASK);
++    nseg->attrib.dpl = __SHIFTOUT(attrib, DESC_DPL_MASK);
++    nseg->attrib.p = __SHIFTOUT(attrib, DESC_P_MASK);
++    nseg->attrib.avl = __SHIFTOUT(attrib, DESC_AVL_MASK);
++    nseg->attrib.l = __SHIFTOUT(attrib, DESC_L_MASK);
++    nseg->attrib.def = __SHIFTOUT(attrib, DESC_B_MASK);
++    nseg->attrib.g = __SHIFTOUT(attrib, DESC_G_MASK);
++}
++
++static void
++nvmm_set_registers(CPUState *cpu)
++{
++    struct CPUX86State *env = (CPUArchState *)cpu->env_ptr;
++    struct nvmm_machine *mach = get_nvmm_mach();
++    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
++    struct nvmm_vcpu *vcpu = &qcpu->vcpu;
++    struct nvmm_x64_state *state = vcpu->state;
++    uint64_t bitmap;
++    size_t i;
++    int ret;
++
++    assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
++
++    /* GPRs. */
++    state->gprs[NVMM_X64_GPR_RAX] = env->regs[R_EAX];
++    state->gprs[NVMM_X64_GPR_RCX] = env->regs[R_ECX];
++    state->gprs[NVMM_X64_GPR_RDX] = env->regs[R_EDX];
++    state->gprs[NVMM_X64_GPR_RBX] = env->regs[R_EBX];
++    state->gprs[NVMM_X64_GPR_RSP] = env->regs[R_ESP];
++    state->gprs[NVMM_X64_GPR_RBP] = env->regs[R_EBP];
++    state->gprs[NVMM_X64_GPR_RSI] = env->regs[R_ESI];
++    state->gprs[NVMM_X64_GPR_RDI] = env->regs[R_EDI];
++#ifdef TARGET_X86_64
++    state->gprs[NVMM_X64_GPR_R8]  = env->regs[R_R8];
++    state->gprs[NVMM_X64_GPR_R9]  = env->regs[R_R9];
++    state->gprs[NVMM_X64_GPR_R10] = env->regs[R_R10];
++    state->gprs[NVMM_X64_GPR_R11] = env->regs[R_R11];
++    state->gprs[NVMM_X64_GPR_R12] = env->regs[R_R12];
++    state->gprs[NVMM_X64_GPR_R13] = env->regs[R_R13];
++    state->gprs[NVMM_X64_GPR_R14] = env->regs[R_R14];
++    state->gprs[NVMM_X64_GPR_R15] = env->regs[R_R15];
++#endif
++
++    /* RIP and RFLAGS. */
++    state->gprs[NVMM_X64_GPR_RIP] = env->eip;
++    state->gprs[NVMM_X64_GPR_RFLAGS] = env->eflags;
++
++    /* Segments. */
++    nvmm_set_segment(&state->segs[NVMM_X64_SEG_CS], &env->segs[R_CS]);
++    nvmm_set_segment(&state->segs[NVMM_X64_SEG_DS], &env->segs[R_DS]);
++    nvmm_set_segment(&state->segs[NVMM_X64_SEG_ES], &env->segs[R_ES]);
++    nvmm_set_segment(&state->segs[NVMM_X64_SEG_FS], &env->segs[R_FS]);
++    nvmm_set_segment(&state->segs[NVMM_X64_SEG_GS], &env->segs[R_GS]);
++    nvmm_set_segment(&state->segs[NVMM_X64_SEG_SS], &env->segs[R_SS]);
++
++    /* Special segments. */
++    nvmm_set_segment(&state->segs[NVMM_X64_SEG_GDT], &env->gdt);
++    nvmm_set_segment(&state->segs[NVMM_X64_SEG_LDT], &env->ldt);
++    nvmm_set_segment(&state->segs[NVMM_X64_SEG_TR], &env->tr);
++    nvmm_set_segment(&state->segs[NVMM_X64_SEG_IDT], &env->idt);
++
++    /* Control registers. */
++    state->crs[NVMM_X64_CR_CR0] = env->cr[0];
++    state->crs[NVMM_X64_CR_CR2] = env->cr[2];
++    state->crs[NVMM_X64_CR_CR3] = env->cr[3];
++    state->crs[NVMM_X64_CR_CR4] = env->cr[4];
++    state->crs[NVMM_X64_CR_CR8] = qcpu->tpr;
++    state->crs[NVMM_X64_CR_XCR0] = env->xcr0;
++
++    /* Debug registers. */
++    state->drs[NVMM_X64_DR_DR0] = env->dr[0];
++    state->drs[NVMM_X64_DR_DR1] = env->dr[1];
++    state->drs[NVMM_X64_DR_DR2] = env->dr[2];
++    state->drs[NVMM_X64_DR_DR3] = env->dr[3];
++    state->drs[NVMM_X64_DR_DR6] = env->dr[6];
++    state->drs[NVMM_X64_DR_DR7] = env->dr[7];
++
++    /* FPU. */
++    state->fpu.fx_cw = env->fpuc;
++    state->fpu.fx_sw = (env->fpus & ~0x3800) | ((env->fpstt & 0x7) << 11);
++    state->fpu.fx_tw = 0;
++    for (i = 0; i < 8; i++) {
++        state->fpu.fx_tw |= (!env->fptags[i]) << i;
++    }
++    state->fpu.fx_opcode = env->fpop;
++    state->fpu.fx_ip.fa_64 = env->fpip;
++    state->fpu.fx_dp.fa_64 = env->fpdp;
++    state->fpu.fx_mxcsr = env->mxcsr;
++    state->fpu.fx_mxcsr_mask = 0x0000FFFF;
++    assert(sizeof(state->fpu.fx_87_ac) == sizeof(env->fpregs));
++    memcpy(state->fpu.fx_87_ac, env->fpregs, sizeof(env->fpregs));
++    for (i = 0; i < CPU_NB_REGS; i++) {
++        memcpy(&state->fpu.fx_xmm[i].xmm_bytes[0],
++            &env->xmm_regs[i].ZMM_Q(0), 8);
++        memcpy(&state->fpu.fx_xmm[i].xmm_bytes[8],
++            &env->xmm_regs[i].ZMM_Q(1), 8);
++    }
++
++    /* MSRs. */
++    state->msrs[NVMM_X64_MSR_EFER] = env->efer;
++    state->msrs[NVMM_X64_MSR_STAR] = env->star;
++#ifdef TARGET_X86_64
++    state->msrs[NVMM_X64_MSR_LSTAR] = env->lstar;
++    state->msrs[NVMM_X64_MSR_CSTAR] = env->cstar;
++    state->msrs[NVMM_X64_MSR_SFMASK] = env->fmask;
++    state->msrs[NVMM_X64_MSR_KERNELGSBASE] = env->kernelgsbase;
++#endif
++    state->msrs[NVMM_X64_MSR_SYSENTER_CS]  = env->sysenter_cs;
++    state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = env->sysenter_esp;
++    state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = env->sysenter_eip;
++    state->msrs[NVMM_X64_MSR_PAT] = env->pat;
++    state->msrs[NVMM_X64_MSR_TSC] = env->tsc;
++
++    bitmap =
++        NVMM_X64_STATE_SEGS |
++        NVMM_X64_STATE_GPRS |
++        NVMM_X64_STATE_CRS  |
++        NVMM_X64_STATE_DRS  |
++        NVMM_X64_STATE_MSRS |
++        NVMM_X64_STATE_FPU;
++
++    ret = nvmm_vcpu_setstate(mach, vcpu, bitmap);
++    if (ret == -1) {
++        error_report("NVMM: Failed to set virtual processor context,"
++            " error=%d", errno);
++    }
++}
++
++static void
++nvmm_get_segment(SegmentCache *qseg, const struct nvmm_x64_state_seg *nseg)
++{
++    qseg->selector = nseg->selector;
++    qseg->limit = nseg->limit;
++    qseg->base = nseg->base;
++
++    qseg->flags =
++        __SHIFTIN((uint32_t)nseg->attrib.type, DESC_TYPE_MASK) |
++        __SHIFTIN((uint32_t)nseg->attrib.s, DESC_S_MASK) |
++        __SHIFTIN((uint32_t)nseg->attrib.dpl, DESC_DPL_MASK) |
++        __SHIFTIN((uint32_t)nseg->attrib.p, DESC_P_MASK) |
++        __SHIFTIN((uint32_t)nseg->attrib.avl, DESC_AVL_MASK) |
++        __SHIFTIN((uint32_t)nseg->attrib.l, DESC_L_MASK) |
++        __SHIFTIN((uint32_t)nseg->attrib.def, DESC_B_MASK) |
++        __SHIFTIN((uint32_t)nseg->attrib.g, DESC_G_MASK);
++}
++
++static void
++nvmm_get_registers(CPUState *cpu)
++{
++    struct CPUX86State *env = (CPUArchState *)cpu->env_ptr;
++    struct nvmm_machine *mach = get_nvmm_mach();
++    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
++    struct nvmm_vcpu *vcpu = &qcpu->vcpu;
++    X86CPU *x86_cpu = X86_CPU(cpu);
++    struct nvmm_x64_state *state = vcpu->state;
++    uint64_t bitmap, tpr;
++    size_t i;
++    int ret;
++
++    assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
++
++    bitmap =
++        NVMM_X64_STATE_SEGS |
++        NVMM_X64_STATE_GPRS |
++        NVMM_X64_STATE_CRS  |
++        NVMM_X64_STATE_DRS  |
++        NVMM_X64_STATE_MSRS |
++        NVMM_X64_STATE_FPU;
++
++    ret = nvmm_vcpu_getstate(mach, vcpu, bitmap);
++    if (ret == -1) {
++        error_report("NVMM: Failed to get virtual processor context,"
++            " error=%d", errno);
++    }
++
++    /* GPRs. */
++    env->regs[R_EAX] = state->gprs[NVMM_X64_GPR_RAX];
++    env->regs[R_ECX] = state->gprs[NVMM_X64_GPR_RCX];
++    env->regs[R_EDX] = state->gprs[NVMM_X64_GPR_RDX];
++    env->regs[R_EBX] = state->gprs[NVMM_X64_GPR_RBX];
++    env->regs[R_ESP] = state->gprs[NVMM_X64_GPR_RSP];
++    env->regs[R_EBP] = state->gprs[NVMM_X64_GPR_RBP];
++    env->regs[R_ESI] = state->gprs[NVMM_X64_GPR_RSI];
++    env->regs[R_EDI] = state->gprs[NVMM_X64_GPR_RDI];
++#ifdef TARGET_X86_64
++    env->regs[R_R8]  = state->gprs[NVMM_X64_GPR_R8];
++    env->regs[R_R9]  = state->gprs[NVMM_X64_GPR_R9];
++    env->regs[R_R10] = state->gprs[NVMM_X64_GPR_R10];
++    env->regs[R_R11] = state->gprs[NVMM_X64_GPR_R11];
++    env->regs[R_R12] = state->gprs[NVMM_X64_GPR_R12];
++    env->regs[R_R13] = state->gprs[NVMM_X64_GPR_R13];
++    env->regs[R_R14] = state->gprs[NVMM_X64_GPR_R14];
++    env->regs[R_R15] = state->gprs[NVMM_X64_GPR_R15];
++#endif
++
++    /* RIP and RFLAGS. */
++    env->eip = state->gprs[NVMM_X64_GPR_RIP];
++    env->eflags = state->gprs[NVMM_X64_GPR_RFLAGS];
++
++    /* Segments. */
++    nvmm_get_segment(&env->segs[R_ES], &state->segs[NVMM_X64_SEG_ES]);
++    nvmm_get_segment(&env->segs[R_CS], &state->segs[NVMM_X64_SEG_CS]);
++    nvmm_get_segment(&env->segs[R_SS], &state->segs[NVMM_X64_SEG_SS]);
++    nvmm_get_segment(&env->segs[R_DS], &state->segs[NVMM_X64_SEG_DS]);
++    nvmm_get_segment(&env->segs[R_FS], &state->segs[NVMM_X64_SEG_FS]);
++    nvmm_get_segment(&env->segs[R_GS], &state->segs[NVMM_X64_SEG_GS]);
++
++    /* Special segments. */
++    nvmm_get_segment(&env->gdt, &state->segs[NVMM_X64_SEG_GDT]);
++    nvmm_get_segment(&env->ldt, &state->segs[NVMM_X64_SEG_LDT]);
++    nvmm_get_segment(&env->tr, &state->segs[NVMM_X64_SEG_TR]);
++    nvmm_get_segment(&env->idt, &state->segs[NVMM_X64_SEG_IDT]);
++
++    /* Control registers. */
++    env->cr[0] = state->crs[NVMM_X64_CR_CR0];
++    env->cr[2] = state->crs[NVMM_X64_CR_CR2];
++    env->cr[3] = state->crs[NVMM_X64_CR_CR3];
++    env->cr[4] = state->crs[NVMM_X64_CR_CR4];
++    tpr = state->crs[NVMM_X64_CR_CR8];
++    if (tpr != qcpu->tpr) {
++        qcpu->tpr = tpr;
++        cpu_set_apic_tpr(x86_cpu->apic_state, tpr);
++    }
++    env->xcr0 = state->crs[NVMM_X64_CR_XCR0];
++
++    /* Debug registers. */
++    env->dr[0] = state->drs[NVMM_X64_DR_DR0];
++    env->dr[1] = state->drs[NVMM_X64_DR_DR1];
++    env->dr[2] = state->drs[NVMM_X64_DR_DR2];
++    env->dr[3] = state->drs[NVMM_X64_DR_DR3];
++    env->dr[6] = state->drs[NVMM_X64_DR_DR6];
++    env->dr[7] = state->drs[NVMM_X64_DR_DR7];
++
++    /* FPU. */
++    env->fpuc = state->fpu.fx_cw;
++    env->fpstt = (state->fpu.fx_sw >> 11) & 0x7;
++    env->fpus = state->fpu.fx_sw & ~0x3800;
++    for (i = 0; i < 8; i++) {
++        env->fptags[i] = !((state->fpu.fx_tw >> i) & 1);
++    }
++    env->fpop = state->fpu.fx_opcode;
++    env->fpip = state->fpu.fx_ip.fa_64;
++    env->fpdp = state->fpu.fx_dp.fa_64;
++    env->mxcsr = state->fpu.fx_mxcsr;
++    assert(sizeof(state->fpu.fx_87_ac) == sizeof(env->fpregs));
++    memcpy(env->fpregs, state->fpu.fx_87_ac, sizeof(env->fpregs));
++    for (i = 0; i < CPU_NB_REGS; i++) {
++        memcpy(&env->xmm_regs[i].ZMM_Q(0),
++            &state->fpu.fx_xmm[i].xmm_bytes[0], 8);
++        memcpy(&env->xmm_regs[i].ZMM_Q(1),
++            &state->fpu.fx_xmm[i].xmm_bytes[8], 8);
++    }
++
++    /* MSRs. */
++    env->efer = state->msrs[NVMM_X64_MSR_EFER];
++    env->star = state->msrs[NVMM_X64_MSR_STAR];
++#ifdef TARGET_X86_64
++    env->lstar = state->msrs[NVMM_X64_MSR_LSTAR];
++    env->cstar = state->msrs[NVMM_X64_MSR_CSTAR];
++    env->fmask = state->msrs[NVMM_X64_MSR_SFMASK];
++    env->kernelgsbase = state->msrs[NVMM_X64_MSR_KERNELGSBASE];
++#endif
++    env->sysenter_cs  = state->msrs[NVMM_X64_MSR_SYSENTER_CS];
++    env->sysenter_esp = state->msrs[NVMM_X64_MSR_SYSENTER_ESP];
++    env->sysenter_eip = state->msrs[NVMM_X64_MSR_SYSENTER_EIP];
++    env->pat = state->msrs[NVMM_X64_MSR_PAT];
++    env->tsc = state->msrs[NVMM_X64_MSR_TSC];
++
++    x86_update_hflags(env);
++}
++
++static bool
++nvmm_can_take_int(CPUState *cpu)
++{
++    struct CPUX86State *env = (CPUArchState *)cpu->env_ptr;
++    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
++    struct nvmm_vcpu *vcpu = &qcpu->vcpu;
++    struct nvmm_machine *mach = get_nvmm_mach();
++
++    if (qcpu->int_window_exit) {
++        return false;
++    }
++
++    if (qcpu->int_shadow || !(env->eflags & IF_MASK)) {
++        struct nvmm_x64_state *state = vcpu->state;
++
++        /* Exit on interrupt window. */
++        nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_INTR);
++        state->intr.int_window_exiting = 1;
++        nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_INTR);
++
++        return false;
++    }
++
++    return true;
++}
++
++static bool
++nvmm_can_take_nmi(CPUState *cpu)
++{
++    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
++
++    /*
++     * Contrary to INTs, NMIs always schedule an exit when they are
++     * completed. Therefore, if window-exiting is enabled, it means
++     * NMIs are blocked.
++     */
++    if (qcpu->nmi_window_exit) {
++        return false;
++    }
++
++    return true;
++}
++
++/*
++ * Called before the VCPU is run. We inject events generated by the I/O
++ * thread, and synchronize the guest TPR.
++ */
++static void
++nvmm_vcpu_pre_run(CPUState *cpu)
++{
++    struct CPUX86State *env = (CPUArchState *)cpu->env_ptr;
++    struct nvmm_machine *mach = get_nvmm_mach();
++    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
++    struct nvmm_vcpu *vcpu = &qcpu->vcpu;
++    X86CPU *x86_cpu = X86_CPU(cpu);
++    struct nvmm_x64_state *state = vcpu->state;
++    struct nvmm_vcpu_event *event = vcpu->event;
++    bool has_event = false;
++    bool sync_tpr = false;
++    uint8_t tpr;
++    int ret;
++
++    qemu_mutex_lock_iothread();
++
++    tpr = cpu_get_apic_tpr(x86_cpu->apic_state);
++    if (tpr != qcpu->tpr) {
++        qcpu->tpr = tpr;
++        sync_tpr = true;
++    }
++
++    /*
++     * Force the VCPU out of its inner loop to process any INIT requests
++     * or commit pending TPR access.
++     */
++    if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
++        cpu->exit_request = 1;
++    }
++
++    if (!has_event && (cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
++        if (nvmm_can_take_nmi(cpu)) {
++            cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
++            event->type = NVMM_VCPU_EVENT_INTR;
++            event->vector = 2;
++            has_event = true;
++        }
++    }
++
++    if (!has_event && (cpu->interrupt_request & CPU_INTERRUPT_HARD)) {
++        if (nvmm_can_take_int(cpu)) {
++            cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
++            event->type = NVMM_VCPU_EVENT_INTR;
++            event->vector = cpu_get_pic_interrupt(env);
++            has_event = true;
++        }
++    }
++
++    /* Don't want SMIs. */
++    if (cpu->interrupt_request & CPU_INTERRUPT_SMI) {
++        cpu->interrupt_request &= ~CPU_INTERRUPT_SMI;
++    }
++
++    if (sync_tpr) {
++        ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_CRS);
++        if (ret == -1) {
++            error_report("NVMM: Failed to get CPU state,"
++                " error=%d", errno);
++        }
++
++        state->crs[NVMM_X64_CR_CR8] = qcpu->tpr;
++
++        ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_CRS);
++        if (ret == -1) {
++            error_report("NVMM: Failed to set CPU state,"
++                " error=%d", errno);
++        }
++    }
++
++    if (has_event) {
++        ret = nvmm_vcpu_inject(mach, vcpu);
++        if (ret == -1) {
++            error_report("NVMM: Failed to inject event,"
++                " error=%d", errno);
++        }
++    }
++
++    qemu_mutex_unlock_iothread();
++}
++
++/*
++ * Called after the VCPU ran. We synchronize the host view of the TPR and
++ * RFLAGS.
++ */
++static void
++nvmm_vcpu_post_run(CPUState *cpu, struct nvmm_vcpu_exit *exit)
++{
++    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
++    struct CPUX86State *env = (CPUArchState *)cpu->env_ptr;
++    X86CPU *x86_cpu = X86_CPU(cpu);
++    uint64_t tpr;
++
++    env->eflags = exit->exitstate.rflags;
++    qcpu->int_shadow = exit->exitstate.int_shadow;
++    qcpu->int_window_exit = exit->exitstate.int_window_exiting;
++    qcpu->nmi_window_exit = exit->exitstate.nmi_window_exiting;
++
++    tpr = exit->exitstate.cr8;
++    if (qcpu->tpr != tpr) {
++        qcpu->tpr = tpr;
++        qemu_mutex_lock_iothread();
++        cpu_set_apic_tpr(x86_cpu->apic_state, qcpu->tpr);
++        qemu_mutex_unlock_iothread();
++    }
++}
++
++/* -------------------------------------------------------------------------- */
++
++static void
++nvmm_io_callback(struct nvmm_io *io)
++{
++    MemTxAttrs attrs = { 0 };
++    int ret;
++
++    ret = address_space_rw(&address_space_io, io->port, attrs, io->data,
++        io->size, !io->in);
++    if (ret != MEMTX_OK) {
++        error_report("NVMM: I/O Transaction Failed "
++            "[%s, port=%u, size=%zu]", (io->in ? "in" : "out"),
++            io->port, io->size);
++    }
++
++    /* Needed, otherwise infinite loop. */
++    current_cpu->vcpu_dirty = false;
++}
++
++static void
++nvmm_mem_callback(struct nvmm_mem *mem)
++{
++    cpu_physical_memory_rw(mem->gpa, mem->data, mem->size, mem->write);
++
++    /* XXX Needed, otherwise infinite loop. */
++    current_cpu->vcpu_dirty = false;
++}
++
++static struct nvmm_assist_callbacks nvmm_callbacks = {
++    .io = nvmm_io_callback,
++    .mem = nvmm_mem_callback
++};
++
++/* -------------------------------------------------------------------------- */
++
++static int
++nvmm_handle_mem(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu)
++{
++    int ret;
++
++    ret = nvmm_assist_mem(mach, vcpu);
++    if (ret == -1) {
++        error_report("NVMM: Mem Assist Failed [gpa=%p]",
++            (void *)vcpu->exit->u.mem.gpa);
++    }
++
++    return ret;
++}
++
++static int
++nvmm_handle_io(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu)
++{
++    int ret;
++
++    ret = nvmm_assist_io(mach, vcpu);
++    if (ret == -1) {
++        error_report("NVMM: I/O Assist Failed [port=%d]",
++            (int)vcpu->exit->u.io.port);
++    }
++
++    return ret;
++}
++
++static int
++nvmm_handle_rdmsr(struct nvmm_machine *mach, CPUState *cpu,
++    struct nvmm_vcpu_exit *exit)
++{
++    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
++    struct nvmm_vcpu *vcpu = &qcpu->vcpu;
++    X86CPU *x86_cpu = X86_CPU(cpu);
++    struct nvmm_x64_state *state = vcpu->state;
++    uint64_t val;
++    int ret;
++
++    switch (exit->u.rdmsr.msr) {
++    case MSR_IA32_APICBASE:
++        val = cpu_get_apic_base(x86_cpu->apic_state);
++        break;
++    case MSR_MTRRcap:
++    case MSR_MTRRdefType:
++    case MSR_MCG_CAP:
++    case MSR_MCG_STATUS:
++        val = 0;
++        break;
++    default: /* More MSRs to add? */
++        val = 0;
++        error_report("NVMM: Unexpected RDMSR 0x%x, ignored",
++            exit->u.rdmsr.msr);
++        break;
++    }
++
++    ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_GPRS);
++    if (ret == -1) {
++        return -1;
++    }
++
++    state->gprs[NVMM_X64_GPR_RAX] = (val & 0xFFFFFFFF);
++    state->gprs[NVMM_X64_GPR_RDX] = (val >> 32);
++    state->gprs[NVMM_X64_GPR_RIP] = exit->u.rdmsr.npc;
++
++    ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS);
++    if (ret == -1) {
++        return -1;
++    }
++
++    return 0;
++}
++
++static int
++nvmm_handle_wrmsr(struct nvmm_machine *mach, CPUState *cpu,
++    struct nvmm_vcpu_exit *exit)
++{
++    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
++    struct nvmm_vcpu *vcpu = &qcpu->vcpu;
++    X86CPU *x86_cpu = X86_CPU(cpu);
++    struct nvmm_x64_state *state = vcpu->state;
++    uint64_t val;
++    int ret;
++
++    val = exit->u.wrmsr.val;
++
++    switch (exit->u.wrmsr.msr) {
++    case MSR_IA32_APICBASE:
++        cpu_set_apic_base(x86_cpu->apic_state, val);
++        break;
++    case MSR_MTRRdefType:
++    case MSR_MCG_STATUS:
++        break;
++    default: /* More MSRs to add? */
++        error_report("NVMM: Unexpected WRMSR 0x%x [val=0x%lx], ignored",
++            exit->u.wrmsr.msr, val);
++        break;
++    }
++
++    ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_GPRS);
++    if (ret == -1) {
++        return -1;
++    }
++
++    state->gprs[NVMM_X64_GPR_RIP] = exit->u.wrmsr.npc;
++
++    ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS);
++    if (ret == -1) {
++        return -1;
++    }
++
++    return 0;
++}
++
++static int
++nvmm_handle_halted(struct nvmm_machine *mach, CPUState *cpu,
++    struct nvmm_vcpu_exit *exit)
++{
++    struct CPUX86State *env = (CPUArchState *)cpu->env_ptr;
++    int ret = 0;
++
++    qemu_mutex_lock_iothread();
++
++    if (!((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
++          (env->eflags & IF_MASK)) &&
++        !(cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
++        cpu->exception_index = EXCP_HLT;
++        cpu->halted = true;
++        ret = 1;
++    }
++
++    qemu_mutex_unlock_iothread();
++
++    return ret;
++}
++
++static int
++nvmm_inject_ud(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu)
++{
++    struct nvmm_vcpu_event *event = vcpu->event;
++
++    event->type = NVMM_VCPU_EVENT_EXCP;
++    event->vector = 6;
++    event->u.excp.error = 0;
++
++    return nvmm_vcpu_inject(mach, vcpu);
++}
++
++static int
++nvmm_vcpu_loop(CPUState *cpu)
++{
++    struct CPUX86State *env = (CPUArchState *)cpu->env_ptr;
++    struct nvmm_machine *mach = get_nvmm_mach();
++    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
++    struct nvmm_vcpu *vcpu = &qcpu->vcpu;
++    X86CPU *x86_cpu = X86_CPU(cpu);
++    struct nvmm_vcpu_exit *exit = vcpu->exit;
++    int ret;
++
++    /*
++     * Some asynchronous events must be handled outside of the inner
++     * VCPU loop. They are handled here.
++     */
++    if (cpu->interrupt_request & CPU_INTERRUPT_INIT) {
++        nvmm_cpu_synchronize_state(cpu);
++        do_cpu_init(x86_cpu);
++        /* set int/nmi windows back to the reset state */
++    }
++    if (cpu->interrupt_request & CPU_INTERRUPT_POLL) {
++        cpu->interrupt_request &= ~CPU_INTERRUPT_POLL;
++        apic_poll_irq(x86_cpu->apic_state);
++    }
++    if (((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
++         (env->eflags & IF_MASK)) ||
++        (cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
++        cpu->halted = false;
++    }
++    if (cpu->interrupt_request & CPU_INTERRUPT_SIPI) {
++        nvmm_cpu_synchronize_state(cpu);
++        do_cpu_sipi(x86_cpu);
++    }
++    if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
++        cpu->interrupt_request &= ~CPU_INTERRUPT_TPR;
++        nvmm_cpu_synchronize_state(cpu);
++        apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip,
++            env->tpr_access_type);
++    }
++
++    if (cpu->halted) {
++        cpu->exception_index = EXCP_HLT;
++        qatomic_set(&cpu->exit_request, false);
++        return 0;
++    }
++
++    qemu_mutex_unlock_iothread();
++    cpu_exec_start(cpu);
++
++    /*
++     * Inner VCPU loop.
++     */
++    do {
++        if (cpu->vcpu_dirty) {
++            nvmm_set_registers(cpu);
++            cpu->vcpu_dirty = false;
++        }
++
++        if (qcpu->stop) {
++            cpu->exception_index = EXCP_INTERRUPT;
++            qcpu->stop = false;
++            ret = 1;
++            break;
++        }
++
++        nvmm_vcpu_pre_run(cpu);
++
++        if (qatomic_read(&cpu->exit_request)) {
++            qemu_cpu_kick_self();
++        }
++
++        ret = nvmm_vcpu_run(mach, vcpu);
++        if (ret == -1) {
++            error_report("NVMM: Failed to exec a virtual processor,"
++                " error=%d", errno);
++            break;
++        }
++
++        nvmm_vcpu_post_run(cpu, exit);
++
++        switch (exit->reason) {
++        case NVMM_VCPU_EXIT_NONE:
++            break;
++        case NVMM_VCPU_EXIT_MEMORY:
++            ret = nvmm_handle_mem(mach, vcpu);
++            break;
++        case NVMM_VCPU_EXIT_IO:
++            ret = nvmm_handle_io(mach, vcpu);
++            break;
++        case NVMM_VCPU_EXIT_INT_READY:
++        case NVMM_VCPU_EXIT_NMI_READY:
++        case NVMM_VCPU_EXIT_TPR_CHANGED:
++            break;
++        case NVMM_VCPU_EXIT_HALTED:
++            ret = nvmm_handle_halted(mach, cpu, exit);
++            break;
++        case NVMM_VCPU_EXIT_SHUTDOWN:
++            qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
++            cpu->exception_index = EXCP_INTERRUPT;
++            ret = 1;
++            break;
++        case NVMM_VCPU_EXIT_RDMSR:
++            ret = nvmm_handle_rdmsr(mach, cpu, exit);
++            break;
++        case NVMM_VCPU_EXIT_WRMSR:
++            ret = nvmm_handle_wrmsr(mach, cpu, exit);
++            break;
++        case NVMM_VCPU_EXIT_MONITOR:
++        case NVMM_VCPU_EXIT_MWAIT:
++            ret = nvmm_inject_ud(mach, vcpu);
++            break;
++        default:
++            error_report("NVMM: Unexpected VM exit code 0x%lx [hw=0x%lx]",
++                exit->reason, exit->u.inv.hwcode);
++            nvmm_get_registers(cpu);
++            qemu_mutex_lock_iothread();
++            qemu_system_guest_panicked(cpu_get_crash_info(cpu));
++            qemu_mutex_unlock_iothread();
++            ret = -1;
++            break;
++        }
++    } while (ret == 0);
++
++    cpu_exec_end(cpu);
++    qemu_mutex_lock_iothread();
++    current_cpu = cpu;
++
++    qatomic_set(&cpu->exit_request, false);
++
++    return ret < 0;
++}
++
++/* -------------------------------------------------------------------------- */
++
++static void
++do_nvmm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
++{
++    nvmm_get_registers(cpu);
++    cpu->vcpu_dirty = true;
++}
++
++static void
++do_nvmm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg)
++{
++    nvmm_set_registers(cpu);
++    cpu->vcpu_dirty = false;
++}
++
++static void
++do_nvmm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg)
++{
++    nvmm_set_registers(cpu);
++    cpu->vcpu_dirty = false;
++}
++
++static void
++do_nvmm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg)
++{
++    cpu->vcpu_dirty = true;
++}
++
++void nvmm_cpu_synchronize_state(CPUState *cpu)
++{
++    if (!cpu->vcpu_dirty) {
++        run_on_cpu(cpu, do_nvmm_cpu_synchronize_state, RUN_ON_CPU_NULL);
++    }
++}
++
++void nvmm_cpu_synchronize_post_reset(CPUState *cpu)
++{
++    run_on_cpu(cpu, do_nvmm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
++}
++
++void nvmm_cpu_synchronize_post_init(CPUState *cpu)
++{
++    run_on_cpu(cpu, do_nvmm_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
++}
++
++void nvmm_cpu_synchronize_pre_loadvm(CPUState *cpu)
++{
++    run_on_cpu(cpu, do_nvmm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
++}
++
++/* -------------------------------------------------------------------------- */
++
++static Error *nvmm_migration_blocker;
++
++static void
++nvmm_ipi_signal(int sigcpu)
++{
++    struct qemu_vcpu *qcpu;
++
++    if (current_cpu) {
++        qcpu = get_qemu_vcpu(current_cpu);
++        qcpu->stop = true;
++    }
++}
++
++static void
++nvmm_init_cpu_signals(void)
++{
++    struct sigaction sigact;
++    sigset_t set;
++
++    /* Install the IPI handler. */
++    memset(&sigact, 0, sizeof(sigact));
++    sigact.sa_handler = nvmm_ipi_signal;
++    sigaction(SIG_IPI, &sigact, NULL);
++
++    /* Allow IPIs on the current thread. */
++    sigprocmask(SIG_BLOCK, NULL, &set);
++    sigdelset(&set, SIG_IPI);
++    pthread_sigmask(SIG_SETMASK, &set, NULL);
++}
++
++int
++nvmm_init_vcpu(CPUState *cpu)
++{
++    struct nvmm_machine *mach = get_nvmm_mach();
++    struct nvmm_vcpu_conf_cpuid cpuid;
++    struct nvmm_vcpu_conf_tpr tpr;
++    Error *local_error = NULL;
++    struct qemu_vcpu *qcpu;
++    int ret, err;
++
++    nvmm_init_cpu_signals();
++
++    if (nvmm_migration_blocker == NULL) {
++        error_setg(&nvmm_migration_blocker,
++            "NVMM: Migration not supported");
++
++        (void)migrate_add_blocker(nvmm_migration_blocker, &local_error);
++        if (local_error) {
++            error_report_err(local_error);
++            migrate_del_blocker(nvmm_migration_blocker);
++            error_free(nvmm_migration_blocker);
++            return -EINVAL;
++        }
++    }
++
++    qcpu = g_malloc0(sizeof(*qcpu));
++    if (qcpu == NULL) {
++        error_report("NVMM: Failed to allocate VCPU context.");
++        return -ENOMEM;
++    }
++
++    ret = nvmm_vcpu_create(mach, cpu->cpu_index, &qcpu->vcpu);
++    if (ret == -1) {
++        err = errno;
++        error_report("NVMM: Failed to create a virtual processor,"
++            " error=%d", err);
++        g_free(qcpu);
++        return -err;
++    }
++
++    memset(&cpuid, 0, sizeof(cpuid));
++    cpuid.mask = 1;
++    cpuid.leaf = 0x00000001;
++    cpuid.u.mask.set.edx = CPUID_MCE | CPUID_MCA | CPUID_MTRR;
++    ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_CPUID,
++        &cpuid);
++    if (ret == -1) {
++        err = errno;
++        error_report("NVMM: Failed to configure a virtual processor,"
++            " error=%d", err);
++        g_free(qcpu);
++        return -err;
++    }
++
++    ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_CALLBACKS,
++        &nvmm_callbacks);
++    if (ret == -1) {
++        err = errno;
++        error_report("NVMM: Failed to configure a virtual processor,"
++            " error=%d", err);
++        g_free(qcpu);
++        return -err;
++    }
++
++    if (qemu_mach.cap.arch.vcpu_conf_support & NVMM_CAP_ARCH_VCPU_CONF_TPR) {
++        memset(&tpr, 0, sizeof(tpr));
++        tpr.exit_changed = 1;
++        ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_TPR, &tpr);
++        if (ret == -1) {
++            err = errno;
++            error_report("NVMM: Failed to configure a virtual processor,"
++                " error=%d", err);
++            g_free(qcpu);
++            return -err;
++        }
++    }
++
++    cpu->vcpu_dirty = true;
++    cpu->hax_vcpu = (struct hax_vcpu_state *)qcpu;
++
++    return 0;
++}
++
++int
++nvmm_vcpu_exec(CPUState *cpu)
++{
++    int ret, fatal;
++
++    while (1) {
++        if (cpu->exception_index >= EXCP_INTERRUPT) {
++            ret = cpu->exception_index;
++            cpu->exception_index = -1;
++            break;
++        }
++
++        fatal = nvmm_vcpu_loop(cpu);
++
++        if (fatal) {
++            error_report("NVMM: Failed to execute a VCPU.");
++            abort();
++        }
++    }
++
++    return ret;
++}
++
++void
++nvmm_destroy_vcpu(CPUState *cpu)
++{
++    struct nvmm_machine *mach = get_nvmm_mach();
++    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
++
++    nvmm_vcpu_destroy(mach, &qcpu->vcpu);
++    g_free(cpu->hax_vcpu);
++}
++
++/* -------------------------------------------------------------------------- */
++
++static void
++nvmm_update_mapping(hwaddr start_pa, ram_addr_t size, uintptr_t hva,
++    bool add, bool rom, const char *name)
++{
++    struct nvmm_machine *mach = get_nvmm_mach();
++    int ret, prot;
++
++    if (add) {
++        prot = PROT_READ | PROT_EXEC;
++        if (!rom) {
++            prot |= PROT_WRITE;
++        }
++        ret = nvmm_gpa_map(mach, hva, start_pa, size, prot);
++    } else {
++        ret = nvmm_gpa_unmap(mach, hva, start_pa, size);
++    }
++
++    if (ret == -1) {
++        error_report("NVMM: Failed to %s GPA range '%s' PA:%p, "
++            "Size:%p bytes, HostVA:%p, error=%d",
++            (add ? "map" : "unmap"), name, (void *)(uintptr_t)start_pa,
++            (void *)size, (void *)hva, errno);
++    }
++}
++
++static void
++nvmm_process_section(MemoryRegionSection *section, int add)
++{
++    MemoryRegion *mr = section->mr;
++    hwaddr start_pa = section->offset_within_address_space;
++    ram_addr_t size = int128_get64(section->size);
++    unsigned int delta;
++    uintptr_t hva;
++
++    if (!memory_region_is_ram(mr)) {
++        return;
++    }
++
++    /* Adjust start_pa and size so that they are page-aligned. */
++    delta = qemu_real_host_page_size - (start_pa & ~qemu_real_host_page_mask);
++    delta &= ~qemu_real_host_page_mask;
++    if (delta > size) {
++        return;
++    }
++    start_pa += delta;
++    size -= delta;
++    size &= qemu_real_host_page_mask;
++    if (!size || (start_pa & ~qemu_real_host_page_mask)) {
++        return;
++    }
++
++    hva = (uintptr_t)memory_region_get_ram_ptr(mr) +
++        section->offset_within_region + delta;
++
++    nvmm_update_mapping(start_pa, size, hva, add,
++        memory_region_is_rom(mr), mr->name);
++}
++
++static void
++nvmm_region_add(MemoryListener *listener, MemoryRegionSection *section)
++{
++    memory_region_ref(section->mr);
++    nvmm_process_section(section, 1);
++}
++
++static void
++nvmm_region_del(MemoryListener *listener, MemoryRegionSection *section)
++{
++    nvmm_process_section(section, 0);
++    memory_region_unref(section->mr);
++}
++
++static void
++nvmm_transaction_begin(MemoryListener *listener)
++{
++    /* nothing */
++}
++
++static void
++nvmm_transaction_commit(MemoryListener *listener)
++{
++    /* nothing */
++}
++
++static void
++nvmm_log_sync(MemoryListener *listener, MemoryRegionSection *section)
++{
++    MemoryRegion *mr = section->mr;
++
++    if (!memory_region_is_ram(mr)) {
++        return;
++    }
++
++    memory_region_set_dirty(mr, 0, int128_get64(section->size));
++}
++
++static MemoryListener nvmm_memory_listener = {
++    .begin = nvmm_transaction_begin,
++    .commit = nvmm_transaction_commit,
++    .region_add = nvmm_region_add,
++    .region_del = nvmm_region_del,
++    .log_sync = nvmm_log_sync,
++    .priority = 10,
++};
++
++static void
++nvmm_ram_block_added(RAMBlockNotifier *n, void *host, size_t size)
++{
++    struct nvmm_machine *mach = get_nvmm_mach();
++    uintptr_t hva = (uintptr_t)host;
++    int ret;
++
++    ret = nvmm_hva_map(mach, hva, size);
++
++    if (ret == -1) {
++        error_report("NVMM: Failed to map HVA, HostVA:%p "
++            "Size:%p bytes, error=%d",
++            (void *)hva, (void *)size, errno);
++    }
++}
++
++static struct RAMBlockNotifier nvmm_ram_notifier = {
++    .ram_block_added = nvmm_ram_block_added
++};
++
++/* -------------------------------------------------------------------------- */
++
++static int
++nvmm_accel_init(MachineState *ms)
++{
++    int ret, err;
++
++    ret = nvmm_init();
++    if (ret == -1) {
++        err = errno;
++        error_report("NVMM: Initialization failed, error=%d", errno);
++        return -err;
++    }
++
++    ret = nvmm_capability(&qemu_mach.cap);
++    if (ret == -1) {
++        err = errno;
++        error_report("NVMM: Unable to fetch capability, error=%d", errno);
++        return -err;
++    }
++    if (qemu_mach.cap.version != 1) {
++        error_report("NVMM: Unsupported version %u", qemu_mach.cap.version);
++        return -EPROGMISMATCH;
++    }
++    if (qemu_mach.cap.state_size != sizeof(struct nvmm_x64_state)) {
++        error_report("NVMM: Wrong state size %u", qemu_mach.cap.state_size);
++        return -EPROGMISMATCH;
++    }
++
++    ret = nvmm_machine_create(&qemu_mach.mach);
++    if (ret == -1) {
++        err = errno;
++        error_report("NVMM: Machine creation failed, error=%d", errno);
++        return -err;
++    }
++
++    memory_listener_register(&nvmm_memory_listener, &address_space_memory);
++    ram_block_notifier_add(&nvmm_ram_notifier);
++
++    cpus_register_accel(&nvmm_cpus);
++
++    printf("NetBSD Virtual Machine Monitor accelerator is operational\n");
++    return 0;
++}
++
++int
++nvmm_enabled(void)
++{
++    return nvmm_allowed;
++}
++
++static void
++nvmm_accel_class_init(ObjectClass *oc, void *data)
++{
++    AccelClass *ac = ACCEL_CLASS(oc);
++    ac->name = "NVMM";
++    ac->init_machine = nvmm_accel_init;
++    ac->allowed = &nvmm_allowed;
++}
++
++static const TypeInfo nvmm_accel_type = {
++    .name = ACCEL_CLASS_NAME("nvmm"),
++    .parent = TYPE_ACCEL,
++    .class_init = nvmm_accel_class_init,
++};
++
++static void
++nvmm_type_init(void)
++{
++    type_register_static(&nvmm_accel_type);
++}
++
++type_init(nvmm_type_init);
diff --git a/emulators/qemu/patches/patch-target_i386_nvmm_cpus.c b/emulators/qemu/patches/patch-target_i386_nvmm_cpus.c
new file mode 100644
index 00000000000..aef0f6bf074
--- /dev/null
+++ b/emulators/qemu/patches/patch-target_i386_nvmm_cpus.c
@@ -0,0 +1,105 @@
+$NetBSD: patch-target_i386_nvmm_cpus.c,v 1.1 2021/03/06 11:19:34 reinoud Exp $
+
+--- target/i386/nvmm-cpus.c.orig	2021-03-05 16:55:06.648916883 +0000
++++ target/i386/nvmm-cpus.c
+@@ -0,0 +1,100 @@
++/*
++ * Copyright (c) 2018-2019 Maxime Villard, All rights reserved.
++ *
++ * NetBSD Virtual Machine Monitor (NVMM) accelerator for QEMU.
++ *
++ * This work is licensed under the terms of the GNU GPL, version 2 or later.
++ * See the COPYING file in the top-level directory.
++ */
++
++#include "qemu/osdep.h"
++#include "sysemu/kvm_int.h"
++#include "qemu/main-loop.h"
++#include "sysemu/cpus.h"
++#include "qemu/guest-random.h"
++
++#include "sysemu/nvmm.h"
++#include "nvmm-cpus.h"
++
++static void *qemu_nvmm_cpu_thread_fn(void *arg)
++{
++    CPUState *cpu = arg;
++    int r;
++
++    assert(nvmm_enabled());
++
++    rcu_register_thread();
++
++    qemu_mutex_lock_iothread();
++    qemu_thread_get_self(cpu->thread);
++    cpu->thread_id = qemu_get_thread_id();
++    current_cpu = cpu;
++
++    r = nvmm_init_vcpu(cpu);
++    if (r < 0) {
++        fprintf(stderr, "nvmm_init_vcpu failed: %s\n", strerror(-r));
++        exit(1);
++    }
++
++    /* signal CPU creation */
++    cpu_thread_signal_created(cpu);
++    qemu_guest_random_seed_thread_part2(cpu->random_seed);
++
++    do {
++        if (cpu_can_run(cpu)) {
++            r = nvmm_vcpu_exec(cpu);
++            if (r == EXCP_DEBUG) {
++                cpu_handle_guest_debug(cpu);
++            }
++        }
++        while (cpu_thread_is_idle(cpu)) {
++            qemu_cond_wait_iothread(cpu->halt_cond);
++        }
++        qemu_wait_io_event_common(cpu);
++    } while (!cpu->unplug || cpu_can_run(cpu));
++
++    nvmm_destroy_vcpu(cpu);
++    cpu_thread_signal_destroyed(cpu);
++    qemu_mutex_unlock_iothread();
++    rcu_unregister_thread();
++    return NULL;
++}
++
++static void nvmm_start_vcpu_thread(CPUState *cpu)
++{
++    char thread_name[VCPU_THREAD_NAME_SIZE];
++
++    cpu->thread = g_malloc0(sizeof(QemuThread));
++    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
++    qemu_cond_init(cpu->halt_cond);
++    snprintf(thread_name, VCPU_THREAD_NAME_SIZE, "CPU %d/NVMM",
++             cpu->cpu_index);
++    qemu_thread_create(cpu->thread, thread_name, qemu_nvmm_cpu_thread_fn,
++                       cpu, QEMU_THREAD_JOINABLE);
++}
++
++/*
++ * Abort the call to run the virtual processor by another thread, and to
++ * return the control to that thread.
++ */
++static void nvmm_kick_vcpu_thread(CPUState *cpu)
++{
++    /*                                                                                                         
++     * FIXME: race condition with the exit_request check in 
++     * hax_vcpu_hax_exec
++     */
++
++    // if (!qemu_cpu_is_self(cpu)) {
++    cpu->exit_request = 1;
++    cpus_kick_thread(cpu);
++    //}
++}
++
++const CpusAccel nvmm_cpus = {
++    .create_vcpu_thread = nvmm_start_vcpu_thread,
++    .kick_vcpu_thread = nvmm_kick_vcpu_thread,
++    .synchronize_post_reset = nvmm_cpu_synchronize_post_reset,
++    .synchronize_post_init = nvmm_cpu_synchronize_post_init,
++    .synchronize_state = nvmm_cpu_synchronize_state,
++    .synchronize_pre_loadvm = nvmm_cpu_synchronize_pre_loadvm,
++};
diff --git a/emulators/qemu/patches/patch-target_i386_nvmm_cpus.h b/emulators/qemu/patches/patch-target_i386_nvmm_cpus.h
new file mode 100644
index 00000000000..185cbca1ac9
--- /dev/null
+++ b/emulators/qemu/patches/patch-target_i386_nvmm_cpus.h
@@ -0,0 +1,32 @@
+$NetBSD: patch-target_i386_nvmm_cpus.h,v 1.1 2021/03/06 11:19:34 reinoud Exp $
+
+--- target/i386/nvmm-cpus.h.orig	2021-03-04 12:38:27.666319632 +0000
++++ target/i386/nvmm-cpus.h
+@@ -0,0 +1,27 @@
++/*
++ * Copyright (c) 2018-2019 Maxime Villard, All rights reserved.
++ *
++ * NetBSD Virtual Machine Monitor (NVMM) accelerator for QEMU.
++ *
++ * This work is licensed under the terms of the GNU GPL, version 2 or later.
++ * See the COPYING file in the top-level directory.
++ */
++
++#ifndef NVMM_CPUS_H
++#define NVMM_CPUS_H
++
++#include "sysemu/cpus.h"
++
++int nvmm_init_vcpu(CPUState *cpu);
++int nvmm_vcpu_exec(CPUState *cpu);
++void nvmm_destroy_vcpu(CPUState *cpu);
++void nvmm_vcpu_kick(CPUState *cpu);
++
++void nvmm_cpu_synchronize_state(CPUState *cpu);
++void nvmm_cpu_synchronize_post_reset(CPUState *cpu);
++void nvmm_cpu_synchronize_post_init(CPUState *cpu);
++void nvmm_cpu_synchronize_pre_loadvm(CPUState *cpu);
++
++extern const CpusAccel nvmm_cpus;
++
++#endif /* NVMM_CPUS_H */