OS-7419 bhyve: want switch to start VM in suspended state

Reviewed by: John Levon <john.levon@joyent.com>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Approved by: John Levon <john.levon@joyent.com>

3 files changed, 35 insertions, 3 deletions

diff --git a/usr/src/cmd/bhyve/bhyverun.c b/usr/src/cmd/bhyve/bhyverun.c
index 405accda35..c85c834e9f 100644
--- a/usr/src/cmd/bhyve/bhyverun.c
+++ b/usr/src/cmd/bhyve/bhyverun.c
@@ -182,6 +182,9 @@ usage(int code)
 		"       -A: create ACPI tables\n"
 		"       -c: number of cpus and/or topology specification\n"
 		"       -C: include guest memory in core file\n"
+#ifndef __FreeBSD__
+	        "       -d: suspend cpu at boot\n"
+#endif
 		"       -e: exit on unhandled I/O access\n"
 		"       -g: gdb port\n"
 		"       -h: help\n"
@@ -431,8 +434,14 @@ fbsdrun_start_thread(void *param)
 	return (NULL);
 }
 
+#ifdef __FreeBSD__
 void
 fbsdrun_addcpu(struct vmctx *ctx, int fromcpu, int newcpu, uint64_t rip)
+#else
+void
+fbsdrun_addcpu(struct vmctx *ctx, int fromcpu, int newcpu, uint64_t rip,
+    bool suspend)
+#endif
 {
 	int error;
 
@@ -450,6 +459,11 @@ fbsdrun_addcpu(struct vmctx *ctx, int fromcpu, int newcpu, uint64_t rip)
 
 	CPU_SET_ATOMIC(newcpu, &cpumask);
 
+#ifndef __FreeBSD__
+	if (suspend)
+		(void) vm_suspend_cpu(ctx, newcpu);
+#endif
+
 	/*
 	 * Set up the vmexit struct to allow execution to start
 	 * at the given RIP
@@ -1003,6 +1017,9 @@ main(int argc, char *argv[])
 	int max_vcpus, mptgen, memflags;
 	int rtc_localtime;
 	bool gdb_stop;
+#ifndef __FreeBSD__
+	bool suspend;
+#endif
 	struct vmctx *ctx;
 	uint64_t rip;
 	size_t memsize;
@@ -1024,7 +1041,7 @@ main(int argc, char *argv[])
 #ifdef	__FreeBSD__
 	optstr = "abehuwxACHIPSWYp:g:G:c:s:m:l:B:U:";
 #else
-	optstr = "abehuwxACHIPSWYg:G:c:s:m:l:B:U:";
+	optstr = "abdehuwxACHIPSWYg:G:c:s:m:l:B:U:";
 #endif
 	while ((c = getopt(argc, argv, optstr)) != -1) {
 		switch (c) {
@@ -1043,7 +1060,11 @@ main(int argc, char *argv[])
 				    "configuration '%s'", optarg);
 			}
 			break;
-#ifdef	__FreeBSD__
+#ifndef	__FreeBSD__
+		case 'd':
+			suspend = true;
+			break;
+#else
 		case 'p':
 			if (pincpu_parse(optarg) != 0) {
 				errx(EX_USAGE, "invalid vcpu pinning "
@@ -1277,9 +1298,11 @@ main(int argc, char *argv[])
 	/*
 	 * Add CPU 0
 	 */
+#ifdef __FreeBSD__
 	fbsdrun_addcpu(ctx, BSP, BSP, rip);
+#else
+	fbsdrun_addcpu(ctx, BSP, BSP, rip, suspend);
 
-#ifndef __FreeBSD__
 	mark_provisioned();
 #endif
 
diff --git a/usr/src/cmd/bhyve/bhyverun.h b/usr/src/cmd/bhyve/bhyverun.h
index 1d275d5375..cdde04862c 100644
--- a/usr/src/cmd/bhyve/bhyverun.h
+++ b/usr/src/cmd/bhyve/bhyverun.h
@@ -60,7 +60,12 @@ extern pthread_cond_t bcons_wait_done;
 void *paddr_guest2host(struct vmctx *ctx, uintptr_t addr, size_t len);
 
 void fbsdrun_set_capabilities(struct vmctx *ctx, int cpu);
+#ifdef __FreeBSD__
 void fbsdrun_addcpu(struct vmctx *ctx, int fromcpu, int newcpu, uint64_t rip);
+#else
+void fbsdrun_addcpu(struct vmctx *ctx, int fromcpu, int newcpu, uint64_t rip,
+    bool suspend);
+#endif
 int  fbsdrun_muxed(void);
 int  fbsdrun_vmexit_on_hlt(void);
 int  fbsdrun_vmexit_on_pause(void);
diff --git a/usr/src/cmd/bhyve/spinup_ap.c b/usr/src/cmd/bhyve/spinup_ap.c
index 7c4186f5ed..ecdd05694c 100644
--- a/usr/src/cmd/bhyve/spinup_ap.c
+++ b/usr/src/cmd/bhyve/spinup_ap.c
@@ -100,7 +100,11 @@ spinup_ap(struct vmctx *ctx, int vcpu, int newcpu, uint64_t rip)
 
 	spinup_ap_realmode(ctx, newcpu, &rip);
 
+#ifdef __FreeBSD__
 	fbsdrun_addcpu(ctx, vcpu, newcpu, rip);
+#else
+	fbsdrun_addcpu(ctx, vcpu, newcpu, rip, false);
+#endif
 
 	return (newcpu);
 }