OS-6882 zhyve should re-exec() for args

Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Reviewed by: Mike Gerdts <mike.gerdts@joyent.com>
Approved by: Mike Gerdts <mike.gerdts@joyent.com>

5 files changed, 76 insertions, 71 deletions

diff --git a/manifest b/manifest
index b4028fcaa7..56b24ae1a9 100644
--- a/manifest
+++ b/manifest
@@ -5102,7 +5102,7 @@ f usr/lib/brand/bhyve/uninstall 0555 root sys
 f usr/lib/brand/bhyve/config.xml 0444 root sys
 f usr/lib/brand/bhyve/platform.xml 0444 root sys
 f usr/lib/brand/bhyve/statechange 0555 root sys
-h usr/lib/brand/bhyve/zhyve=usr/sbin/bhyve
+f usr/lib/brand/bhyve/zhyve 0555 root sys
 d usr/lib/brand/lx 0755 root bin
 s usr/lib/brand/lx/64=amd64
 d usr/lib/brand/lx/amd64 0755 root bin
diff --git a/usr/src/cmd/bhyve/Makefile b/usr/src/cmd/bhyve/Makefile
index a32e5968bc..181ddf2946 100644
--- a/usr/src/cmd/bhyve/Makefile
+++ b/usr/src/cmd/bhyve/Makefile
@@ -15,7 +15,6 @@
 #
 
 PROG =		bhyve
-ZHYVE =		$(ROOT)/usr/lib/brand/bhyve/zhyve
 
 include ../Makefile.cmd
 include ../Makefile.cmd.64
@@ -76,18 +75,21 @@ SRCS =	acpi.c			\
 	vmm_instruction_emul.c	\
 	xmsr.c			\
 	spinup_ap.c		\
-	bhyve_sol_glue.c	\
-	zhyve.c
+	bhyve_sol_glue.c
 
 OBJS = $(SRCS:.c=.o)
 
-CLEANFILES =	$(PROG)
 CLOBBERFILES =	$(ROOTUSRSBINPROG) $(ZHYVE)
 
+ZHYVE_DIR =	$(ROOT)/usr/lib/brand/bhyve
+ZHYVE_PROG = 	zhyve
+ZHYVE =		$(ZHYVE_DIR)/$(ZHYVE_PROG)
+
 MEVENT_TEST_PROG = mevent_test
 MEVENT_TEST_SRCS = mevent.c mevent_test.c
 MEVENT_TEST_OBJS = $(MEVENT_TEST_SRCS:.c=.o)
-CLEANFILES +=	$(MEVENT_TEST_PROG) $(MEVENT_TEST_OBJS)
+
+CLEANFILES =	$(PROG) $(ZHYVE_PROG) $(MEVENT_TEST_PROG) $(MEVENT_TEST_OBJS)
 
 CFLAGS +=	$(CCVERBOSE) -_gcc=-Wimplicit-function-declaration -_gcc=-Wno-parentheses
 CPPFLAGS =	-I$(COMPAT)/freebsd -I$(CONTRIB)/freebsd \
@@ -102,19 +104,19 @@ CPPFLAGS =	-I$(COMPAT)/freebsd -I$(CONTRIB)/freebsd \
 		-I$(SRC)/uts/i86pc \
 		-I$(SRC)/lib/libdladm/common \
 		-DWITHOUT_CAPSICUM
-LDLIBS +=	-lsocket -lnsl -ldlpi -ldladm -lmd -luuid -lvmmapi -lz -lnvpair
 
 # Disable the crypto code until it is wired up
 CPPFLAGS +=	-DNO_OPENSSL
 
+LDLIBS +=	-lsocket -lnsl -ldlpi -ldladm -lmd -luuid -lvmmapi -lz
+
+zhyve := LDLIBS = -lnvpair
+
 POST_PROCESS += ; $(GENSETDEFS) $@
 
 .KEEP_STATE:
 
-# Real main is in zhyve.c
-bhyverun.o :=	CPPFLAGS += -Dmain=bhyve_main
-
-all: $(PROG) $(MEVENT_TEST_PROG) $(SUBDIRS)
+all: $(PROG) $(MEVENT_TEST_PROG) $(ZHYVE_PROG) $(SUBDIRS)
 
 $(PROG): $(OBJS)
 	$(LINK.c) -o $@ $(OBJS) $(LDFLAGS) $(LDLIBS)
@@ -123,9 +125,7 @@ $(PROG): $(OBJS)
 $(MEVENT_TEST_PROG): $(MEVENT_TEST_OBJS)
 	$(LINK.c) -o $@ $(MEVENT_TEST_OBJS) -lsocket
 
-install: all $(ROOTUSRSBINPROG) $(SUBDIRS)
-	$(RM) $(ZHYVE)
-	$(LN) $(ROOTUSRSBIN)/$(PROG) $(ZHYVE)
+install: all $(ZHYVE) $(ROOTUSRSBINPROG) $(SUBDIRS)
 
 clean: $(SUBDIRS)
 	$(RM) $(OBJS) $(CLEANFILES)
@@ -142,6 +142,9 @@ FRC:
 
 include ../Makefile.targ
 
+$(ZHYVE_DIR)/%: %
+	$(INS.file)
+
 %.o: $(SRC)/uts/i86pc/io/vmm/%.c
 	$(COMPILE.c) $<
 	$(POST_PROCESS_O)
diff --git a/usr/src/cmd/bhyve/bhyverun.c b/usr/src/cmd/bhyve/bhyverun.c
index b34b3ffb7c..949e537738 100644
--- a/usr/src/cmd/bhyve/bhyverun.c
+++ b/usr/src/cmd/bhyve/bhyverun.c
@@ -77,6 +77,10 @@ __FBSDID("$FreeBSD$");
 #endif
 #include <vmmapi.h>
 
+#ifndef __FreeBSD__
+#include <sys/stat.h>
+#endif
+
 #include "bhyverun.h"
 #include "acpi.h"
 #include "atkbdc.h"
@@ -128,7 +132,6 @@ int bcons_wait = 0;
 int bcons_connected = 0;
 pthread_mutex_t bcons_wait_lock = PTHREAD_MUTEX_INITIALIZER;
 pthread_cond_t bcons_wait_done = PTHREAD_COND_INITIALIZER;
-void (*vm_started_cb)(void) = NULL;
 #endif
 
 static cpuset_t cpumask;
@@ -860,6 +863,28 @@ do_open(const char *vmname)
 	return (ctx);
 }
 
+#ifndef __FreeBSD__
+
+#define	FILE_PROVISIONING	"/var/svc/provisioning"
+#define	FILE_PROVISION_SUCCESS	"/var/svc/provision_success"
+
+static void
+mark_provisioned(void)
+{
+	struct stat stbuf;
+
+	if (lstat(FILE_PROVISIONING, &stbuf) != 0)
+		return;
+
+	if (rename(FILE_PROVISIONING, FILE_PROVISION_SUCCESS) != 0) {
+		(void) fprintf(stderr, "Cannot rename %s to %s: %s\n",
+		    FILE_PROVISIONING, FILE_PROVISION_SUCCESS,
+		    strerror(errno));
+	}
+}
+
+#endif
+
 int
 main(int argc, char *argv[])
 {
@@ -1121,9 +1146,7 @@ main(int argc, char *argv[])
 	fbsdrun_addcpu(ctx, BSP, BSP, rip);
 
 #ifndef __FreeBSD__
-	if (vm_started_cb != NULL) {
-		vm_started_cb();
-	}
+	mark_provisioned();
 #endif
 
 	/*
diff --git a/usr/src/cmd/bhyve/zhyve.c b/usr/src/cmd/bhyve/zhyve.c
index de94ea97bd..d3e764b14d 100644
--- a/usr/src/cmd/bhyve/zhyve.c
+++ b/usr/src/cmd/bhyve/zhyve.c
@@ -9,11 +9,16 @@
  * http://www.illumos.org/license/CDDL.
  */
 
-
 /*
  * Copyright (c) 2018, Joyent, Inc.
  */
 
+/*
+ * This small 'zhyve' stub is init for the zone: we therefore need to pick up
+ * our command-line arguments placed in ZHYVE_CMD_FILE by the boot stub, do a
+ * little administration, and exec the real bhyve binary.
+ */
+
 #include <assert.h>
 #include <errno.h>
 #include <fcntl.h>
@@ -29,28 +34,6 @@
 
 #define	ZHYVE_CMD_FILE	"/var/run/bhyve/zhyve.cmd"
 
-#define	FILE_PROVISIONING	"/var/svc/provisioning"
-#define	FILE_PROVISION_SUCCESS	"/var/svc/provision_success"
-
-extern int bhyve_main(int, char **);
-void (*vm_started_cb)(void);
-const char *cmdname;
-
-/*
- * Much like basename() but does not alter the path passed to it.
- */
-static void
-get_cmdname(const char *path)
-{
-	cmdname = strrchr(path, '/');
-	if (cmdname == NULL) {
-		cmdname = path;
-		return;
-	}
-	assert(*cmdname == '/');
-	cmdname++;
-}
-
 /*
  * Do a read of the specified size or return an error.  Returns 0 on success
  * and -1 on error.  Sets errno to EINVAL if EOF is encountered.  For other
@@ -101,7 +84,7 @@ parse_options_file(const char *path, uint_t *argcp, char ***argvp)
 	    (buf = malloc(stbuf.st_size)) == NULL ||
 	    full_read(fd, buf, stbuf.st_size) != 0 ||
 	    nvlist_unpack(buf, stbuf.st_size, &nvl, 0) != 0 ||
-	    nvlist_lookup_string_array(nvl, "zhyve_args", argvp, argcp) != 0) {
+	    nvlist_lookup_string_array(nvl, "bhyve_args", argvp, argcp) != 0) {
 		nvlist_free(nvl);
 		ret = -1;
 	} else {
@@ -117,16 +100,6 @@ parse_options_file(const char *path, uint_t *argcp, char ***argvp)
 	return (ret);
 }
 
-static void
-mark_provisioned(void)
-{
-	if (rename(FILE_PROVISIONING, FILE_PROVISION_SUCCESS) != 0) {
-		(void) fprintf(stderr, "Cannot rename %s to %s: %s\n",
-		    FILE_PROVISIONING, FILE_PROVISION_SUCCESS,
-		    strerror(errno));
-	}
-}
-
 /*
  * Setup to suppress core dumps within the zone.
  */
@@ -139,15 +112,10 @@ config_core_dumps()
 int
 main(int argc, char **argv)
 {
+	char **tmpargs;
 	uint_t zargc;
 	char **zargv;
 	int fd;
-	struct stat stbuf;
-
-	get_cmdname(argv[0]);
-	if (strcmp(cmdname, "zhyve") != 0) {
-		return (bhyve_main(argc, argv));
-	}
 
 	config_core_dumps();
 
@@ -176,13 +144,24 @@ main(int argc, char **argv)
 
 	if (parse_options_file(ZHYVE_CMD_FILE, &zargc, &zargv) != 0) {
 		(void) fprintf(stderr, "%s: failed to parse %s: %s\n",
-		    cmdname, ZHYVE_CMD_FILE, strerror(errno));
-		return (1);
+		    argv[0], ZHYVE_CMD_FILE, strerror(errno));
+		return (EXIT_FAILURE);
 	}
 
-	if (lstat(FILE_PROVISIONING, &stbuf) == 0) {
-		vm_started_cb = mark_provisioned;
+	/*
+	 * Annoyingly, we need a NULL at the end.
+	 */
+
+	if ((tmpargs = malloc(sizeof (*zargv) * (zargc + 1))) == NULL) {
+		perror("malloc failed");
+		return (EXIT_FAILURE);
 	}
 
-	return (bhyve_main(zargc, zargv));
+	memcpy(tmpargs, zargv, sizeof (*zargv) * zargc);
+	tmpargs[zargc] = NULL;
+
+	(void) execv("/usr/sbin/bhyve", tmpargs);
+
+	perror("execv failed");
+	return (EXIT_FAILURE);
 }
diff --git a/usr/src/lib/brand/bhyve/zone/boot.c b/usr/src/lib/brand/bhyve/zone/boot.c
index b8c4fa062b..5b071a3974 100644
--- a/usr/src/lib/brand/bhyve/zone/boot.c
+++ b/usr/src/lib/brand/bhyve/zone/boot.c
@@ -110,6 +110,12 @@ add_smbios(int *argc, char **argv)
 	const char *version;
 	const char *uuid;
 
+	if ((uuid = getenv("_ZONECFG_uuid")) != NULL) {
+		if (add_arg(argc, argv, "-U") != 0 ||
+		    add_arg(argc, argv, uuid) != 0)
+			return (1);
+	}
+
 	/*
 	 * Look for something like joyent_20180329T120303Z.  A little mucky, but
 	 * it's exactly what sysinfo does.
@@ -132,12 +138,6 @@ add_smbios(int *argc, char **argv)
 	    add_arg(argc, argv, smbios) != 0)
 		return (1);
 
-	if ((uuid = getenv("_ZONECFG_uuid")) != NULL) {
-		if (add_arg(argc, argv, "-U") != 0 ||
-		    add_arg(argc, argv, uuid) != 0)
-			return (1);
-	}
-
 	return (0);
 }
 
@@ -537,7 +537,7 @@ main(int argc, char **argv)
 {
 	int fd, err;
 	char *zhargv[ZH_MAXARGS] = {
-		"zhyve",	/* Squats on argv[0] */
+		"bhyve",	/* Squats on argv[0] */
 		"-H",		/* vmexit on halt isns */
 		NULL };
 	int zhargc = 2;
@@ -577,7 +577,7 @@ main(int argc, char **argv)
 	 * exit.
 	 */
 	if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0 ||
-	    nvlist_add_string_array(nvl, "zhyve_args", zhargv, zhargc) != 0) {
+	    nvlist_add_string_array(nvl, "bhyve_args", zhargv, zhargc) != 0) {
 		(void) printf("Error: failed to create nvlist: %s\n",
 		    strerror(errno));
 		return (1);