OS-8108 Support vnc password for BHYVE (#256)release-20200213

Reviewed by: Mike Zeller <mike.zeller@joyent.com>
Reviewed by: Brian Bennett <brian.bennett@joyent.com>
Approved by: Mike Zeller <mike.zeller@joyent.com>

4 files changed, 136 insertions, 24 deletions

diff --git a/usr/src/cmd/bhyve/Makefile b/usr/src/cmd/bhyve/Makefile
index 2257487c0d..3e02ce4a62 100644
--- a/usr/src/cmd/bhyve/Makefile
+++ b/usr/src/cmd/bhyve/Makefile
@@ -11,7 +11,7 @@
 
 #
 # Copyright 2014 Pluribus Networks Inc.
-# Copyright 2019 Joyent, Inc.
+# Copyright 2020 Joyent, Inc.
 #
 
 PROG =		bhyve
@@ -115,9 +115,6 @@ CPPFLAGS =	-I$(COMPAT)/freebsd -I$(CONTRIB)/freebsd \
 		-I$(SRC)/lib/libdladm/common \
 		-DWITHOUT_CAPSICUM
 
-# Disable the crypto code until it is wired up
-CPPFLAGS +=	-DNO_OPENSSL
-
 pci_nvme.o := CERRWARN += -_gcc=-Wno-pointer-sign
 
 SMOFF += all_func_returns,leaks,no_if_block
@@ -125,9 +122,17 @@ SMOFF += all_func_returns,leaks,no_if_block
 # Force c99 for everything
 CSTD=		$(CSTD_GNU99)
 C99MODE=	-xc99=%all
-C99LMODE=	-Xc99=%all
 
-$(PROG) := LDLIBS += -lsocket -lnsl -ldlpi -ldladm -lmd -luuid -lvmmapi -lz
+$(PROG) := LDLIBS +=	\
+	-lsocket	\
+	-lnsl		\
+	-ldlpi		\
+	-ldladm		\
+	-lmd		\
+	-lsunw_crypto	\
+	-luuid		\
+	-lvmmapi	\
+	-lz
 $(ZHYVE_PROG) := LDLIBS += -lnvpair
 $(MEVENT_TEST_PROG) := LDLIBS += -lsocket
 
@@ -150,8 +155,6 @@ clean: $(SUBDIRS)
 clobber: clean $(SUBDIRS)
 	$(RM) $(CLOBBERFILES)
 
-lint:	lint_SRCS $(SUBDIRS)
-
 $(SUBDIRS): FRC
 	@cd $@; pwd; $(MAKE) $(TARGET)
 
diff --git a/usr/src/cmd/bhyve/rfb.c b/usr/src/cmd/bhyve/rfb.c
index 39ea1611f9..f9cc9ed9c3 100644
--- a/usr/src/cmd/bhyve/rfb.c
+++ b/usr/src/cmd/bhyve/rfb.c
@@ -3,7 +3,7 @@
  *
  * Copyright (c) 2015 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
  * Copyright (c) 2015 Leon Dang
- * Copyright 2018 Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -803,7 +803,7 @@ rfb_handle(struct rfb_softc *rc, int cfd)
 		 * The client then sends the resulting 16-bytes response.
 		 */
 #ifndef NO_OPENSSL
-		strncpy(keystr, rc->password, PASSWD_LENGTH);
+		strncpy((char *)keystr, rc->password, PASSWD_LENGTH);
 
 		/* VNC clients encrypts the challenge with all the bit fields
 		 * in each byte of the password mirrored.
@@ -838,7 +838,8 @@ rfb_handle(struct rfb_softc *rc, int cfd)
 				&ks, DES_ENCRYPT);
 
 		if (memcmp(crypt_expected, buf, AUTH_LENGTH) != 0) {
-			message = "Auth Failed: Invalid Password.";
+			message =
+			    (unsigned char *)"Auth Failed: Invalid Password.";
 			sres = htonl(1);
 		} else
 			sres = 0;
diff --git a/usr/src/lib/brand/bhyve/zone/Makefile b/usr/src/lib/brand/bhyve/zone/Makefile
index d7e957072c..196e31d575 100644
--- a/usr/src/lib/brand/bhyve/zone/Makefile
+++ b/usr/src/lib/brand/bhyve/zone/Makefile
@@ -8,7 +8,7 @@
 # source.  A copy of the CDDL is also available via the Internet at
 # http://www.illumos.org/license/CDDL.
 #
-# Copyright (c) 2018, Joyent, Inc.
+# Copyright 2020 Joyent, Inc.
 #
 
 include $(SRC)/cmd/Makefile.cmd
@@ -22,7 +22,7 @@ PROG2 =		bhhwcompat
 PROGS +=	$(PROG1) $(PROG2)
 CLEANFILES +=	$(PROG1) $(PROG2)
 
-$(PROG1) :=	LDLIBS += -lnvpair
+$(PROG1) :=	LDLIBS += -lnvpair -lcustr
 $(PROG2) :=	CPPFLAGS = -I$(COMPAT)/freebsd -I$(CONTRIB)/freebsd \
 	$(CPPFLAGS.master) -I$(SRC)/uts/i86pc \
 	-I$(COMPAT)/freebsd/amd64 -I$(CONTRIB)/freebsd/amd64
@@ -42,6 +42,4 @@ install: $(PROGS) $(ROOTPROGS) $(ROOTXMLDOCS) $(ROOTTEMPLATES)
 clean:
 	$(RM) $(CLEANFILES)
 
-lint:
-
 include $(SRC)/cmd/Makefile.targ
diff --git a/usr/src/lib/brand/bhyve/zone/boot.c b/usr/src/lib/brand/bhyve/zone/boot.c
index 51aab83daf..ceada201ac 100644
--- a/usr/src/lib/brand/bhyve/zone/boot.c
+++ b/usr/src/lib/brand/bhyve/zone/boot.c
@@ -10,7 +10,7 @@
  */
 
 /*
- * Copyright (c) 2019, Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  */
 
 /*
@@ -23,9 +23,11 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <libnvpair.h>
+#include <libcustr.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/debug.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/utsname.h>
@@ -553,6 +555,88 @@ add_bhyve_extra_opts(int *argc, char **argv)
 	return (0);
 }
 
+#define	INVALID_CHAR	(char)(255)
+
+static char
+decode_char(char encoded)
+{
+	if (encoded >= 'A' && encoded <= 'Z')
+		return (encoded - 'A');
+	if (encoded >= 'a' && encoded <= 'z')
+		return (encoded - 'a' + 26);
+	if (encoded >= '0' && encoded <= '9')
+		return (encoded - '0' + 52);
+	if (encoded == '+')
+		return (62);
+	if (encoded == '/')
+		return (63);
+	if (encoded == '=')
+		return (0);
+	return (INVALID_CHAR);
+}
+
+static int
+add_base64(custr_t *cus, const char *b64)
+{
+	size_t b64len = strlen(b64);
+
+	if (b64len == 0 || b64len % 4 != 0)
+		return (-1);
+
+	while (b64len > 0) {
+		uint_t padding = 0;
+		char c0 = decode_char(b64[0]);
+		char c1 = decode_char(b64[1]);
+		char c2 = decode_char(b64[2]);
+		char c3 = decode_char(b64[3]);
+
+		if (c0 == INVALID_CHAR || c1 == INVALID_CHAR ||
+		    c2 == INVALID_CHAR || c3 == INVALID_CHAR) {
+			(void) printf("Error: base64 value contains invalid "
+			    "character(s)\n");
+			return (-1);
+		}
+
+		/*
+		 * For each block of 4 input characters, an '=' should
+		 * only appear as the last two characters.
+		 */
+		if (b64[0] == '=' || b64[1] == '=') {
+			(void) printf("Error: base64 value contains invalid "
+			    "padding\n");
+			return (-1);
+		}
+
+		if (b64len == 4) {
+			/*
+			 * We can end with '==' or '=', but never '='
+			 * followed by something else.
+			 */
+			if (b64[2] == '=') {
+				if (b64[3] != '=') {
+					(void) printf("Error: base64 value "
+					    "contains invalid padding\n");
+					return (-1);
+				}
+				padding = 2;
+			} else if (b64[3] == '=') {
+				padding = 1;
+			}
+		}
+
+		VERIFY0(custr_appendc(cus, c0 << 2 | c1 >> 4));
+		if (padding < 2)
+			VERIFY0(custr_appendc(cus, c1 << 4 | c2 >> 2));
+		if (padding < 1)
+			VERIFY0(custr_appendc(cus, c2 << 6 | c3));
+
+		b64len -= 4;
+		b64 += 4;
+	}
+
+	return (0);
+}
+
 /*
  * Adds the frame buffer and an xhci tablet to help with the pointer.
  */
@@ -560,7 +644,8 @@ static int
 add_fbuf(int *argc, char **argv)
 {
 	char conf[MAXPATHLEN];
-	int len;
+	custr_t *cconf = NULL;
+	char *password = NULL;
 
 	/*
 	 * Do not add a frame buffer or tablet if VNC is disabled.
@@ -569,24 +654,49 @@ add_fbuf(int *argc, char **argv)
 		return (0);
 	}
 
-	len = snprintf(conf, sizeof (conf),
-	    "%d:0,fbuf,vga=off,unix=/tmp/vm.vnc", PCI_SLOT_FBUF);
-	assert(len < sizeof (conf));
+	if (custr_alloc_buf(&cconf, conf, sizeof (conf)) != 0) {
+		return (-1);
+	}
+
+	VERIFY0(custr_append_printf(cconf, "%d:0,fbuf,vga=off,unix=/tmp/vm.vnc",
+	    PCI_SLOT_FBUF));
+
+	password = get_zcfg_var("attr", "vnc_password", NULL);
+	if (password != NULL) {
+		VERIFY0(custr_append(cconf, ",password="));
+
+		if (add_base64(cconf, password) != 0) {
+			goto fail;
+		}
+	}
 
 	if (add_arg(argc, argv, "-s") != 0 ||
 	    add_arg(argc, argv, conf) != 0) {
-		return (-1);
+		goto fail;
 	}
 
-	len = snprintf(conf, sizeof (conf), "%d:1,xhci,tablet", PCI_SLOT_FBUF);
-	assert(len < sizeof (conf));
+	custr_reset(cconf);
+	VERIFY0(custr_append_printf(cconf, "%d:1,xhci,tablet", PCI_SLOT_FBUF));
 
 	if (add_arg(argc, argv, "-s") != 0 ||
 	    add_arg(argc, argv, conf) != 0) {
-		return (-1);
+		goto fail;
 	}
 
+	/*
+	 * Since cconf was allocated using custr_alloc_buf() where 'conf'
+	 * is the underlying fixed buffer for cconf, we can free cconf
+	 * which in this instance will just free cconf, but _not_ the
+	 * underlying fixed buffer (conf) which is left unchanged by
+	 * custr_free().
+	 */
+
+	custr_free(cconf);
 	return (0);
+
+fail:
+	custr_free(cconf);
+	return (-1);
 }
 
 /* Must be called last */