OS-6630 bhyve should support vnc

Reviewed by: Jorge Schrauwen <sjorge@blackdot.be>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Approved by: Jerry Jelinek <jerry.jelinek@joyent.com>

4 files changed, 151 insertions, 1 deletions

diff --git a/usr/src/cmd/bhyve/pci_fbuf.c b/usr/src/cmd/bhyve/pci_fbuf.c
index 5361c7b61b..8478f6e531 100644
--- a/usr/src/cmd/bhyve/pci_fbuf.c
+++ b/usr/src/cmd/bhyve/pci_fbuf.c
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2015 Nahanni Systems, Inc.
+ * Copyright 2018 Joyent, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -95,6 +96,9 @@ struct pci_fbuf_softc {
 	char      *rfb_host;
 	char      *rfb_password;
 	int       rfb_port;
+#ifndef __FreeBSD__
+	char	  *rfb_unix;
+#endif
 	int       rfb_wait;
 	int       vga_enabled;
 	int	  vga_full;
@@ -256,6 +260,10 @@ pci_fbuf_parse_opts(struct pci_fbuf_softc *sc, char *opts)
 				sc->rfb_port = atoi(config);
 				sc->rfb_host = tmpstr;
 			}
+#ifndef __FreeBSD__
+		} else if (!strcmp(xopts, "unix")) {
+			sc->rfb_unix = config;
+#endif
 	        } else if (!strcmp(xopts, "vga")) {
 			if (!strcmp(config, "off")) {
 				sc->vga_enabled = 0;
@@ -409,7 +417,17 @@ pci_fbuf_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 
 	memset((void *)sc->fb_base, 0, FB_SIZE);
 
+#ifdef __FreeBSD__
 	error = rfb_init(sc->rfb_host, sc->rfb_port, sc->rfb_wait, sc->rfb_password);
+#else
+	if (sc->rfb_unix != NULL) {
+		error = rfb_init_unix(sc->rfb_unix, sc->rfb_wait,
+		    sc->rfb_password);
+	} else {
+		error = rfb_init(sc->rfb_host, sc->rfb_port, sc->rfb_wait,
+		    sc->rfb_password);
+	}
+#endif
 done:
 	if (error)
 		free(sc);
diff --git a/usr/src/cmd/bhyve/rfb.c b/usr/src/cmd/bhyve/rfb.c
index cbcdb08f85..96712a6acc 100644
--- a/usr/src/cmd/bhyve/rfb.c
+++ b/usr/src/cmd/bhyve/rfb.c
@@ -1,6 +1,7 @@
 /*-
  * Copyright (c) 2015 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
  * Copyright (c) 2015 Leon Dang
+ * Copyright 2018 Joyent, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -56,6 +57,10 @@ __FBSDID("$FreeBSD$");
 
 #include <zlib.h>
 
+#ifndef __FreeBSD__
+#include <sys/debug.h>
+#endif
+
 #include "bhyvegc.h"
 #include "console.h"
 #include "rfb.h"
@@ -1060,3 +1065,86 @@ rfb_init(char *hostname, int port, int wait, char *password)
 
 	return (0);
 }
+
+#ifndef __FreeBSD__
+int
+rfb_init_unix(char *path, int wait, char *password)
+{
+	struct rfb_softc *rc;
+	struct sockaddr_un sock;
+
+	if ((rc = calloc(1, sizeof (struct rfb_softc))) == NULL) {
+		perror("calloc");
+		return (-1);
+	}
+	rc->sfd = -1;
+
+	if ((rc->crc = calloc(howmany(RFB_MAX_WIDTH * RFB_MAX_HEIGHT, 32),
+	    sizeof (uint32_t))) == NULL) {
+		perror("calloc");
+		goto fail;
+	}
+	if ((rc->crc_tmp = calloc(howmany(RFB_MAX_WIDTH * RFB_MAX_HEIGHT, 32),
+	    sizeof (uint32_t))) == NULL) {
+		perror("calloc");
+		goto fail;
+	}
+	rc->crc_width = RFB_MAX_WIDTH;
+	rc->crc_height = RFB_MAX_HEIGHT;
+
+	rc->password = password;
+
+	rc->sfd = socket(PF_UNIX, SOCK_STREAM, 0);
+	if (rc->sfd < 0) {
+		perror("socket");
+		goto fail;
+	}
+
+	sock.sun_family = AF_UNIX;
+	if (strlcpy(sock.sun_path, path, sizeof (sock.sun_path)) >=
+	    sizeof (sock.sun_path)) {
+		(void) fprintf(stderr, "socket path '%s' too long\n", path);
+		goto fail;
+	}
+
+	(void) unlink(path);
+	if (bind(rc->sfd, (struct sockaddr *)&sock, sizeof (sock)) < 0) {
+		perror("bind");
+		goto fail;
+	}
+
+	if (listen(rc->sfd, 1) < 0) {
+		perror("listen");
+		goto fail;
+	}
+
+	rc->hw_crc = sse42_supported();
+
+	rc->conn_wait = wait;
+	if (wait) {
+		VERIFY3S(pthread_mutex_init(&rc->mtx, NULL), ==, 0);
+		VERIFY3S(pthread_cond_init(&rc->cond, NULL), ==, 0);
+	}
+
+	VERIFY3S(pthread_create(&rc->tid, NULL, rfb_thr, rc), ==, 0);
+	pthread_set_name_np(rc->tid, "rfb");
+
+	if (wait) {
+		DPRINTF(("Waiting for rfb client...\n"));
+		VERIFY3S(pthread_mutex_lock(&rc->mtx), ==, 0);
+		VERIFY3S(pthread_cond_wait(&rc->cond, &rc->mtx), ==, 0);
+		VERIFY3S(pthread_mutex_unlock(&rc->mtx), ==, 0);
+	}
+
+	return (0);
+
+fail:
+	if (rc->sfd != -1) {
+		VERIFY3S(close(rc->sfd), ==, 0);
+	}
+	free(rc->crc);
+	free(rc->crc_tmp);
+	free(rc);
+	return (-1);
+}
+#endif
diff --git a/usr/src/cmd/bhyve/rfb.h b/usr/src/cmd/bhyve/rfb.h
index 1bd2eca3f7..94d937e5b8 100644
--- a/usr/src/cmd/bhyve/rfb.h
+++ b/usr/src/cmd/bhyve/rfb.h
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2015 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
+ * Copyright 2018 Joyent, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -32,5 +33,8 @@
 #define	RFB_PORT	5900
 
 int	rfb_init(char *hostname, int port, int wait, char *password);
+#ifndef __FreeBSD__
+int	rfb_init_unix(char *path, int wait, char *password);
+#endif
 
 #endif /* _RFB_H_ */
diff --git a/usr/src/lib/brand/bhyve/zone/boot.c b/usr/src/lib/brand/bhyve/zone/boot.c
index bc05a33ec4..29c09193e9 100644
--- a/usr/src/lib/brand/bhyve/zone/boot.c
+++ b/usr/src/lib/brand/bhyve/zone/boot.c
@@ -46,7 +46,8 @@ typedef enum {
 	PCI_SLOT_CD,
 	PCI_SLOT_BOOT_DISK,
 	PCI_SLOT_OTHER_DISKS,
-	PCI_SLOT_NICS
+	PCI_SLOT_NICS,
+	PCI_SLOT_FBUF = 30,
 } pci_slot_t;
 
 static boolean_t debug;
@@ -465,6 +466,44 @@ add_bhyve_extra_opts(int *argc, char **argv)
 	return (0);
 }
 
+/*
+ * Adds the frame buffer and an xhci tablet to help with the pointer.
+ */
+static int
+add_fbuf(int *argc, char **argv)
+{
+	char *val;
+	char conf[MAXPATHLEN];
+	int len;
+
+	/*
+	 * Do not add a frame buffer or tablet if VNC is disabled.
+	 */
+	if ((val = get_zcfg_var("attr", "vnc_port", NULL)) != NULL &&
+	    strcmp(val, "-1") == 0) {
+		return (0);
+	}
+
+	len = snprintf(conf, sizeof (conf),
+	    "%d:0,fbuf,vga=off,unix=/tmp/vm.vnc", PCI_SLOT_FBUF);
+	assert(len < sizeof (conf));
+
+	if (add_arg(argc, argv, "-s") != 0 ||
+	    add_arg(argc, argv, conf) != 0) {
+		return (-1);
+	}
+
+	len = snprintf(conf, sizeof (conf), "%d:1,xhci,tablet", PCI_SLOT_FBUF);
+	assert(len < sizeof (conf));
+
+	if (add_arg(argc, argv, "-s") != 0 ||
+	    add_arg(argc, argv, conf) != 0) {
+		return (-1);
+	}
+
+	return (0);
+}
+
 /* Must be called last */
 static int
 add_vmname(int *argc, char **argv)
@@ -580,6 +619,7 @@ main(int argc, char **argv)
 	    add_devices(&zhargc, (char **)&zhargv) != 0 ||
 	    add_nets(&zhargc, (char **)&zhargv) != 0 ||
 	    add_bhyve_extra_opts(&zhargc, (char **)&zhargv) != 0 ||
+	    add_fbuf(&zhargc, (char **)&zhargv) != 0 ||
 	    add_vmname(&zhargc, (char **)&zhargv) != 0) {
 		return (1);
 	}