9024 rework PV-HVM disk device handling

Reviewed by: Evan Layton <evan.layton@nexenta.com>
Reviewed by: Sanjay Nadkarni <sanjay.nadkarni@nexenta.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
Approved by: Richard Lowe <richlowe@richlowe.net>

3 files changed, 72 insertions, 81 deletions

diff --git a/usr/src/uts/common/io/scsi/targets/sd.c b/usr/src/uts/common/io/scsi/targets/sd.c
index b5b4a8216d..64b3b1a6dd 100644
--- a/usr/src/uts/common/io/scsi/targets/sd.c
+++ b/usr/src/uts/common/io/scsi/targets/sd.c
@@ -1764,7 +1764,6 @@ struct dev_ops sd_ops = {
  */
 #include <sys/modctl.h>
 
-#ifndef XPV_HVM_DRIVER
 static struct modldrv modldrv = {
 	&mod_driverops,		/* Type of module. This one is a driver */
 	SD_MODULE_NAME,		/* Module name. */
@@ -1775,18 +1774,6 @@ static struct modlinkage modlinkage = {
 	MODREV_1, &modldrv, NULL
 };
 
-#else /* XPV_HVM_DRIVER */
-static struct modlmisc modlmisc = {
-	&mod_miscops,		/* Type of module. This one is a misc */
-	"HVM " SD_MODULE_NAME,		/* Module name. */
-};
-
-static struct modlinkage modlinkage = {
-	MODREV_1, &modlmisc, NULL
-};
-
-#endif /* XPV_HVM_DRIVER */
-
 static cmlb_tg_ops_t sd_tgops = {
 	TG_DK_OPS_VERSION_1,
 	sd_tg_rdwr,
@@ -2421,20 +2408,12 @@ _init(void)
 	/* establish driver name from module name */
 	sd_label = (char *)mod_modname(&modlinkage);
 
-#ifndef XPV_HVM_DRIVER
 	err = ddi_soft_state_init(&sd_state, sizeof (struct sd_lun),
 	    SD_MAXUNIT);
 	if (err != 0) {
 		return (err);
 	}
 
-#else /* XPV_HVM_DRIVER */
-	/* Remove the leading "hvm_" from the module name */
-	ASSERT(strncmp(sd_label, "hvm_", strlen("hvm_")) == 0);
-	sd_label += strlen("hvm_");
-
-#endif /* XPV_HVM_DRIVER */
-
 	mutex_init(&sd_detach_mutex, NULL, MUTEX_DRIVER, NULL);
 	mutex_init(&sd_log_mutex,    NULL, MUTEX_DRIVER, NULL);
 	mutex_init(&sd_label_mutex,  NULL, MUTEX_DRIVER, NULL);
@@ -2474,9 +2453,8 @@ _init(void)
 
 		sd_scsi_target_lun_fini();
 
-#ifndef XPV_HVM_DRIVER
 		ddi_soft_state_fini(&sd_state);
-#endif /* !XPV_HVM_DRIVER */
+
 		return (err);
 	}
 
@@ -2517,9 +2495,7 @@ _fini(void)
 	cv_destroy(&sd_tr.srq_resv_reclaim_cv);
 	cv_destroy(&sd_tr.srq_inprocess_cv);
 
-#ifndef XPV_HVM_DRIVER
 	ddi_soft_state_fini(&sd_state);
-#endif /* !XPV_HVM_DRIVER */
 
 	return (err);
 }
@@ -2723,9 +2699,7 @@ sdprobe(dev_info_t *devi)
 {
 	struct scsi_device	*devp;
 	int			rval;
-#ifndef XPV_HVM_DRIVER
 	int			instance = ddi_get_instance(devi);
-#endif /* !XPV_HVM_DRIVER */
 
 	/*
 	 * if it wasn't for pln, sdprobe could actually be nulldev
@@ -2742,11 +2716,9 @@ sdprobe(dev_info_t *devi)
 		return (DDI_PROBE_FAILURE);
 	}
 
-#ifndef XPV_HVM_DRIVER
 	if (ddi_get_soft_state(sd_state, instance) != NULL) {
 		return (DDI_PROBE_PARTIAL);
 	}
-#endif /* !XPV_HVM_DRIVER */
 
 	/*
 	 * Call the SCSA utility probe routine to see if we actually
@@ -7310,11 +7282,9 @@ sd_unit_attach(dev_info_t *devi)
 	 * this routine will have a value of zero.
 	 */
 	instance = ddi_get_instance(devp->sd_dev);
-#ifndef XPV_HVM_DRIVER
 	if (ddi_soft_state_zalloc(sd_state, instance) != DDI_SUCCESS) {
 		goto probe_failed;
 	}
-#endif /* !XPV_HVM_DRIVER */
 
 	/*
 	 * Retrieve a pointer to the newly-allocated soft state.
@@ -8580,9 +8550,7 @@ sd_unit_detach(dev_info_t *devi)
 	int			tgt;
 	dev_t			dev;
 	dev_info_t		*pdip = ddi_get_parent(devi);
-#ifndef XPV_HVM_DRIVER
 	int			instance = ddi_get_instance(devi);
-#endif /* !XPV_HVM_DRIVER */
 
 	mutex_enter(&sd_detach_mutex);
 
@@ -9011,9 +8979,8 @@ sd_unit_detach(dev_info_t *devi)
 	devp->sd_private = NULL;
 
 	bzero(un, sizeof (struct sd_lun));
-#ifndef XPV_HVM_DRIVER
+
 	ddi_soft_state_free(sd_state, instance);
-#endif /* !XPV_HVM_DRIVER */
 
 	mutex_exit(&sd_detach_mutex);
 
diff --git a/usr/src/uts/common/xen/io/xdf.c b/usr/src/uts/common/xen/io/xdf.c
index ea8f4d0bb1..7dfffaed41 100644
--- a/usr/src/uts/common/xen/io/xdf.c
+++ b/usr/src/uts/common/xen/io/xdf.c
@@ -26,6 +26,7 @@
 
 /*
  * Copyright (c) 2014, 2017 by Delphix. All rights reserved.
+ * Copyright 2017 Nexenta Systems, Inc.
  */
 
 /*
@@ -34,8 +35,8 @@
  *	- support alternate block size (currently only DEV_BSIZE supported)
  *	- revalidate geometry for removable devices
  *
- * This driver export solaris disk device nodes, accepts IO requests from
- * those nodes, and services those requests by talking to a backend device
+ * This driver exports disk device nodes, accepts IO requests from those
+ * nodes, and services those requests by talking to a backend device
  * in another domain.
  *
  * Communication with the backend device is done via a ringbuffer (which is
@@ -53,9 +54,9 @@
  * the device label and if flush operations are supported.  Once this is
  * done we enter the XD_READY state and can process any IO operations.
  *
- * We recieve notifications of xenbus state changes for the backend device
+ * We receive notifications of xenbus state changes for the backend device
  * (aka, the "other end") via the xdf_oe_change() callback.  This callback
- * is single threaded, meaning that we can't recieve new notification of
+ * is single threaded, meaning that we can't receive new notification of
  * other end state changes while we're processing an outstanding
  * notification of an other end state change.  There for we can't do any
  * blocking operations from the xdf_oe_change() callback.  This is why we
@@ -81,10 +82,10 @@
 #include <sys/mach_mmu.h>
 #ifdef XPV_HVM_DRIVER
 #include <sys/xpv_support.h>
-#include <sys/sunndi.h>
 #else /* !XPV_HVM_DRIVER */
 #include <sys/evtchn_impl.h>
 #endif /* !XPV_HVM_DRIVER */
+#include <sys/sunndi.h>
 #include <public/io/xenbus.h>
 #include <xen/sys/xenbus_impl.h>
 #include <sys/scsi/generic/inquiry.h>
@@ -143,7 +144,7 @@ static int		xdf_maxphys = XB_MAXPHYS;
 static diskaddr_t	xdf_flush_block = DEFAULT_FLUSH_BLOCK;
 static int		xdf_fbrewrites;	/* flush block re-write count */
 
-/* misc public functions (used by xdf_shell.c) */
+/* misc public functions */
 int xdf_lb_rdwr(dev_info_t *, uchar_t, void *, diskaddr_t, size_t, void *);
 int xdf_lb_getinfo(dev_info_t *, int, void *, void *);
 
@@ -601,11 +602,11 @@ xdf_cmlb_attach(xdf_t *vdp)
 	    XD_IS_RM(vdp),
 	    B_TRUE,
 	    XD_IS_CD(vdp) ? DDI_NT_CD_XVMD : DDI_NT_BLOCK_XVMD,
-#if defined(XPV_HVM_DRIVER)
+#ifdef XPV_HVM_DRIVER
 	    (XD_IS_CD(vdp) ? 0 : CMLB_CREATE_ALTSLICE_VTOC_16_DTYPE_DIRECT),
-#else /* !XPV_HVM_DRIVER */
+#else /* XPV_HVM_DRIVER */
 	    0,
-#endif /* !XPV_HVM_DRIVER */
+#endif /* XPV_HVM_DRIVER */
 	    vdp->xdf_vd_lbl, NULL));
 }
 
@@ -688,14 +689,13 @@ xdf_kstat_runq_to_waitq(xdf_t *vdp, buf_t *bp)
 }
 
 int
-xdf_kstat_create(dev_info_t *dip, char *ks_module, int instance)
+xdf_kstat_create(dev_info_t *dip)
 {
 	xdf_t		*vdp = (xdf_t *)ddi_get_driver_private(dip);
 	kstat_t		*kstat;
 	buf_t		*bp;
 
-	if ((kstat = kstat_create(
-	    ks_module, instance, NULL, "disk",
+	if ((kstat = kstat_create("xdf", ddi_get_instance(dip), NULL, "disk",
 	    KSTAT_TYPE_IO, 1, KSTAT_FLAG_PERSISTENT)) == NULL)
 		return (-1);
 
@@ -799,7 +799,7 @@ xdf_kstat_delete(dev_info_t *dip)
  * Add an IO requests onto the active queue.
  *
  * We have to detect IOs generated by xdf_ready_tq_thread.  These IOs
- * are used to establish a connection to the backend, so they recieve
+ * are used to establish a connection to the backend, so they receive
  * priority over all other IOs.  Since xdf_ready_tq_thread only does
  * synchronous IO, there can only be one xdf_ready_tq_thread request at any
  * given time and we record the buf associated with that request in
@@ -1109,7 +1109,7 @@ xdf_ring_destroy(xdf_t *vdp)
 	}
 
 	/*
-	 * We don't want to recieve async notifications from the backend
+	 * We don't want to receive async notifications from the backend
 	 * when it finishes processing ring entries.
 	 */
 #ifdef XPV_HVM_DRIVER
@@ -1551,7 +1551,7 @@ xdf_setstate_init(xdf_t *vdp)
 
 	/*
 	 * Sanity check for the existance of the xenbus device-type property.
-	 * This property might not exist if we our xenbus device nodes was
+	 * This property might not exist if our xenbus device nodes were
 	 * force destroyed while we were still connected to the backend.
 	 */
 	if (xenbus_read_str(xsname, XBP_DEV_TYPE, &str) != 0)
@@ -1898,7 +1898,7 @@ xdf_setstate_connected(xdf_t *vdp)
 	 * but we can't initiate IO from the other end change callback thread
 	 * (which is the current context we're executing in.)  This is because
 	 * if the other end disconnects while we're doing IO from the callback
-	 * thread, then we can't recieve that disconnect event and we hang
+	 * thread, then we can't receive that disconnect event and we hang
 	 * waiting for an IO that can never complete.
 	 */
 	(void) ddi_taskq_dispatch(vdp->xdf_ready_tq, xdf_setstate_ready, vdp,
@@ -2081,7 +2081,7 @@ xdf_iorestart(caddr_t arg)
 	return (DDI_INTR_CLAIMED);
 }
 
-#if defined(XPV_HVM_DRIVER)
+#ifdef XPV_HVM_DRIVER
 
 typedef struct xdf_hvm_entry {
 	list_node_t	xdf_he_list;
@@ -2436,6 +2436,14 @@ xdf_lb_rdwr(dev_info_t *dip, uchar_t cmd, void *bufp,
 	/* We don't allow IO from the oe_change callback thread */
 	ASSERT(curthread != vdp->xdf_oe_change_thread);
 
+	/*
+	 * Having secsize of 0 means that device isn't connected yet.
+	 * FIXME This happens for CD devices, and there's nothing we
+	 * can do about it at the moment.
+	 */
+	if (vdp->xdf_xdev_secsize == 0)
+		return (EIO);
+
 	if ((start + ((reqlen / (vdp->xdf_xdev_secsize / DEV_BSIZE))
 	    >> DEV_BSHIFT)) > vdp->xdf_pgeom.g_capacity)
 		return (EINVAL);
@@ -3249,8 +3257,7 @@ err:
 }
 
 /*
- * xdf_devid_fabricate() is a local copy of xdfs_devid_fabricate() that has been
- * modified to use xdf functions, and uses the in-memory devid if one exists.
+ * Uses the in-memory devid if one exists.
  *
  * Create a devid and write it on the first block of the last track of
  * the last cylinder.
@@ -3416,8 +3423,8 @@ xdf_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 	}
 	/* DDI_ATTACH */
 
-	if (((xsname = xvdi_get_xsname(dip)) == NULL) ||
-	    ((oename = xvdi_get_oename(dip)) == NULL))
+	if ((xsname = xvdi_get_xsname(dip)) == NULL ||
+	    (oename = xvdi_get_oename(dip)) == NULL)
 		return (DDI_FAILURE);
 
 	/*
@@ -3503,20 +3510,23 @@ xdf_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 	vdp->xdf_pgeom_fixed = B_FALSE;
 
 	/*
-	 * create default device minor nodes: non-removable disk
-	 * we will adjust minor nodes after we are connected w/ backend
+	 * Create default device minor nodes: non-removable disk.
+	 * We will adjust minor nodes after we are connected w/ backend.
+	 *
+	 * FIXME creating device minor nodes is currently disabled for CD
+	 * devices, re-enable once the issues with xdf CD devices are fixed.
 	 */
-	cmlb_alloc_handle(&vdp->xdf_vd_lbl);
-	if (xdf_cmlb_attach(vdp) != 0) {
-		cmn_err(CE_WARN,
-		    "xdf@%s: attach failed, cmlb attach failed",
-		    ddi_get_name_addr(dip));
-		goto errout0;
+	if (!dev_iscd) {
+		cmlb_alloc_handle(&vdp->xdf_vd_lbl);
+		if (xdf_cmlb_attach(vdp) != 0) {
+			cmn_err(CE_WARN,
+			    "xdf@%s: attach failed, cmlb attach failed",
+			    ddi_get_name_addr(dip));
+			goto errout0;
+		}
 	}
 
-	/*
-	 * We ship with cache-enabled disks
-	 */
+	/* We ship with cache-enabled disks */
 	vdp->xdf_wce = B_TRUE;
 
 	mutex_enter(&vdp->xdf_cb_lk);
@@ -3534,6 +3544,12 @@ xdf_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 		goto errout1;
 	}
 
+	/* Nothing else to do for CD devices */
+	if (dev_iscd) {
+		mutex_exit(&vdp->xdf_cb_lk);
+		goto done;
+	}
+
 	/*
 	 * In order to do cmlb_validate, we have to wait for the disk to
 	 * acknowledge the attach, so we can query the backend for the disk
@@ -3546,8 +3562,8 @@ xdf_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 	 */
 	timeout = ddi_get_lbolt() + drv_usectohz(XDF_STATE_TIMEOUT);
 	while (vdp->xdf_state != XD_CONNECTED && vdp->xdf_state != XD_READY) {
-		if (cv_timedwait(&vdp->xdf_dev_cv, &vdp->xdf_cb_lk, timeout) <
-		    0) {
+		if (cv_timedwait(&vdp->xdf_dev_cv, &vdp->xdf_cb_lk,
+		    timeout) < 0) {
 			cmn_err(CE_WARN, "xdf@%s: disk failed to connect",
 			    ddi_get_name_addr(dip));
 			mutex_exit(&vdp->xdf_cb_lk);
@@ -3598,9 +3614,8 @@ xdf_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 		}
 	}
 
-
-#if defined(XPV_HVM_DRIVER)
-
+done:
+#ifdef XPV_HVM_DRIVER
 	xdf_hvm_add(dip);
 
 	/* Report our version to dom0.  */
@@ -3610,13 +3625,24 @@ xdf_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 
 #endif /* XPV_HVM_DRIVER */
 
-	/* create kstat for iostat(1M) */
-	if (xdf_kstat_create(dip, "xdf", instance) != 0) {
+	/* Create kstat for iostat(1M) */
+	if (xdf_kstat_create(dip) != 0) {
 		cmn_err(CE_WARN, "xdf@%s: failed to create kstat",
 		    ddi_get_name_addr(dip));
 		goto errout1;
 	}
 
+	/*
+	 * Don't bother with getting real device identification
+	 * strings (is it even possible?), they are unlikely to
+	 * change often (if at all).
+	 */
+	(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, INQUIRY_VENDOR_ID,
+	    "Xen");
+	(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, INQUIRY_PRODUCT_ID,
+	    dev_iscd ? "Virtual CD" : "Virtual disk");
+	(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, INQUIRY_REVISION_ID,
+	    "1.0");
 
 	ddi_report_dev(dip);
 	DPRINTF(DDI_DBG, ("xdf@%s: attached\n", vdp->xdf_addr));
@@ -3716,7 +3742,7 @@ xdf_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 
 	ASSERT(!ISDMACBON(vdp));
 
-#if defined(XPV_HVM_DRIVER)
+#ifdef XPV_HVM_DRIVER
 	xdf_hvm_rm(dip);
 #endif /* XPV_HVM_DRIVER */
 
@@ -3821,12 +3847,12 @@ _init(void)
 	xdf_gs_cache = kmem_cache_create("xdf_gs_cache",
 	    sizeof (ge_slot_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
 
-#if defined(XPV_HVM_DRIVER)
+#ifdef XPV_HVM_DRIVER
 	xdf_hvm_init();
 #endif /* XPV_HVM_DRIVER */
 
 	if ((rc = mod_install(&xdf_modlinkage)) != 0) {
-#if defined(XPV_HVM_DRIVER)
+#ifdef XPV_HVM_DRIVER
 		xdf_hvm_fini();
 #endif /* XPV_HVM_DRIVER */
 		kmem_cache_destroy(xdf_vreq_cache);
@@ -3845,7 +3871,7 @@ _fini(void)
 	if ((err = mod_remove(&xdf_modlinkage)) != 0)
 		return (err);
 
-#if defined(XPV_HVM_DRIVER)
+#ifdef XPV_HVM_DRIVER
 	xdf_hvm_fini();
 #endif /* XPV_HVM_DRIVER */
 
diff --git a/usr/src/uts/common/xen/io/xdf.h b/usr/src/uts/common/xen/io/xdf.h
index 59a6ff0564..dbe012c30c 100644
--- a/usr/src/uts/common/xen/io/xdf.h
+++ b/usr/src/uts/common/xen/io/xdf.h
@@ -314,7 +314,7 @@ typedef struct xdf {
 #define	SUSRES_DBG	0x40
 #define	LBL_DBG		0x80
 
-#if defined(XPV_HVM_DRIVER)
+#ifdef XPV_HVM_DRIVER
 extern int xdf_lb_getinfo(dev_info_t *, int, void *, void *);
 extern int xdf_lb_rdwr(dev_info_t *, uchar_t, void *, diskaddr_t, size_t,
     void *);
@@ -322,8 +322,6 @@ extern void xdfmin(struct buf *bp);
 extern dev_info_t *xdf_hvm_hold(const char *);
 extern boolean_t xdf_hvm_connect(dev_info_t *);
 extern int xdf_hvm_setpgeom(dev_info_t *, cmlb_geom_t *);
-extern int xdf_kstat_create(dev_info_t *, char *, int);
-extern void xdf_kstat_delete(dev_info_t *);
 extern boolean_t xdf_is_cd(dev_info_t *);
 extern boolean_t xdf_is_rm(dev_info_t *);
 extern boolean_t xdf_media_req_supported(dev_info_t *);