OS-6738 bhyve ppt should not use /dev/mem

Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Approved by: Jerry Jelinek <jerry.jelinek@joyent.com>

2 files changed, 65 insertions, 23 deletions

diff --git a/usr/src/cmd/bhyve/pci_passthru.c b/usr/src/cmd/bhyve/pci_passthru.c
index d133c61849..2ed490c71a 100644
--- a/usr/src/cmd/bhyve/pci_passthru.c
+++ b/usr/src/cmd/bhyve/pci_passthru.c
@@ -62,10 +62,6 @@ __FBSDID("$FreeBSD$");
 #include "pci_emul.h"
 #include "mem.h"
 
-#ifndef _PATH_MEM
-#define	_PATH_MEM	"/dev/mem"
-#endif
-
 #define	LEGACY_SUPPORT	1
 
 #define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1)
@@ -541,18 +537,6 @@ init_msix_table(struct vmctx *ctx, struct passthru_softc *sc, uint64_t base)
 			pi->pi_msix.pba_page = NULL;
 			pi->pi_msix.pba_page_offset = 0;
 		} else {
-			int memfd;
-
-			/*
-			 * This cannot work in a zone and should be replaced
-			 * with a better interface offered by the ppt driver.
-			 */
-			memfd = open(_PATH_MEM, O_RDWR, 0);
-			if (memfd < 0) {
-				warn("failed to open %s", _PATH_MEM);
-				return (-1);
-			}
-
 			/*
 			 * The PBA overlaps with either the first or last
 			 * page of the MSI-X table region.  Map the
@@ -564,9 +548,8 @@ init_msix_table(struct vmctx *ctx, struct passthru_softc *sc, uint64_t base)
 				pi->pi_msix.pba_page_offset = table_offset +
 				    table_size - 4096;
 			pi->pi_msix.pba_page = mmap(NULL, 4096, PROT_READ |
-			    PROT_WRITE, MAP_SHARED, memfd, start +
+			    PROT_WRITE, MAP_SHARED, sc->pptfd,
 			    pi->pi_msix.pba_page_offset);
-			(void) close(memfd);
 			if (pi->pi_msix.pba_page == MAP_FAILED) {
 				warn("Failed to map PBA page for MSI-X on %d",
 				    sc->pptfd);
diff --git a/usr/src/uts/i86pc/io/vmm/io/ppt.c b/usr/src/uts/i86pc/io/vmm/io/ppt.c
index 6b808b6c5d..d36c360e03 100644
--- a/usr/src/uts/i86pc/io/vmm/io/ppt.c
+++ b/usr/src/uts/i86pc/io/vmm/io/ppt.c
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/pci_cap.h>
 #include <sys/ppt_dev.h>
 #include <sys/mkdev.h>
+#include <sys/sysmacros.h>
 
 #include "vmm_lapic.h"
 #include "vmm_ktr.h"
@@ -318,6 +319,64 @@ ppt_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
 	return (0);
 }
 
+static int
+ppt_find_pba_bar(struct pptdev *ppt)
+{
+	uint16_t base;
+	uint32_t pba_off;
+
+	if (PCI_CAP_LOCATE(ppt->pptd_cfg, PCI_CAP_ID_MSI_X, &base) !=
+	    DDI_SUCCESS)
+		return (-1);
+
+	pba_off = pci_config_get32(ppt->pptd_cfg, base + PCI_MSIX_PBA_OFFSET);
+
+	if (pba_off == PCI_EINVAL32)
+		return (-1);
+
+	return (pba_off & PCI_MSIX_PBA_BIR_MASK);
+}
+
+static int
+ppt_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len,
+    size_t *maplen, uint_t model)
+{
+	minor_t minor;
+	struct pptdev *ppt;
+	int err;
+	int bar;
+
+	minor = getminor(dev);
+
+	if ((ppt = ddi_get_soft_state(ppt_state, minor)) == NULL)
+		return (ENXIO);
+
+#ifdef _MULTI_DATAMODEL
+	if (ddi_model_convert_from(model) != DDI_MODEL_NONE)
+		return (ENXIO);
+#endif
+
+	if (off < 0 || off != P2ALIGN(off, PAGESIZE))
+		return (EINVAL);
+
+	if ((bar = ppt_find_pba_bar(ppt)) == -1)
+		return (EINVAL);
+
+	/*
+	 * Add 1 to the BAR number to get the register number used by DDI.
+	 * Register 0 corresponds to PCI config space, the PCI BARs start at 1.
+	 */
+	bar += 1;
+
+	err = devmap_devmem_setup(dhp, ppt->pptd_dip, NULL, bar, off, len,
+	    PROT_USER | PROT_READ | PROT_WRITE, IOMEM_DATA_CACHED, &ppt_attr);
+
+	if (err == DDI_SUCCESS)
+		*maplen = len;
+
+	return (err);
+}
+
 
 static void
 ppt_bar_wipe(struct pptdev *ppt)
@@ -448,7 +507,6 @@ ppt_ddi_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 	if (ppt_bar_crawl(ppt) != 0) {
 		goto fail;
 	}
-
 	if (ddi_create_minor_node(dip, PPT_MINOR_NAME, S_IFCHR, inst,
 	    DDI_PSEUDO, 0) != DDI_SUCCESS) {
 		goto fail;
@@ -539,13 +597,14 @@ static struct cb_ops ppt_cb_ops = {
 	nodev,		/* read */
 	nodev,		/* write */
 	ppt_ioctl,
-	nodev,		/* devmap */
-	nodev,		/* mmap */
-	nodev,		/* segmap */
+	ppt_devmap,	/* devmap */
+	NULL,		/* mmap */
+	NULL,		/* segmap */
 	nochpoll,	/* poll */
 	ddi_prop_op,
 	NULL,
-	D_NEW | D_MP | D_DEVMAP
+	D_NEW | D_MP | D_64BIT | D_DEVMAP,
+	CB_REV
 };
 
 static struct dev_ops ppt_ops = {