[illumos-gate merge]

commit b6bc2fd4673eae6c96e2aea9e16105dd32a66b7b
    7364 NVMe driver performance can be improved by caching nvme_dma_t structs for PRPL. (fix lint)
commit 8834f7ac63a18ac48dfacf20506346d82b04fc14
    7364 NVMe driver performance can be improved by caching nvme_dma_t structs for PRPL.

2 files changed, 82 insertions, 15 deletions

diff --git a/usr/src/uts/common/io/nvme/nvme.c b/usr/src/uts/common/io/nvme/nvme.c
index 16f2d6101b..1ea72fc792 100644
--- a/usr/src/uts/common/io/nvme/nvme.c
+++ b/usr/src/uts/common/io/nvme/nvme.c
@@ -257,6 +257,9 @@ static int nvme_bd_write(void *, bd_xfer_t *);
 static int nvme_bd_sync(void *, bd_xfer_t *);
 static int nvme_bd_devid(void *, dev_info_t *, ddi_devid_t *);
 
+static int nvme_prp_dma_constructor(void *, void *, int);
+static void nvme_prp_dma_destructor(void *, void *);
+
 static void nvme_prepare_devid(nvme_t *, uint32_t);
 
 static void *nvme_state;
@@ -488,7 +491,7 @@ nvme_check_dma_hdl(nvme_dma_t *dma)
 }
 
 static void
-nvme_free_dma(nvme_dma_t *dma)
+nvme_free_dma_common(nvme_dma_t *dma)
 {
 	if (dma->nd_dmah != NULL)
 		(void) ddi_dma_unbind_handle(dma->nd_dmah);
@@ -496,15 +499,28 @@ nvme_free_dma(nvme_dma_t *dma)
 		ddi_dma_mem_free(&dma->nd_acch);
 	if (dma->nd_dmah != NULL)
 		ddi_dma_free_handle(&dma->nd_dmah);
-	kmem_free(dma, sizeof (nvme_dma_t));
 }
 
-static int
-nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags,
-    ddi_dma_attr_t *dma_attr, nvme_dma_t **ret)
+static void
+nvme_free_dma(nvme_dma_t *dma)
 {
-	nvme_dma_t *dma = kmem_zalloc(sizeof (nvme_dma_t), KM_SLEEP);
+	nvme_free_dma_common(dma);
+	kmem_free(dma, sizeof (*dma));
+}
+
+/* ARGSUSED */
+static void
+nvme_prp_dma_destructor(void *buf, void *private)
+{
+	nvme_dma_t *dma = (nvme_dma_t *)buf;
 
+	nvme_free_dma_common(dma);
+}
+
+static int
+nvme_alloc_dma_common(nvme_t *nvme, nvme_dma_t *dma,
+    size_t len, uint_t flags, ddi_dma_attr_t *dma_attr)
+{
 	if (ddi_dma_alloc_handle(nvme->n_dip, dma_attr, DDI_DMA_SLEEP, NULL,
 	    &dma->nd_dmah) != DDI_SUCCESS) {
 		/*
@@ -531,8 +547,23 @@ nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags,
 		dev_err(nvme->n_dip, CE_WARN,
 		    "!failed to bind DMA memory");
 		atomic_inc_32(&nvme->n_dma_bind_err);
+		nvme_free_dma_common(dma);
+		return (DDI_FAILURE);
+	}
+
+	return (DDI_SUCCESS);
+}
+
+static int
+nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags,
+    ddi_dma_attr_t *dma_attr, nvme_dma_t **ret)
+{
+	nvme_dma_t *dma = kmem_zalloc(sizeof (nvme_dma_t), KM_SLEEP);
+
+	if (nvme_alloc_dma_common(nvme, dma, len, flags, dma_attr) !=
+	    DDI_SUCCESS) {
 		*ret = NULL;
-		nvme_free_dma(dma);
+		kmem_free(dma, sizeof (nvme_dma_t));
 		return (DDI_FAILURE);
 	}
 
@@ -542,6 +573,28 @@ nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags,
 	return (DDI_SUCCESS);
 }
 
+/* ARGSUSED */
+static int
+nvme_prp_dma_constructor(void *buf, void *private, int flags)
+{
+	nvme_dma_t *dma = (nvme_dma_t *)buf;
+	nvme_t *nvme = (nvme_t *)private;
+
+	dma->nd_dmah = NULL;
+	dma->nd_acch = NULL;
+
+	if (nvme_alloc_dma_common(nvme, dma, nvme->n_pagesize,
+	    DDI_DMA_READ, &nvme->n_prp_dma_attr) != DDI_SUCCESS) {
+		return (-1);
+	}
+
+	ASSERT(dma->nd_ncookie == 1);
+
+	dma->nd_cached = B_TRUE;
+
+	return (0);
+}
+
 static int
 nvme_zalloc_queue_dma(nvme_t *nvme, uint32_t nentry, uint16_t qe_len,
     uint_t flags, nvme_dma_t **dma)
@@ -660,7 +713,11 @@ static void
 nvme_free_cmd(nvme_cmd_t *cmd)
 {
 	if (cmd->nc_dma) {
-		nvme_free_dma(cmd->nc_dma);
+		if (cmd->nc_dma->nd_cached)
+			kmem_cache_free(cmd->nc_nvme->n_prp_cache,
+			    cmd->nc_dma);
+		else
+			nvme_free_dma(cmd->nc_dma);
 		cmd->nc_dma = NULL;
 	}
 
@@ -2447,6 +2504,14 @@ nvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 		goto fail;
 	}
 
+	/*
+	 * Create PRP DMA cache
+	 */
+	(void) snprintf(name, sizeof (name), "%s%d_prp_cache",
+	    ddi_driver_name(dip), ddi_get_instance(dip));
+	nvme->n_prp_cache = kmem_cache_create(name, sizeof (nvme_dma_t),
+	    0, nvme_prp_dma_constructor, nvme_prp_dma_destructor,
+	    NULL, (void *)nvme, NULL, 0);
 
 	if (nvme_init(nvme) != DDI_SUCCESS)
 		goto fail;
@@ -2539,6 +2604,10 @@ nvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 		    (nvme->n_ioq_count + 1));
 	}
 
+	if (nvme->n_prp_cache != NULL) {
+		kmem_cache_destroy(nvme->n_prp_cache);
+	}
+
 	if (nvme->n_progress & NVME_REGS_MAPPED) {
 		nvme_shutdown(nvme, NVME_CC_SHN_NORMAL, B_FALSE);
 		(void) nvme_reset(nvme, B_FALSE);
@@ -2635,15 +2704,10 @@ nvme_fill_prp(nvme_cmd_t *cmd, bd_xfer_t *xfer)
 	 */
 	VERIFY(nprp == 1);
 
-	if (nvme_zalloc_dma(nvme, nvme->n_pagesize * nprp, DDI_DMA_READ,
-	    &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) {
-		dev_err(nvme->n_dip, CE_WARN, "!%s: nvme_zalloc_dma failed",
-		    __func__);
-		return (DDI_FAILURE);
-	}
+	cmd->nc_dma = kmem_cache_alloc(nvme->n_prp_cache, KM_SLEEP);
+	bzero(cmd->nc_dma->nd_memp, cmd->nc_dma->nd_len);
 
 	cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_dma->nd_cookie.dmac_laddress;
-	ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, &cmd->nc_dma->nd_cookie);
 
 	/*LINTED: E_PTR_BAD_CAST_ALIGN*/
 	for (prp = (uint64_t *)cmd->nc_dma->nd_memp;
diff --git a/usr/src/uts/common/io/nvme/nvme_var.h b/usr/src/uts/common/io/nvme/nvme_var.h
index f23e63b5db..409ae4841e 100644
--- a/usr/src/uts/common/io/nvme/nvme_var.h
+++ b/usr/src/uts/common/io/nvme/nvme_var.h
@@ -11,6 +11,7 @@
 
 /*
  * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
+ * Copyright 2016 The MathWorks, Inc. All rights reserved.
  */
 
 #ifndef _NVME_VAR_H
@@ -57,6 +58,7 @@ struct nvme_dma {
 	uint_t nd_ncookie;
 	caddr_t nd_memp;
 	size_t nd_len;
+	boolean_t nd_cached;
 };
 
 struct nvme_cmd {
@@ -108,6 +110,7 @@ struct nvme {
 	ddi_acc_handle_t n_regh;
 
 	kmem_cache_t *n_cmd_cache;
+	kmem_cache_t *n_prp_cache;
 
 	size_t n_inth_sz;
 	ddi_intr_handle_t *n_inth;