11203 Support for NVMe drive firmware updates

Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Hans Rosenfeld <rosenfeld@grumpf.hope-2000.org>
Reviewed by: Gergő Mihály Doma <domag02@gmail.com>
Reviewed by: C Fraire <cfraire@me.com>
Approved by: Gordon Ross <gwr@nexenta.com>

9 files changed, 840 insertions, 94 deletions

diff --git a/usr/src/cmd/nvmeadm/nvmeadm.c b/usr/src/cmd/nvmeadm/nvmeadm.c
index 00931bf77e..f53475db5b 100644
--- a/usr/src/cmd/nvmeadm/nvmeadm.c
+++ b/usr/src/cmd/nvmeadm/nvmeadm.c
@@ -12,6 +12,7 @@
 /*
  * Copyright 2016 Nexenta Systems, Inc.
  * Copyright 2017 Joyent, Inc.
+ * Copyright 2019 Western Digital Corporation.
  */
 
 /*
@@ -29,14 +30,14 @@
  *		get-param ...
  *		set-param ...
  *		load-firmware ...
+ *		commit-firmware ...
  *		activate-firmware ...
- *		write-uncorrectable ...
- *		compare ...
- *		compare-and-write ...
  */
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
 #include <strings.h>
 #include <ctype.h>
 #include <err.h>
@@ -112,6 +113,9 @@ static int do_get_features(int, const nvme_process_arg_t *);
 static int do_format(int, const nvme_process_arg_t *);
 static int do_secure_erase(int, const nvme_process_arg_t *);
 static int do_attach_detach(int, const nvme_process_arg_t *);
+static int do_firmware_load(int, const nvme_process_arg_t *);
+static int do_firmware_commit(int, const nvme_process_arg_t *);
+static int do_firmware_activate(int, const nvme_process_arg_t *);
 
 static void usage_list(const char *);
 static void usage_identify(const char *);
@@ -120,6 +124,9 @@ static void usage_get_features(const char *);
 static void usage_format(const char *);
 static void usage_secure_erase(const char *);
 static void usage_attach_detach(const char *);
+static void usage_firmware_load(const char *);
+static void usage_firmware_commit(const char *);
+static void usage_firmware_activate(const char *);
 
 int verbose;
 int debug;
@@ -175,6 +182,24 @@ static const nvmeadm_cmd_t nvmeadm_cmds[] = {
 		do_attach_detach, usage_attach_detach, B_FALSE
 	},
 	{
+		"load-firmware",
+		"load firmware to a controller",
+		NULL,
+		do_firmware_load, usage_firmware_load, B_FALSE
+	},
+	{
+		"commit-firmware",
+		"commit downloaded firmware to a slot of a controller",
+		NULL,
+		do_firmware_commit, usage_firmware_commit, B_FALSE
+	},
+	{
+		"activate-firmware",
+		"activate a firmware slot of a controller",
+		NULL,
+		do_firmware_activate, usage_firmware_activate, B_FALSE
+	},
+	{
 		NULL, NULL, NULL,
 		NULL, NULL, B_FALSE
 	}
@@ -361,7 +386,7 @@ usage(const nvmeadm_cmd_t *cmd)
 		(void) fprintf(stderr, "\ncommands:\n");
 
 		for (cmd = &nvmeadm_cmds[0]; cmd->c_name != NULL; cmd++)
-			(void) fprintf(stderr, "  %-15s - %s\n",
+			(void) fprintf(stderr, "  %-18s - %s\n",
 			    cmd->c_name, cmd->c_desc);
 	}
 	(void) fprintf(stderr, "\nflags:\n"
@@ -1009,3 +1034,202 @@ do_attach_detach(int fd, const nvme_process_arg_t *npa)
 
 	return (0);
 }
+
+static void
+usage_firmware_load(const char *c_name)
+{
+	(void) fprintf(stderr, "%s <ctl> <image file> [<offset>]\n\n"
+	    "  Load firmware <image file> to offset <offset>.\n"
+	    "  The firmware needs to be committed to a slot using "
+	    "\"nvmeadm commit-firmware\"\n  command.\n", c_name);
+}
+
+/*
+ * Read exactly len bytes, or until eof.
+ */
+static ssize_t
+read_block(int fd, char *buf, size_t len)
+{
+	size_t remain;
+	ssize_t bytes;
+
+	remain = len;
+	while (remain > 0) {
+		bytes = read(fd, buf, remain);
+		if (bytes == 0)
+			break;
+
+		if (bytes < 0) {
+			if (errno == EINTR)
+				continue;
+
+			return (-1);
+		}
+
+		buf += bytes;
+		remain -= bytes;
+	}
+
+	return (len - remain);
+}
+
+/*
+ * Convert a string to a valid firmware upload offset (in bytes).
+ */
+static offset_t
+get_fw_offsetb(char *str)
+{
+	longlong_t offsetb;
+	char *valend;
+
+	errno = 0;
+	offsetb = strtoll(str, &valend, 0);
+	if (errno != 0 || *valend != '\0' || offsetb < 0 ||
+	    offsetb > NVME_FW_OFFSETB_MAX)
+		errx(-1, "Offset must be numeric and in the range of 0 to %llu",
+		    NVME_FW_OFFSETB_MAX);
+
+	if ((offsetb & NVME_DWORD_MASK) != 0)
+		errx(-1, "Offset must be multiple of %d", NVME_DWORD_SIZE);
+
+	return ((offset_t)offsetb);
+}
+
+#define	FIRMWARE_READ_BLKSIZE	(64 * 1024)		/* 64K */
+
+static int
+do_firmware_load(int fd, const nvme_process_arg_t *npa)
+{
+	int fw_fd;
+	ssize_t len;
+	offset_t offset = 0;
+	size_t size;
+	char buf[FIRMWARE_READ_BLKSIZE];
+
+	if (npa->npa_argc > 2)
+		errx(-1, "Too many arguments");
+
+	if (npa->npa_argc == 0)
+		errx(-1, "Requires firmware file name, and an "
+		    "optional offset");
+
+	if (npa->npa_argc == 2)
+		offset = get_fw_offsetb(npa->npa_argv[1]);
+
+	fw_fd = open(npa->npa_argv[0], O_RDONLY);
+	if (fw_fd < 0)
+		errx(-1, "Failed to open \"%s\": %s", npa->npa_argv[0],
+		    strerror(errno));
+
+	size = 0;
+	do {
+		len = read_block(fw_fd, buf, sizeof (buf));
+
+		if (len < 0)
+			errx(-1, "Error reading \"%s\": %s", npa->npa_argv[0],
+			    strerror(errno));
+
+		if (len == 0)
+			break;
+
+		if (!nvme_firmware_load(fd, buf, len, offset))
+			errx(-1, "Error loading \"%s\": %s", npa->npa_argv[0],
+			    strerror(errno));
+
+		offset += len;
+		size += len;
+	} while (len == sizeof (buf));
+
+	close(fw_fd);
+
+	if (verbose)
+		(void) printf("%zu bytes downloaded.\n", size);
+
+	return (0);
+}
+
+/*
+ * Convert str to a valid firmware slot number.
+ */
+static uint_t
+get_slot_number(char *str)
+{
+	longlong_t slot;
+	char *valend;
+
+	errno = 0;
+	slot = strtoll(str, &valend, 0);
+	if (errno != 0 || *valend != '\0' ||
+	    slot < NVME_FW_SLOT_MIN || slot > NVME_FW_SLOT_MAX)
+		errx(-1, "Slot must be numeric and in the range of %d to %d",
+		    NVME_FW_SLOT_MIN, NVME_FW_SLOT_MAX);
+
+	return ((uint_t)slot);
+}
+
+static void
+usage_firmware_commit(const char *c_name)
+{
+	(void) fprintf(stderr, "%s <ctl> <slot>\n\n"
+	    "  Commit previously downloaded firmware to slot <slot>.\n"
+	    "  The firmware is only activated after a "
+	    "\"nvmeadm activate-firmware\" command.\n", c_name);
+}
+
+static int
+do_firmware_commit(int fd, const nvme_process_arg_t *npa)
+{
+	uint_t slot;
+	uint16_t sct, sc;
+
+	if (npa->npa_argc > 1)
+		errx(-1, "Too many arguments");
+
+	if (npa->npa_argc == 0)
+		errx(-1, "Firmware slot number is required");
+
+	slot = get_slot_number(npa->npa_argv[0]);
+
+	if (!nvme_firmware_commit(fd, slot, NVME_FWC_SAVE, &sct, &sc))
+		errx(-1, "Failed to commit firmware to slot %u: %s",
+		    slot, nvme_str_error(sct, sc));
+
+	if (verbose)
+		(void) printf("Firmware committed to slot %u.\n", slot);
+
+	return (0);
+}
+
+static void
+usage_firmware_activate(const char *c_name)
+{
+	(void) fprintf(stderr, "%s <ctl> <slot>\n\n"
+	    "  Activate firmware in slot <slot>.\n"
+	    "  The firmware will be in use after the next system reset.\n",
+	    c_name);
+}
+
+static int
+do_firmware_activate(int fd, const nvme_process_arg_t *npa)
+{
+	uint_t slot;
+	uint16_t sct, sc;
+
+	if (npa->npa_argc > 1)
+		errx(-1, "Too many arguments");
+
+	if (npa->npa_argc == 0)
+		errx(-1, "Firmware slot number is required");
+
+	slot = get_slot_number(npa->npa_argv[0]);
+
+	if (!nvme_firmware_commit(fd, slot, NVME_FWC_ACTIVATE, &sct, &sc))
+		errx(-1, "Failed to activate slot %u: %s", slot,
+		    nvme_str_error(sct, sc));
+
+	if (verbose)
+		printf("Slot %u activated: %s.\n", slot,
+		    nvme_str_error(sct, sc));
+
+	return (0);
+}
diff --git a/usr/src/cmd/nvmeadm/nvmeadm.h b/usr/src/cmd/nvmeadm/nvmeadm.h
index 4464350ace..a542f98be7 100644
--- a/usr/src/cmd/nvmeadm/nvmeadm.h
+++ b/usr/src/cmd/nvmeadm/nvmeadm.h
@@ -11,6 +11,7 @@
 
 /*
  * Copyright 2016 Nexenta Systems, Inc.
+ * Copyright 2019 Western Digital Corporation
  */
 
 #ifndef _NVMEADM_H
@@ -64,6 +65,7 @@ extern void nvme_print_feat_auto_pst(uint64_t, void *, size_t,
     nvme_identify_ctrl_t *);
 extern void nvme_print_feat_progress(uint64_t, void *, size_t,
     nvme_identify_ctrl_t *);
+extern const char *nvme_str_error(int, int);
 
 /* device node functions */
 extern int nvme_open(di_minor_t);
@@ -79,6 +81,8 @@ extern int nvme_intr_cnt(int);
 extern boolean_t nvme_format_nvm(int, uint8_t, uint8_t);
 extern boolean_t nvme_detach(int);
 extern boolean_t nvme_attach(int);
+extern boolean_t nvme_firmware_load(int, void *, size_t, offset_t);
+extern boolean_t nvme_firmware_commit(int fd, int, int, uint16_t *, uint16_t *);
 
 #ifdef __cplusplus
 }
diff --git a/usr/src/cmd/nvmeadm/nvmeadm_dev.c b/usr/src/cmd/nvmeadm/nvmeadm_dev.c
index 9126672e34..f808517556 100644
--- a/usr/src/cmd/nvmeadm/nvmeadm_dev.c
+++ b/usr/src/cmd/nvmeadm/nvmeadm_dev.c
@@ -11,6 +11,7 @@
 
 /*
  * Copyright 2016 Nexenta Systems, Inc.
+ * Copyright 2019 Western Digital Corporation
  */
 
 #include <sys/types.h>
@@ -31,40 +32,44 @@ nvme_ioctl(int fd, int ioc, size_t *bufsize, void **buf, uint64_t arg,
     uint64_t *res)
 {
 	nvme_ioctl_t nioc = { 0 };
-
-	if (buf != NULL)
-		*buf = NULL;
+	void *ptr = NULL;
+	int ret;
 
 	if (res != NULL)
 		*res = ~0ULL;
 
 	if (bufsize != NULL && *bufsize != 0) {
-		void *ptr;
-
 		assert(buf != NULL);
 
-		ptr = calloc(1, *bufsize);
-		if (ptr == NULL)
-			err(-1, "nvme_ioctl()");
+		if (*buf != NULL) {
+			nioc.n_buf = (uintptr_t)*buf;
+		} else {
+			ptr = calloc(1, *bufsize);
+			if (ptr == NULL)
+				err(-1, "nvme_ioctl()");
+
+			nioc.n_buf = (uintptr_t)ptr;
+		}
 
-		nioc.n_buf = (uintptr_t)ptr;
 		nioc.n_len = *bufsize;
 	}
 
 	nioc.n_arg = arg;
 
-	if (ioctl(fd, ioc, &nioc) != 0) {
+	ret = ioctl(fd, ioc, &nioc);
+
+	if (res != NULL)
+		*res = nioc.n_arg;
+
+	if (ret != 0) {
 		if (debug)
 			warn("nvme_ioctl()");
-		if (nioc.n_buf != 0)
-			free((void *)nioc.n_buf);
+		if (ptr != NULL)
+			free(ptr);
 
 		return (B_FALSE);
 	}
 
-	if (res != NULL)
-		*res = nioc.n_arg;
-
 	if (bufsize != NULL)
 		*bufsize = nioc.n_len;
 
@@ -143,7 +148,9 @@ nvme_intr_cnt(int fd)
 {
 	uint64_t res = 0;
 
-	(void) nvme_ioctl(fd, NVME_IOC_INTR_CNT, NULL, NULL, 0, &res);
+	if (!nvme_ioctl(fd, NVME_IOC_INTR_CNT, NULL, NULL, 0, &res))
+		return (-1);
+
 	return ((int)res);
 }
 
@@ -170,6 +177,30 @@ nvme_attach(int fd)
 	return (nvme_ioctl(fd, NVME_IOC_ATTACH, NULL, NULL, 0, NULL));
 }
 
+boolean_t
+nvme_firmware_load(int fd, void *buf, size_t len, offset_t offset)
+{
+	return (nvme_ioctl(fd, NVME_IOC_FIRMWARE_DOWNLOAD, &len, &buf, offset,
+	    NULL));
+}
+
+boolean_t
+nvme_firmware_commit(int fd, int slot, int action, uint16_t *sct, uint16_t *sc)
+{
+	boolean_t rv;
+	uint64_t res;
+
+	rv = nvme_ioctl(fd, NVME_IOC_FIRMWARE_COMMIT, NULL, NULL,
+	    ((uint64_t)action << 32) | slot, &res);
+
+	if (sct != NULL)
+		*sct = (uint16_t)(res >> 16);
+	if (sc != NULL)
+		*sc = (uint16_t)res;
+
+	return (rv);
+}
+
 int
 nvme_open(di_minor_t minor)
 {
diff --git a/usr/src/cmd/nvmeadm/nvmeadm_print.c b/usr/src/cmd/nvmeadm/nvmeadm_print.c
index 582a849a3e..0eb81a4e91 100644
--- a/usr/src/cmd/nvmeadm/nvmeadm_print.c
+++ b/usr/src/cmd/nvmeadm/nvmeadm_print.c
@@ -11,6 +11,7 @@
 
 /*
  * Copyright 2016 Nexenta Systems, Inc.
+ * Copyright 2019 Western Digital Corporation
  */
 
 /*
@@ -853,11 +854,13 @@ nvme_print_fwslot_log(nvme_fwslot_log_t *fwlog)
 
 	nvme_print(0, "Firmware Slot Information", -1, NULL);
 	nvme_print_uint64(2, "Active Firmware Slot", fwlog->fw_afi, NULL, NULL);
+	if (fwlog->fw_next != 0)
+		nvme_print_uint64(2, "Next Firmware Slot", fwlog->fw_next,
+		    NULL, NULL);
 
 	for (i = 0; i != ARRAYSIZE(fwlog->fw_frs); i++) {
-		if (fwlog->fw_frs[i][0] == '\0')
-			break;
 		nvme_print_str(2, "Firmware Revision for Slot", i + 1,
+		    fwlog->fw_frs[i][0] == '\0' ? "<Unused>" :
 		    fwlog->fw_frs[i], sizeof (fwlog->fw_frs[i]));
 	}
 }
@@ -1136,3 +1139,57 @@ nvme_print_feat_progress(uint64_t res, void *b, size_t s,
 	nvme_print_uint64(4, "Pre-Boot Software Load Count",
 	    spm.b.spm_pbslc, NULL, NULL);
 }
+
+static const char *
+nvme_str_generic_error(int sc)
+{
+	switch (sc) {
+	case NVME_CQE_SC_GEN_SUCCESS:
+		return ("Success");
+	default:
+		return ("See message log (usually /var/adm/messages) "
+		    "for details");
+	}
+}
+
+static const char *
+nvme_str_specific_error(int sc)
+{
+	switch (sc) {
+	case NVME_CQE_SC_SPC_INV_FW_SLOT:
+		return ("Invalid firmware slot");
+	case NVME_CQE_SC_SPC_INV_FW_IMG:
+		return ("Invalid firmware image");
+	case NVME_CQE_SC_SPC_FW_RESET:
+		return ("Conventional reset required - use "
+		    "'reboot -p' or similar");
+	case NVME_CQE_SC_SPC_FW_NSSR:
+		return ("NVM subsystem reset required - power cycle "
+		    "your system");
+	case NVME_CQE_SC_SPC_FW_NEXT_RESET:
+		return ("Image will be activated at next reset");
+	case NVME_CQE_SC_SPC_FW_MTFA:
+		return ("Activation requires maxmimum time violation");
+	case NVME_CQE_SC_SPC_FW_PROHIBITED:
+		return ("Activation prohibited");
+	default:
+		return ("See message log (usually /var/adm/messages) "
+		    "for details");
+	}
+}
+
+const char *
+nvme_str_error(int sct, int sc)
+{
+	switch (sct) {
+	case NVME_CQE_SCT_GENERIC:
+		return (nvme_str_generic_error(sc));
+
+	case NVME_CQE_SCT_SPECIFIC:
+		return (nvme_str_specific_error(sc));
+
+	default:
+		return ("See message log (usually /var/adm/messages) "
+		    "for details");
+	}
+}
diff --git a/usr/src/man/man1m/nvmeadm.1m b/usr/src/man/man1m/nvmeadm.1m
index 505a75030b..feb699dd79 100644
--- a/usr/src/man/man1m/nvmeadm.1m
+++ b/usr/src/man/man1m/nvmeadm.1m
@@ -10,8 +10,9 @@
 .\"
 .\"
 .\" Copyright 2016 Nexenta Systems, Inc.  All rights reserved.
+.\" Copyright 2019 Western Digital Corporation.
 .\"
-.Dd January 19, 2018
+.Dd June 27, 2019
 .Dt NVMEADM 1M
 .Os
 .Sh NAME
@@ -57,6 +58,22 @@
 .Op Fl dv
 .Cm attach
 .Ar ctl[/ns]
+.Nm
+.Op Fl dv
+.Cm load-firmware
+.Ar ctl
+.Ar firmware-file
+.Op Ar offset
+.Nm
+.Op Fl dv
+.Cm commit-firmware
+.Ar ctl
+.Ar slot
+.Nm
+.Op Fl dv
+.Cm activate-firmware
+.Ar ctl
+.Ar slot
 .Sh DESCRIPTION
 The
 .Nm
@@ -133,6 +150,30 @@ with the
 .Nm
 .Cm identify
 command.
+.It Ar firmware-file
+Specifies the name of a firmware file to be loaded into the controller
+using the
+.Cm load-firmware
+command.
+.It Ar offset
+Specifies the byte offset at which to load
+.Ar firmware-file
+within the controller's upload buffer.
+Vendors may require multiple images to be loaded at different offsets
+before a firmware set is committed to a
+.Ar slot .
+.It Ar slot
+Specifies the firmware slot into which a firmware set is committed
+using the
+.Cm commit-firmware
+command, and subsequently activated with the
+.Cm activate-firmware
+command.
+Slots and their contents can be printed using
+.Cm nvmeadm get-logpage
+to request the
+.Ar firmware
+logpage.
 .El
 .Sh COMMANDS
 .Bl -tag -width ""
@@ -303,6 +344,41 @@ previous
 .Nm
 .Cm detach
 command.
+.It Xo
+.Nm
+.Cm load-firmware
+.Ar ctl
+.Ar firmware-file
+.Op Ar offset
+.Xc
+Loads
+.Ar firmware-file
+into the controller's upload memory at
+.Ar offset ,
+the default is 0. A vendor may require multiple files to be loaded
+at different offsets before the firmware is committed to a
+.Ar slot .
+.It Xo
+.Nm
+.Cm commit-firmware
+.Ar ctl
+.Ar slot
+.Xc
+Commits firmware previously loaded by the
+.Cm load-firmware
+command to
+.Ar slot .
+.It Xo
+.Nm
+.Cm activate-firmware
+.Ar ctl
+.Ar slot
+.Xc
+Activates the firmware in slot
+.Ar slot .
+The firmware image in
+.Ar slot
+is activated at the next NVM controller reset.
 .El
 .Sh EXIT STATUS
 .Ex -std
@@ -393,6 +469,41 @@ nvme4: Get Features
   Power Management
     Power State:                            0
 .Ed
+.It Sy Example 5: Load and activate firmware
+.Bd -literal
+# nvmeadm get-logpage nvme3 firmware
+nvme3: Firmware Slot Information
+  Active Firmware Slot:                     4
+  Next Firmware Slot:                       4
+  Firmware Revision for Slot 1:             KNGND110
+  Firmware Revision for Slot 2:             KNGND110
+  Firmware Revision for Slot 3:             KNGND110
+  Firmware Revision for Slot 4:             KNGND112
+  Firmware Revision for Slot 5:             KNGND110
+  Firmware Revision for Slot 6:             <Unused>
+  Firmware Revision for Slot 7:             <Unused>
+
+# nvmeadm -v load-firmware nvme3 KNGND113.bin
+1740544 bytes downloaded.
+
+# nvmeadm -v commit-firmware nvme3 5
+Firmware committed to slot 5.
+
+# nvmeadm -v activate-firmware nvme3 5
+Slot 5 activated: NVM subsystem reset required - power cycle your system.
+
+# nvmeadm get-logpage nvme3 firmware
+nvme3: Firmware Slot Information
+  Active Firmware Slot:                     4
+  Next Firmware Slot:                       5
+  Firmware Revision for Slot 1:             KNGND110
+  Firmware Revision for Slot 2:             KNGND110
+  Firmware Revision for Slot 3:             KNGND110
+  Firmware Revision for Slot 4:             KNGND112
+  Firmware Revision for Slot 5:             KNGND113
+  Firmware Revision for Slot 6:             <Unused>
+  Firmware Revision for Slot 7:             <Unused>
+.Ed
 .El
 .Sh INTERFACE STABILITY
 The command line interface of
diff --git a/usr/src/uts/common/io/nvme/nvme.c b/usr/src/uts/common/io/nvme/nvme.c
index 8a3af7d7d7..5af89e3874 100644
--- a/usr/src/uts/common/io/nvme/nvme.c
+++ b/usr/src/uts/common/io/nvme/nvme.c
@@ -232,7 +232,6 @@
  * - support for media formatting and hard partitioning into namespaces
  * - support for big-endian systems
  * - support for fast reboot
- * - support for firmware updates
  * - support for NVMe Subsystem Reset (1.1)
  * - support for Scatter/Gather lists (1.1)
  * - support for Reservations (1.1)
@@ -305,6 +304,9 @@ int nvme_admin_cmd_timeout = 1;
 /* tunable for FORMAT NVM command timeout in seconds, default is 600s */
 int nvme_format_cmd_timeout = 600;
 
+/* tunable for firmware commit with NVME_FWC_SAVE, default is 15s */
+int nvme_commit_save_cmd_timeout = 15;
+
 static int nvme_attach(dev_info_t *, ddi_attach_cmd_t);
 static int nvme_detach(dev_info_t *, ddi_detach_cmd_t);
 static int nvme_quiesce(dev_info_t *);
@@ -1448,6 +1450,46 @@ nvme_check_specific_cmd_status(nvme_cmd_t *cmd)
 			bd_error(cmd->nc_xfer, BD_ERR_ILLRQ);
 		return (EROFS);
 
+	case NVME_CQE_SC_SPC_INV_FW_SLOT:
+		/* Invalid Firmware Slot */
+		ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE);
+		return (EINVAL);
+
+	case NVME_CQE_SC_SPC_INV_FW_IMG:
+		/* Invalid Firmware Image */
+		ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE);
+		return (EINVAL);
+
+	case NVME_CQE_SC_SPC_FW_RESET:
+		/* Conventional Reset Required */
+		ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE);
+		return (0);
+
+	case NVME_CQE_SC_SPC_FW_NSSR:
+		/* NVMe Subsystem Reset Required */
+		ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE);
+		return (0);
+
+	case NVME_CQE_SC_SPC_FW_NEXT_RESET:
+		/* Activation Requires Reset */
+		ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE);
+		return (0);
+
+	case NVME_CQE_SC_SPC_FW_MTFA:
+		/* Activation Requires Maximum Time Violation */
+		ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE);
+		return (EAGAIN);
+
+	case NVME_CQE_SC_SPC_FW_PROHIBITED:
+		/* Activation Prohibited */
+		ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE);
+		return (EINVAL);
+
+	case NVME_CQE_SC_SPC_FW_OVERLAP:
+		/* Overlapping Firmware Ranges */
+		ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_IMAGE_LOAD);
+		return (EINVAL);
+
 	default:
 		return (nvme_check_unknown_cmd_status(cmd));
 	}
@@ -3871,6 +3913,129 @@ nvme_ioctl_identify(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode,
 	return (rv);
 }
 
+/*
+ * Execute commands on behalf of the various ioctls.
+ */
+static int
+nvme_ioc_cmd(nvme_t *nvme, nvme_sqe_t *sqe, boolean_t is_admin, void *data_addr,
+    uint32_t data_len, int rwk, nvme_cqe_t *cqe, uint_t timeout)
+{
+	nvme_cmd_t *cmd;
+	nvme_qpair_t *ioq;
+	int rv = 0;
+
+	cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
+	if (is_admin) {
+		cmd->nc_sqid = 0;
+		ioq = nvme->n_adminq;
+	} else {
+		cmd->nc_sqid = (CPU->cpu_id % nvme->n_ioq_count) + 1;
+		ASSERT(cmd->nc_sqid <= nvme->n_ioq_count);
+		ioq = nvme->n_ioq[cmd->nc_sqid];
+	}
+
+	cmd->nc_callback = nvme_wakeup_cmd;
+	cmd->nc_sqe = *sqe;
+
+	if ((rwk & (FREAD | FWRITE)) != 0) {
+		if (data_addr == NULL) {
+			rv = EINVAL;
+			goto free_cmd;
+		}
+
+		/*
+		 * Because we use PRPs and haven't implemented PRP
+		 * lists here, the maximum data size is restricted to
+		 * 2 pages.
+		 */
+		if (data_len > 2 * nvme->n_pagesize) {
+			dev_err(nvme->n_dip, CE_WARN, "!Data size %u is too "
+			    "large for nvme_ioc_cmd(). Limit is 2 pages "
+			    "(%u bytes)", data_len,  2 * nvme->n_pagesize);
+
+			rv = EINVAL;
+			goto free_cmd;
+		}
+
+		if (nvme_zalloc_dma(nvme, data_len, DDI_DMA_READ,
+		    &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) {
+			dev_err(nvme->n_dip, CE_WARN,
+			    "!nvme_zalloc_dma failed for nvme_ioc_cmd()");
+
+			rv = ENOMEM;
+			goto free_cmd;
+		}
+
+		if (cmd->nc_dma->nd_ncookie > 2) {
+			dev_err(nvme->n_dip, CE_WARN,
+			    "!too many DMA cookies for nvme_ioc_cmd()");
+			atomic_inc_32(&nvme->n_too_many_cookies);
+
+			rv = E2BIG;
+			goto free_cmd;
+		}
+
+		cmd->nc_sqe.sqe_dptr.d_prp[0] =
+		    cmd->nc_dma->nd_cookie.dmac_laddress;
+
+		if (cmd->nc_dma->nd_ncookie > 1) {
+			ddi_dma_nextcookie(cmd->nc_dma->nd_dmah,
+			    &cmd->nc_dma->nd_cookie);
+			cmd->nc_sqe.sqe_dptr.d_prp[1] =
+			    cmd->nc_dma->nd_cookie.dmac_laddress;
+		}
+
+		if ((rwk & FWRITE) != 0) {
+			if (ddi_copyin(data_addr, cmd->nc_dma->nd_memp,
+			    data_len, rwk & FKIOCTL) != 0) {
+				rv = EFAULT;
+				goto free_cmd;
+			}
+		}
+	}
+
+	if (is_admin) {
+		nvme_admin_cmd(cmd, timeout);
+	} else {
+		mutex_enter(&cmd->nc_mutex);
+
+		rv = nvme_submit_io_cmd(ioq, cmd);
+
+		if (rv == EAGAIN) {
+			mutex_exit(&cmd->nc_mutex);
+			dev_err(cmd->nc_nvme->n_dip, CE_WARN,
+			    "!nvme_ioc_cmd() failed, I/O Q full");
+			goto free_cmd;
+		}
+
+		nvme_wait_cmd(cmd, timeout);
+
+		mutex_exit(&cmd->nc_mutex);
+	}
+
+	if (cqe != NULL)
+		*cqe = cmd->nc_cqe;
+
+	if ((rv = nvme_check_cmd_status(cmd)) != 0) {
+		dev_err(nvme->n_dip, CE_WARN,
+		    "!nvme_ioc_cmd() failed with sct = %x, sc = %x",
+		    cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
+
+		goto free_cmd;
+	}
+
+	if ((rwk & FREAD) != 0) {
+		if (ddi_copyout(cmd->nc_dma->nd_memp,
+		    data_addr, data_len, rwk & FKIOCTL) != 0)
+			rv = EFAULT;
+	}
+
+free_cmd:
+	nvme_free_cmd(cmd);
+
+	return (rv);
+}
+
 static int
 nvme_ioctl_capabilities(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc,
     int mode, cred_t *cred_p)
@@ -4190,6 +4355,109 @@ nvme_ioctl_attach(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode,
 }
 
 static int
+nvme_ioctl_firmware_download(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc,
+    int mode, cred_t *cred_p)
+{
+	int rv = 0;
+	size_t len, copylen;
+	offset_t offset;
+	uintptr_t buf;
+	nvme_sqe_t sqe = {
+	    .sqe_opc	= NVME_OPC_FW_IMAGE_LOAD
+	};
+
+	if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0)
+		return (EPERM);
+
+	if (nsid != 0)
+		return (EINVAL);
+
+	/*
+	 * The offset (in n_len) is restricted to the number of DWORDs in
+	 * 32 bits.
+	 */
+	if (nioc->n_len > NVME_FW_OFFSETB_MAX)
+		return (EINVAL);
+
+	/* Confirm that both offset and length are a multiple of DWORD bytes */
+	if ((nioc->n_len & NVME_DWORD_MASK) != 0 ||
+	    (nioc->n_arg & NVME_DWORD_MASK) != 0)
+		return (EINVAL);
+
+	len = nioc->n_len;
+	offset = nioc->n_arg;
+	buf = (uintptr_t)nioc->n_buf;
+	while (len > 0 && rv == 0) {
+		/*
+		 * nvme_ioc_cmd() does not use SGLs or PRP lists.
+		 * It is limited to 2 PRPs per NVM command, so limit
+		 * the size of the data to 2 pages.
+		 */
+		copylen = MIN(2 * nvme->n_pagesize, len);
+
+		sqe.sqe_cdw10 = (uint32_t)(copylen >> NVME_DWORD_SHIFT) - 1;
+		sqe.sqe_cdw11 = (uint32_t)(offset >> NVME_DWORD_SHIFT);
+
+		rv = nvme_ioc_cmd(nvme, &sqe, B_TRUE, (void *)buf, copylen,
+		    FWRITE, NULL, nvme_admin_cmd_timeout);
+
+		buf += copylen;
+		offset += copylen;
+		len -= copylen;
+	}
+
+	return (rv);
+}
+
+static int
+nvme_ioctl_firmware_commit(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc,
+    int mode, cred_t *cred_p)
+{
+	nvme_firmware_commit_dw10_t fc_dw10 = { 0 };
+	uint32_t slot = nioc->n_arg & 0xffffffff;
+	uint32_t action = nioc->n_arg >> 32;
+	nvme_cqe_t cqe = { 0 };
+	nvme_sqe_t sqe = {
+	    .sqe_opc	= NVME_OPC_FW_ACTIVATE
+	};
+	int timeout;
+	int rv;
+
+	if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0)
+		return (EPERM);
+
+	if (nsid != 0)
+		return (EINVAL);
+
+	/* Validate slot is in range. */
+	if (slot < NVME_FW_SLOT_MIN || slot > NVME_FW_SLOT_MAX)
+		return (EINVAL);
+
+	switch (action) {
+	case NVME_FWC_SAVE:
+	case NVME_FWC_SAVE_ACTIVATE:
+		timeout = nvme_commit_save_cmd_timeout;
+		break;
+	case NVME_FWC_ACTIVATE:
+	case NVME_FWC_ACTIVATE_IMMED:
+		timeout = nvme_admin_cmd_timeout;
+		break;
+	default:
+		return (EINVAL);
+	}
+
+	fc_dw10.b.fc_slot = slot;
+	fc_dw10.b.fc_action = action;
+	sqe.sqe_cdw10 = fc_dw10.r;
+
+	rv = nvme_ioc_cmd(nvme, &sqe, B_TRUE, NULL, 0, 0, &cqe, timeout);
+
+	nioc->n_arg = ((uint64_t)cqe.cqe_sf.sf_sct << 16) | cqe.cqe_sf.sf_sc;
+
+	return (rv);
+}
+
+static int
 nvme_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred_p,
     int *rval_p)
 {
@@ -4213,7 +4481,9 @@ nvme_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred_p,
 		nvme_ioctl_version,
 		nvme_ioctl_format,
 		nvme_ioctl_detach,
-		nvme_ioctl_attach
+		nvme_ioctl_attach,
+		nvme_ioctl_firmware_download,
+		nvme_ioctl_firmware_commit
 	};
 
 	if (nvme == NULL)
diff --git a/usr/src/uts/common/io/nvme/nvme_reg.h b/usr/src/uts/common/io/nvme/nvme_reg.h
index b332863876..e291c4f506 100644
--- a/usr/src/uts/common/io/nvme/nvme_reg.h
+++ b/usr/src/uts/common/io/nvme/nvme_reg.h
@@ -12,6 +12,7 @@
 /*
  * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
  * Copyright (c) 2018, Joyent, Inc.
+ * Copyright 2019 Western Digital Corporation
  */
 
 /*
@@ -321,70 +322,6 @@ typedef struct {
 	nvme_cqe_sf_t cqe_sf;		/* Status Field */
 } nvme_cqe_t;
 
-/* NVMe completion status code type */
-#define	NVME_CQE_SCT_GENERIC	0	/* Generic Command Status */
-#define	NVME_CQE_SCT_SPECIFIC	1	/* Command Specific Status */
-#define	NVME_CQE_SCT_INTEGRITY	2	/* Media and Data Integrity Errors */
-#define	NVME_CQE_SCT_VENDOR	7	/* Vendor Specific */
-
-/* NVMe completion status code (generic) */
-#define	NVME_CQE_SC_GEN_SUCCESS		0x0	/* Successful Completion */
-#define	NVME_CQE_SC_GEN_INV_OPC		0x1	/* Invalid Command Opcode */
-#define	NVME_CQE_SC_GEN_INV_FLD		0x2	/* Invalid Field in Command */
-#define	NVME_CQE_SC_GEN_ID_CNFL		0x3	/* Command ID Conflict */
-#define	NVME_CQE_SC_GEN_DATA_XFR_ERR	0x4	/* Data Transfer Error */
-#define	NVME_CQE_SC_GEN_ABORT_PWRLOSS	0x5	/* Cmds Aborted / Pwr Loss */
-#define	NVME_CQE_SC_GEN_INTERNAL_ERR	0x6	/* Internal Error */
-#define	NVME_CQE_SC_GEN_ABORT_REQUEST	0x7	/* Command Abort Requested */
-#define	NVME_CQE_SC_GEN_ABORT_SQ_DEL	0x8	/* Cmd Aborted / SQ deletion */
-#define	NVME_CQE_SC_GEN_ABORT_FUSE_FAIL	0x9	/* Cmd Aborted / Failed Fused */
-#define	NVME_CQE_SC_GEN_ABORT_FUSE_MISS	0xa	/* Cmd Aborted / Missing Fusd */
-#define	NVME_CQE_SC_GEN_INV_NS		0xb	/* Inval Namespace or Format */
-#define	NVME_CQE_SC_GEN_CMD_SEQ_ERR	0xc	/* Command Sequence Error */
-#define	NVME_CQE_SC_GEN_INV_SGL_LAST	0xd	/* Inval SGL Last Seg Desc */
-#define	NVME_CQE_SC_GEN_INV_SGL_NUM	0xe	/* Inval Number of SGL Desc */
-#define	NVME_CQE_SC_GEN_INV_DSGL_LEN	0xf	/* Data SGL Length Invalid */
-#define	NVME_CQE_SC_GEN_INV_MSGL_LEN	0x10	/* Metadata SGL Length Inval */
-#define	NVME_CQE_SC_GEN_INV_SGL_DESC	0x11	/* SGL Descriptor Type Inval */
-
-/* NVMe completion status code (generic NVM commands) */
-#define	NVME_CQE_SC_GEN_NVM_LBA_RANGE	0x80	/* LBA Out Of Range */
-#define	NVME_CQE_SC_GEN_NVM_CAP_EXC	0x81	/* Capacity Exceeded */
-#define	NVME_CQE_SC_GEN_NVM_NS_NOTRDY	0x82	/* Namespace Not Ready */
-#define	NVME_CQE_SC_GEN_NVM_RSV_CNFLCT	0x83	/* Reservation Conflict */
-
-/* NVMe completion status code (command specific) */
-#define	NVME_CQE_SC_SPC_INV_CQ		0x0	/* Completion Queue Invalid */
-#define	NVME_CQE_SC_SPC_INV_QID		0x1	/* Invalid Queue Identifier */
-#define	NVME_CQE_SC_SPC_MAX_QSZ_EXC	0x2	/* Max Queue Size Exceeded */
-#define	NVME_CQE_SC_SPC_ABRT_CMD_EXC	0x3	/* Abort Cmd Limit Exceeded */
-#define	NVME_CQE_SC_SPC_ASYNC_EVREQ_EXC	0x5	/* Async Event Request Limit */
-#define	NVME_CQE_SC_SPC_INV_FW_SLOT	0x6	/* Invalid Firmware Slot */
-#define	NVME_CQE_SC_SPC_INV_FW_IMG	0x7	/* Invalid Firmware Image */
-#define	NVME_CQE_SC_SPC_INV_INT_VECT	0x8	/* Invalid Interrupt Vector */
-#define	NVME_CQE_SC_SPC_INV_LOG_PAGE	0x9	/* Invalid Log Page */
-#define	NVME_CQE_SC_SPC_INV_FORMAT	0xa	/* Invalid Format */
-#define	NVME_CQE_SC_SPC_FW_RESET	0xb	/* FW Application Reset Reqd */
-#define	NVME_CQE_SC_SPC_INV_Q_DEL	0xc	/* Invalid Queue Deletion */
-#define	NVME_CQE_SC_SPC_FEAT_SAVE	0xd	/* Feature Id Not Saveable */
-#define	NVME_CQE_SC_SPC_FEAT_CHG	0xe	/* Feature Not Changeable */
-#define	NVME_CQE_SC_SPC_FEAT_NS_SPEC	0xf	/* Feature Not Namespace Spec */
-#define	NVME_CQE_SC_SPC_FW_NSSR		0x10	/* FW Application NSSR Reqd */
-
-/* NVMe completion status code (NVM command specific */
-#define	NVME_CQE_SC_SPC_NVM_CNFL_ATTR	0x80	/* Conflicting Attributes */
-#define	NVME_CQE_SC_SPC_NVM_INV_PROT	0x81	/* Invalid Protection */
-#define	NVME_CQE_SC_SPC_NVM_READONLY	0x82	/* Write to Read Only Range */
-
-/* NVMe completion status code (data / metadata integrity) */
-#define	NVME_CQE_SC_INT_NVM_WRITE	0x80	/* Write Fault */
-#define	NVME_CQE_SC_INT_NVM_READ	0x81	/* Unrecovered Read Error */
-#define	NVME_CQE_SC_INT_NVM_GUARD	0x82	/* Guard Check Error */
-#define	NVME_CQE_SC_INT_NVM_APPL_TAG	0x83	/* Application Tag Check Err */
-#define	NVME_CQE_SC_INT_NVM_REF_TAG	0x84	/* Reference Tag Check Err */
-#define	NVME_CQE_SC_INT_NVM_COMPARE	0x85	/* Compare Failure */
-#define	NVME_CQE_SC_INT_NVM_ACCESS	0x86	/* Access Denied */
-
 /*
  * NVMe Asynchronous Event Request
  */
diff --git a/usr/src/uts/common/io/nvme/nvme_var.h b/usr/src/uts/common/io/nvme/nvme_var.h
index 780205d145..6f3b53d3ec 100644
--- a/usr/src/uts/common/io/nvme/nvme_var.h
+++ b/usr/src/uts/common/io/nvme/nvme_var.h
@@ -275,7 +275,6 @@ struct nvme_task_arg {
 	nvme_cmd_t *nt_cmd;
 };
 
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/usr/src/uts/common/sys/nvme.h b/usr/src/uts/common/sys/nvme.h
index 98f203ef10..0c090da173 100644
--- a/usr/src/uts/common/sys/nvme.h
+++ b/usr/src/uts/common/sys/nvme.h
@@ -12,6 +12,7 @@
 /*
  * Copyright 2016 Nexenta Systems, Inc.
  * Copyright (c) 2018, Joyent, Inc.
+ * Copyright 2019 Western Digital Corporation
  */
 
 #ifndef _SYS_NVME_H
@@ -48,7 +49,9 @@ extern "C" {
 #define	NVME_IOC_FORMAT			(NVME_IOC | 8)
 #define	NVME_IOC_DETACH			(NVME_IOC | 9)
 #define	NVME_IOC_ATTACH			(NVME_IOC | 10)
-#define	NVME_IOC_MAX			NVME_IOC_ATTACH
+#define	NVME_IOC_FIRMWARE_DOWNLOAD	(NVME_IOC | 11)
+#define	NVME_IOC_FIRMWARE_COMMIT	(NVME_IOC | 12)
+#define	NVME_IOC_MAX			NVME_IOC_FIRMWARE_COMMIT
 
 #define	IS_NVME_IOC(x)			((x) > NVME_IOC && (x) <= NVME_IOC_MAX)
 #define	NVME_IOC_CMD(x)			((x) & 0xff)
@@ -435,10 +438,12 @@ typedef struct {
 
 typedef struct {
 	uint8_t fw_afi:3;		/* Active Firmware Slot */
-	uint8_t fw_rsvd1:5;
-	uint8_t fw_rsvd2[7];
+	uint8_t fw_rsvd1:1;
+	uint8_t fw_next:3;		/* Next Active Firmware Slot */
+	uint8_t fw_rsvd2:1;
+	uint8_t fw_rsvd3[7];
 	char fw_frs[7][8];		/* Firmware Revision / Slot */
-	uint8_t fw_rsvd3[512 - 64];
+	uint8_t fw_rsvd4[512 - 64];
 } nvme_fwslot_log_t;
 
 
@@ -636,8 +641,116 @@ typedef union {
 	uint32_t r;
 } nvme_software_progress_marker_t;
 
+/*
+ * Firmware Commit - Command Dword 10
+ */
+#define	NVME_FWC_SAVE		0x0	/* Save image only */
+#define	NVME_FWC_SAVE_ACTIVATE	0x1	/* Save and activate at next reset */
+#define	NVME_FWC_ACTIVATE	0x2	/* Activate slot at next reset */
+#define	NVME_FWC_ACTIVATE_IMMED	0x3	/* Activate slot immediately */
+
+/*
+ * Firmware slot number is only 3 bits, and zero is not allowed.
+ * Valid range is 1 to 7.
+ */
+#define	NVME_FW_SLOT_MIN	1	/* lowest allowable slot number ... */
+#define	NVME_FW_SLOT_MAX	7	/* ... and highest */
+
+/*
+ * Some constants to make verification of DWORD variables and arguments easier.
+ * A DWORD is 4 bytes.
+ */
+#define	NVME_DWORD_SHIFT	2
+#define	NVME_DWORD_SIZE		(1 << NVME_DWORD_SHIFT)
+#define	NVME_DWORD_MASK		(NVME_DWORD_SIZE - 1)
+
+/*
+ * Maximum offset a firmware image can be load at is the number of
+ * DWORDS in a 32 bit field. Expressed in bytes its is:
+ */
+#define	NVME_FW_OFFSETB_MAX	((u_longlong_t)UINT32_MAX << NVME_DWORD_SHIFT)
+
+typedef union {
+	struct {
+		uint32_t fc_slot:3;	/* Firmware slot */
+		uint32_t fc_action:3;	/* Commit action */
+		uint32_t fc_rsvd:26;
+	} b;
+	uint32_t r;
+} nvme_firmware_commit_dw10_t;
+
 #pragma pack() /* pack(1) */
 
+/* NVMe completion status code type */
+#define	NVME_CQE_SCT_GENERIC	0	/* Generic Command Status */
+#define	NVME_CQE_SCT_SPECIFIC	1	/* Command Specific Status */
+#define	NVME_CQE_SCT_INTEGRITY	2	/* Media and Data Integrity Errors */
+#define	NVME_CQE_SCT_VENDOR	7	/* Vendor Specific */
+
+/* NVMe completion status code (generic) */
+#define	NVME_CQE_SC_GEN_SUCCESS		0x0	/* Successful Completion */
+#define	NVME_CQE_SC_GEN_INV_OPC		0x1	/* Invalid Command Opcode */
+#define	NVME_CQE_SC_GEN_INV_FLD		0x2	/* Invalid Field in Command */
+#define	NVME_CQE_SC_GEN_ID_CNFL		0x3	/* Command ID Conflict */
+#define	NVME_CQE_SC_GEN_DATA_XFR_ERR	0x4	/* Data Transfer Error */
+#define	NVME_CQE_SC_GEN_ABORT_PWRLOSS	0x5	/* Cmds Aborted / Pwr Loss */
+#define	NVME_CQE_SC_GEN_INTERNAL_ERR	0x6	/* Internal Error */
+#define	NVME_CQE_SC_GEN_ABORT_REQUEST	0x7	/* Command Abort Requested */
+#define	NVME_CQE_SC_GEN_ABORT_SQ_DEL	0x8	/* Cmd Aborted / SQ deletion */
+#define	NVME_CQE_SC_GEN_ABORT_FUSE_FAIL	0x9	/* Cmd Aborted / Failed Fused */
+#define	NVME_CQE_SC_GEN_ABORT_FUSE_MISS	0xa	/* Cmd Aborted / Missing Fusd */
+#define	NVME_CQE_SC_GEN_INV_NS		0xb	/* Inval Namespace or Format */
+#define	NVME_CQE_SC_GEN_CMD_SEQ_ERR	0xc	/* Command Sequence Error */
+#define	NVME_CQE_SC_GEN_INV_SGL_LAST	0xd	/* Inval SGL Last Seg Desc */
+#define	NVME_CQE_SC_GEN_INV_SGL_NUM	0xe	/* Inval Number of SGL Desc */
+#define	NVME_CQE_SC_GEN_INV_DSGL_LEN	0xf	/* Data SGL Length Invalid */
+#define	NVME_CQE_SC_GEN_INV_MSGL_LEN	0x10	/* Metadata SGL Length Inval */
+#define	NVME_CQE_SC_GEN_INV_SGL_DESC	0x11	/* SGL Descriptor Type Inval */
+#define	NVME_CQE_SC_GEN_INV_USE_CMB	0x12	/* Inval use of Ctrl Mem Buf */
+#define	NVME_CQE_SC_GEN_INV_PRP_OFF	0x13	/* PRP Offset Invalid */
+#define	NVME_CQE_SC_GEN_AWU_EXCEEDED	0x14	/* Atomic Write Unit Exceeded */
+
+/* NVMe completion status code (generic NVM commands) */
+#define	NVME_CQE_SC_GEN_NVM_LBA_RANGE	0x80	/* LBA Out Of Range */
+#define	NVME_CQE_SC_GEN_NVM_CAP_EXC	0x81	/* Capacity Exceeded */
+#define	NVME_CQE_SC_GEN_NVM_NS_NOTRDY	0x82	/* Namespace Not Ready */
+#define	NVME_CQE_SC_GEN_NVM_RSV_CNFLCT	0x83	/* Reservation Conflict */
+
+/* NVMe completion status code (command specific) */
+#define	NVME_CQE_SC_SPC_INV_CQ		0x0	/* Completion Queue Invalid */
+#define	NVME_CQE_SC_SPC_INV_QID		0x1	/* Invalid Queue Identifier */
+#define	NVME_CQE_SC_SPC_MAX_QSZ_EXC	0x2	/* Max Queue Size Exceeded */
+#define	NVME_CQE_SC_SPC_ABRT_CMD_EXC	0x3	/* Abort Cmd Limit Exceeded */
+#define	NVME_CQE_SC_SPC_ASYNC_EVREQ_EXC	0x5	/* Async Event Request Limit */
+#define	NVME_CQE_SC_SPC_INV_FW_SLOT	0x6	/* Invalid Firmware Slot */
+#define	NVME_CQE_SC_SPC_INV_FW_IMG	0x7	/* Invalid Firmware Image */
+#define	NVME_CQE_SC_SPC_INV_INT_VECT	0x8	/* Invalid Interrupt Vector */
+#define	NVME_CQE_SC_SPC_INV_LOG_PAGE	0x9	/* Invalid Log Page */
+#define	NVME_CQE_SC_SPC_INV_FORMAT	0xa	/* Invalid Format */
+#define	NVME_CQE_SC_SPC_FW_RESET	0xb	/* FW Application Reset Reqd */
+#define	NVME_CQE_SC_SPC_INV_Q_DEL	0xc	/* Invalid Queue Deletion */
+#define	NVME_CQE_SC_SPC_FEAT_SAVE	0xd	/* Feature Id Not Saveable */
+#define	NVME_CQE_SC_SPC_FEAT_CHG	0xe	/* Feature Not Changeable */
+#define	NVME_CQE_SC_SPC_FEAT_NS_SPEC	0xf	/* Feature Not Namespace Spec */
+#define	NVME_CQE_SC_SPC_FW_NSSR		0x10	/* FW Application NSSR Reqd */
+#define	NVME_CQE_SC_SPC_FW_NEXT_RESET	0x11	/* FW Application Next Reqd */
+#define	NVME_CQE_SC_SPC_FW_MTFA		0x12	/* FW Application Exceed MTFA */
+#define	NVME_CQE_SC_SPC_FW_PROHIBITED	0x13	/* FW Application Prohibited */
+#define	NVME_CQE_SC_SPC_FW_OVERLAP	0x14	/* Overlapping FW ranges */
+
+/* NVMe completion status code (NVM command specific */
+#define	NVME_CQE_SC_SPC_NVM_CNFL_ATTR	0x80	/* Conflicting Attributes */
+#define	NVME_CQE_SC_SPC_NVM_INV_PROT	0x81	/* Invalid Protection */
+#define	NVME_CQE_SC_SPC_NVM_READONLY	0x82	/* Write to Read Only Range */
+
+/* NVMe completion status code (data / metadata integrity) */
+#define	NVME_CQE_SC_INT_NVM_WRITE	0x80	/* Write Fault */
+#define	NVME_CQE_SC_INT_NVM_READ	0x81	/* Unrecovered Read Error */
+#define	NVME_CQE_SC_INT_NVM_GUARD	0x82	/* Guard Check Error */
+#define	NVME_CQE_SC_INT_NVM_APPL_TAG	0x83	/* Application Tag Check Err */
+#define	NVME_CQE_SC_INT_NVM_REF_TAG	0x84	/* Reference Tag Check Err */
+#define	NVME_CQE_SC_INT_NVM_COMPARE	0x85	/* Compare Failure */
+#define	NVME_CQE_SC_INT_NVM_ACCESS	0x86	/* Access Denied */
 
 #ifdef __cplusplus
 }