11203 Support for NVMe drive firmware updates

Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Hans Rosenfeld <rosenfeld@grumpf.hope-2000.org>
Reviewed by: Gergő Mihály Doma <domag02@gmail.com>
Reviewed by: C Fraire <cfraire@me.com>
Approved by: Gordon Ross <gwr@nexenta.com>

4 files changed, 338 insertions, 22 deletions

diff --git a/usr/src/cmd/nvmeadm/nvmeadm.c b/usr/src/cmd/nvmeadm/nvmeadm.c
index 00931bf77e..f53475db5b 100644
--- a/usr/src/cmd/nvmeadm/nvmeadm.c
+++ b/usr/src/cmd/nvmeadm/nvmeadm.c
@@ -12,6 +12,7 @@
 /*
  * Copyright 2016 Nexenta Systems, Inc.
  * Copyright 2017 Joyent, Inc.
+ * Copyright 2019 Western Digital Corporation.
  */
 
 /*
@@ -29,14 +30,14 @@
  *		get-param ...
  *		set-param ...
  *		load-firmware ...
+ *		commit-firmware ...
  *		activate-firmware ...
- *		write-uncorrectable ...
- *		compare ...
- *		compare-and-write ...
  */
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
 #include <strings.h>
 #include <ctype.h>
 #include <err.h>
@@ -112,6 +113,9 @@ static int do_get_features(int, const nvme_process_arg_t *);
 static int do_format(int, const nvme_process_arg_t *);
 static int do_secure_erase(int, const nvme_process_arg_t *);
 static int do_attach_detach(int, const nvme_process_arg_t *);
+static int do_firmware_load(int, const nvme_process_arg_t *);
+static int do_firmware_commit(int, const nvme_process_arg_t *);
+static int do_firmware_activate(int, const nvme_process_arg_t *);
 
 static void usage_list(const char *);
 static void usage_identify(const char *);
@@ -120,6 +124,9 @@ static void usage_get_features(const char *);
 static void usage_format(const char *);
 static void usage_secure_erase(const char *);
 static void usage_attach_detach(const char *);
+static void usage_firmware_load(const char *);
+static void usage_firmware_commit(const char *);
+static void usage_firmware_activate(const char *);
 
 int verbose;
 int debug;
@@ -175,6 +182,24 @@ static const nvmeadm_cmd_t nvmeadm_cmds[] = {
 		do_attach_detach, usage_attach_detach, B_FALSE
 	},
 	{
+		"load-firmware",
+		"load firmware to a controller",
+		NULL,
+		do_firmware_load, usage_firmware_load, B_FALSE
+	},
+	{
+		"commit-firmware",
+		"commit downloaded firmware to a slot of a controller",
+		NULL,
+		do_firmware_commit, usage_firmware_commit, B_FALSE
+	},
+	{
+		"activate-firmware",
+		"activate a firmware slot of a controller",
+		NULL,
+		do_firmware_activate, usage_firmware_activate, B_FALSE
+	},
+	{
 		NULL, NULL, NULL,
 		NULL, NULL, B_FALSE
 	}
@@ -361,7 +386,7 @@ usage(const nvmeadm_cmd_t *cmd)
 		(void) fprintf(stderr, "\ncommands:\n");
 
 		for (cmd = &nvmeadm_cmds[0]; cmd->c_name != NULL; cmd++)
-			(void) fprintf(stderr, "  %-15s - %s\n",
+			(void) fprintf(stderr, "  %-18s - %s\n",
 			    cmd->c_name, cmd->c_desc);
 	}
 	(void) fprintf(stderr, "\nflags:\n"
@@ -1009,3 +1034,202 @@ do_attach_detach(int fd, const nvme_process_arg_t *npa)
 
 	return (0);
 }
+
+static void
+usage_firmware_load(const char *c_name)
+{
+	(void) fprintf(stderr, "%s <ctl> <image file> [<offset>]\n\n"
+	    "  Load firmware <image file> to offset <offset>.\n"
+	    "  The firmware needs to be committed to a slot using "
+	    "\"nvmeadm commit-firmware\"\n  command.\n", c_name);
+}
+
+/*
+ * Read exactly len bytes, or until eof.
+ */
+static ssize_t
+read_block(int fd, char *buf, size_t len)
+{
+	size_t remain;
+	ssize_t bytes;
+
+	remain = len;
+	while (remain > 0) {
+		bytes = read(fd, buf, remain);
+		if (bytes == 0)
+			break;
+
+		if (bytes < 0) {
+			if (errno == EINTR)
+				continue;
+
+			return (-1);
+		}
+
+		buf += bytes;
+		remain -= bytes;
+	}
+
+	return (len - remain);
+}
+
+/*
+ * Convert a string to a valid firmware upload offset (in bytes).
+ */
+static offset_t
+get_fw_offsetb(char *str)
+{
+	longlong_t offsetb;
+	char *valend;
+
+	errno = 0;
+	offsetb = strtoll(str, &valend, 0);
+	if (errno != 0 || *valend != '\0' || offsetb < 0 ||
+	    offsetb > NVME_FW_OFFSETB_MAX)
+		errx(-1, "Offset must be numeric and in the range of 0 to %llu",
+		    NVME_FW_OFFSETB_MAX);
+
+	if ((offsetb & NVME_DWORD_MASK) != 0)
+		errx(-1, "Offset must be multiple of %d", NVME_DWORD_SIZE);
+
+	return ((offset_t)offsetb);
+}
+
+#define	FIRMWARE_READ_BLKSIZE	(64 * 1024)		/* 64K */
+
+static int
+do_firmware_load(int fd, const nvme_process_arg_t *npa)
+{
+	int fw_fd;
+	ssize_t len;
+	offset_t offset = 0;
+	size_t size;
+	char buf[FIRMWARE_READ_BLKSIZE];
+
+	if (npa->npa_argc > 2)
+		errx(-1, "Too many arguments");
+
+	if (npa->npa_argc == 0)
+		errx(-1, "Requires firmware file name, and an "
+		    "optional offset");
+
+	if (npa->npa_argc == 2)
+		offset = get_fw_offsetb(npa->npa_argv[1]);
+
+	fw_fd = open(npa->npa_argv[0], O_RDONLY);
+	if (fw_fd < 0)
+		errx(-1, "Failed to open \"%s\": %s", npa->npa_argv[0],
+		    strerror(errno));
+
+	size = 0;
+	do {
+		len = read_block(fw_fd, buf, sizeof (buf));
+
+		if (len < 0)
+			errx(-1, "Error reading \"%s\": %s", npa->npa_argv[0],
+			    strerror(errno));
+
+		if (len == 0)
+			break;
+
+		if (!nvme_firmware_load(fd, buf, len, offset))
+			errx(-1, "Error loading \"%s\": %s", npa->npa_argv[0],
+			    strerror(errno));
+
+		offset += len;
+		size += len;
+	} while (len == sizeof (buf));
+
+	close(fw_fd);
+
+	if (verbose)
+		(void) printf("%zu bytes downloaded.\n", size);
+
+	return (0);
+}
+
+/*
+ * Convert str to a valid firmware slot number.
+ */
+static uint_t
+get_slot_number(char *str)
+{
+	longlong_t slot;
+	char *valend;
+
+	errno = 0;
+	slot = strtoll(str, &valend, 0);
+	if (errno != 0 || *valend != '\0' ||
+	    slot < NVME_FW_SLOT_MIN || slot > NVME_FW_SLOT_MAX)
+		errx(-1, "Slot must be numeric and in the range of %d to %d",
+		    NVME_FW_SLOT_MIN, NVME_FW_SLOT_MAX);
+
+	return ((uint_t)slot);
+}
+
+static void
+usage_firmware_commit(const char *c_name)
+{
+	(void) fprintf(stderr, "%s <ctl> <slot>\n\n"
+	    "  Commit previously downloaded firmware to slot <slot>.\n"
+	    "  The firmware is only activated after a "
+	    "\"nvmeadm activate-firmware\" command.\n", c_name);
+}
+
+static int
+do_firmware_commit(int fd, const nvme_process_arg_t *npa)
+{
+	uint_t slot;
+	uint16_t sct, sc;
+
+	if (npa->npa_argc > 1)
+		errx(-1, "Too many arguments");
+
+	if (npa->npa_argc == 0)
+		errx(-1, "Firmware slot number is required");
+
+	slot = get_slot_number(npa->npa_argv[0]);
+
+	if (!nvme_firmware_commit(fd, slot, NVME_FWC_SAVE, &sct, &sc))
+		errx(-1, "Failed to commit firmware to slot %u: %s",
+		    slot, nvme_str_error(sct, sc));
+
+	if (verbose)
+		(void) printf("Firmware committed to slot %u.\n", slot);
+
+	return (0);
+}
+
+static void
+usage_firmware_activate(const char *c_name)
+{
+	(void) fprintf(stderr, "%s <ctl> <slot>\n\n"
+	    "  Activate firmware in slot <slot>.\n"
+	    "  The firmware will be in use after the next system reset.\n",
+	    c_name);
+}
+
+static int
+do_firmware_activate(int fd, const nvme_process_arg_t *npa)
+{
+	uint_t slot;
+	uint16_t sct, sc;
+
+	if (npa->npa_argc > 1)
+		errx(-1, "Too many arguments");
+
+	if (npa->npa_argc == 0)
+		errx(-1, "Firmware slot number is required");
+
+	slot = get_slot_number(npa->npa_argv[0]);
+
+	if (!nvme_firmware_commit(fd, slot, NVME_FWC_ACTIVATE, &sct, &sc))
+		errx(-1, "Failed to activate slot %u: %s", slot,
+		    nvme_str_error(sct, sc));
+
+	if (verbose)
+		printf("Slot %u activated: %s.\n", slot,
+		    nvme_str_error(sct, sc));
+
+	return (0);
+}
diff --git a/usr/src/cmd/nvmeadm/nvmeadm.h b/usr/src/cmd/nvmeadm/nvmeadm.h
index 4464350ace..a542f98be7 100644
--- a/usr/src/cmd/nvmeadm/nvmeadm.h
+++ b/usr/src/cmd/nvmeadm/nvmeadm.h
@@ -11,6 +11,7 @@
 
 /*
  * Copyright 2016 Nexenta Systems, Inc.
+ * Copyright 2019 Western Digital Corporation
  */
 
 #ifndef _NVMEADM_H
@@ -64,6 +65,7 @@ extern void nvme_print_feat_auto_pst(uint64_t, void *, size_t,
     nvme_identify_ctrl_t *);
 extern void nvme_print_feat_progress(uint64_t, void *, size_t,
     nvme_identify_ctrl_t *);
+extern const char *nvme_str_error(int, int);
 
 /* device node functions */
 extern int nvme_open(di_minor_t);
@@ -79,6 +81,8 @@ extern int nvme_intr_cnt(int);
 extern boolean_t nvme_format_nvm(int, uint8_t, uint8_t);
 extern boolean_t nvme_detach(int);
 extern boolean_t nvme_attach(int);
+extern boolean_t nvme_firmware_load(int, void *, size_t, offset_t);
+extern boolean_t nvme_firmware_commit(int fd, int, int, uint16_t *, uint16_t *);
 
 #ifdef __cplusplus
 }
diff --git a/usr/src/cmd/nvmeadm/nvmeadm_dev.c b/usr/src/cmd/nvmeadm/nvmeadm_dev.c
index 9126672e34..f808517556 100644
--- a/usr/src/cmd/nvmeadm/nvmeadm_dev.c
+++ b/usr/src/cmd/nvmeadm/nvmeadm_dev.c
@@ -11,6 +11,7 @@
 
 /*
  * Copyright 2016 Nexenta Systems, Inc.
+ * Copyright 2019 Western Digital Corporation
  */
 
 #include <sys/types.h>
@@ -31,40 +32,44 @@ nvme_ioctl(int fd, int ioc, size_t *bufsize, void **buf, uint64_t arg,
     uint64_t *res)
 {
 	nvme_ioctl_t nioc = { 0 };
-
-	if (buf != NULL)
-		*buf = NULL;
+	void *ptr = NULL;
+	int ret;
 
 	if (res != NULL)
 		*res = ~0ULL;
 
 	if (bufsize != NULL && *bufsize != 0) {
-		void *ptr;
-
 		assert(buf != NULL);
 
-		ptr = calloc(1, *bufsize);
-		if (ptr == NULL)
-			err(-1, "nvme_ioctl()");
+		if (*buf != NULL) {
+			nioc.n_buf = (uintptr_t)*buf;
+		} else {
+			ptr = calloc(1, *bufsize);
+			if (ptr == NULL)
+				err(-1, "nvme_ioctl()");
+
+			nioc.n_buf = (uintptr_t)ptr;
+		}
 
-		nioc.n_buf = (uintptr_t)ptr;
 		nioc.n_len = *bufsize;
 	}
 
 	nioc.n_arg = arg;
 
-	if (ioctl(fd, ioc, &nioc) != 0) {
+	ret = ioctl(fd, ioc, &nioc);
+
+	if (res != NULL)
+		*res = nioc.n_arg;
+
+	if (ret != 0) {
 		if (debug)
 			warn("nvme_ioctl()");
-		if (nioc.n_buf != 0)
-			free((void *)nioc.n_buf);
+		if (ptr != NULL)
+			free(ptr);
 
 		return (B_FALSE);
 	}
 
-	if (res != NULL)
-		*res = nioc.n_arg;
-
 	if (bufsize != NULL)
 		*bufsize = nioc.n_len;
 
@@ -143,7 +148,9 @@ nvme_intr_cnt(int fd)
 {
 	uint64_t res = 0;
 
-	(void) nvme_ioctl(fd, NVME_IOC_INTR_CNT, NULL, NULL, 0, &res);
+	if (!nvme_ioctl(fd, NVME_IOC_INTR_CNT, NULL, NULL, 0, &res))
+		return (-1);
+
 	return ((int)res);
 }
 
@@ -170,6 +177,30 @@ nvme_attach(int fd)
 	return (nvme_ioctl(fd, NVME_IOC_ATTACH, NULL, NULL, 0, NULL));
 }
 
+boolean_t
+nvme_firmware_load(int fd, void *buf, size_t len, offset_t offset)
+{
+	return (nvme_ioctl(fd, NVME_IOC_FIRMWARE_DOWNLOAD, &len, &buf, offset,
+	    NULL));
+}
+
+boolean_t
+nvme_firmware_commit(int fd, int slot, int action, uint16_t *sct, uint16_t *sc)
+{
+	boolean_t rv;
+	uint64_t res;
+
+	rv = nvme_ioctl(fd, NVME_IOC_FIRMWARE_COMMIT, NULL, NULL,
+	    ((uint64_t)action << 32) | slot, &res);
+
+	if (sct != NULL)
+		*sct = (uint16_t)(res >> 16);
+	if (sc != NULL)
+		*sc = (uint16_t)res;
+
+	return (rv);
+}
+
 int
 nvme_open(di_minor_t minor)
 {
diff --git a/usr/src/cmd/nvmeadm/nvmeadm_print.c b/usr/src/cmd/nvmeadm/nvmeadm_print.c
index 582a849a3e..0eb81a4e91 100644
--- a/usr/src/cmd/nvmeadm/nvmeadm_print.c
+++ b/usr/src/cmd/nvmeadm/nvmeadm_print.c
@@ -11,6 +11,7 @@
 
 /*
  * Copyright 2016 Nexenta Systems, Inc.
+ * Copyright 2019 Western Digital Corporation
  */
 
 /*
@@ -853,11 +854,13 @@ nvme_print_fwslot_log(nvme_fwslot_log_t *fwlog)
 
 	nvme_print(0, "Firmware Slot Information", -1, NULL);
 	nvme_print_uint64(2, "Active Firmware Slot", fwlog->fw_afi, NULL, NULL);
+	if (fwlog->fw_next != 0)
+		nvme_print_uint64(2, "Next Firmware Slot", fwlog->fw_next,
+		    NULL, NULL);
 
 	for (i = 0; i != ARRAYSIZE(fwlog->fw_frs); i++) {
-		if (fwlog->fw_frs[i][0] == '\0')
-			break;
 		nvme_print_str(2, "Firmware Revision for Slot", i + 1,
+		    fwlog->fw_frs[i][0] == '\0' ? "<Unused>" :
 		    fwlog->fw_frs[i], sizeof (fwlog->fw_frs[i]));
 	}
 }
@@ -1136,3 +1139,57 @@ nvme_print_feat_progress(uint64_t res, void *b, size_t s,
 	nvme_print_uint64(4, "Pre-Boot Software Load Count",
 	    spm.b.spm_pbslc, NULL, NULL);
 }
+
+static const char *
+nvme_str_generic_error(int sc)
+{
+	switch (sc) {
+	case NVME_CQE_SC_GEN_SUCCESS:
+		return ("Success");
+	default:
+		return ("See message log (usually /var/adm/messages) "
+		    "for details");
+	}
+}
+
+static const char *
+nvme_str_specific_error(int sc)
+{
+	switch (sc) {
+	case NVME_CQE_SC_SPC_INV_FW_SLOT:
+		return ("Invalid firmware slot");
+	case NVME_CQE_SC_SPC_INV_FW_IMG:
+		return ("Invalid firmware image");
+	case NVME_CQE_SC_SPC_FW_RESET:
+		return ("Conventional reset required - use "
+		    "'reboot -p' or similar");
+	case NVME_CQE_SC_SPC_FW_NSSR:
+		return ("NVM subsystem reset required - power cycle "
+		    "your system");
+	case NVME_CQE_SC_SPC_FW_NEXT_RESET:
+		return ("Image will be activated at next reset");
+	case NVME_CQE_SC_SPC_FW_MTFA:
+		return ("Activation requires maxmimum time violation");
+	case NVME_CQE_SC_SPC_FW_PROHIBITED:
+		return ("Activation prohibited");
+	default:
+		return ("See message log (usually /var/adm/messages) "
+		    "for details");
+	}
+}
+
+const char *
+nvme_str_error(int sct, int sc)
+{
+	switch (sct) {
+	case NVME_CQE_SCT_GENERIC:
+		return (nvme_str_generic_error(sc));
+
+	case NVME_CQE_SCT_SPECIFIC:
+		return (nvme_str_specific_error(sc));
+
+	default:
+		return ("See message log (usually /var/adm/messages) "
+		    "for details");
+	}
+}