6235 want NVMe management utility

Reviewed by: Josef 'Jeff' Sipek <josef.sipek@nexenta.com>
Reviewed by: Rick McNeal <rick.mcneal@nexenta.com>
Reviewed by: Gordon Ross <gordon.ross@nexenta.com>
Reviewed by: Dan Fields <dan.fields@nexenta.com>
Reviewed by: Sanjay Nadkarni <sanjay.nadkarni@nexenta.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
Approved by: Rich Lowe <richlowe@richlowe.net>

6 files changed, 2475 insertions, 0 deletions

diff --git a/usr/src/cmd/Makefile b/usr/src/cmd/Makefile
index 08841eb06d..e33e3643f9 100644
--- a/usr/src/cmd/Makefile
+++ b/usr/src/cmd/Makefile
@@ -480,6 +480,7 @@ i386_SUBDIRS=		\
 	addbadsec	\
 	biosdev		\
 	diskscan	\
+	nvmeadm		\
 	rtc		\
 	ucodeadm	\
 	xvm
diff --git a/usr/src/cmd/nvmeadm/Makefile b/usr/src/cmd/nvmeadm/Makefile
new file mode 100644
index 0000000000..c042d4075f
--- /dev/null
+++ b/usr/src/cmd/nvmeadm/Makefile
@@ -0,0 +1,43 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright 2015 Nexenta Systems, Inc.
+#
+
+
+PROG= nvmeadm
+
+OBJS= nvmeadm.o nvmeadm_dev.o nvmeadm_print.o
+SRCS= $(OBJS:%.o=%.c)
+
+include ../Makefile.cmd
+
+.KEEP_STATE:
+
+CFLAGS +=	$(CCVERBOSE)
+LDLIBS +=	-ldevinfo
+C99MODE=	$(C99_ENABLE)
+
+all: $(PROG)
+
+$(PROG): $(OBJS)
+	$(LINK.c) -o $@ $(OBJS) $(LDLIBS)
+	$(POST_PROCESS)
+
+install: all $(ROOTUSRSBINPROG)
+
+clean:
+	$(RM) $(OBJS) $(PROG)
+
+lint:	lint_SRCS
+
+include ../Makefile.targ
diff --git a/usr/src/cmd/nvmeadm/nvmeadm.c b/usr/src/cmd/nvmeadm/nvmeadm.c
new file mode 100644
index 0000000000..13cace3ead
--- /dev/null
+++ b/usr/src/cmd/nvmeadm/nvmeadm.c
@@ -0,0 +1,1005 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2016 Nexenta Systems, Inc.
+ */
+
+/*
+ * nvmeadm -- NVMe administration utility
+ *
+ * nvmeadm [-v] [-d] [-h] <command> [<ctl>[/<ns>][,...]] [args]
+ * commands:	list
+ *		identify
+ *		get-logpage <logpage name>
+ *		get-features <feature>[,...]
+ *		format ...
+ *		secure-erase ...
+ *		detach ...
+ *		attach ...
+ *		get-param ...
+ *		set-param ...
+ *		load-firmware ...
+ *		activate-firmware ...
+ *		write-uncorrectable ...
+ *		compare ...
+ *		compare-and-write ...
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <strings.h>
+#include <ctype.h>
+#include <err.h>
+#include <sys/sunddi.h>
+#include <libdevinfo.h>
+
+#include <sys/nvme.h>
+
+#include "nvmeadm.h"
+
+typedef struct nvme_process_arg nvme_process_arg_t;
+typedef struct nvme_feature nvme_feature_t;
+typedef struct nvmeadm_cmd nvmeadm_cmd_t;
+
+struct nvme_process_arg {
+	int npa_argc;
+	char **npa_argv;
+	char *npa_name;
+	uint32_t npa_nsid;
+	boolean_t npa_isns;
+	const nvmeadm_cmd_t *npa_cmd;
+	di_node_t npa_node;
+	di_minor_t npa_minor;
+	char *npa_path;
+	char *npa_dsk;
+	nvme_identify_ctrl_t *npa_idctl;
+	nvme_identify_nsid_t *npa_idns;
+	nvme_version_t *npa_version;
+};
+
+struct nvme_feature {
+	char *f_name;
+	char *f_short;
+	uint8_t f_feature;
+	size_t f_bufsize;
+	uint_t f_getflags;
+	int (*f_get)(int, const nvme_feature_t *, nvme_identify_ctrl_t *);
+	void (*f_print)(uint64_t, void *, size_t, nvme_identify_ctrl_t *);
+};
+
+#define	NVMEADM_CTRL	1
+#define	NVMEADM_NS	2
+#define	NVMEADM_BOTH	(NVMEADM_CTRL | NVMEADM_NS)
+
+struct nvmeadm_cmd {
+	char *c_name;
+	char *c_desc;
+	char *c_flagdesc;
+	int (*c_func)(int, const nvme_process_arg_t *);
+	void (*c_usage)(const char *);
+	boolean_t c_multi;
+};
+
+
+static void usage(const nvmeadm_cmd_t *);
+static void nvme_walk(nvme_process_arg_t *, di_node_t);
+static boolean_t nvme_match(nvme_process_arg_t *);
+
+static int nvme_process(di_node_t, di_minor_t, void *);
+
+static int do_list(int, const nvme_process_arg_t *);
+static int do_identify(int, const nvme_process_arg_t *);
+static int do_get_logpage_error(int, const nvme_process_arg_t *);
+static int do_get_logpage_health(int, const nvme_process_arg_t *);
+static int do_get_logpage_fwslot(int, const nvme_process_arg_t *);
+static int do_get_logpage(int, const nvme_process_arg_t *);
+static int do_get_feat_common(int, const nvme_feature_t *,
+    nvme_identify_ctrl_t *);
+static int do_get_feat_intr_vect(int, const nvme_feature_t *,
+    nvme_identify_ctrl_t *);
+static int do_get_features(int, const nvme_process_arg_t *);
+static int do_format(int, const nvme_process_arg_t *);
+static int do_secure_erase(int, const nvme_process_arg_t *);
+static int do_attach_detach(int, const nvme_process_arg_t *);
+
+static void usage_list(const char *);
+static void usage_identify(const char *);
+static void usage_get_logpage(const char *);
+static void usage_get_features(const char *);
+static void usage_format(const char *);
+static void usage_secure_erase(const char *);
+static void usage_attach_detach(const char *);
+
+int verbose;
+int debug;
+int found;
+static int exitcode;
+
+static const nvmeadm_cmd_t nvmeadm_cmds[] = {
+	{
+		"list",
+		"list controllers and namespaces",
+		NULL,
+		do_list, usage_list, B_TRUE
+	},
+	{
+		"identify",
+		"identify controllers and/or namespaces",
+		NULL,
+		do_identify, usage_identify, B_TRUE
+	},
+	{
+		"get-logpage",
+		"get a log page from controllers and/or namespaces",
+		NULL,
+		do_get_logpage, usage_get_logpage, B_TRUE
+	},
+	{
+		"get-features",
+		"get features from controllers and/or namespaces",
+		NULL,
+		do_get_features, usage_get_features, B_TRUE
+	},
+	{
+		"format",
+		"format namespace(s) of a controller",
+		NULL,
+		do_format, usage_format, B_FALSE
+	},
+	{
+		"secure-erase",
+		"secure erase namespace(s) of a controller",
+		"  -c  Do a cryptographic erase.",
+		do_secure_erase, usage_secure_erase, B_FALSE
+	},
+	{
+		"detach",
+		"detach blkdev(7d) from namespace(s) of a controller",
+		NULL,
+		do_attach_detach, usage_attach_detach, B_FALSE
+	},
+	{
+		"attach",
+		"attach blkdev(7d) to namespace(s) of a controller",
+		NULL,
+		do_attach_detach, usage_attach_detach, B_FALSE
+	},
+	{
+		NULL, NULL, NULL,
+		NULL, NULL, B_FALSE
+	}
+};
+
+static const nvme_feature_t features[] = {
+	{ "Arbitration", "",
+	    NVME_FEAT_ARBITRATION, 0, NVMEADM_CTRL,
+	    do_get_feat_common, nvme_print_feat_arbitration },
+	{ "Power Management", "",
+	    NVME_FEAT_POWER_MGMT, 0, NVMEADM_CTRL,
+	    do_get_feat_common, nvme_print_feat_power_mgmt },
+	{ "LBA Range Type", "range",
+	    NVME_FEAT_LBA_RANGE, NVME_LBA_RANGE_BUFSIZE, NVMEADM_NS,
+	    do_get_feat_common, nvme_print_feat_lba_range },
+	{ "Temperature Threshold", "",
+	    NVME_FEAT_TEMPERATURE, 0, NVMEADM_CTRL,
+	    do_get_feat_common, nvme_print_feat_temperature },
+	{ "Error Recovery", "",
+	    NVME_FEAT_ERROR, 0, NVMEADM_CTRL,
+	    do_get_feat_common, nvme_print_feat_error },
+	{ "Volatile Write Cache", "cache",
+	    NVME_FEAT_WRITE_CACHE, 0, NVMEADM_CTRL,
+	    do_get_feat_common, nvme_print_feat_write_cache },
+	{ "Number of Queues", "queues",
+	    NVME_FEAT_NQUEUES, 0, NVMEADM_CTRL,
+	    do_get_feat_common, nvme_print_feat_nqueues },
+	{ "Interrupt Coalescing", "coalescing",
+	    NVME_FEAT_INTR_COAL, 0, NVMEADM_CTRL,
+	    do_get_feat_common, nvme_print_feat_intr_coal },
+	{ "Interrupt Vector Configuration", "vector",
+	    NVME_FEAT_INTR_VECT, 0, NVMEADM_CTRL,
+	    do_get_feat_intr_vect, nvme_print_feat_intr_vect },
+	{ "Write Atomicity", "atomicity",
+	    NVME_FEAT_WRITE_ATOM, 0, NVMEADM_CTRL,
+	    do_get_feat_common, nvme_print_feat_write_atom },
+	{ "Asynchronous Event Configuration", "event",
+	    NVME_FEAT_ASYNC_EVENT, 0, NVMEADM_CTRL,
+	    do_get_feat_common, nvme_print_feat_async_event },
+	{ "Autonomous Power State Transition", "",
+	    NVME_FEAT_AUTO_PST, NVME_AUTO_PST_BUFSIZE, NVMEADM_CTRL,
+	    do_get_feat_common, nvme_print_feat_auto_pst },
+	{ "Software Progress Marker", "progress",
+	    NVME_FEAT_PROGRESS, 0, NVMEADM_CTRL,
+	    do_get_feat_common, nvme_print_feat_progress },
+	{ NULL, NULL, 0, 0, B_FALSE, NULL }
+};
+
+
+int
+main(int argc, char **argv)
+{
+	int c;
+	extern int optind;
+	const nvmeadm_cmd_t *cmd;
+	di_node_t node;
+	nvme_process_arg_t npa = { 0 };
+	int help = 0;
+	char *tmp, *lasts = NULL;
+
+	while ((c = getopt(argc, argv, "dhv")) != -1) {
+		switch (c) {
+		case 'd':
+			debug++;
+			break;
+		case 'v':
+			verbose++;
+			break;
+		case 'h':
+			help++;
+			break;
+		case '?':
+			usage(NULL);
+			exit(-1);
+		}
+	}
+
+	if (optind == argc) {
+		usage(NULL);
+		if (help)
+			exit(0);
+		else
+			exit(-1);
+	}
+
+	/* Look up the specified command in the command table. */
+	for (cmd = &nvmeadm_cmds[0]; cmd->c_name != NULL; cmd++)
+		if (strcmp(cmd->c_name, argv[optind]) == 0)
+			break;
+
+	if (cmd->c_name == NULL) {
+		usage(NULL);
+		exit(-1);
+	}
+
+	if (help) {
+		usage(cmd);
+		exit(0);
+	}
+
+	npa.npa_cmd = cmd;
+
+	optind++;
+
+	/*
+	 * All commands but "list" require a ctl/ns argument.
+	 */
+	if ((optind == argc || (strncmp(argv[optind], "nvme", 4) != 0)) &&
+	    cmd->c_func != do_list) {
+		warnx("missing controller/namespace name");
+		usage(cmd);
+		exit(-1);
+	}
+
+
+	/* Store the remaining arguments for use by the command. */
+	npa.npa_argc = argc - optind - 1;
+	npa.npa_argv = &argv[optind + 1];
+
+	/*
+	 * Make sure we're not running commands on multiple controllers that
+	 * aren't allowed to do that.
+	 */
+	if (argv[optind] != NULL && strchr(argv[optind], ',') != NULL &&
+	    cmd->c_multi == B_FALSE) {
+		warnx("%s not allowed on multiple controllers",
+		    cmd->c_name);
+		usage(cmd);
+		exit(-1);
+	}
+
+	/*
+	 * Get controller/namespace arguments and run command.
+	 */
+	npa.npa_name = strtok_r(argv[optind], ",", &lasts);
+	do {
+		if (npa.npa_name != NULL) {
+			tmp = strchr(npa.npa_name, '/');
+			if (tmp != NULL) {
+				unsigned long nsid;
+				*tmp++ = '\0';
+				errno = 0;
+				nsid = strtoul(tmp, NULL, 10);
+				if (nsid >= UINT32_MAX || errno != 0) {
+					warn("invalid namespace %s", tmp);
+					exitcode--;
+					continue;
+				}
+				if (nsid == 0) {
+					warnx("invalid namespace %s", tmp);
+					exitcode--;
+					continue;
+				}
+				npa.npa_nsid = nsid;
+				npa.npa_isns = B_TRUE;
+			}
+		}
+
+		if ((node = di_init("/", DINFOSUBTREE | DINFOMINOR)) == NULL)
+			err(-1, "failed to initialize libdevinfo");
+		nvme_walk(&npa, node);
+		di_fini(node);
+
+		if (found == 0) {
+			if (npa.npa_name != NULL) {
+				warnx("%s%.*s%.*d: no such controller or "
+				    "namespace", npa.npa_name,
+				    npa.npa_nsid > 0 ? -1 : 0, "/",
+				    npa.npa_nsid > 0 ? -1 : 0, npa.npa_nsid);
+			} else {
+				warnx("no controllers found");
+			}
+			exitcode--;
+		}
+		found = 0;
+		npa.npa_name = strtok_r(NULL, ",", &lasts);
+	} while (npa.npa_name != NULL);
+
+	exit(exitcode);
+}
+
+static void
+usage(const nvmeadm_cmd_t *cmd)
+{
+	(void) fprintf(stderr, "usage:\n");
+	(void) fprintf(stderr, "  %s -h %s\n", getprogname(),
+	    cmd != NULL ? cmd->c_name : "[<command>]");
+	(void) fprintf(stderr, "  %s [-dv] ", getprogname());
+
+	if (cmd != NULL) {
+		cmd->c_usage(cmd->c_name);
+	} else {
+		(void) fprintf(stderr,
+		    "<command> <ctl>[/<ns>][,...] [<args>]\n");
+		(void) fprintf(stderr,
+		    "\n  Manage NVMe controllers and namespaces.\n");
+		(void) fprintf(stderr, "\ncommands:\n");
+
+		for (cmd = &nvmeadm_cmds[0]; cmd->c_name != NULL; cmd++)
+			(void) fprintf(stderr, "  %-15s - %s\n",
+			    cmd->c_name, cmd->c_desc);
+	}
+	(void) fprintf(stderr, "\nflags:\n"
+	    "  -h  print usage information\n"
+	    "  -d  print information useful for debugging %s\n"
+	    "  -v  print verbose information\n", getprogname());
+	if (cmd != NULL && cmd->c_flagdesc != NULL)
+		(void) fprintf(stderr, "%s\n", cmd->c_flagdesc);
+}
+
+static boolean_t
+nvme_match(nvme_process_arg_t *npa)
+{
+	char *name;
+	uint32_t nsid = 0;
+
+	if (npa->npa_name == NULL)
+		return (B_TRUE);
+
+	if (asprintf(&name, "%s%d", di_driver_name(npa->npa_node),
+	    di_instance(npa->npa_node)) < 0)
+		err(-1, "nvme_match()");
+
+	if (strcmp(name, npa->npa_name) != 0) {
+		free(name);
+		return (B_FALSE);
+	}
+
+	free(name);
+
+	if (npa->npa_isns) {
+		if (npa->npa_nsid == 0)
+			return (B_TRUE);
+		nsid = strtoul(di_minor_name(npa->npa_minor), NULL, 10);
+	}
+
+	if (npa->npa_isns && npa->npa_nsid != nsid)
+		return (B_FALSE);
+
+	return (B_TRUE);
+}
+
+char *
+nvme_dskname(const nvme_process_arg_t *npa)
+{
+	char *path = NULL;
+	di_node_t child;
+	di_dim_t dim;
+	char *addr;
+
+	dim = di_dim_init();
+
+	for (child = di_child_node(npa->npa_node);
+	    child != DI_NODE_NIL;
+	    child = di_sibling_node(child)) {
+		addr = di_bus_addr(child);
+		if (addr == NULL)
+			continue;
+
+		if (addr[0] == 'w')
+			addr++;
+
+		if (strncasecmp(addr, di_minor_name(npa->npa_minor),
+		    strchrnul(addr, ',') - addr) != 0)
+			continue;
+
+		path = di_dim_path_dev(dim, di_driver_name(child),
+		    di_instance(child), "c");
+
+		if (path != NULL) {
+			path[strlen(path) - 2] = '\0';
+			path = strrchr(path, '/') + 1;
+			if (path != NULL) {
+				path = strdup(path);
+				if (path == NULL)
+					err(-1, "nvme_dskname");
+			}
+		}
+
+		break;
+	}
+
+	di_dim_fini(dim);
+	return (path);
+}
+
+static int
+nvme_process(di_node_t node, di_minor_t minor, void *arg)
+{
+	nvme_process_arg_t *npa = arg;
+	int fd;
+
+	npa->npa_node = node;
+	npa->npa_minor = minor;
+
+	if (!nvme_match(npa))
+		return (DI_WALK_CONTINUE);
+
+	if ((fd = nvme_open(minor)) < 0)
+		return (DI_WALK_CONTINUE);
+
+	found++;
+
+	npa->npa_path = di_devfs_path(node);
+	if (npa->npa_path == NULL)
+		goto out;
+
+	npa->npa_version = nvme_version(fd);
+	if (npa->npa_version == NULL)
+		goto out;
+
+	npa->npa_idctl = nvme_identify_ctrl(fd);
+	if (npa->npa_idctl == NULL)
+		goto out;
+
+	npa->npa_idns = nvme_identify_nsid(fd);
+	if (npa->npa_idns == NULL)
+		goto out;
+
+	if (npa->npa_isns)
+		npa->npa_dsk = nvme_dskname(npa);
+
+	exitcode += npa->npa_cmd->c_func(fd, npa);
+
+out:
+	di_devfs_path_free(npa->npa_path);
+	free(npa->npa_dsk);
+	free(npa->npa_version);
+	free(npa->npa_idctl);
+	free(npa->npa_idns);
+
+	npa->npa_version = NULL;
+	npa->npa_idctl = NULL;
+	npa->npa_idns = NULL;
+
+	nvme_close(fd);
+
+	return (DI_WALK_CONTINUE);
+}
+
+static void
+nvme_walk(nvme_process_arg_t *npa, di_node_t node)
+{
+	char *minor_nodetype = DDI_NT_NVME_NEXUS;
+
+	if (npa->npa_isns)
+		minor_nodetype = DDI_NT_NVME_ATTACHMENT_POINT;
+
+	(void) di_walk_minor(node, minor_nodetype, 0, npa, nvme_process);
+}
+
+static void
+usage_list(const char *c_name)
+{
+	(void) fprintf(stderr, "%s [<ctl>[/<ns>][,...]\n\n"
+	    "  List NVMe controllers and their namespaces. If no "
+	    "controllers and/or name-\n  spaces are specified, all "
+	    "controllers and namespaces in the system will be\n  "
+	    "listed.\n", c_name);
+}
+
+static int
+do_list_nsid(int fd, const nvme_process_arg_t *npa)
+{
+	_NOTE(ARGUNUSED(fd));
+
+	(void) printf("  %s/%s (%s): ", npa->npa_name,
+	    di_minor_name(npa->npa_minor),
+	    npa->npa_dsk != NULL ? npa->npa_dsk : "unattached");
+	nvme_print_nsid_summary(npa->npa_idns);
+
+	return (0);
+}
+
+static int
+do_list(int fd, const nvme_process_arg_t *npa)
+{
+	_NOTE(ARGUNUSED(fd));
+
+	nvme_process_arg_t ns_npa = { 0 };
+	nvmeadm_cmd_t cmd = { 0 };
+	char *name;
+
+	if (asprintf(&name, "%s%d", di_driver_name(npa->npa_node),
+	    di_instance(npa->npa_node)) < 0)
+		err(-1, "do_list()");
+
+	(void) printf("%s: ", name);
+	nvme_print_ctrl_summary(npa->npa_idctl, npa->npa_version);
+
+	ns_npa.npa_name = name;
+	ns_npa.npa_isns = B_TRUE;
+	ns_npa.npa_nsid = npa->npa_nsid;
+	cmd = *(npa->npa_cmd);
+	cmd.c_func = do_list_nsid;
+	ns_npa.npa_cmd = &cmd;
+
+	nvme_walk(&ns_npa, npa->npa_node);
+
+	free(name);
+
+	return (exitcode);
+}
+
+static void
+usage_identify(const char *c_name)
+{
+	(void) fprintf(stderr, "%s <ctl>[/<ns>][,...]\n\n"
+	    "  Print detailed information about the specified NVMe "
+	    "controllers and/or name-\n  spaces.\n", c_name);
+}
+
+static int
+do_identify(int fd, const nvme_process_arg_t *npa)
+{
+	if (npa->npa_nsid == 0) {
+		nvme_capabilities_t *cap;
+
+		cap = nvme_capabilities(fd);
+		if (cap == NULL)
+			return (-1);
+
+		(void) printf("%s: ", npa->npa_name);
+		nvme_print_identify_ctrl(npa->npa_idctl, cap,
+		    npa->npa_version);
+
+		free(cap);
+	} else {
+		(void) printf("%s/%s: ", npa->npa_name,
+		    di_minor_name(npa->npa_minor));
+		nvme_print_identify_nsid(npa->npa_idns,
+		    npa->npa_version);
+	}
+
+	return (0);
+}
+
+static void
+usage_get_logpage(const char *c_name)
+{
+	(void) fprintf(stderr, "%s <ctl>[/<ns>][,...] <logpage>\n\n"
+	    "  Print the specified log page of the specified NVMe "
+	    "controllers and/or name-\n  spaces. Supported log pages "
+	    "are error, health, and firmware.\n", c_name);
+}
+
+static int
+do_get_logpage_error(int fd, const nvme_process_arg_t *npa)
+{
+	int nlog = npa->npa_idctl->id_elpe + 1;
+	size_t bufsize = sizeof (nvme_error_log_entry_t) * nlog;
+	nvme_error_log_entry_t *elog;
+
+	if (npa->npa_nsid != 0)
+		errx(-1, "Error Log not available on a per-namespace basis");
+
+	elog = nvme_get_logpage(fd, NVME_LOGPAGE_ERROR, &bufsize);
+
+	if (elog == NULL)
+		return (-1);
+
+	nlog = bufsize / sizeof (nvme_error_log_entry_t);
+
+	(void) printf("%s: ", npa->npa_name);
+	nvme_print_error_log(nlog, elog);
+
+	free(elog);
+
+	return (0);
+}
+
+static int
+do_get_logpage_health(int fd, const nvme_process_arg_t *npa)
+{
+	size_t bufsize = sizeof (nvme_health_log_t);
+	nvme_health_log_t *hlog;
+
+	if (npa->npa_nsid != 0) {
+		if (npa->npa_idctl->id_lpa.lp_smart == 0)
+			errx(-1, "SMART/Health information not available "
+			    "on a per-namespace basis on this controller");
+	}
+
+	hlog = nvme_get_logpage(fd, NVME_LOGPAGE_HEALTH, &bufsize);
+
+	if (hlog == NULL)
+		return (-1);
+
+	(void) printf("%s: ", npa->npa_name);
+	nvme_print_health_log(hlog, npa->npa_idctl);
+
+	free(hlog);
+
+	return (0);
+}
+
+static int
+do_get_logpage_fwslot(int fd, const nvme_process_arg_t *npa)
+{
+	size_t bufsize = sizeof (nvme_fwslot_log_t);
+	nvme_fwslot_log_t *fwlog;
+
+	if (npa->npa_nsid != 0)
+		errx(-1, "Firmware Slot information not available on a "
+		    "per-namespace basis");
+
+	fwlog = nvme_get_logpage(fd, NVME_LOGPAGE_FWSLOT, &bufsize);
+
+	if (fwlog == NULL)
+		return (-1);
+
+	(void) printf("%s: ", npa->npa_name);
+	nvme_print_fwslot_log(fwlog);
+
+	free(fwlog);
+
+	return (0);
+}
+
+static int
+do_get_logpage(int fd, const nvme_process_arg_t *npa)
+{
+	int ret = 0;
+	int (*func)(int, const nvme_process_arg_t *);
+
+	if (npa->npa_argc < 1) {
+		warnx("missing logpage name");
+		usage(npa->npa_cmd);
+		exit(-1);
+	}
+
+	if (strcmp(npa->npa_argv[0], "error") == 0)
+		func = do_get_logpage_error;
+	else if (strcmp(npa->npa_argv[0], "health") == 0)
+		func = do_get_logpage_health;
+	else if (strcmp(npa->npa_argv[0], "firmware") == 0)
+		func = do_get_logpage_fwslot;
+	else
+		errx(-1, "invalid log page: %s", npa->npa_argv[0]);
+
+	ret = func(fd, npa);
+	return (ret);
+}
+
+static void
+usage_get_features(const char *c_name)
+{
+	const nvme_feature_t *feat;
+
+	(void) fprintf(stderr, "%s <ctl>[/<ns>][,...] [<feature>[,...]]\n\n"
+	    "  Print the specified features of the specified NVMe controllers "
+	    "and/or\n  namespaces. Supported features are:\n\n", c_name);
+	(void) fprintf(stderr, "    %-35s %-14s %s\n",
+	    "FEATURE NAME", "SHORT NAME", "CONTROLLER/NAMESPACE");
+	for (feat = &features[0]; feat->f_feature != 0; feat++) {
+		char *type;
+
+		if ((feat->f_getflags & NVMEADM_BOTH) == NVMEADM_BOTH)
+			type = "both";
+		else if ((feat->f_getflags & NVMEADM_CTRL) != 0)
+			type = "controller only";
+		else
+			type = "namespace only";
+
+		(void) fprintf(stderr, "    %-35s %-14s %s\n",
+		    feat->f_name, feat->f_short, type);
+	}
+
+}
+
+static int
+do_get_feat_common(int fd, const nvme_feature_t *feat,
+    nvme_identify_ctrl_t *idctl)
+{
+	void *buf = NULL;
+	size_t bufsize = feat->f_bufsize;
+	uint64_t res;
+
+	if (nvme_get_feature(fd, feat->f_feature, 0, &res, &bufsize, &buf)
+	    == B_FALSE)
+		return (EINVAL);
+
+	nvme_print(2, feat->f_name, -1, NULL);
+	feat->f_print(res, buf, bufsize, idctl);
+	free(buf);
+
+	return (0);
+}
+
+static int
+do_get_feat_intr_vect(int fd, const nvme_feature_t *feat,
+    nvme_identify_ctrl_t *idctl)
+{
+	uint64_t res;
+	uint64_t arg;
+	int intr_cnt;
+
+	intr_cnt = nvme_intr_cnt(fd);
+
+	if (intr_cnt == -1)
+		return (EINVAL);
+
+	nvme_print(2, feat->f_name, -1, NULL);
+
+	for (arg = 0; arg < intr_cnt; arg++) {
+		if (nvme_get_feature(fd, feat->f_feature, arg, &res, NULL, NULL)
+		    == B_FALSE)
+			return (EINVAL);
+
+		feat->f_print(res, NULL, 0, idctl);
+	}
+
+	return (0);
+}
+
+static int
+do_get_features(int fd, const nvme_process_arg_t *npa)
+{
+	const nvme_feature_t *feat;
+	char *f, *flist, *lasts;
+	boolean_t header_printed = B_FALSE;
+
+	if (npa->npa_argc > 1)
+		errx(-1, "unexpected arguments");
+
+	/*
+	 * No feature list given, print all supported features.
+	 */
+	if (npa->npa_argc == 0) {
+		(void) printf("%s: Get Features\n", npa->npa_name);
+		for (feat = &features[0]; feat->f_feature != 0; feat++) {
+			if ((npa->npa_nsid != 0 &&
+			    (feat->f_getflags & NVMEADM_NS) == 0) ||
+			    (npa->npa_nsid == 0 &&
+			    (feat->f_getflags & NVMEADM_CTRL) == 0))
+				continue;
+
+			(void) feat->f_get(fd, feat, npa->npa_idctl);
+		}
+
+		return (0);
+	}
+
+	/*
+	 * Process feature list.
+	 */
+	flist = strdup(npa->npa_argv[0]);
+	if (flist == NULL)
+		err(-1, "do_get_features");
+
+	for (f = strtok_r(flist, ",", &lasts);
+	    f != NULL;
+	    f = strtok_r(NULL, ",", &lasts)) {
+		while (isspace(*f))
+			f++;
+
+		for (feat = &features[0]; feat->f_feature != 0; feat++) {
+			if (strncasecmp(feat->f_name, f, strlen(f)) == 0 ||
+			    strncasecmp(feat->f_short, f, strlen(f)) == 0)
+				break;
+		}
+
+		if (feat->f_feature == 0) {
+			warnx("unknown feature %s", f);
+			continue;
+		}
+
+		if ((npa->npa_nsid != 0 &&
+		    (feat->f_getflags & NVMEADM_NS) == 0) ||
+		    (npa->npa_nsid == 0 &&
+		    (feat->f_getflags & NVMEADM_CTRL) == 0)) {
+			warnx("feature %s %s supported for namespaces",
+			    feat->f_name, (feat->f_getflags & NVMEADM_NS) != 0 ?
+			    "only" : "not");
+			continue;
+		}
+
+		if (!header_printed) {
+			(void) printf("%s: Get Features\n", npa->npa_name);
+			header_printed = B_TRUE;
+		}
+
+		if (feat->f_get(fd, feat, npa->npa_idctl) != 0) {
+			warnx("unsupported feature: %s", feat->f_name);
+			continue;
+		}
+	}
+
+	free(flist);
+	return (0);
+}
+
+static int
+do_format_common(int fd, const nvme_process_arg_t *npa, unsigned long lbaf,
+    unsigned long ses)
+{
+	nvme_process_arg_t ns_npa = { 0 };
+	nvmeadm_cmd_t cmd = { 0 };
+
+	cmd = *(npa->npa_cmd);
+	cmd.c_func = do_attach_detach;
+	cmd.c_name = "detach";
+	ns_npa = *npa;
+	ns_npa.npa_cmd = &cmd;
+
+	if (do_attach_detach(fd, &ns_npa) != 0)
+		return (exitcode);
+	if (nvme_format_nvm(fd, lbaf, ses) == B_FALSE) {
+		warn("%s failed", npa->npa_cmd->c_name);
+		exitcode += -1;
+	}
+	cmd.c_name = "attach";
+	exitcode += do_attach_detach(fd, &ns_npa);
+
+	return (exitcode);
+}
+
+static void
+usage_format(const char *c_name)
+{
+	(void) fprintf(stderr, "%s <ctl>[/<ns>] [<lba-format>]\n\n"
+	    "  Format one or all namespaces of the specified NVMe "
+	    "controller. Supported LBA\n  formats can be queried with "
+	    "the \"%s identify\" command on the namespace\n  to be "
+	    "formatted.\n", c_name, getprogname());
+}
+
+static int
+do_format(int fd, const nvme_process_arg_t *npa)
+{
+	unsigned long lbaf;
+
+	if (npa->npa_idctl->id_oacs.oa_format == 0)
+		errx(-1, "%s not supported", npa->npa_cmd->c_name);
+
+	if (npa->npa_isns && npa->npa_idctl->id_fna.fn_format != 0)
+		errx(-1, "%s not supported on individual namespace",
+		    npa->npa_cmd->c_name);
+
+
+	if (npa->npa_argc > 0) {
+		errno = 0;
+		lbaf = strtoul(npa->npa_argv[0], NULL, 10);
+
+		if (errno != 0 || lbaf > NVME_FRMT_MAX_LBAF)
+			errx(-1, "invalid LBA format %d", lbaf + 1);
+
+		if (npa->npa_idns->id_lbaf[lbaf].lbaf_ms != 0)
+			errx(-1, "LBA formats with metadata not supported");
+	} else {
+		lbaf = npa->npa_idns->id_flbas.lba_format;
+	}
+
+	return (do_format_common(fd, npa, lbaf, 0));
+}
+
+static void
+usage_secure_erase(const char *c_name)
+{
+	(void) fprintf(stderr, "%s <ctl>[/<ns>] [-c]\n\n"
+	    "  Secure-Erase one or all namespaces of the specified "
+	    "NVMe controller.\n", c_name);
+}
+
+static int
+do_secure_erase(int fd, const nvme_process_arg_t *npa)
+{
+	unsigned long lbaf;
+	uint8_t ses = NVME_FRMT_SES_USER;
+
+	if (npa->npa_idctl->id_oacs.oa_format == 0)
+		errx(-1, "%s not supported", npa->npa_cmd->c_name);
+
+	if (npa->npa_isns && npa->npa_idctl->id_fna.fn_sec_erase != 0)
+		errx(-1, "%s not supported on individual namespace",
+		    npa->npa_cmd->c_name);
+
+	if (npa->npa_argc > 0) {
+		if (strcmp(npa->npa_argv[0], "-c") == 0)
+			ses = NVME_FRMT_SES_CRYPTO;
+		else
+			usage(npa->npa_cmd);
+	}
+
+	if (ses == NVME_FRMT_SES_CRYPTO &&
+	    npa->npa_idctl->id_fna.fn_crypt_erase == 0)
+		errx(-1, "cryptographic %s not supported",
+		    npa->npa_cmd->c_name);
+
+	lbaf = npa->npa_idns->id_flbas.lba_format;
+
+	return (do_format_common(fd, npa, lbaf, ses));
+}
+
+static void
+usage_attach_detach(const char *c_name)
+{
+	(void) fprintf(stderr, "%s <ctl>[/<ns>]\n\n"
+	    "  %c%s blkdev(7d) %s one or all namespaces of the "
+	    "specified NVMe controller.\n",
+	    c_name, toupper(c_name[0]), &c_name[1],
+	    c_name[0] == 'd' ? "from" : "to");
+}
+
+static int
+do_attach_detach(int fd, const nvme_process_arg_t *npa)
+{
+	char *c_name = npa->npa_cmd->c_name;
+
+	if (!npa->npa_isns) {
+		nvme_process_arg_t ns_npa = { 0 };
+
+		ns_npa.npa_name = npa->npa_name;
+		ns_npa.npa_isns = B_TRUE;
+		ns_npa.npa_cmd = npa->npa_cmd;
+
+		nvme_walk(&ns_npa, npa->npa_node);
+
+		return (exitcode);
+	} else {
+		if ((c_name[0] == 'd' ? nvme_detach : nvme_attach)(fd)
+		    == B_FALSE) {
+			warn("%s failed", c_name);
+			return (-1);
+		}
+	}
+
+	return (0);
+}
diff --git a/usr/src/cmd/nvmeadm/nvmeadm.h b/usr/src/cmd/nvmeadm/nvmeadm.h
new file mode 100644
index 0000000000..4464350ace
--- /dev/null
+++ b/usr/src/cmd/nvmeadm/nvmeadm.h
@@ -0,0 +1,87 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2016 Nexenta Systems, Inc.
+ */
+
+#ifndef _NVMEADM_H
+#define	_NVMEADM_H
+
+#include <stdio.h>
+#include <libdevinfo.h>
+#include <sys/nvme.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int verbose;
+extern int debug;
+
+/* printing functions */
+extern void nvme_print(int, char *, int, const char *, ...);
+extern void nvme_print_ctrl_summary(nvme_identify_ctrl_t *, nvme_version_t *);
+extern void nvme_print_nsid_summary(nvme_identify_nsid_t *);
+extern void nvme_print_identify_ctrl(nvme_identify_ctrl_t *,
+    nvme_capabilities_t *, nvme_version_t *);
+extern void nvme_print_identify_nsid(nvme_identify_nsid_t *, nvme_version_t *);
+extern void nvme_print_error_log(int, nvme_error_log_entry_t *);
+extern void nvme_print_health_log(nvme_health_log_t *, nvme_identify_ctrl_t *);
+extern void nvme_print_fwslot_log(nvme_fwslot_log_t *);
+
+extern void nvme_print_feat_arbitration(uint64_t, void *, size_t,
+    nvme_identify_ctrl_t *);
+extern void nvme_print_feat_power_mgmt(uint64_t, void *, size_t,
+    nvme_identify_ctrl_t *);
+extern void nvme_print_feat_lba_range(uint64_t, void *, size_t,
+    nvme_identify_ctrl_t *);
+extern void nvme_print_feat_temperature(uint64_t, void *, size_t,
+    nvme_identify_ctrl_t *);
+extern void nvme_print_feat_error(uint64_t, void *, size_t,
+    nvme_identify_ctrl_t *);
+extern void nvme_print_feat_write_cache(uint64_t, void *, size_t,
+    nvme_identify_ctrl_t *);
+extern void nvme_print_feat_nqueues(uint64_t, void *, size_t,
+    nvme_identify_ctrl_t *);
+extern void nvme_print_feat_intr_coal(uint64_t, void *, size_t,
+    nvme_identify_ctrl_t *);
+extern void nvme_print_feat_intr_vect(uint64_t, void *, size_t,
+    nvme_identify_ctrl_t *);
+extern void nvme_print_feat_write_atom(uint64_t, void *, size_t,
+    nvme_identify_ctrl_t *);
+extern void nvme_print_feat_async_event(uint64_t, void *, size_t,
+    nvme_identify_ctrl_t *);
+extern void nvme_print_feat_auto_pst(uint64_t, void *, size_t,
+    nvme_identify_ctrl_t *);
+extern void nvme_print_feat_progress(uint64_t, void *, size_t,
+    nvme_identify_ctrl_t *);
+
+/* device node functions */
+extern int nvme_open(di_minor_t);
+extern void nvme_close(int);
+extern nvme_version_t *nvme_version(int);
+extern nvme_capabilities_t *nvme_capabilities(int);
+extern nvme_identify_ctrl_t *nvme_identify_ctrl(int);
+extern nvme_identify_nsid_t *nvme_identify_nsid(int);
+extern void *nvme_get_logpage(int, uint8_t, size_t *);
+extern boolean_t nvme_get_feature(int, uint8_t, uint32_t, uint64_t *, size_t *,
+    void **);
+extern int nvme_intr_cnt(int);
+extern boolean_t nvme_format_nvm(int, uint8_t, uint8_t);
+extern boolean_t nvme_detach(int);
+extern boolean_t nvme_attach(int);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _NVMEADM_H */
diff --git a/usr/src/cmd/nvmeadm/nvmeadm_dev.c b/usr/src/cmd/nvmeadm/nvmeadm_dev.c
new file mode 100644
index 0000000000..2ac3946a5d
--- /dev/null
+++ b/usr/src/cmd/nvmeadm/nvmeadm_dev.c
@@ -0,0 +1,201 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2016 Nexenta Systems, Inc.
+ */
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stropts.h>
+#include <err.h>
+#include <libdevinfo.h>
+#include <sys/nvme.h>
+#include <assert.h>
+
+#include "nvmeadm.h"
+
+
+static boolean_t
+nvme_ioctl(int fd, int ioc, size_t *bufsize, void **buf, uint64_t arg,
+    uint64_t *res)
+{
+	nvme_ioctl_t nioc = { 0 };
+
+	if (buf != NULL)
+		*buf = NULL;
+
+	if (res != NULL)
+		*res = ~0ULL;
+
+	if (bufsize != NULL && *bufsize != 0) {
+		assert(buf != NULL);
+
+		if ((nioc.n_buf = (uintptr_t)calloc(*bufsize, 1)) == NULL)
+			err(-1, "nvme_ioctl()");
+
+		nioc.n_len = *bufsize;
+	}
+
+	nioc.n_arg = arg;
+
+	if (ioctl(fd, ioc, &nioc) != 0) {
+		if (debug)
+			warn("nvme_ioctl()");
+		if (nioc.n_buf != 0)
+			free((void *)nioc.n_buf);
+
+		return (B_FALSE);
+	}
+
+	if (res != NULL)
+		*res = nioc.n_arg;
+
+	if (bufsize != NULL)
+		*bufsize = nioc.n_len;
+
+	if (buf != NULL)
+		*buf = (void *)nioc.n_buf;
+
+	return (B_TRUE);
+}
+
+nvme_capabilities_t *
+nvme_capabilities(int fd)
+{
+	void *cap = NULL;
+	size_t bufsize = sizeof (nvme_capabilities_t);
+
+	(void) nvme_ioctl(fd, NVME_IOC_CAPABILITIES, &bufsize, &cap, 0, NULL);
+
+	return (cap);
+}
+
+nvme_version_t *
+nvme_version(int fd)
+{
+	void *vs = NULL;
+	size_t bufsize = sizeof (nvme_version_t);
+
+	(void) nvme_ioctl(fd, NVME_IOC_VERSION, &bufsize, &vs, 0, NULL);
+
+	return (vs);
+}
+
+nvme_identify_ctrl_t *
+nvme_identify_ctrl(int fd)
+{
+	void *idctl = NULL;
+	size_t bufsize = NVME_IDENTIFY_BUFSIZE;
+
+	(void) nvme_ioctl(fd, NVME_IOC_IDENTIFY_CTRL, &bufsize, &idctl, 0,
+	    NULL);
+
+	return (idctl);
+}
+
+nvme_identify_nsid_t *
+nvme_identify_nsid(int fd)
+{
+	void *idns = NULL;
+	size_t bufsize = NVME_IDENTIFY_BUFSIZE;
+
+	(void) nvme_ioctl(fd, NVME_IOC_IDENTIFY_NSID, &bufsize, &idns, 0, NULL);
+
+	return (idns);
+}
+
+void *
+nvme_get_logpage(int fd, uint8_t logpage, size_t *bufsize)
+{
+	void *buf = NULL;
+
+	(void) nvme_ioctl(fd, NVME_IOC_GET_LOGPAGE, bufsize, &buf, logpage,
+	    NULL);
+
+	return (buf);
+}
+
+boolean_t
+nvme_get_feature(int fd, uint8_t feature, uint32_t arg, uint64_t *res,
+    size_t *bufsize, void **buf)
+{
+	return (nvme_ioctl(fd, NVME_IOC_GET_FEATURES, bufsize, buf,
+	    (uint64_t)feature << 32 | arg, res));
+}
+
+int
+nvme_intr_cnt(int fd)
+{
+	uint64_t res = 0;
+
+	(void) nvme_ioctl(fd, NVME_IOC_INTR_CNT, NULL, NULL, 0, &res);
+	return ((int)res);
+}
+
+boolean_t
+nvme_format_nvm(int fd, uint8_t lbaf, uint8_t ses)
+{
+	nvme_format_nvm_t frmt = { 0 };
+
+	frmt.b.fm_lbaf = lbaf & 0xf;
+	frmt.b.fm_ses = ses & 0x7;
+
+	return (nvme_ioctl(fd, NVME_IOC_FORMAT, NULL, NULL, frmt.r, NULL));
+}
+
+boolean_t
+nvme_detach(int fd)
+{
+	return (nvme_ioctl(fd, NVME_IOC_DETACH, NULL, NULL, 0, NULL));
+}
+
+boolean_t
+nvme_attach(int fd)
+{
+	return (nvme_ioctl(fd, NVME_IOC_ATTACH, NULL, NULL, 0, NULL));
+}
+
+int
+nvme_open(di_minor_t minor)
+{
+	char *devpath, *path;
+	int fd;
+
+	if ((devpath = di_devfs_minor_path(minor)) == NULL)
+		err(-1, "nvme_open()");
+
+	if (asprintf(&path, "/devices%s", devpath) < 0) {
+		di_devfs_path_free(devpath);
+		err(-1, "nvme_open()");
+	}
+
+	di_devfs_path_free(devpath);
+
+	fd = open(path, O_RDWR);
+	free(path);
+
+	if (fd < 0) {
+		if (debug)
+			warn("nvme_open(%s)", path);
+		return (-1);
+	}
+
+	return (fd);
+}
+
+void
+nvme_close(int fd)
+{
+	(void) close(fd);
+}
diff --git a/usr/src/cmd/nvmeadm/nvmeadm_print.c b/usr/src/cmd/nvmeadm/nvmeadm_print.c
new file mode 100644
index 0000000000..582a849a3e
--- /dev/null
+++ b/usr/src/cmd/nvmeadm/nvmeadm_print.c
@@ -0,0 +1,1138 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2016 Nexenta Systems, Inc.
+ */
+
+/*
+ * functions for printing of NVMe data structures and their members
+ */
+
+#include <sys/byteorder.h>
+#include <sys/types.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <strings.h>
+#include <stdarg.h>
+#include <err.h>
+#include <assert.h>
+
+#include "nvmeadm.h"
+
+static int nvme_strlen(const char *, int);
+
+static void nvme_print_str(int, char *, int, const char *, int);
+static void nvme_print_double(int, char *, double, int, char *);
+static void nvme_print_uint64(int, char *, uint64_t, const char *, char *);
+static void nvme_print_uint128(int, char *, nvme_uint128_t, char *, int, int);
+static void nvme_print_bit(int, char *, int, char *, char *);
+
+#define	ARRAYSIZE(x)		(sizeof (x) / sizeof (*(x)))
+
+static const char *generic_status_codes[] = {
+	"Successful Completion",
+	"Invalid Command Opcode",
+	"Invalid Field in Command",
+	"Command ID Conflict",
+	"Data Transfer Error",
+	"Commands Aborted due to Power Loss Notification",
+	"Internal Error",
+	"Command Abort Requested",
+	"Command Aborted due to SQ Deletion",
+	"Command Aborted due to Failed Fused Command",
+	"Command Aborted due to Missing Fused Command",
+	"Invalid Namespace or Format",
+	"Command Sequence Error",
+	/* NVMe 1.1 */
+	"Invalid SGL Segment Descriptor",
+	"Invalid Number of SGL Descriptors",
+	"Data SGL Length Invalid",
+	"Metadata SGL Length Invalid",
+	"SGL Descriptor Type Invalid",
+	/* NVMe 1.2 */
+	"Invalid Use of Controller Memory Buffer",
+	"PRP Offset Invalid",
+	"Atomic Write Unit Exceeded"
+};
+
+static const char *specific_status_codes[] = {
+	"Completion Queue Invalid",
+	"Invalid Queue Identifier",
+	"Invalid Queue Size",
+	"Abort Command Limit Exceeded",
+	"Reserved",
+	"Asynchronous Event Request Limit Exceeded",
+	"Invalid Firmware Slot",
+	"Invalid Firmware Image",
+	"Invalid Interrupt Vector",
+	"Invalid Log Page",
+	"Invalid Format",
+	"Firmware Activation Requires Conventional Reset",
+	"Invalid Queue Deletion",
+	/* NVMe 1.1 */
+	"Feature Identifier Not Saveable",
+	"Feature Not Changeable",
+	"Feature Not Namespace Specific",
+	"Firmware Activation Requires NVM Subsystem Reset",
+	/* NVMe 1.2 */
+	"Firmware Activation Requires Reset",
+	"Firmware Activation Requires Maximum Time Violation",
+	"Firmware Activation Prohibited",
+	"Overlapping Range",
+	"Namespace Insufficient Capacity",
+	"Namespace Identifier Unavailable",
+	"Reserved",
+	"Namespace Already Attached",
+	"Namespace Is Private",
+	"Namespace Not Attached",
+	"Thin Provisioning Not Supported",
+	"Controller List Invalid"
+};
+
+static const char *generic_nvm_status_codes[] = {
+	"LBA Out Of Range",
+	"Capacity Exceeded",
+	"Namespace Not Ready",
+	/* NVMe 1.1 */
+	"Reservation Conflict",
+	/* NVMe 1.2 */
+	"Format In Progress",
+};
+
+static const char *specific_nvm_status_codes[] = {
+	"Conflicting Attributes",
+	"Invalid Protection Information",
+	"Attempted Write to Read Only Range"
+};
+
+static const char *media_nvm_status_codes[] = {
+	"Write Fault",
+	"Unrecovered Read Error",
+	"End-to-End Guard Check Error",
+	"End-to-End Application Tag Check Error",
+	"End-to-End Reference Tag Check Error",
+	"Compare Failure",
+	"Access Denied",
+	/* NVMe 1.2 */
+	"Deallocated or Unwritten Logical Block"
+};
+
+static const char *status_code_types[] = {
+	"Generic Command Status",
+	"Command Specific Status",
+	"Media Errors",
+	"Reserved",
+	"Reserved",
+	"Reserved",
+	"Reserved",
+	"Vendor Specific"
+};
+
+static const char *lbaf_relative_performance[] = {
+	"Best", "Better", "Good", "Degraded"
+};
+
+static const char *lba_range_types[] = {
+	"Reserved", "Filesystem", "RAID", "Cache", "Page/Swap File"
+};
+
+/*
+ * nvme_print
+ *
+ * This function prints a string indented by the specified number of spaces,
+ * optionally followed by the specified index if it is >= 0. If a format string
+ * is specified, a single colon and the required number of spaces for alignment
+ * are printed before the format string and any remaining arguments are passed
+ * vprintf.
+ *
+ * NVME_PRINT_ALIGN was chosen so that all values will be lined up nicely even
+ * for the longest name at its default indentation.
+ */
+
+#define	NVME_PRINT_ALIGN	43
+
+void
+nvme_print(int indent, char *name, int index, const char *fmt, ...)
+{
+	int align = NVME_PRINT_ALIGN - (indent + strlen(name) + 1);
+	va_list ap;
+
+	if (index >= 0)
+		align -= snprintf(NULL, 0, " %d", index);
+
+	if (align < 0)
+		align = 0;
+
+	va_start(ap, fmt);
+
+	(void) printf("%*s%s", indent, "", name);
+
+	if (index >= 0)
+		(void) printf(" %d", index);
+
+	if (fmt != NULL) {
+		(void) printf(": %*s", align, "");
+		(void) vprintf(fmt, ap);
+	}
+
+	(void) printf("\n");
+	va_end(ap);
+}
+
+/*
+ * nvme_strlen -- return length of string without trailing whitespace
+ */
+static int
+nvme_strlen(const char *str, int len)
+{
+	if (len < 0)
+		return (0);
+
+	while (str[--len] == ' ')
+		;
+
+	return (++len);
+}
+
+/*
+ * nvme_print_str -- print a string up to the specified length
+ */
+static void
+nvme_print_str(int indent, char *name, int index, const char *value, int len)
+{
+	if (len == 0)
+		len = strlen(value);
+
+	nvme_print(indent, name, index, "%.*s", nvme_strlen(value, len), value);
+}
+
+/*
+ * nvme_print_double -- print a double up to a specified number of places with
+ * optional unit
+ */
+static void
+nvme_print_double(int indent, char *name, double value, int places, char *unit)
+{
+	if (unit == NULL)
+		unit = "";
+
+	nvme_print(indent, name, -1, "%.*g%s", places, value, unit);
+}
+
+/*
+ * nvme_print_uint64 -- print uint64_t with optional unit in decimal or another
+ * format specified
+ */
+static void
+nvme_print_uint64(int indent, char *name, uint64_t value, const char *fmt,
+    char *unit)
+{
+	char *tmp_fmt;
+
+	if (unit == NULL)
+		unit = "";
+
+	if (fmt == NULL)
+		fmt = "%"PRId64;
+
+	if (asprintf(&tmp_fmt, "%s%%s", fmt) < 0)
+		err(-1, "nvme_print_uint64()");
+
+	nvme_print(indent, name, -1, tmp_fmt, value, unit);
+
+	free(tmp_fmt);
+}
+
+/*
+ * nvme_print_uint128 -- print a 128bit uint with optional unit, after applying
+ * binary and/or decimal shifting
+ */
+static void
+nvme_print_uint128(int indent, char *name, nvme_uint128_t value, char *unit,
+    int scale_bits, int scale_tens)
+{
+	const char hex[] = "0123456789abcdef";
+	uint8_t o[(128 + scale_bits) / 3];
+	char p[sizeof (o) * 2];
+	char *pp = &p[0];
+	int i, x;
+	uint64_t rem = 0;
+
+	if (unit == NULL)
+		unit = "";
+
+	/*
+	 * Don't allow binary shifting by more than 64 bits to keep the
+	 * arithmetic simple. Also limit decimal shifting based on the size
+	 * of any possible remainder from binary shifting.
+	 */
+	assert(scale_bits <= 64);
+	assert(scale_tens <= (64 - scale_bits) / 3);
+
+	bzero(o, sizeof (o));
+	bzero(p, sizeof (p));
+
+	/*
+	 * Convert the two 64-bit numbers into a series of BCD digits using
+	 * a double-dabble algorithm. By using more or less iterations than
+	 * 128 we can do a binary shift in either direction.
+	 */
+	for (x = 0; x != 128 - scale_bits; x++) {
+		for (i = 0; i != sizeof (o); i++) {
+			if ((o[i] & 0xf0) > 0x40)
+				o[i] += 0x30;
+
+			if ((o[i] & 0xf) > 4)
+				o[i] += 3;
+		}
+
+		for (i = 0; i != sizeof (o) - 1; i++)
+			o[i] = (o[i] << 1) + (o[i+1] >> 7);
+
+		o[i] = (o[i] << 1) + (value.hi >> 63);
+
+		value.hi = (value.hi << 1) + (value.lo >> 63);
+		value.lo = (value.lo << 1);
+	}
+
+	/*
+	 * If we're supposed to do a decimal left shift (* 10^x), too,
+	 * calculate the remainder of the previous binary shift operation.
+	 */
+	if (scale_tens > 0) {
+		rem = value.hi >> (64 - scale_bits);
+
+		for (i = 0; i != scale_tens; i++)
+			rem *= 10;
+
+		rem >>= scale_bits;
+	}
+
+	/*
+	 * Construct the decimal number for printing. Skip leading zeros.
+	 */
+	for (i = 0; i < sizeof (o); i++)
+		if (o[i] != 0)
+			break;
+
+	if (i == sizeof (o)) {
+		/*
+		 * The converted number is 0. Just print the calculated
+		 * remainder and return.
+		 */
+		nvme_print(indent, name, -1, "%"PRId64"%s", rem, unit);
+		return;
+	} else {
+		if (o[i] > 0xf)
+			*pp++ = hex[o[i] >> 4];
+
+		*pp++ = hex[o[i] & 0xf];
+
+		for (i++; i < sizeof (o); i++) {
+			*pp++ = hex[o[i] >> 4];
+			*pp++ = hex[o[i] & 0xf];
+		}
+	}
+
+	/*
+	 * For negative decimal scaling, use the printf precision specifier to
+	 * truncate the results according to the requested decimal scaling. For
+	 * positive decimal scaling we print the remainder padded with 0.
+	 */
+	nvme_print(indent, name, -1, "%.*s%0.*"PRId64"%s",
+	    strlen(p) + scale_tens, p,
+	    scale_tens > 0 ? scale_tens : 0, rem,
+	    unit);
+}
+
+/*
+ * nvme_print_bit -- print a bit with optional names for both states
+ */
+static void
+nvme_print_bit(int indent, char *name, int value, char *s_true, char *s_false)
+{
+	if (s_true == NULL)
+		s_true = "supported";
+	if (s_false == NULL)
+		s_false = "unsupported";
+
+	nvme_print(indent, name, -1, "%s", value ? s_true : s_false);
+}
+
+/*
+ * nvme_print_ctrl_summary -- print a 1-line summary of the IDENTIFY CONTROLLER
+ * data structure
+ */
+void
+nvme_print_ctrl_summary(nvme_identify_ctrl_t *idctl, nvme_version_t *version)
+{
+	(void) printf("model: %.*s, serial: %.*s, FW rev: %.*s, NVMe v%d.%d\n",
+	    nvme_strlen(idctl->id_model, sizeof (idctl->id_model)),
+	    idctl->id_model,
+	    nvme_strlen(idctl->id_serial, sizeof (idctl->id_serial)),
+	    idctl->id_serial,
+	    nvme_strlen(idctl->id_fwrev, sizeof (idctl->id_fwrev)),
+	    idctl->id_fwrev,
+	    version->v_major, version->v_minor);
+}
+
+/*
+ * nvme_print_nsid_summary -- print a 1-line summary of the IDENTIFY NAMESPACE
+ * data structure
+ */
+void
+nvme_print_nsid_summary(nvme_identify_nsid_t *idns)
+{
+	int bsize = 1 << idns->id_lbaf[idns->id_flbas.lba_format].lbaf_lbads;
+
+	(void) printf("Size = %"PRId64" MB, "
+	    "Capacity = %"PRId64" MB, "
+	    "Used = %"PRId64" MB\n",
+	    idns->id_nsize * bsize / 1024 / 1024,
+	    idns->id_ncap * bsize / 1024 / 1024,
+	    idns->id_nuse * bsize / 1024 / 1024);
+
+}
+
+/*
+ * nvme_print_identify_ctrl
+ *
+ * This function pretty-prints the structure returned by the IDENTIFY CONTROLLER
+ * command.
+ */
+void
+nvme_print_identify_ctrl(nvme_identify_ctrl_t *idctl,
+    nvme_capabilities_t *cap, nvme_version_t *version)
+{
+	int i;
+
+	nvme_print(0, "Identify Controller", -1, NULL);
+	nvme_print(2, "Controller Capabilities and Features", -1, NULL);
+	nvme_print_str(4, "Model", -1,
+	    idctl->id_model, sizeof (idctl->id_model));
+	nvme_print_str(4, "Serial", -1,
+	    idctl->id_serial, sizeof (idctl->id_serial));
+	nvme_print_str(4, "Firmware Revision", -1,
+	    idctl->id_fwrev, sizeof (idctl->id_fwrev));
+	if (verbose) {
+		nvme_print_uint64(4, "PCI vendor ID",
+		    idctl->id_vid, "0x%0.4"PRIx64, NULL);
+		nvme_print_uint64(4, "subsystem vendor ID",
+		    idctl->id_ssvid, "0x%0.4"PRIx64, NULL);
+		nvme_print_uint64(4, "Recommended Arbitration Burst",
+		    idctl->id_rab, NULL, NULL);
+		nvme_print(4, "Vendor IEEE OUI", -1, "%0.2X-%0.2X-%0.2X",
+		    idctl->id_oui[0], idctl->id_oui[1], idctl->id_oui[2]);
+	}
+	nvme_print(4, "Multi-Interface Capabilities", -1, NULL);
+	nvme_print_bit(6, "Multiple PCI Express ports",
+	    idctl->id_mic.m_multi_pci, NULL, NULL);
+
+	if (NVME_VERSION_ATLEAST(version, 1, 1)) {
+		nvme_print_bit(6, "Multiple Controllers",
+		    idctl->id_mic.m_multi_ctrl, NULL, NULL);
+		nvme_print_bit(6, "Is SR-IOV virtual function",
+		    idctl->id_mic.m_sr_iov, "yes", "no");
+	}
+	if (idctl->id_mdts > 0)
+		nvme_print_uint64(4, "Maximum Data Transfer Size",
+		    (1 << idctl->id_mdts) * cap->mpsmin / 1024, NULL, "kB");
+	else
+		nvme_print_str(4, "Maximum Data Transfer Size", -1,
+		    "unlimited", 0);
+
+	if (NVME_VERSION_ATLEAST(version, 1, 1)) {
+		nvme_print_uint64(4, "Unique Controller Identifier",
+		    idctl->id_cntlid, "0x%0.4"PRIx64, NULL);
+	}
+
+	nvme_print(2, "Admin Command Set Attributes", -1, NULL);
+	nvme_print(4, "Optional Admin Command Support", -1, NULL);
+	nvme_print_bit(6, "Security Send & Receive",
+	    idctl->id_oacs.oa_security, NULL, NULL);
+	nvme_print_bit(6, "Format NVM",
+	    idctl->id_oacs.oa_format, NULL, NULL);
+	nvme_print_bit(6, "Firmware Activate & Download",
+	    idctl->id_oacs.oa_firmware, NULL, NULL);
+	if (verbose) {
+		nvme_print_uint64(4, "Abort Command Limit",
+		    (uint16_t)idctl->id_acl + 1, NULL, NULL);
+		nvme_print_uint64(4, "Asynchronous Event Request Limit",
+		    (uint16_t)idctl->id_aerl + 1, NULL, NULL);
+	}
+	nvme_print(4, "Firmware Updates", -1, NULL);
+	nvme_print_bit(6, "Firmware Slot 1",
+	    idctl->id_frmw.fw_readonly, "read-only", "writable");
+	nvme_print_uint64(6, "No. of Firmware Slots",
+	    idctl->id_frmw.fw_nslot, NULL, NULL);
+	nvme_print(2, "Log Page Attributes", -1, NULL);
+	nvme_print_bit(6, "per Namespace SMART/Health info",
+	    idctl->id_lpa.lp_smart, NULL, NULL);
+	nvme_print_uint64(4, "Error Log Page Entries",
+	    (uint16_t)idctl->id_elpe + 1, NULL, NULL);
+	nvme_print_uint64(4, "Number of Power States",
+	    (uint16_t)idctl->id_npss + 1, NULL, NULL);
+	if (verbose) {
+		nvme_print_bit(4, "Admin Vendor-specific Command Format",
+		    idctl->id_avscc.av_spec, "standard", "vendor-specific");
+	}
+
+	if (NVME_VERSION_ATLEAST(version, 1, 1)) {
+		nvme_print_bit(4, "Autonomous Power State Transitions",
+		    idctl->id_apsta.ap_sup, NULL, NULL);
+	}
+
+	nvme_print(2, "NVM Command Set Attributes", -1, NULL);
+	if (verbose) {
+		nvme_print(4, "Submission Queue Entry Size", -1,
+		    "min %d, max %d",
+		    1 << idctl->id_sqes.qes_min, 1 << idctl->id_sqes.qes_max);
+		nvme_print(4, "Completion Queue Entry Size", -1,
+		    "min %d, max %d",
+		    1 << idctl->id_cqes.qes_min, 1 << idctl->id_cqes.qes_max);
+	}
+	nvme_print_uint64(4, "Number of Namespaces",
+	    idctl->id_nn, NULL, NULL);
+	nvme_print(4, "Optional NVM Command Support", -1, NULL);
+	nvme_print_bit(6, "Compare",
+	    idctl->id_oncs.on_compare, NULL, NULL);
+	nvme_print_bit(6, "Write Uncorrectable",
+	    idctl->id_oncs.on_wr_unc, NULL, NULL);
+	nvme_print_bit(6, "Dataset Management",
+	    idctl->id_oncs.on_dset_mgmt, NULL, NULL);
+
+	if (NVME_VERSION_ATLEAST(version, 1, 1)) {
+		nvme_print_bit(6, "Write Zeros",
+		    idctl->id_oncs.on_wr_zero, NULL, NULL);
+		nvme_print_bit(6, "Save/Select in Get/Set Features",
+		    idctl->id_oncs.on_save, NULL, NULL);
+		nvme_print_bit(6, "Reservations",
+		    idctl->id_oncs.on_reserve, NULL, NULL);
+	}
+
+	nvme_print(4, "Fused Operation Support", -1, NULL);
+	nvme_print_bit(6, "Compare and Write",
+	    idctl->id_fuses.f_cmp_wr, NULL, NULL);
+	nvme_print(4, "Format NVM Attributes", -1, NULL);
+	nvme_print_bit(6, "per Namespace Format",
+	    idctl->id_fna.fn_format == 0, NULL, NULL);
+	nvme_print_bit(6, "per Namespace Secure Erase",
+	    idctl->id_fna.fn_sec_erase == 0, NULL, NULL);
+	nvme_print_bit(6, "Cryptographic Erase",
+	    idctl->id_fna.fn_crypt_erase, NULL, NULL);
+	nvme_print_bit(4, "Volatile Write Cache",
+	    idctl->id_vwc.vwc_present, "present", "not present");
+	nvme_print_uint64(4, "Atomic Write Unit Normal",
+	    (uint32_t)idctl->id_awun + 1, NULL,
+	    idctl->id_awun == 0 ? " block" : " blocks");
+	nvme_print_uint64(4, "Atomic Write Unit Power Fail",
+	    (uint32_t)idctl->id_awupf + 1, NULL,
+	    idctl->id_awupf == 0 ? " block" : " blocks");
+
+	if (verbose != 0)
+		nvme_print_bit(4, "NVM Vendor-specific Command Format",
+		    idctl->id_nvscc.nv_spec, "standard", "vendor-specific");
+
+	if (NVME_VERSION_ATLEAST(version, 1, 1)) {
+		nvme_print_uint64(4, "Atomic Compare & Write Size",
+		    (uint32_t)idctl->id_acwu + 1, NULL,
+		    idctl->id_acwu == 0 ? " block" : " blocks");
+		nvme_print(4, "SGL Support", -1, NULL);
+		nvme_print_bit(6, "SGLs in NVM commands",
+		    idctl->id_sgls.sgl_sup, NULL, NULL);
+		nvme_print_bit(6, "SGL Bit Bucket Descriptor",
+		    idctl->id_sgls.sgl_bucket, NULL, NULL);
+	}
+
+	for (i = 0; i != idctl->id_npss + 1; i++) {
+		double scale = 0.01;
+		double power = 0;
+		int places = 2;
+		char *unit = "W";
+
+		if (NVME_VERSION_ATLEAST(version, 1, 1) &&
+		    idctl->id_psd[i].psd_mps == 1) {
+			scale = 0.0001;
+			places = 4;
+		}
+
+		power = (double)idctl->id_psd[i].psd_mp * scale;
+		if (power < 1.0) {
+			power *= 1000.0;
+			unit = "mW";
+		}
+
+		nvme_print(4, "Power State Descriptor", i, NULL);
+		nvme_print_double(6, "Maximum Power", power, places, unit);
+		nvme_print_bit(6, "Non-Operational State",
+		    idctl->id_psd[i].psd_nops, "yes", "no");
+		nvme_print_uint64(6, "Entry Latency",
+		    idctl->id_psd[i].psd_enlat, NULL, "us");
+		nvme_print_uint64(6, "Exit Latency",
+		    idctl->id_psd[i].psd_exlat, NULL, "us");
+		nvme_print_uint64(6, "Relative Read Throughput (0 = best)",
+		    idctl->id_psd[i].psd_rrt, NULL, NULL);
+		nvme_print_uint64(6, "Relative Read Latency (0 = best)",
+		    idctl->id_psd[i].psd_rrl, NULL, NULL);
+		nvme_print_uint64(6, "Relative Write Throughput (0 = best)",
+		    idctl->id_psd[i].psd_rwt, NULL, NULL);
+		nvme_print_uint64(6, "Relative Write Latency (0 = best)",
+		    idctl->id_psd[i].psd_rwl, NULL, NULL);
+	}
+}
+
+/*
+ * nvme_print_identify_nsid
+ *
+ * This function pretty-prints the structure returned by the IDENTIFY NAMESPACE
+ * command.
+ */
+void
+nvme_print_identify_nsid(nvme_identify_nsid_t *idns, nvme_version_t *version)
+{
+	int bsize = 1 << idns->id_lbaf[idns->id_flbas.lba_format].lbaf_lbads;
+	int i;
+
+	nvme_print(0, "Identify Namespace", -1, NULL);
+	nvme_print(2, "Namespace Capabilities and Features", -1, NULL);
+	nvme_print_uint64(4, "Namespace Size",
+	    idns->id_nsize * bsize / 1024 / 1024, NULL, "MB");
+	nvme_print_uint64(4, "Namespace Capacity",
+	    idns->id_ncap * bsize / 1024 / 1024, NULL, "MB");
+	nvme_print_uint64(4, "Namespace Utilization",
+	    idns->id_nuse * bsize / 1024 / 1024, NULL, "MB");
+	nvme_print(4, "Namespace Features", -1, NULL);
+	nvme_print_bit(6, "Thin Provisioning",
+	    idns->id_nsfeat.f_thin, NULL, NULL);
+	nvme_print_uint64(4, "Number of LBA Formats",
+	    (uint16_t)idns->id_nlbaf + 1, NULL, NULL);
+	nvme_print(4, "Formatted LBA Size", -1, NULL);
+	nvme_print_uint64(6, "LBA Format",
+	    (uint16_t)idns->id_flbas.lba_format, NULL, NULL);
+	nvme_print_bit(6, "Extended Data LBA",
+	    idns->id_flbas.lba_extlba, "yes", "no");
+	nvme_print(4, "Metadata Capabilities", -1, NULL);
+	nvme_print_bit(6, "Extended Data LBA",
+	    idns->id_mc.mc_extlba, NULL, NULL);
+	nvme_print_bit(6, "Separate Metadata",
+	    idns->id_mc.mc_separate, NULL, NULL);
+	nvme_print(4, "End-to-End Data Protection Capabilities", -1, NULL);
+	nvme_print_bit(6, "Protection Information Type 1",
+	    idns->id_dpc.dp_type1, NULL, NULL);
+	nvme_print_bit(6, "Protection Information Type 2",
+	    idns->id_dpc.dp_type2, NULL, NULL);
+	nvme_print_bit(6, "Protection Information Type 3",
+	    idns->id_dpc.dp_type3, NULL, NULL);
+	nvme_print_bit(6, "Protection Information first",
+	    idns->id_dpc.dp_first, NULL, NULL);
+	nvme_print_bit(6, "Protection Information last",
+	    idns->id_dpc.dp_last, NULL, NULL);
+	nvme_print(4, "End-to-End Data Protection Settings", -1, NULL);
+	if (idns->id_dps.dp_pinfo == 0)
+		nvme_print_str(6, "Protection Information", -1,
+		    "disabled", 0);
+	else
+		nvme_print_uint64(6, "Protection Information Type",
+		    idns->id_dps.dp_pinfo, NULL, NULL);
+	nvme_print_bit(6, "Protection Information in Metadata",
+	    idns->id_dps.dp_first, "first 8 bytes", "last 8 bytes");
+
+	if (NVME_VERSION_ATLEAST(version, 1, 1)) {
+		nvme_print(4, "Namespace Multi-Path I/O and Namespace Sharing "
+		    "Capabilities", -1, NULL);
+		nvme_print_bit(6, "Namespace is shared",
+		    idns->id_nmic.nm_shared, "yes", "no");
+		nvme_print(2, "Reservation Capabilities", -1, NULL);
+		nvme_print_bit(6, "Persist Through Power Loss",
+		    idns->id_rescap.rc_persist, NULL, NULL);
+		nvme_print_bit(6, "Write Exclusive",
+		    idns->id_rescap.rc_wr_excl, NULL, NULL);
+		nvme_print_bit(6, "Exclusive Access",
+		    idns->id_rescap.rc_excl, NULL, NULL);
+		nvme_print_bit(6, "Write Exclusive - Registrants Only",
+		    idns->id_rescap.rc_wr_excl_r, NULL, NULL);
+		nvme_print_bit(6, "Exclusive Access - Registrants Only",
+		    idns->id_rescap.rc_excl_r, NULL, NULL);
+		nvme_print_bit(6, "Write Exclusive - All Registrants",
+		    idns->id_rescap.rc_wr_excl_a, NULL, NULL);
+		nvme_print_bit(6, "Exclusive Access - All Registrants",
+		    idns->id_rescap.rc_excl_a, NULL, NULL);
+
+		nvme_print(4, "IEEE Extended Unique Identifier", -1,
+		    "%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X",
+		    idns->id_eui64[0], idns->id_eui64[1],
+		    idns->id_eui64[2], idns->id_eui64[3],
+		    idns->id_eui64[4], idns->id_eui64[5],
+		    idns->id_eui64[6], idns->id_eui64[7]);
+	}
+
+	for (i = 0; i <= idns->id_nlbaf; i++) {
+		if (verbose == 0 && idns->id_lbaf[i].lbaf_ms != 0)
+			continue;
+
+		nvme_print(4, "LBA Format", i, NULL);
+		nvme_print_uint64(6, "Metadata Size",
+		    idns->id_lbaf[i].lbaf_ms, NULL, " bytes");
+		nvme_print_uint64(6, "LBA Data Size",
+		    1 << idns->id_lbaf[i].lbaf_lbads, NULL, " bytes");
+		nvme_print_str(6, "Relative Performance", -1,
+		    lbaf_relative_performance[idns->id_lbaf[i].lbaf_rp], 0);
+	}
+}
+
+/*
+ * nvme_print_error_log
+ *
+ * This function pretty-prints all non-zero error log entries, or all entries
+ * if verbose is set.
+ */
+void
+nvme_print_error_log(int nlog, nvme_error_log_entry_t *elog)
+{
+	int i;
+
+	nvme_print(0, "Error Log", -1, NULL);
+	for (i = 0; i != nlog; i++)
+		if (elog[i].el_count == 0)
+			break;
+	nvme_print_uint64(2, "Number of Error Log Entries", i, NULL, NULL);
+
+	for (i = 0; i != nlog; i++) {
+		int sc = elog[i].el_sf.sf_sc;
+		const char *sc_str = "";
+
+		if (elog[i].el_count == 0 && verbose == 0)
+			break;
+
+		switch (elog[i].el_sf.sf_sct) {
+		case 0: /* Generic Command Status */
+			if (sc < ARRAYSIZE(generic_status_codes))
+				sc_str = generic_status_codes[sc];
+			else if (sc >= 0x80 &&
+			    sc - 0x80 < ARRAYSIZE(generic_nvm_status_codes))
+				sc_str = generic_nvm_status_codes[sc - 0x80];
+			break;
+		case 1: /* Specific Command Status */
+			if (sc < ARRAYSIZE(specific_status_codes))
+				sc_str = specific_status_codes[sc];
+			else if (sc >= 0x80 &&
+			    sc - 0x80 < ARRAYSIZE(specific_nvm_status_codes))
+				sc_str = specific_nvm_status_codes[sc - 0x80];
+			break;
+		case 2: /* Media Errors */
+			if (sc >= 0x80 &&
+			    sc - 0x80 < ARRAYSIZE(media_nvm_status_codes))
+				sc_str = media_nvm_status_codes[sc - 0x80];
+			break;
+		case 7: /* Vendor Specific */
+			sc_str = "Unknown Vendor Specific";
+			break;
+		default:
+			sc_str = "Reserved";
+			break;
+		}
+
+		nvme_print(2, "Entry", i, NULL);
+		nvme_print_uint64(4, "Error Count",
+		    elog[i].el_count, NULL, NULL);
+		nvme_print_uint64(4, "Submission Queue ID",
+		    elog[i].el_sqid, NULL, NULL);
+		nvme_print_uint64(4, "Command ID",
+		    elog[i].el_cid, NULL, NULL);
+		nvme_print(4, "Status Field", -1, NULL);
+		nvme_print_uint64(6, "Phase Tag",
+		    elog[i].el_sf.sf_p, NULL, NULL);
+		nvme_print(6, "Status Code", -1, "0x%0.2x (%s)",
+		    sc, sc_str);
+		nvme_print(6, "Status Code Type", -1, "0x%x (%s)",
+		    elog[i].el_sf.sf_sct,
+		    status_code_types[elog[i].el_sf.sf_sct]);
+		nvme_print_bit(6, "More",
+		    elog[i].el_sf.sf_m, "yes", "no");
+		nvme_print_bit(6, "Do Not Retry",
+		    elog[i].el_sf.sf_m, "yes", "no");
+		nvme_print_uint64(4, "Parameter Error Location byte",
+		    elog[i].el_byte, "0x%0.2"PRIx64, NULL);
+		nvme_print_uint64(4, "Parameter Error Location bit",
+		    elog[i].el_bit, NULL, NULL);
+		nvme_print_uint64(4, "Logical Block Address",
+		    elog[i].el_lba, NULL, NULL);
+		nvme_print(4, "Namespace ID", -1, "%d",
+		    elog[i].el_nsid == 0xffffffff ?
+		    0 : elog[i].el_nsid);
+		nvme_print_uint64(4,
+		    "Vendor Specifc Information Available",
+		    elog[i].el_vendor, NULL, NULL);
+	}
+}
+
+/*
+ * nvme_print_health_log
+ *
+ * This function pretty-prints a summary of the SMART/Health log, or all
+ * of the log if verbose is set.
+ */
+void
+nvme_print_health_log(nvme_health_log_t *hlog, nvme_identify_ctrl_t *idctl)
+{
+	nvme_print(0, "SMART/Health Information", -1, NULL);
+	nvme_print(2, "Critical Warnings", -1, NULL);
+	nvme_print_bit(4, "Available Space",
+	    hlog->hl_crit_warn.cw_avail, "low", "OK");
+	nvme_print_bit(4, "Temperature",
+	    hlog->hl_crit_warn.cw_temp, "too high", "OK");
+	nvme_print_bit(4, "Device Reliability",
+	    hlog->hl_crit_warn.cw_reliab, "degraded", "OK");
+	nvme_print_bit(4, "Media",
+	    hlog->hl_crit_warn.cw_readonly, "read-only", "OK");
+	if (idctl->id_vwc.vwc_present != 0)
+		nvme_print_bit(4, "Volatile Memory Backup",
+		    hlog->hl_crit_warn.cw_volatile, "failed", "OK");
+
+	nvme_print_uint64(2, "Temperature",
+	    hlog->hl_temp - 273, NULL, "C");
+	nvme_print_uint64(2, "Available Spare Capacity",
+	    hlog->hl_avail_spare, NULL, "%");
+
+	if (verbose != 0)
+		nvme_print_uint64(2, "Available Spare Threshold",
+		    hlog->hl_avail_spare_thr, NULL, "%");
+
+	nvme_print_uint64(2, "Device Life Used",
+	    hlog->hl_used, NULL, "%");
+
+	if (verbose == 0)
+		return;
+
+	/*
+	 * The following two fields are in 1000 512 byte units. Convert that to
+	 * GB by doing binary shifts (9 left and 30 right) and muliply by 10^3.
+	 */
+	nvme_print_uint128(2, "Data Read",
+	    hlog->hl_data_read, "GB", 30 - 9, 3);
+	nvme_print_uint128(2, "Data Written",
+	    hlog->hl_data_write, "GB", 30 - 9, 3);
+
+	nvme_print_uint128(2, "Read Commands",
+	    hlog->hl_host_read, NULL, 0, 0);
+	nvme_print_uint128(2, "Write Commands",
+	    hlog->hl_host_write, NULL, 0, 0);
+	nvme_print_uint128(2, "Controller Busy",
+	    hlog->hl_ctrl_busy, "min", 0, 0);
+	nvme_print_uint128(2, "Power Cycles",
+	    hlog->hl_power_cycles, NULL, 0, 0);
+	nvme_print_uint128(2, "Power On",
+	    hlog->hl_power_on_hours, "h", 0, 0);
+	nvme_print_uint128(2, "Unsafe Shutdowns",
+	    hlog->hl_unsafe_shutdn, NULL, 0, 0);
+	nvme_print_uint128(2, "Uncorrectable Media Errors",
+	    hlog->hl_media_errors, NULL, 0, 0);
+	nvme_print_uint128(2, "Errors Logged",
+	    hlog->hl_errors_logged, NULL, 0, 0);
+}
+
+/*
+ * nvme_print_fwslot_log
+ *
+ * This function pretty-prints the firmware slot information.
+ */
+void
+nvme_print_fwslot_log(nvme_fwslot_log_t *fwlog)
+{
+	int i;
+
+	nvme_print(0, "Firmware Slot Information", -1, NULL);
+	nvme_print_uint64(2, "Active Firmware Slot", fwlog->fw_afi, NULL, NULL);
+
+	for (i = 0; i != ARRAYSIZE(fwlog->fw_frs); i++) {
+		if (fwlog->fw_frs[i][0] == '\0')
+			break;
+		nvme_print_str(2, "Firmware Revision for Slot", i + 1,
+		    fwlog->fw_frs[i], sizeof (fwlog->fw_frs[i]));
+	}
+}
+
+/*
+ * nvme_print_feat_*
+ *
+ * These functions pretty-print the data structures returned by GET FEATURES.
+ */
+void
+nvme_print_feat_arbitration(uint64_t res, void *b, size_t s,
+    nvme_identify_ctrl_t *id)
+{
+	_NOTE(ARGUNUSED(b));
+	_NOTE(ARGUNUSED(s));
+	_NOTE(ARGUNUSED(id));
+	nvme_arbitration_t arb;
+
+	arb.r = (uint32_t)res;
+	if (arb.b.arb_ab != 7)
+		nvme_print_uint64(4, "Arbitration Burst",
+		    1 << arb.b.arb_ab, NULL, NULL);
+	else
+		nvme_print_str(4, "Arbitration Burst", 0,
+		    "no limit", 0);
+	nvme_print_uint64(4, "Low Priority Weight",
+	    (uint16_t)arb.b.arb_lpw + 1, NULL, NULL);
+	nvme_print_uint64(4, "Medium Priority Weight",
+	    (uint16_t)arb.b.arb_mpw + 1, NULL, NULL);
+	nvme_print_uint64(4, "High Priority Weight",
+	    (uint16_t)arb.b.arb_hpw + 1, NULL, NULL);
+}
+
+void
+nvme_print_feat_power_mgmt(uint64_t res, void *b, size_t s,
+    nvme_identify_ctrl_t *id)
+{
+	_NOTE(ARGUNUSED(b));
+	_NOTE(ARGUNUSED(s));
+	_NOTE(ARGUNUSED(id));
+	nvme_power_mgmt_t pm;
+
+	pm.r = (uint32_t)res;
+	nvme_print_uint64(4, "Power State", (uint8_t)pm.b.pm_ps,
+	    NULL, NULL);
+}
+
+void
+nvme_print_feat_lba_range(uint64_t res, void *buf, size_t bufsize,
+    nvme_identify_ctrl_t *id)
+{
+	_NOTE(ARGUNUSED(id));
+
+	nvme_lba_range_type_t lrt;
+	nvme_lba_range_t *lr;
+	size_t n_lr;
+	int i;
+
+	if (buf == NULL)
+		return;
+
+	lrt.r = res;
+	lr = buf;
+
+	n_lr = bufsize / sizeof (nvme_lba_range_t);
+	if (n_lr > lrt.b.lr_num + 1)
+		n_lr = lrt.b.lr_num + 1;
+
+	nvme_print_uint64(4, "Number of LBA Ranges",
+	    (uint8_t)lrt.b.lr_num + 1, NULL, NULL);
+
+	for (i = 0; i != n_lr; i++) {
+		if (verbose == 0 && lr[i].lr_nlb == 0)
+			continue;
+
+		nvme_print(4, "LBA Range", i, NULL);
+		if (lr[i].lr_type < ARRAYSIZE(lba_range_types))
+			nvme_print_str(6, "Type", -1,
+			    lba_range_types[lr[i].lr_type], 0);
+		else
+			nvme_print_uint64(6, "Type",
+			    lr[i].lr_type, NULL, NULL);
+		nvme_print(6, "Attributes", -1, NULL);
+		nvme_print_bit(8, "Writable",
+		    lr[i].lr_attr.lr_write, "yes", "no");
+		nvme_print_bit(8, "Hidden",
+		    lr[i].lr_attr.lr_hidden, "yes", "no");
+		nvme_print_uint64(6, "Starting LBA",
+		    lr[i].lr_slba, NULL, NULL);
+		nvme_print_uint64(6, "Number of Logical Blocks",
+		    lr[i].lr_nlb, NULL, NULL);
+		nvme_print(6, "Unique Identifier", -1,
+		    "%.2x%.2x%.2x%.2x%.2x%.2x%.2x%.2x"
+		    "%.2x%.2x%.2x%.2x%.2x%.2x%.2x%.2x",
+		    lr[i].lr_guid[0], lr[i].lr_guid[1],
+		    lr[i].lr_guid[2], lr[i].lr_guid[3],
+		    lr[i].lr_guid[4], lr[i].lr_guid[5],
+		    lr[i].lr_guid[6], lr[i].lr_guid[7],
+		    lr[i].lr_guid[8], lr[i].lr_guid[9],
+		    lr[i].lr_guid[10], lr[i].lr_guid[11],
+		    lr[i].lr_guid[12], lr[i].lr_guid[13],
+		    lr[i].lr_guid[14], lr[i].lr_guid[15]);
+	}
+}
+
+void
+nvme_print_feat_temperature(uint64_t res, void *b, size_t s,
+    nvme_identify_ctrl_t *id)
+{
+	_NOTE(ARGUNUSED(b));
+	_NOTE(ARGUNUSED(s));
+	_NOTE(ARGUNUSED(id));
+	nvme_temp_threshold_t tt;
+
+	tt.r = (uint32_t)res;
+	nvme_print_uint64(4, "Temperature Threshold", tt.b.tt_tmpth - 273,
+	    NULL, "C");
+}
+
+void
+nvme_print_feat_error(uint64_t res, void *b, size_t s,
+    nvme_identify_ctrl_t *id)
+{
+	_NOTE(ARGUNUSED(b));
+	_NOTE(ARGUNUSED(s));
+	_NOTE(ARGUNUSED(id));
+	nvme_error_recovery_t er;
+
+	er.r = (uint32_t)res;
+	if (er.b.er_tler > 0)
+		nvme_print_uint64(4, "Time Limited Error Recovery",
+		    (uint32_t)er.b.er_tler * 100, NULL, "ms");
+	else
+		nvme_print_str(4, "Time Limited Error Recovery", -1,
+		    "no time limit", 0);
+}
+
+void
+nvme_print_feat_write_cache(uint64_t res, void *b, size_t s,
+    nvme_identify_ctrl_t *id)
+{
+	_NOTE(ARGUNUSED(b));
+	_NOTE(ARGUNUSED(s));
+	_NOTE(ARGUNUSED(id));
+	nvme_write_cache_t wc;
+
+	wc.r = (uint32_t)res;
+	nvme_print_bit(4, "Volatile Write Cache",
+	    wc.b.wc_wce, "enabled", "disabled");
+}
+
+void
+nvme_print_feat_nqueues(uint64_t res, void *b, size_t s,
+    nvme_identify_ctrl_t *id)
+{
+	_NOTE(ARGUNUSED(b));
+	_NOTE(ARGUNUSED(s));
+	_NOTE(ARGUNUSED(id));
+	nvme_nqueues_t nq;
+
+	nq.r = (uint32_t)res;
+	nvme_print_uint64(4, "Number of Submission Queues",
+	    nq.b.nq_nsq + 1, NULL, NULL);
+	nvme_print_uint64(4, "Number of Completion Queues",
+	    nq.b.nq_ncq + 1, NULL, NULL);
+}
+
+void
+nvme_print_feat_intr_coal(uint64_t res, void *b, size_t s,
+    nvme_identify_ctrl_t *id)
+{
+	_NOTE(ARGUNUSED(b));
+	_NOTE(ARGUNUSED(s));
+	_NOTE(ARGUNUSED(id));
+	nvme_intr_coal_t ic;
+
+	ic.r = (uint32_t)res;
+	nvme_print_uint64(4, "Aggregation Threshold",
+	    ic.b.ic_thr + 1, NULL, NULL);
+	nvme_print_uint64(4, "Aggregation Time",
+	    (uint16_t)ic.b.ic_time * 100, NULL, "us");
+}
+void
+nvme_print_feat_intr_vect(uint64_t res, void *b, size_t s,
+    nvme_identify_ctrl_t *id)
+{
+	_NOTE(ARGUNUSED(b));
+	_NOTE(ARGUNUSED(s));
+	_NOTE(ARGUNUSED(id));
+	nvme_intr_vect_t iv;
+	char *tmp;
+
+	iv.r = (uint32_t)res;
+	if (asprintf(&tmp, "Vector %d Coalescing Disable", iv.b.iv_iv) < 0)
+		err(-1, "nvme_print_feat_common()");
+
+	nvme_print_bit(4, tmp, iv.b.iv_cd, "yes", "no");
+}
+
+void
+nvme_print_feat_write_atom(uint64_t res, void *b, size_t s,
+    nvme_identify_ctrl_t *id)
+{
+	_NOTE(ARGUNUSED(b));
+	_NOTE(ARGUNUSED(s));
+	_NOTE(ARGUNUSED(id));
+	nvme_write_atomicity_t wa;
+
+	wa.r = (uint32_t)res;
+	nvme_print_bit(4, "Disable Normal", wa.b.wa_dn, "yes", "no");
+}
+
+void
+nvme_print_feat_async_event(uint64_t res, void *b, size_t s,
+    nvme_identify_ctrl_t *idctl)
+{
+	_NOTE(ARGUNUSED(b));
+	_NOTE(ARGUNUSED(s));
+	nvme_async_event_conf_t aec;
+
+	aec.r = (uint32_t)res;
+	nvme_print_bit(4, "Available Space below threshold",
+	    aec.b.aec_avail, "enabled", "disabled");
+	nvme_print_bit(4, "Temperature above threshold",
+	    aec.b.aec_temp, "enabled", "disabled");
+	nvme_print_bit(4, "Device Reliability compromised",
+	    aec.b.aec_reliab, "enabled", "disabled");
+	nvme_print_bit(4, "Media read-only",
+	    aec.b.aec_readonly, "enabled", "disabled");
+	if (idctl->id_vwc.vwc_present != 0)
+		nvme_print_bit(4, "Volatile Memory Backup failed",
+		    aec.b.aec_volatile, "enabled", "disabled");
+}
+
+void
+nvme_print_feat_auto_pst(uint64_t res, void *buf, size_t bufsize,
+    nvme_identify_ctrl_t *id)
+{
+	_NOTE(ARGUNUSED(id));
+
+	nvme_auto_power_state_trans_t apst;
+	nvme_auto_power_state_t *aps;
+	int i;
+	int cnt = bufsize / sizeof (nvme_auto_power_state_t);
+
+	if (buf == NULL)
+		return;
+
+	apst.r = res;
+	aps = buf;
+
+	nvme_print_bit(4, "Autonomous Power State Transition",
+	    apst.b.apst_apste, "enabled", "disabled");
+	for (i = 0; i != cnt; i++) {
+		if (aps[i].apst_itps == 0 && aps[i].apst_itpt == 0)
+			break;
+
+		nvme_print(4, "Power State", i, NULL);
+		nvme_print_uint64(6, "Idle Transition Power State",
+		    (uint16_t)aps[i].apst_itps, NULL, NULL);
+		nvme_print_uint64(6, "Idle Time Prior to Transition",
+		    aps[i].apst_itpt, NULL, "ms");
+	}
+}
+
+void
+nvme_print_feat_progress(uint64_t res, void *b, size_t s,
+    nvme_identify_ctrl_t *id)
+{
+	_NOTE(ARGUNUSED(b));
+	_NOTE(ARGUNUSED(s));
+	_NOTE(ARGUNUSED(id));
+	nvme_software_progress_marker_t spm;
+
+	spm.r = (uint32_t)res;
+	nvme_print_uint64(4, "Pre-Boot Software Load Count",
+	    spm.b.spm_pbslc, NULL, NULL);
+}