11958 need topo maps for the SMCI,SYS-2028U-E1CNRT+

11959 extend disk topo plugin to enumerate nvme devices
Reviewed by: Robert Mustacchi <rm@fingolfin.org>
Approved by: Dan McDonald <danmcd@joyent.com>

19 files changed, 1176 insertions, 86 deletions

diff --git a/usr/src/lib/fm/topo/libtopo/common/hc.c b/usr/src/lib/fm/topo/libtopo/common/hc.c
index 3101d12d56..cb53f87a51 100644
--- a/usr/src/lib/fm/topo/libtopo/common/hc.c
+++ b/usr/src/lib/fm/topo/libtopo/common/hc.c
@@ -22,7 +22,7 @@
 
 /*
  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2019 Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  */
 
 #include <stdio.h>
@@ -179,6 +179,7 @@ static const hcc_t hc_canon[] = {
 	{ MOTHERBOARD, TOPO_STABILITY_PRIVATE },
 	{ NIU, TOPO_STABILITY_PRIVATE },
 	{ NIUFN, TOPO_STABILITY_PRIVATE },
+	{ NVME, TOPO_STABILITY_PRIVATE },
 	{ PCI_BUS, TOPO_STABILITY_PRIVATE },
 	{ PCI_DEVICE, TOPO_STABILITY_PRIVATE },
 	{ PCI_FUNCTION, TOPO_STABILITY_PRIVATE },
diff --git a/usr/src/lib/fm/topo/libtopo/common/libtopo.h b/usr/src/lib/fm/topo/libtopo/common/libtopo.h
index 62ad109808..06f473a3b1 100644
--- a/usr/src/lib/fm/topo/libtopo/common/libtopo.h
+++ b/usr/src/lib/fm/topo/libtopo/common/libtopo.h
@@ -23,7 +23,7 @@
  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 /*
- * Copyright 2019 Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  */
 
 #ifndef _LIBTOPO_H
@@ -1018,7 +1018,8 @@ typedef enum topo_led_type {
 
 typedef enum topo_slot_type {
 	TOPO_SLOT_TYPE_DIMM = 1,
-	TOPO_SLOT_TYPE_UFM
+	TOPO_SLOT_TYPE_UFM,
+	TOPO_SLOT_TYPE_M2
 } topo_slot_type_t;
 
 /*
diff --git a/usr/src/lib/fm/topo/libtopo/common/topo_hc.h b/usr/src/lib/fm/topo/libtopo/common/topo_hc.h
index 88f955bd68..593f8946fb 100644
--- a/usr/src/lib/fm/topo/libtopo/common/topo_hc.h
+++ b/usr/src/lib/fm/topo/libtopo/common/topo_hc.h
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2019 Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  */
 
 #ifndef _TOPO_HC_H
@@ -68,6 +68,7 @@ extern "C" {
 #define	MOTHERBOARD	"motherboard"
 #define	NIU		"niu"
 #define	NIUFN		"niufn"
+#define	NVME		"nvme"
 #define	PCI_BUS		"pcibus"
 #define	PCI_DEVICE	"pcidev"
 #define	PCI_FUNCTION    "pcifn"
@@ -138,6 +139,7 @@ extern "C" {
 #define	TOPO_BINDING_ENCLOSURE	"enclosure"
 #define	TOPO_BINDING_SLOT	"slot"
 #define	TOPO_BINDING_PORT	"port"
+#define	TOPO_BINDING_PARENT_DEV	"parent-device"
 
 #define	TOPO_PGROUP_STORAGE	"storage"
 #define	TOPO_STORAGE_INITIATOR_PORT	"initiator-port"
@@ -283,6 +285,9 @@ extern "C" {
 #define	TOPO_PGROUP_DATALINK_LINK_DUPLEX_UNKNOWN	"unknown"
 #define	TOPO_PGROUP_DATALINK_LINK_NAME		"link-name"
 
+#define	TOPO_PGROUP_NVME		"nvme-properties"
+#define	TOPO_PROP_NVME_VER		"nvme-version"
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/usr/src/lib/fm/topo/libtopo/common/topo_xml.c b/usr/src/lib/fm/topo/libtopo/common/topo_xml.c
index 58743077bd..648d5f3467 100644
--- a/usr/src/lib/fm/topo/libtopo/common/topo_xml.c
+++ b/usr/src/lib/fm/topo/libtopo/common/topo_xml.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2019 Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  */
 
 #include <libxml/parser.h>
@@ -1346,14 +1346,19 @@ pad_process(topo_mod_t *mp, tf_rdata_t *rd, xmlNodePtr pxn, tnode_t *ptn,
 		*rpad = new;
 	}
 
-	if (new->tpad_dcnt > 0)
-		if (dependents_create(mp, rd->rd_finfo, new, pxn, ptn) < 0)
-			return (-1);
-
+	/*
+	 * We need to process the property groups before enumerating any
+	 * dependents as that enuemration can itself have dependencies on
+	 * properties set on the parent node.
+	 */
 	if (new->tpad_pgcnt > 0)
 		if (pgroups_create(mp, new, ptn) < 0)
 			return (-1);
 
+	if (new->tpad_dcnt > 0)
+		if (dependents_create(mp, rd->rd_finfo, new, pxn, ptn) < 0)
+			return (-1);
+
 	return (0);
 }
 
diff --git a/usr/src/lib/fm/topo/maps/Makefile b/usr/src/lib/fm/topo/maps/Makefile
index 04fc0d4b7d..db3ec89603 100644
--- a/usr/src/lib/fm/topo/maps/Makefile
+++ b/usr/src/lib/fm/topo/maps/Makefile
@@ -22,7 +22,7 @@
 #
 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
-# Copyright (c) 2018, Joyent, Inc.
+# Copyright 2020 Joyent, Inc.
 # Copyright 2019 Peter Tribble.
 #
 
@@ -54,6 +54,7 @@ i386_SUBDIRS = i86pc \
 		Joyent,Joyent-Compute-Platform-330x \
 		Joyent,Joyent-Storage-Platform-7001 \
 		SMCI,SSG-2028R-ACR24L \
+		SMCI,SYS-2028U-E1CNRT+ \
 		SMCI,SSG-6049P-E1CR36L \
 		SMCI,SSG-2029P-ACR24L
 
diff --git a/usr/src/lib/fm/topo/maps/SMCI,SYS-2028U-E1CNRT+/Makefile b/usr/src/lib/fm/topo/maps/SMCI,SYS-2028U-E1CNRT+/Makefile
new file mode 100644
index 0000000000..c63c1bc113
--- /dev/null
+++ b/usr/src/lib/fm/topo/maps/SMCI,SYS-2028U-E1CNRT+/Makefile
@@ -0,0 +1,29 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright 2020 Joyent, Inc.
+#
+ARCH = i86pc
+CLASS = arch
+DTDFILE = topology.dtd.1
+
+TOPOFILE = \
+	SYS-2028U-E1CNRT+-hc-topology.xml \
+	SYS-2028U-E1CNRT+-chassis-hc-topology.xml \
+	SYS-2028U-E1CNRT+-usb.usbtopo
+
+SRCDIR = ../SMCI,SYS-2028U-E1CNRT+
+
+PLATFORM = SYS-2028U-E1CNRT+
+
+include ../Makefile.map
+
diff --git a/usr/src/lib/fm/topo/maps/SMCI,SYS-2028U-E1CNRT+/SYS-2028U-E1CNRT+-chassis-hc-topology.xml b/usr/src/lib/fm/topo/maps/SMCI,SYS-2028U-E1CNRT+/SYS-2028U-E1CNRT+-chassis-hc-topology.xml
new file mode 100644
index 0000000000..a1261c28bb
--- /dev/null
+++ b/usr/src/lib/fm/topo/maps/SMCI,SYS-2028U-E1CNRT+/SYS-2028U-E1CNRT+-chassis-hc-topology.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0"?>
+<!DOCTYPE topology SYSTEM "/usr/share/lib/xml/dtd/topology.dtd.1">
+<!--
+ This file and its contents are supplied under the terms of the
+ Common Development and Distribution License ("CDDL"), version 1.0.
+ You may only use this file in accordance with the terms of version
+ 1.0 of the CDDL.
+
+ A full copy of the text of the CDDL should have accompanied this
+ source.  A copy of the CDDL is also available via the Internet at
+ http://www.illumos.org/license/CDDL.
+
+ Copyright 2020 Joyent, Inc.
+-->
+
+<topology name='chassis' scheme='hc'>
+  <range name='chassis' min='0' max='0'>
+    <node instance='0'>
+      <fac-enum provider='fac_prov_ipmi' />
+      <!--
+        chassis locate LED
+      -->
+      <facility name='locate' type='indicator' provider='fac_prov_ipmi' >
+          <propgroup name='facility' version='1' name-stability='Private'
+             data-stability='Private' >
+                <propval name='type' type='uint32' value='1' />
+                <propmethod name='chassis_ident_mode' version='0'
+                    propname='mode' proptype='uint32' mutable='1' />
+          </propgroup>
+      </facility>
+      <propgroup name='ipmi' version='1'
+          name-stability='Private' data-stability='Private' >
+         <propval name='entity-list' type='string_array' >
+           <propitem value='Chassis Intru' />
+         </propval>
+      </propgroup>
+    </node>
+  </range>
+</topology>
diff --git a/usr/src/lib/fm/topo/maps/SMCI,SYS-2028U-E1CNRT+/SYS-2028U-E1CNRT+-hc-topology.xml b/usr/src/lib/fm/topo/maps/SMCI,SYS-2028U-E1CNRT+/SYS-2028U-E1CNRT+-hc-topology.xml
new file mode 100644
index 0000000000..2431c371ca
--- /dev/null
+++ b/usr/src/lib/fm/topo/maps/SMCI,SYS-2028U-E1CNRT+/SYS-2028U-E1CNRT+-hc-topology.xml
@@ -0,0 +1,161 @@
+<?xml version="1.0"?>
+<!DOCTYPE topology SYSTEM "/usr/share/lib/xml/dtd/topology.dtd.1">
+<!--
+
+ This file and its contents are supplied under the terms of the
+ Common Development and Distribution License ("CDDL"), version 1.0.
+ You may only use this file in accordance with the terms of version
+ 1.0 of the CDDL.
+
+ A full copy of the text of the CDDL should have accompanied this
+ source.  A copy of the CDDL is also available via the Internet at
+ http://www.illumos.org/license/CDDL.
+
+ Copyright 2020 Joyent, Inc.
+
+-->
+
+<topology name='i86pc' scheme='hc'>
+
+  <range name='motherboard' min='0' max='0'>
+    <enum-method name='smbios' version='1' />
+    <node instance='0' static='true'>
+        <fac-enum provider='fac_prov_ipmi' />
+        <propgroup name='protocol' version='1'
+            name-stability='Private' data-stability='Private' >
+            <propval name='label' type='string' value='MB' />
+        </propgroup>
+        <propgroup name='ipmi' version='1'
+            name-stability='Private' data-stability='Private' >
+            <propval name='entity-list' type='string_array' >
+                <propitem value='CPU1 Temp' />
+                <propitem value='CPU2 Temp' />
+                <propitem value='MB_NIC_Temp1' />
+                <propitem value='MB_NIC_Temp2' />
+                <propitem value='PCH Temp' />
+                <propitem value='Peripheral Temp' />
+                <propitem value='5VSB' />
+                <propitem value='5VCC' />
+                <propitem value='3.3VSB' />
+                <propitem value='3.3VCC' />
+                <propitem value='1.5V PCH' />
+                <propitem value='1.2V BMC' />
+                <propitem value='1.05V PCH' />
+                <propitem value='12V' />
+                <propitem value='5VCC' />
+                <propitem value='5VSB' />
+                <propitem value='NVMe_SSD Temp' />
+                <propitem value='VBAT' />
+                <propitem value='Vcpu1' />
+                <propitem value='Vcpu2' />
+                <propitem value='VDIMMAB' />
+                <propitem value='VDIMMCD' />
+                <propitem value='VDIMMEF' />
+                <propitem value='VDIMMGH' />
+                <propitem value='VmemABVRM' />
+                <propitem value='VmemCDVRM' />
+                <propitem value='VmemEFVRM' />
+                <propitem value='VmemGHVRM' />
+           </propval>
+        </propgroup>
+    </node>
+
+    <dependents grouping='children'>
+      <range name='chip' min='0' max='1'>
+        <enum-method name='chip' version='1' />
+      </range>
+      <range name='hostbridge' min='0' max='254'>
+        <enum-method name='hostbridge' version='1' />
+      </range>
+      <range name='sp' min='0' max='0'>
+        <enum-method name='ipmi' version='1' />
+      </range>
+      <range name='usb-mobo' min='0' max='256'>
+        <enum-method name='usb' version='1' />
+      </range>
+    </dependents>
+
+  </range>
+
+  <range name='chassis' min='0' max='0'>
+    <propmap name='SYS-2028U-E1CNRT+-chassis' />
+
+    <dependents grouping='children'>
+
+       <range name='psu' min='0' max='1'>
+          <enum-method name='ipmi' version='1' />
+       </range>
+       <range name='fan' min='0' max='8'>
+          <enum-method name='ipmi' version='1' />
+       </range>
+      <range name='usb-chassis' min='0' max='256'>
+        <enum-method name='usb' version='1' />
+      </range>
+
+    </dependents>
+
+  </range>
+
+  <range name='ses-enclosure' min='0' max='0'>
+      <enum-method name='ses' version='1' />
+      <node instance='0' static='true'>
+      <dependents grouping='children'>
+      <range name='bay' min='0' max='23'>
+        <node instance='20' static='true'>
+          <propgroup name='binding' version='1'
+              name-stability='Private' data-stability='Private' >
+              <propval name='driver' type='string' value='nvme' />
+              <propval name='parent-device' type='string'
+                value='/pci@0,0/pci8086,6f08@3' />
+          </propgroup>
+          <dependents grouping='children'>
+          <range name='nvme' min='0' max='0'>
+            <enum-method name='disk' version='1' />
+          </range>
+          </dependents>
+        </node>
+        <node instance='21' static='true'>
+          <propgroup name='binding' version='1'
+              name-stability='Private' data-stability='Private' >
+              <propval name='driver' type='string' value='nvme' />
+              <propval name='parent-device' type='string'
+                value='/pci@0,0/pci8086,6f09@3,1' />
+          </propgroup>
+          <dependents grouping='children'>
+          <range name='nvme' min='0' max='0'>
+            <enum-method name='disk' version='1' />
+          </range>
+          </dependents>
+        </node>
+        <node instance='22' static='true'>
+          <propgroup name='binding' version='1'
+            name-stability='Private' data-stability='Private' >
+              <propval name='driver' type='string' value='nvme' />
+              <propval name='parent-device' type='string'
+                value='/pci@0,0/pci8086,6f0a@3,2' />
+          </propgroup>
+          <dependents grouping='children'>
+          <range name='nvme' min='0' max='0'>
+            <enum-method name='disk' version='1' />
+          </range>
+          </dependents>
+        </node>
+        <node instance='23' static='true'>
+          <propgroup name='binding' version='1'
+              name-stability='Private' data-stability='Private' >
+              <propval name='driver' type='string' value='nvme' />
+              <propval name='parent-device' type='string'
+                value='/pci@0,0/pci8086,6f0b@3,3' />
+          </propgroup>
+          <dependents grouping='children'>
+          <range name='nvme' min='0' max='0'>
+            <enum-method name='disk' version='1' />
+          </range>
+          </dependents>
+        </node>
+      </range> <!-- bay -->
+      </dependents>
+    </node> <!-- ses-enclosure=0 -->
+  </range> <!-- ses-enclosure -->
+
+</topology>
diff --git a/usr/src/lib/fm/topo/maps/SMCI,SYS-2028U-E1CNRT+/SYS-2028U-E1CNRT+-usb.usbtopo b/usr/src/lib/fm/topo/maps/SMCI,SYS-2028U-E1CNRT+/SYS-2028U-E1CNRT+-usb.usbtopo
new file mode 100644
index 0000000000..434e7b28fd
--- /dev/null
+++ b/usr/src/lib/fm/topo/maps/SMCI,SYS-2028U-E1CNRT+/SYS-2028U-E1CNRT+-usb.usbtopo
@@ -0,0 +1,72 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright 2020 Joyent, Inc.
+#
+
+#
+# This file describes the USB topology for the SuperMicro
+# SYS-2028U-E1CNRT+ product which uses the Super X10DRU-i+ motherboard.  The
+# board contains two type-A USB 3.0 ports that are exposed externally.  It also
+# has an internal type-A USB 3.0 port on the motherboard.  Finally it has a
+# header for driving two more USB 3.0 ports, but it are not wired up on
+# this platform.
+#
+# For more information on the format see topo_usb_metadata.c.
+#
+
+enable-acpi-match
+port
+        label
+                Rear Upper USB
+        chassis
+        external
+        port-type
+                0x3
+        acpi-path
+                \_SB_.PCI0.XHCI.RHUB.HS12
+        acpi-path
+                \_SB_.PCI0.XHCI.RHUB.SSP3
+        acpi-path
+                \_SB_.PCI0.EHC2.HUBN.PR01.PR16
+end-port
+
+port
+        label
+                Rear Lower USB
+        chassis
+        external
+        port-type
+                0x3
+        acpi-path
+                \_SB_.PCI0.XHCI.RHUB.HS11
+        acpi-path
+                \_SB_.PCI0.XHCI.RHUB.SSP2
+        acpi-path
+                \_SB_.PCI0.EHC2.HUBN.PR01.PR15
+end-port
+
+port
+        label
+                Internal USB
+
+        internal
+        port-type
+                0x3
+        acpi-path
+                \_SB_.PCI0.XHCI.RHUB.HS07
+        acpi-path
+                \_SB_.PCI0.XHCI.RHUB.SSP6
+        acpi-path
+                \_SB_.PCI0.EHC2.HUBN.PR01.PR13
+end-port
+
diff --git a/usr/src/lib/fm/topo/modules/common/disk/Makefile b/usr/src/lib/fm/topo/modules/common/disk/Makefile
index 8e8935be65..9caf0818c9 100644
--- a/usr/src/lib/fm/topo/modules/common/disk/Makefile
+++ b/usr/src/lib/fm/topo/modules/common/disk/Makefile
@@ -22,13 +22,13 @@
 # Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
-# Copyright (c) 2018, Joyent, Inc.
+# Copyright 2020 Joyent, Inc.
 #
 
 MODULE = disk
 CLASS = common
 
-MODULESRCS = disk.c disk_common.c disk_mptsas.c
+MODULESRCS = disk.c disk_common.c disk_mptsas.c disk_nvme.c
 
 include ../../Makefile.plugin
 
diff --git a/usr/src/lib/fm/topo/modules/common/disk/disk.c b/usr/src/lib/fm/topo/modules/common/disk/disk.c
index c16fa28aae..7663ffc8fb 100644
--- a/usr/src/lib/fm/topo/modules/common/disk/disk.c
+++ b/usr/src/lib/fm/topo/modules/common/disk/disk.c
@@ -22,7 +22,7 @@
  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 /*
- * Copyright (c) 2018, Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  */
 
 #include <strings.h>
@@ -53,7 +53,7 @@ disk_declare_driver(topo_mod_t *mod, tnode_t *baynode, topo_list_t *dlistp,
 {
 	int err;
 
-	if (strcmp("mpt_sas", driver) == 0) {
+	if (strcmp(MPTSAS_DRV, driver) == 0) {
 		char *sas_address = NULL;
 		tnode_t *child = NULL;
 
@@ -66,6 +66,11 @@ disk_declare_driver(topo_mod_t *mod, tnode_t *baynode, topo_list_t *dlistp,
 		topo_mod_strfree(mod, sas_address);
 
 		return (err);
+	} else if (strcmp(NVME_DRV, driver) == 0) {
+		if (disk_nvme_enum_disk(mod, baynode) != 0)
+			return (-1);
+
+		return (0);
 	}
 
 	topo_mod_dprintf(mod, "unknown disk driver '%s'\n", driver);
@@ -82,9 +87,10 @@ disk_enum(topo_mod_t *mod, tnode_t *baynode,
 	int		err;
 	topo_list_t	*dlistp = topo_mod_getspecific(mod);
 
-	if (strcmp(name, DISK) != 0) {
-		topo_mod_dprintf(mod, "disk_enum: "
-		    "only know how to enumerate %s components.\n", DISK);
+	if (strcmp(name, DISK) != 0 && strcmp(name, NVME) != 0) {
+		topo_mod_dprintf(mod, "disk_enum: can't enumerate %s nodes - "
+		    "only know how to enumerate %s and %s nodes.", name,
+		    DISK, NVME);
 		return (-1);
 	}
 
@@ -102,7 +108,13 @@ disk_enum(topo_mod_t *mod, tnode_t *baynode,
 			    topo_strerror(err));
 			return (-1);
 		}
-		if (topo_node_fru_set(baynode, fmri, 0, &err) != 0) {
+		/*
+		 * If the disk enumerator module has been run from an XML map
+		 * and the parent bay node was already created by an enumerator
+		 * module (e.g. ses), then the FRU will already be set.
+		 */
+		if (topo_node_fru_set(baynode, fmri, 0, &err) != 0 &&
+		    err != ETOPO_PROP_DEFD) {
 			topo_mod_dprintf(mod, "disk_enum: "
 			    "topo_node_fru error %s\n", topo_strerror(err));
 			nvlist_free(fmri);
@@ -134,7 +146,8 @@ disk_enum(topo_mod_t *mod, tnode_t *baynode,
 	if (topo_prop_get_string(baynode, TOPO_PGROUP_BINDING,
 	    TOPO_BINDING_OCCUPANT, &device, &err) != 0) {
 		topo_mod_dprintf(mod, "disk_enum: "
-		    "binding error %s\n", topo_strerror(err));
+		    "failed to lookup prop %s/%s: %s\n", TOPO_PGROUP_BINDING,
+		    TOPO_BINDING_OCCUPANT, topo_strerror(err));
 		return (-1);
 	}
 
diff --git a/usr/src/lib/fm/topo/modules/common/disk/disk.h b/usr/src/lib/fm/topo/modules/common/disk/disk.h
index 02aa7efc87..d78688f551 100644
--- a/usr/src/lib/fm/topo/modules/common/disk/disk.h
+++ b/usr/src/lib/fm/topo/modules/common/disk/disk.h
@@ -23,12 +23,13 @@
  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 /*
- * Copyright (c) 2018, Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  */
 
 #ifndef _DISK_H
 #define	_DISK_H
 
+#include <sys/fm/protocol.h>
 #include <fm/topo_mod.h>
 #include <libdevinfo.h>
 
@@ -42,6 +43,18 @@ extern "C" {
 /* Max. number of devices for thumper */
 #define	DEVID_MAX		48
 
+/*
+ * Given a /devices path for a whole disk, appending this extension gives the
+ * path to a raw device that can be opened.
+ */
+#if defined(__i386) || defined(__amd64)
+#define	PHYS_EXTN	":q,raw"
+#elif defined(__sparc) || defined(__sparcv9)
+#define	PHYS_EXTN	":c,raw"
+#else
+#error	Unknown architecture
+#endif
+
 /* Properties added to the "storage" pgroup: */
 #define	TOPO_PGROUP_STORAGE		"storage"
 #define	TOPO_STORAGE_LOGICAL_DISK_NAME	"logical-disk"
@@ -52,21 +65,26 @@ extern "C" {
 #define	TOPO_STORAGE_CAPACITY		"capacity-in-bytes"
 #define	TOPO_STORAGE_RPM		"speed-in-rpm"
 
-/*
- * Properties for binding group: The binding group required in platform
- * specific xml that describes 'bay' nodes containing internal disks.
- */
-#define	TOPO_PGROUP_BINDING		"binding"
-#define	TOPO_BINDING_OCCUPANT		"occupant-path"
-#define	TOPO_BINDING_DRIVER		"driver"
-
-/*
- * The binding group required in platform specific xml that describes 'bay'
- * nodes containing disks attached to an HBA using the 'mpt_sas' driver.
- */
-#define	TOPO_BINDING_DEVCTL		"devctl"
-#define	TOPO_BINDING_ENCLOSURE		"enclosure"
-#define	TOPO_BINDING_SLOT		"slot"
+static const topo_pgroup_info_t io_pgroup = {
+	TOPO_PGROUP_IO,
+	TOPO_STABILITY_PRIVATE,
+	TOPO_STABILITY_PRIVATE,
+	1
+};
+
+static const topo_pgroup_info_t disk_auth_pgroup = {
+	FM_FMRI_AUTHORITY,
+	TOPO_STABILITY_PRIVATE,
+	TOPO_STABILITY_PRIVATE,
+	1
+};
+
+static const topo_pgroup_info_t storage_pgroup = {
+	TOPO_PGROUP_STORAGE,
+	TOPO_STABILITY_PRIVATE,
+	TOPO_STABILITY_PRIVATE,
+	1
+};
 
 /*
  * device node information.
diff --git a/usr/src/lib/fm/topo/modules/common/disk/disk_common.c b/usr/src/lib/fm/topo/modules/common/disk/disk_common.c
index ef4d638f94..7c9027517d 100644
--- a/usr/src/lib/fm/topo/modules/common/disk/disk_common.c
+++ b/usr/src/lib/fm/topo/modules/common/disk/disk_common.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2019 Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  */
 
 /*
@@ -62,18 +62,6 @@ typedef struct disk_cbdata {
 } disk_cbdata_t;
 
 /*
- * Given a /devices path for a whole disk, appending this extension gives the
- * path to a raw device that can be opened.
- */
-#if defined(__i386) || defined(__amd64)
-#define	PHYS_EXTN	":q,raw"
-#elif defined(__sparc) || defined(__sparcv9)
-#define	PHYS_EXTN	":c,raw"
-#else
-#error	Unknown architecture
-#endif
-
-/*
  * Methods for disks. This is used by the disk-transport module to
  * generate ereports based off SCSI disk status.
  */
@@ -101,27 +89,6 @@ static const topo_method_t disk_fac_methods[] = {
 	{ NULL }
 };
 
-static const topo_pgroup_info_t io_pgroup = {
-	TOPO_PGROUP_IO,
-	TOPO_STABILITY_PRIVATE,
-	TOPO_STABILITY_PRIVATE,
-	1
-};
-
-static const topo_pgroup_info_t disk_auth_pgroup = {
-	FM_FMRI_AUTHORITY,
-	TOPO_STABILITY_PRIVATE,
-	TOPO_STABILITY_PRIVATE,
-	1
-};
-
-static const topo_pgroup_info_t storage_pgroup = {
-	TOPO_PGROUP_STORAGE,
-	TOPO_STABILITY_PRIVATE,
-	TOPO_STABILITY_PRIVATE,
-	1
-};
-
 /*
  * Set the properties of the disk node, from dev_di_node_t data.
  * Properties include:
diff --git a/usr/src/lib/fm/topo/modules/common/disk/disk_drivers.h b/usr/src/lib/fm/topo/modules/common/disk/disk_drivers.h
index 6851e2fe27..5a8adfaf83 100644
--- a/usr/src/lib/fm/topo/modules/common/disk/disk_drivers.h
+++ b/usr/src/lib/fm/topo/modules/common/disk/disk_drivers.h
@@ -9,7 +9,7 @@
  * http://www.illumos.org/license/CDDL.
  */
 /*
- * Copyright (c) 2013, Joyent, Inc. All rights reserved.
+ * Copyright 2020 Joyent, Inc.
  */
 
 #ifndef _DISK_DRIVERS_H
@@ -22,7 +22,11 @@
 extern "C" {
 #endif
 
+#define	MPTSAS_DRV	"mpt_sas"
+#define	NVME_DRV	"nvme"
+
 int disk_mptsas_find_disk(topo_mod_t *, tnode_t *, char **);
+int disk_nvme_enum_disk(topo_mod_t *, tnode_t *);
 
 #ifdef __cplusplus
 }
diff --git a/usr/src/lib/fm/topo/modules/common/disk/disk_nvme.c b/usr/src/lib/fm/topo/modules/common/disk/disk_nvme.c
new file mode 100644
index 0000000000..a50705070e
--- /dev/null
+++ b/usr/src/lib/fm/topo/modules/common/disk/disk_nvme.c
@@ -0,0 +1,691 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2020 Joyent, Inc.
+ */
+
+/*
+ * This file drives topo node enumeration of NVMe controllers.  A single "nvme"
+ * node is enumerated for each NVMe controller.   Child "disk" nodes are then
+ * enumerated for each configured NVMe namespace.
+ *
+ * nvme nodes are expected to be enumerated under either a "bay" node (for U.2
+ * devices) or a "slot" node (for M.2 devices) or a "pciexfn" node (for AIC
+ * devices).
+ *
+ * Enumeration of NVMe controllers on PCIe add-in cards is automatically driven
+ * by the pcibus topo module.
+ *
+ * In order to allow for associating a given NVMe controller with a physical
+ * location, enumeration of U.2 and M.2 devices should be driven by a
+ * platform-specific topo map which statically sets the following two
+ * properties on the parent "bay" or "slot" node:
+ *
+ * propgroup        property        description
+ * ---------        --------        ------------
+ * binding          driver          "nvme"
+ * binding          parent-device   devpath of parent PCIe device
+ *
+ * for example:
+ *
+ * <propgroup name="binding" version="1" name-stability="Private"
+ *   data-stability="Private">
+ *     <propval name="driver" type="string" value="nvme"/>
+ *     <propval name="parent-device" type="string"
+ *       value="/pci@0,0/pci8086,6f09@3,1"/>
+ * </propgroup>
+ * <dependents grouping="children">
+ *     <range name="nvme" min="0" max="0">
+ *         <enum-method name="disk" version="1"/>
+ *     </range>
+ * </dependents>
+ */
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string.h>
+#include <strings.h>
+
+#include <sys/fm/protocol.h>
+#include <fm/topo_hc.h>
+#include <fm/topo_mod.h>
+
+#include <sys/dkio.h>
+#include <sys/scsi/generic/inquiry.h>
+
+#include <sys/nvme.h>
+#include "disk.h"
+#include "disk_drivers.h"
+
+typedef struct nvme_enum_info {
+	topo_mod_t		*nei_mod;
+	di_node_t		nei_dinode;
+	nvme_identify_ctrl_t	*nei_idctl;
+	nvme_version_t		nei_vers;
+	tnode_t			*nei_parent;
+	tnode_t			*nei_nvme;
+	nvlist_t		*nei_nvme_fmri;
+	const char		*nei_nvme_path;
+	int			nei_fd;
+} nvme_enum_info_t;
+
+typedef struct devlink_arg {
+	topo_mod_t		*dla_mod;
+	char			*dla_logical_disk;
+	uint_t			dla_strsz;
+} devlink_arg_t;
+
+static int
+devlink_cb(di_devlink_t dl, void *arg)
+{
+	devlink_arg_t *dlarg = (devlink_arg_t *)arg;
+	topo_mod_t *mod = dlarg->dla_mod;
+	const char *devpath;
+	char *slice, *ctds;
+
+	if ((devpath = di_devlink_path(dl)) == NULL ||
+	    (dlarg->dla_logical_disk = topo_mod_strdup(mod, devpath)) ==
+	    NULL) {
+		return (DI_WALK_TERMINATE);
+	}
+
+	/*
+	 * We need to keep track of the original string size before we
+	 * truncate it with a NUL, so that we can free the right number of
+	 * bytes when we're done, otherwise libumem will complain.
+	 */
+	dlarg->dla_strsz = strlen(dlarg->dla_logical_disk) + 1;
+
+	/* trim the slice off the public name */
+	if (((ctds = strrchr(dlarg->dla_logical_disk, '/')) != NULL) &&
+	    ((slice = strchr(ctds, 's')) != NULL))
+		*slice = '\0';
+
+	return (DI_WALK_TERMINATE);
+}
+
+static char *
+get_logical_disk(topo_mod_t *mod, const char *devpath, uint_t *bufsz)
+{
+	di_devlink_handle_t devhdl;
+	devlink_arg_t dlarg = { 0 };
+	char *minorpath = NULL;
+
+	if (asprintf(&minorpath, "%s:a", devpath) < 0) {
+		return (NULL);
+	}
+
+	if ((devhdl = di_devlink_init(NULL, 0)) == DI_NODE_NIL) {
+		topo_mod_dprintf(mod, "%s: di_devlink_init failed", __func__);
+		free(minorpath);
+		return (NULL);
+	}
+
+	dlarg.dla_mod = mod;
+
+	(void) di_devlink_walk(devhdl, "^dsk/", minorpath, DI_PRIMARY_LINK,
+	    &dlarg, devlink_cb);
+
+	(void) di_devlink_fini(&devhdl);
+	free(minorpath);
+
+	*bufsz = dlarg.dla_strsz;
+	return (dlarg.dla_logical_disk);
+}
+
+static int
+make_disk_node(nvme_enum_info_t *nvme_info, di_node_t dinode,
+    topo_instance_t inst)
+{
+	topo_mod_t *mod = nvme_info->nei_mod;
+	nvlist_t *auth = NULL, *fmri = NULL;
+	tnode_t *disk;
+	char *rev = NULL, *model = NULL, *serial = NULL, *path;
+	char *logical_disk = NULL, *devid, *manuf, *ctd = NULL;
+	char *cap_bytes_str = NULL, full_path[MAXPATHLEN + 1];
+	char *pname = topo_node_name(nvme_info->nei_parent);
+	topo_instance_t pinst = topo_node_instance(nvme_info->nei_parent);
+	const char **ppaths = NULL;
+	struct dk_minfo minfo;
+	uint64_t cap_bytes;
+	uint_t bufsz;
+	int fd = -1, err, ret = -1, r;
+
+	if ((path = di_devfs_path(dinode)) == NULL) {
+		topo_mod_dprintf(mod, "%s: failed to get dev path", __func__);
+		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
+		return (ret);
+	}
+
+	topo_mod_dprintf(mod, "%s: found nvme namespace: %s", __func__, path);
+
+	/*
+	 * Issue the DKIOCGMEDIAINFO ioctl to get the capacity
+	 */
+	(void) snprintf(full_path, MAXPATHLEN, "/devices%s%s", path,
+	    PHYS_EXTN);
+	if ((fd = open(full_path, O_RDWR)) < 0 ||
+	    ioctl(fd, DKIOCGMEDIAINFO, &minfo) < 0) {
+		topo_mod_dprintf(mod, "failed to get blkdev capacity (%s)",
+		    strerror(errno));
+		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
+		goto error;
+	}
+
+	cap_bytes = minfo.dki_lbsize * minfo.dki_capacity;
+
+	if (asprintf(&cap_bytes_str, "%" PRIu64, cap_bytes) < 0) {
+		topo_mod_dprintf(mod, "%s: failed to alloc string", __func__);
+		(void) topo_mod_seterrno(mod, EMOD_NOMEM);
+		goto error;
+	}
+
+	/*
+	 * Gather the FRU identity information from the devinfo properties
+	 */
+	if (di_prop_lookup_strings(DDI_DEV_T_ANY, dinode, DEVID_PROP_NAME,
+	    &devid) == -1 ||
+	    di_prop_lookup_strings(DDI_DEV_T_ANY, dinode, INQUIRY_VENDOR_ID,
+	    &manuf) == -1 ||
+	    di_prop_lookup_strings(DDI_DEV_T_ANY, dinode, INQUIRY_PRODUCT_ID,
+	    &model) == -1 ||
+	    di_prop_lookup_strings(DDI_DEV_T_ANY, dinode, INQUIRY_REVISION_ID,
+	    &rev) == -1 ||
+	    di_prop_lookup_strings(DDI_DEV_T_ANY, dinode, INQUIRY_SERIAL_NO,
+	    &serial) == -1) {
+		topo_mod_dprintf(mod, "%s: failed to lookup devinfo props on "
+		    "%s", __func__, path);
+		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
+		goto error;
+	}
+
+	model = topo_mod_clean_str(mod, model);
+	rev = topo_mod_clean_str(mod, rev);
+	serial = topo_mod_clean_str(mod, serial);
+
+	/*
+	 * Lookup the /dev/dsk/c#t#d# disk device name from the blkdev path
+	 */
+	if ((logical_disk = get_logical_disk(mod, path, &bufsz)) == NULL) {
+		topo_mod_dprintf(mod, "failed to find logical disk");
+		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
+		goto error;
+	}
+
+	/*
+	 * If we were able to look up the logical disk path for this namespace
+	 * then set ctd to be that pathname, minus the "/dev/dsk/" portion.
+	 */
+	if ((ctd = strrchr(logical_disk, '/')) !=  NULL) {
+		ctd = ctd + 1;
+	} else {
+		topo_mod_dprintf(mod, "malformed logical disk path: %s",
+		    logical_disk);
+		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
+		goto error;
+	}
+
+	/*
+	 * Build the FMRI and then bind the disk node to the parent nvme node.
+	 */
+	auth = topo_mod_auth(mod, nvme_info->nei_nvme);
+	fmri = topo_mod_hcfmri(mod, nvme_info->nei_nvme, FM_HC_SCHEME_VERSION,
+	    DISK, inst, NULL, auth, model, rev, serial);
+
+	if (fmri == NULL) {
+		/* errno set */
+		topo_mod_dprintf(mod, "%s: hcfmri failed for %s=%u/%s=0/%s=%u",
+		    __func__, pname, pinst, NVME, DISK, inst);
+		goto error;
+	}
+	if ((disk = topo_node_bind(mod, nvme_info->nei_nvme, DISK, inst,
+	    fmri)) == NULL) {
+		/* errno set */
+		topo_mod_dprintf(mod, "%s: bind failed for %s=%u/%s=0/%s=%u",
+		    __func__, pname, pinst, NVME, DISK, inst);
+		goto error;
+	}
+
+	/* Create authority and system propgroups */
+	topo_pgroup_hcset(disk, auth);
+
+	/*
+	 * As the "disk" in this case is simply a logical construct
+	 * representing an NVMe namespace, we inherit the FRU from the parent
+	 * node.
+	 */
+	if (topo_node_fru_set(disk, NULL, 0, &err) != 0) {
+		topo_mod_dprintf(mod, "%s: failed to set FRU: %s", __func__,
+		    topo_strerror(err));
+		(void) topo_mod_seterrno(mod, err);
+		goto error;
+	}
+
+	if ((ppaths = topo_mod_zalloc(mod, sizeof (char *))) == NULL) {
+		(void) topo_mod_seterrno(mod, EMOD_NOMEM);
+		goto error;
+	}
+	ppaths[0] = path;
+
+	/*
+	 * Create the "storage" and "io" property groups and then fill them
+	 * with the standard set of properties for "disk" nodes.
+	 */
+	if (topo_pgroup_create(disk, &io_pgroup, &err) != 0 ||
+	    topo_pgroup_create(disk, &storage_pgroup, &err) != 0) {
+		topo_mod_dprintf(mod, "%s: failed to create propgroups: %s",
+		    __func__, topo_strerror(err));
+		(void) topo_mod_seterrno(mod, err);
+		goto error;
+	}
+
+	r = topo_prop_set_string(disk, TOPO_PGROUP_IO, TOPO_IO_DEV_PATH,
+	    TOPO_PROP_IMMUTABLE, path, &err);
+
+	r += topo_prop_set_string_array(disk, TOPO_PGROUP_IO,
+	    TOPO_IO_PHYS_PATH, TOPO_PROP_IMMUTABLE, ppaths, 1, &err);
+
+	r += topo_prop_set_string(disk, TOPO_PGROUP_IO, TOPO_IO_DEVID,
+	    TOPO_PROP_IMMUTABLE, devid, &err);
+
+	r += topo_prop_set_string(disk, TOPO_PGROUP_STORAGE,
+	    TOPO_STORAGE_MANUFACTURER, TOPO_PROP_IMMUTABLE, manuf, &err);
+
+	r += topo_prop_set_string(disk, TOPO_PGROUP_STORAGE,
+	    TOPO_STORAGE_CAPACITY, TOPO_PROP_IMMUTABLE, cap_bytes_str,
+	    &err);
+
+	r += topo_prop_set_string(disk, TOPO_PGROUP_STORAGE,
+	    TOPO_STORAGE_SERIAL_NUM, TOPO_PROP_IMMUTABLE, serial, &err);
+
+	r += topo_prop_set_string(disk, TOPO_PGROUP_STORAGE,
+	    TOPO_STORAGE_MODEL, TOPO_PROP_IMMUTABLE, model, &err);
+
+	r += topo_prop_set_string(disk, TOPO_PGROUP_STORAGE,
+	    TOPO_STORAGE_FIRMWARE_REV, TOPO_PROP_IMMUTABLE, rev, &err);
+
+	r += topo_prop_set_string(disk, TOPO_PGROUP_STORAGE,
+	    TOPO_STORAGE_LOGICAL_DISK_NAME, TOPO_PROP_IMMUTABLE, ctd, &err);
+
+	if (r != 0) {
+		topo_mod_dprintf(mod, "%s: failed to create properties: %s",
+		    __func__, topo_strerror(err));
+		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
+		goto error;
+	}
+
+	ret = 0;
+
+error:
+	free(cap_bytes_str);
+	if (fd > 0)
+		(void) close(fd);
+	if (ppaths != NULL)
+		topo_mod_free(mod, ppaths, sizeof (char *));
+	di_devfs_path_free(path);
+	nvlist_free(auth);
+	nvlist_free(fmri);
+	topo_mod_strfree(mod, rev);
+	topo_mod_strfree(mod, model);
+	topo_mod_strfree(mod, serial);
+	topo_mod_free(mod, logical_disk, bufsz);
+	return (ret);
+}
+
+static const topo_pgroup_info_t nvme_pgroup = {
+	TOPO_PGROUP_NVME,
+	TOPO_STABILITY_PRIVATE,
+	TOPO_STABILITY_PRIVATE,
+	1
+};
+
+
+static int
+make_nvme_node(nvme_enum_info_t *nvme_info)
+{
+	topo_mod_t *mod = nvme_info->nei_mod;
+	nvlist_t *auth = NULL, *fmri = NULL, *fru;
+	tnode_t *nvme;
+	char raw_rev[NVME_FWVER_SZ + 1], raw_model[NVME_MODEL_SZ + 1];
+	char raw_serial[NVME_SERIAL_SZ + 1];
+	char *rev = NULL, *model = NULL, *serial = NULL, *vers = NULL;
+	char *pname = topo_node_name(nvme_info->nei_parent);
+	char *label = NULL;
+	topo_instance_t pinst = topo_node_instance(nvme_info->nei_parent);
+	int err = 0, ret = -1;
+	di_node_t cn;
+	uint_t i;
+
+	/*
+	 * The raw strings returned by the IDENTIFY CONTROLLER command are
+	 * not NUL-terminated, so we fix that up.
+	 */
+	(void) strncpy(raw_rev, nvme_info->nei_idctl->id_fwrev, NVME_FWVER_SZ);
+	raw_rev[NVME_FWVER_SZ] = '\0';
+	(void) strncpy(raw_model, nvme_info->nei_idctl->id_model,
+	    NVME_MODEL_SZ);
+	raw_model[NVME_MODEL_SZ] = '\0';
+	(void) strncpy(raw_serial, nvme_info->nei_idctl->id_serial,
+	    NVME_SERIAL_SZ);
+	raw_serial[NVME_SERIAL_SZ] = '\0';
+
+	/*
+	 * Next we pass the strings through a function that sanitizes them of
+	 * any characters that can't be used in an FMRI string.
+	 */
+	rev = topo_mod_clean_str(mod, raw_rev);
+	model = topo_mod_clean_str(mod, raw_model);
+	serial = topo_mod_clean_str(mod, raw_serial);
+
+	auth = topo_mod_auth(mod, nvme_info->nei_parent);
+	fmri = topo_mod_hcfmri(mod, nvme_info->nei_parent, FM_HC_SCHEME_VERSION,
+	    NVME, 0, NULL, auth, model, rev, serial);
+
+	if (fmri == NULL) {
+		/* errno set */
+		topo_mod_dprintf(mod, "%s: hcfmri failed for %s=%u/%s=0",
+		    __func__, pname, pinst, NVME);
+		goto error;
+	}
+
+	/*
+	 * If our parent is a pciexfn node, then we need to create a nvme range
+	 * underneath it to hold the nvme heirarchy.  For other cases, where
+	 * enumeration is being driven by a topo map file, this range will have
+	 * already been statically defined in the XML.
+	 */
+	if (strcmp(pname, PCIEX_FUNCTION) == 0) {
+		if (topo_node_range_create(mod, nvme_info->nei_parent, NVME, 0,
+		    0) < 0) {
+			/* errno set */
+			topo_mod_dprintf(mod, "%s: error creating %s range",
+			    __func__, NVME);
+			goto error;
+		}
+	}
+
+	/*
+	 * Create a new topo node to represent the NVMe controller and bind it
+	 * to the parent node.
+	 */
+	if ((nvme = topo_node_bind(mod, nvme_info->nei_parent, NVME, 0,
+	    fmri)) == NULL) {
+		/* errno set */
+		topo_mod_dprintf(mod, "%s: bind failed for %s=%u/%s=0",
+		    __func__, pname, pinst, NVME);
+		goto error;
+	}
+	nvme_info->nei_nvme = nvme;
+	nvme_info->nei_nvme_fmri = fmri;
+
+	/*
+	 * If our parent node is a "pciexfn" node then this is a NVMe device on
+	 * a PCIe AIC, so we inherit our parent's FRU.  Otherwise, we set the
+	 * FRU to ourself.
+	 */
+	if (strcmp(topo_node_name(nvme_info->nei_parent), PCIEX_FUNCTION) == 0)
+		fru = NULL;
+	else
+		fru = fmri;
+
+	if (topo_node_fru_set(nvme, fru, 0, &err) != 0) {
+		topo_mod_dprintf(mod, "%s: failed to set FRU: %s", __func__,
+		    topo_strerror(err));
+		(void) topo_mod_seterrno(mod, err);
+		goto error;
+	}
+
+	/*
+	 * Clone the label from our parent node.  We can't inherit the property
+	 * because the label prop is mutable on bay nodes and only immutable
+	 * properties can be inherited.
+	 */
+	if ((topo_node_label(nvme_info->nei_parent, &label, &err) != 0 &&
+	    err != ETOPO_PROP_NOENT) ||
+	    topo_node_label_set(nvme, label, &err) != 0) {
+		topo_mod_dprintf(mod, "%s: failed to set label: %s",
+		    __func__, topo_strerror(err));
+		(void) topo_mod_seterrno(mod, err);
+		goto error;
+	}
+
+	if (topo_pgroup_create(nvme, &nvme_pgroup, &err) != 0) {
+		topo_mod_dprintf(mod, "%s: failed to create %s pgroup: %s",
+		    __func__, TOPO_PGROUP_NVME, topo_strerror(err));
+		(void) topo_mod_seterrno(mod, err);
+		goto error;
+	}
+
+	if (asprintf(&vers, "%u.%u", nvme_info->nei_vers.v_major,
+	    nvme_info->nei_vers.v_minor) < 0) {
+		topo_mod_dprintf(mod, "%s: failed to alloc string", __func__);
+		(void) topo_mod_seterrno(mod, EMOD_NOMEM);
+		goto error;
+	}
+	if (topo_prop_set_string(nvme, TOPO_PGROUP_NVME, TOPO_PROP_NVME_VER,
+	    TOPO_PROP_IMMUTABLE, vers, &err) != 0) {
+		topo_mod_dprintf(mod, "%s: failed to set %s/%s property",
+		    __func__, TOPO_PGROUP_NVME, TOPO_PROP_NVME_VER);
+		(void) topo_mod_seterrno(mod, err);
+		goto error;
+	}
+
+	if (topo_pgroup_create(nvme, &io_pgroup, &err) != 0) {
+		topo_mod_dprintf(mod, "%s: failed to create %s pgroup: %s",
+		    __func__, TOPO_PGROUP_IO, topo_strerror(err));
+		(void) topo_mod_seterrno(mod, err);
+		goto error;
+	}
+	if (topo_prop_set_string(nvme, TOPO_PGROUP_IO, TOPO_IO_DEV_PATH,
+	    TOPO_PROP_IMMUTABLE, nvme_info->nei_nvme_path, &err) != 0) {
+		topo_mod_dprintf(mod, "%s: failed to set %s/%s property",
+		    __func__, TOPO_PGROUP_IO, TOPO_IO_DEV_PATH);
+		(void) topo_mod_seterrno(mod, err);
+		goto error;
+	}
+
+	/*
+	 * Create a child disk node for each namespace.
+	 */
+	if (topo_node_range_create(mod, nvme, DISK, 0,
+	    (nvme_info->nei_idctl->id_nn - 1)) < 0) {
+		/* errno set */
+		topo_mod_dprintf(mod, "%s: error creating %s range", __func__,
+		    DISK);
+		goto error;
+	}
+
+	for (i = 0, cn = di_child_node(nvme_info->nei_dinode);
+	    cn != DI_NODE_NIL;
+	    i++, cn = di_sibling_node(cn)) {
+
+		if (make_disk_node(nvme_info, cn, i) != 0) {
+			char *path = di_devfs_path(cn);
+			/*
+			 * We note the failure, but attempt to forge ahead and
+			 * enumerate any other namespaces.
+			 */
+			topo_mod_dprintf(mod, "%s: make_disk_node() failed "
+			    "for %s\n", __func__,
+			    path ? path : "unknown path");
+			di_devfs_path_free(path);
+		}
+	}
+	ret = 0;
+
+error:
+	free(vers);
+	nvlist_free(auth);
+	nvlist_free(fmri);
+	topo_mod_strfree(mod, rev);
+	topo_mod_strfree(mod, model);
+	topo_mod_strfree(mod, serial);
+	topo_mod_strfree(mod, label);
+	return (ret);
+}
+
+struct diwalk_arg {
+	topo_mod_t	*diwk_mod;
+	tnode_t		*diwk_parent;
+};
+
+/*
+ * This function gathers identity information from the NVMe controller and
+ * stores it in a struct.  This struct is passed to make_nvme_node(), which
+ * does the actual topo node creation.
+ */
+static int
+discover_nvme_ctl(di_node_t node, di_minor_t minor, void *arg)
+{
+	struct diwalk_arg *wkarg = arg;
+	topo_mod_t *mod = wkarg->diwk_mod;
+	char *path = NULL, *devctl = NULL;
+	nvme_ioctl_t nioc = { 0 };
+	nvme_identify_ctrl_t *idctl = NULL;
+	nvme_enum_info_t nvme_info = { 0 };
+	int fd = -1, ret = DI_WALK_TERMINATE;
+
+	if ((path = di_devfs_minor_path(minor)) == NULL) {
+		topo_mod_dprintf(mod, "failed to get minor path");
+		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
+		return (ret);
+	}
+
+	topo_mod_dprintf(mod, "%s=%u: found nvme controller: %s",
+	    topo_node_name(wkarg->diwk_parent),
+	    topo_node_instance(wkarg->diwk_parent), path);
+
+	if (asprintf(&devctl, "/devices%s", path) < 0) {
+		topo_mod_dprintf(mod, "failed to alloc string");
+		(void) topo_mod_seterrno(mod, EMOD_NOMEM);
+		goto error;
+	}
+
+	if ((fd = open(devctl, O_RDWR)) < 0) {
+		topo_mod_dprintf(mod, "failed to open %s", devctl);
+		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
+		goto error;
+	}
+	if ((idctl = topo_mod_zalloc(mod, NVME_IDENTIFY_BUFSIZE)) == NULL) {
+		topo_mod_dprintf(mod, "zalloc failed");
+		(void) topo_mod_seterrno(mod, EMOD_NOMEM);
+		goto error;
+	}
+	nioc.n_len = NVME_IDENTIFY_BUFSIZE;
+	nioc.n_buf = (uintptr_t)idctl;
+
+	if (ioctl(fd, NVME_IOC_IDENTIFY_CTRL, &nioc) != 0) {
+		topo_mod_dprintf(mod, "NVME_IOC_IDENTIFY_CTRL ioctl "
+		    "failed: %s", strerror(errno));
+		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
+		goto error;
+	}
+
+	nioc.n_len = sizeof (nvme_version_t);
+	nioc.n_buf = (uintptr_t)&nvme_info.nei_vers;
+
+	if (ioctl(fd, NVME_IOC_VERSION, &nioc) != 0) {
+		topo_mod_dprintf(mod, "NVME_IOC_VERSION ioctl failed: %s",
+		    strerror(errno));
+		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
+		goto error;
+	}
+
+	nvme_info.nei_mod = mod;
+	nvme_info.nei_nvme_path = path;
+	nvme_info.nei_dinode = node;
+	nvme_info.nei_idctl = idctl;
+	nvme_info.nei_parent = wkarg->diwk_parent;
+	nvme_info.nei_fd = fd;
+
+	if (make_nvme_node(&nvme_info) != 0) {
+		/* errno set */
+		goto error;
+	}
+
+	ret = DI_WALK_CONTINUE;
+
+error:
+	if (fd > 0)
+		(void) close(fd);
+	di_devfs_path_free(path);
+	free(devctl);
+	if (idctl != NULL)
+		topo_mod_free(mod, idctl, NVME_IDENTIFY_BUFSIZE);
+	return (ret);
+}
+
+int
+disk_nvme_enum_disk(topo_mod_t *mod, tnode_t *pnode)
+{
+	char *parent = NULL;
+	int err;
+	di_node_t devtree;
+	di_node_t dnode;
+	struct diwalk_arg wkarg = { 0 };
+	int ret = -1;
+
+	/*
+	 * Lookup a property containing the devfs path of the parent PCIe
+	 * device of the NVMe device we're attempting to enumerate.  This
+	 * property is hard-coded in per-platform topo XML maps that are
+	 * delivered with the OS.  This hard-coded path allows topo to map a
+	 * given NVMe controller to a physical location (bay or slot) on the
+	 * platform, when generating the topo snapshot.
+	 */
+	if (topo_prop_get_string(pnode, TOPO_PGROUP_BINDING,
+	    TOPO_BINDING_PARENT_DEV, &parent, &err) != 0) {
+		topo_mod_dprintf(mod, "parent node was missing nvme binding "
+		    "properties\n");
+		(void) topo_mod_seterrno(mod, err);
+		goto out;
+	}
+	if ((devtree = topo_mod_devinfo(mod)) == DI_NODE_NIL) {
+		topo_mod_dprintf(mod, "failed to get devinfo snapshot");
+		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
+		goto out;
+	}
+
+	/*
+	 * Walk the devinfo tree looking NVMe devices. For each NVMe device,
+	 * check if the devfs path of the parent matches the one specified in
+	 * TOPO_BINDING_PARENT_DEV.
+	 */
+	wkarg.diwk_mod = mod;
+	wkarg.diwk_parent = pnode;
+	dnode = di_drv_first_node(NVME_DRV, devtree);
+	while (dnode != DI_NODE_NIL) {
+		char *path;
+
+		if ((path = di_devfs_path(di_parent_node(dnode))) == NULL) {
+			topo_mod_dprintf(mod, "failed to get dev path");
+			(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
+			goto out;
+		}
+		if (strcmp(parent, path) == 0) {
+			if (di_walk_minor(dnode, DDI_NT_NVME_NEXUS, 0,
+			    &wkarg, discover_nvme_ctl) < 0) {
+				di_devfs_path_free(path);
+				goto out;
+			}
+		}
+		di_devfs_path_free(path);
+		dnode = di_drv_next_node(dnode);
+	}
+	ret = 0;
+
+out:
+	topo_mod_strfree(mod, parent);
+	return (ret);
+}
diff --git a/usr/src/lib/fm/topo/modules/common/pcibus/pcibus.c b/usr/src/lib/fm/topo/modules/common/pcibus/pcibus.c
index b915529033..8927188c12 100644
--- a/usr/src/lib/fm/topo/modules/common/pcibus/pcibus.c
+++ b/usr/src/lib/fm/topo/modules/common/pcibus/pcibus.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2019 Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  */
 
 #include <sys/fm/protocol.h>
@@ -630,11 +630,12 @@ static void
 declare_dev_and_fn(topo_mod_t *mod, tnode_t *bus, tnode_t **dev, di_node_t din,
     int board, int bridge, int rc, int devno, int fnno, int depth)
 {
-	int dcnt = 0, rcnt;
-	char *propstr;
+	int dcnt = 0, rcnt, err;
+	char *propstr, *label = NULL, *pdev = NULL;
 	tnode_t *fn;
 	uint_t class, subclass;
 	uint_t vid, did;
+	uint_t pdev_sz;
 	did_t *dp = NULL;
 
 	if (*dev == NULL) {
@@ -773,6 +774,84 @@ declare_dev_and_fn(topo_mod_t *mod, tnode_t *bus, tnode_t **dev, di_node_t din,
 				pci_receptacle_instantiate(mod, fn, din);
 		}
 	}
+
+	/*
+	 * If this is an NVMe device and if the FRU label indicates it's not an
+	 * onboard device then invoke the disk enumerator to enumerate the NVMe
+	 * controller and associated namespaces.
+	 *
+	 * We skip NVMe devices that appear to be onboard as those are likely
+	 * M.2 or U.2 devices and so should be enumerated via a
+	 * platform-specific XML map so that they can be associated with the
+	 * correct physical bay/slot.  This code is intended to pick up NVMe
+	 * devices that are part of PCIe add-in cards.
+	 */
+	if (topo_node_label(fn, &label, &err) != 0) {
+		topo_mod_dprintf(mod, "%s: failed to lookup FRU label on %s=%d",
+		    __func__, topo_node_name(fn), topo_node_instance(fn));
+		goto out;
+	}
+
+	if (class == PCI_CLASS_MASS && subclass == PCI_MASS_NVME &&
+	    strcmp(label, "MB") != 0) {
+		char *driver = di_driver_name(din);
+		char *slash;
+		topo_pgroup_info_t pgi;
+
+		if (topo_prop_get_string(fn, TOPO_PGROUP_IO, TOPO_IO_DEV,
+		    &pdev, &err) != 0) {
+			topo_mod_dprintf(mod, "%s: failed to lookup %s on "
+			    "%s=%d", __func__, TOPO_IO_DEV, topo_node_name(fn),
+			    topo_node_instance(fn));
+			goto out;
+		}
+
+		/*
+		 * Add the binding properties that are required by the disk
+		 * enumerator to discover the accociated NVMe controller.
+		 */
+		pdev_sz = strlen(pdev) + 1;
+		if ((slash = strrchr(pdev, '/')) == NULL) {
+			topo_mod_dprintf(mod, "%s: malformed dev path\n",
+			    __func__);
+			(void) topo_mod_seterrno(mod, EMOD_PARTIAL_ENUM);
+			goto out;
+		}
+		*slash = '\0';
+
+		pgi.tpi_name = TOPO_PGROUP_BINDING;
+		pgi.tpi_namestab = TOPO_STABILITY_PRIVATE;
+		pgi.tpi_datastab = TOPO_STABILITY_PRIVATE;
+		pgi.tpi_version = TOPO_VERSION;
+		if (topo_pgroup_create(fn, &pgi, &err) != 0 ||
+		    topo_prop_set_string(fn, TOPO_PGROUP_BINDING,
+		    TOPO_BINDING_DRIVER, TOPO_PROP_IMMUTABLE, driver,
+		    &err) != 0 ||
+		    topo_prop_set_string(fn, TOPO_PGROUP_BINDING,
+		    TOPO_BINDING_PARENT_DEV, TOPO_PROP_IMMUTABLE, pdev,
+		    &err) != 0) {
+			topo_mod_dprintf(mod, "%s: failed to set binding "
+			    "props", __func__);
+			(void) topo_mod_seterrno(mod, EMOD_PARTIAL_ENUM);
+			goto out;
+		}
+
+		/*
+		 * Load and invoke the disk enumerator module.
+		 */
+		if (topo_mod_load(mod, DISK, TOPO_VERSION) == NULL) {
+			topo_mod_dprintf(mod, "pcibus enum could not load "
+			    "disk enum\n");
+			(void) topo_mod_seterrno(mod, EMOD_PARTIAL_ENUM);
+			goto out;
+		}
+		(void) topo_mod_enumerate(mod, fn, DISK, NVME, 0, 0, NULL);
+	}
+out:
+	if (pdev != NULL) {
+		topo_mod_free(mod, pdev, pdev_sz);
+	}
+	topo_mod_strfree(mod, label);
 }
 
 int
diff --git a/usr/src/lib/fm/topo/modules/common/ses/ses.c b/usr/src/lib/fm/topo/modules/common/ses/ses.c
index d55835b067..bf92b4d7dd 100644
--- a/usr/src/lib/fm/topo/modules/common/ses/ses.c
+++ b/usr/src/lib/fm/topo/modules/common/ses/ses.c
@@ -23,7 +23,7 @@
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2012 Milan Jurik. All rights reserved.
  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
- * Copyright (c) 2020, Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  */
 
 #include <alloca.h>
@@ -226,14 +226,6 @@ typedef enum {
 	SES_DUP_SUBCHASSIS	= 0x8
 } ses_chassis_type_e;
 
-
-static const topo_pgroup_info_t storage_pgroup = {
-	TOPO_PGROUP_STORAGE,
-	TOPO_STABILITY_PRIVATE,
-	TOPO_STABILITY_PRIVATE,
-	1
-};
-
 static const topo_pgroup_info_t smp_pgroup = {
 	TOPO_PGROUP_SMP,
 	TOPO_STABILITY_PRIVATE,
diff --git a/usr/src/pkg/manifests/service-fault-management.mf b/usr/src/pkg/manifests/service-fault-management.mf
index 18e07dd5fc..19bba3253b 100644
--- a/usr/src/pkg/manifests/service-fault-management.mf
+++ b/usr/src/pkg/manifests/service-fault-management.mf
@@ -21,7 +21,7 @@
 
 #
 # Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
-# Copyright (c) 2019, Joyent, Inc.
+# Copyright 2020 Joyent, Inc.
 # Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
 # Copyright 2019 Peter Tribble.
 #
@@ -735,6 +735,15 @@ $(i386_ONLY)file \
     path=usr/platform/i86pc/lib/fm/topo/maps/SSG-6049P-E1CR36L-usb.usbtopo \
     mode=0444
 $(i386_ONLY)file \
+    path=usr/platform/i86pc/lib/fm/topo/maps/SYS-2028U-E1CNRT+-chassis-hc-topology.xml \
+    mode=0444
+$(i386_ONLY)file \
+    path=usr/platform/i86pc/lib/fm/topo/maps/SYS-2028U-E1CNRT+-hc-topology.xml \
+    mode=0444
+$(i386_ONLY)file \
+    path=usr/platform/i86pc/lib/fm/topo/maps/SYS-2028U-E1CNRT+-usb.usbtopo \
+    mode=0444
+$(i386_ONLY)file \
     path=usr/platform/i86pc/lib/fm/topo/maps/Sun-Fire-X4200-M2-disk-hc-topology.xml \
     mode=0444
 $(i386_ONLY)file \
diff --git a/usr/src/uts/common/sys/nvme.h b/usr/src/uts/common/sys/nvme.h
index f405ea005e..9a0d926dc5 100644
--- a/usr/src/uts/common/sys/nvme.h
+++ b/usr/src/uts/common/sys/nvme.h
@@ -11,7 +11,7 @@
 
 /*
  * Copyright 2016 Nexenta Systems, Inc.
- * Copyright (c) 2018, Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  * Copyright 2019 Western Digital Corporation
  */
 
@@ -137,13 +137,16 @@ typedef struct {
 	uint8_t psd_rsvd10[9];
 } nvme_idctl_psd_t;
 
+#define	NVME_SERIAL_SZ	20
+#define	NVME_MODEL_SZ	40
+
 /* NVMe Identify Controller Data Structure */
 typedef struct {
 	/* Controller Capabilities & Features */
 	uint16_t id_vid;		/* PCI vendor ID */
 	uint16_t id_ssvid;		/* PCI subsystem vendor ID */
-	char id_serial[20];		/* Serial Number */
-	char id_model[40];		/* Model Number */
+	char id_serial[NVME_SERIAL_SZ];	/* Serial Number */
+	char id_model[NVME_MODEL_SZ];	/* Model Number */
 	char id_fwrev[8];		/* Firmware Revision */
 	uint8_t id_rab;			/* Recommended Arbitration Burst */
 	uint8_t id_oui[3];		/* vendor IEEE OUI */