[illumos-gate merge]

commit 02123a497a63fe1acfaf63446b5b19101fa6c347
    12141 libbe cannot handle vfstab updates in NGZ
commit 58e78d166bd2b5a9dda31250b8e8bad7c7548c0e
    12149 mixed up 32 and 64 bit build
commit f7d2fd30ef2e2b3d5779d5b0ad885081307a4512
    12148 libc: _rtbootld.o needs to be built with -m32
commit bfb9edc9bd178b0ce7fa2fbe1fc66e18e316af4e
    12143 scan code should check the return value of zfs_btree_first
commit da9bf00574dd2845efcb9b791f4b44ecf1d1548c
    12078 loader: rewrite zfs vdev initialization

27 files changed, 548 insertions, 310 deletions

diff --git a/usr/src/boot/Makefile.version b/usr/src/boot/Makefile.version
index 707e31612b..7b7e86bbba 100644
--- a/usr/src/boot/Makefile.version
+++ b/usr/src/boot/Makefile.version
@@ -33,4 +33,4 @@ LOADER_VERSION = 1.1
 # Use date like formatting here, YYYY.MM.DD.XX, without leading zeroes.
 # The version is processed from left to right, the version number can only
 # be increased.
-BOOT_VERSION = $(LOADER_VERSION)-2019.12.18.2
+BOOT_VERSION = $(LOADER_VERSION)-2020.01.06.1
diff --git a/usr/src/boot/lib/libstand/zfs/zfsimpl.c b/usr/src/boot/lib/libstand/zfs/zfsimpl.c
index a85f6f5721..212b5faa52 100644
--- a/usr/src/boot/lib/libstand/zfs/zfsimpl.c
+++ b/usr/src/boot/lib/libstand/zfs/zfsimpl.c
@@ -490,12 +490,12 @@ vdev_read_phys(vdev_t *vdev, const blkptr_t *bp, void *buf,
 	}
 
 	rc = vdev->v_phys_read(vdev, vdev->v_read_priv, offset, buf, psize);
-	if (rc)
-		return (rc);
-	if (bp != NULL)
-		return (zio_checksum_verify(vdev->spa, bp, buf));
+	if (rc == 0) {
+		if (bp != NULL)
+			rc = zio_checksum_verify(vdev->v_spa, bp, buf);
+	}
 
-	return (0);
+	return (rc);
 }
 
 typedef struct remap_segment {
@@ -774,8 +774,10 @@ static vdev_t *
 vdev_lookup_top(spa_t *spa, uint64_t vdev)
 {
 	vdev_t *rvd;
+	vdev_list_t *vlist;
 
-	STAILQ_FOREACH(rvd, &spa->spa_vdevs, v_childlink)
+	vlist = &spa->spa_root_vdev->v_children;
+	STAILQ_FOREACH(rvd, vlist, v_childlink)
 		if (rvd->v_id == vdev)
 			break;
 
@@ -838,7 +840,7 @@ static void
 vdev_indirect_remap(vdev_t *vd, uint64_t offset, uint64_t asize, void *arg)
 {
 	list_t stack;
-	spa_t *spa = vd->spa;
+	spa_t *spa = vd->v_spa;
 	zio_t *zio = arg;
 	remap_segment_t *rs;
 
@@ -931,19 +933,20 @@ static int
 vdev_indirect_read(vdev_t *vdev, const blkptr_t *bp, void *buf,
     off_t offset, size_t bytes)
 {
-	zio_t zio = { 0 };
-	spa_t *spa = vdev->spa;
-	indirect_vsd_t *iv = malloc(sizeof (*iv));
+	zio_t zio;
+	spa_t *spa = vdev->v_spa;
+	indirect_vsd_t *iv;
 	indirect_split_t *first;
 	int rc = EIO;
 
+	iv = calloc(1, sizeof (*iv));
 	if (iv == NULL)
 		return (ENOMEM);
-	bzero(iv, sizeof (*iv));
 
 	list_create(&iv->iv_splits,
 	    sizeof (indirect_split_t), offsetof(indirect_split_t, is_node));
 
+	bzero(&zio, sizeof (zio));
 	zio.io_spa = spa;
 	zio.io_bp = (blkptr_t *)bp;
 	zio.io_data = buf;
@@ -1082,40 +1085,73 @@ vdev_create(uint64_t guid, vdev_read_t *vdev_read)
 	vdev_t *vdev;
 	vdev_indirect_config_t *vic;
 
-	vdev = malloc(sizeof (vdev_t));
-	memset(vdev, 0, sizeof (vdev_t));
-	STAILQ_INIT(&vdev->v_children);
-	vdev->v_guid = guid;
-	vdev->v_state = VDEV_STATE_OFFLINE;
-	vdev->v_read = vdev_read;
+	vdev = calloc(1, sizeof (vdev_t));
+	if (vdev != NULL) {
+		STAILQ_INIT(&vdev->v_children);
+		vdev->v_guid = guid;
+		vdev->v_read = vdev_read;
 
-	vic = &vdev->vdev_indirect_config;
-	vic->vic_prev_indirect_vdev = UINT64_MAX;
-	STAILQ_INSERT_TAIL(&zfs_vdevs, vdev, v_alllink);
+		/*
+		 * root vdev has no read function, we use this fact to
+		 * skip setting up data we do not need for root vdev.
+		 * We only point root vdev from spa.
+		 */
+		if (vdev_read != NULL) {
+			vic = &vdev->vdev_indirect_config;
+			vic->vic_prev_indirect_vdev = UINT64_MAX;
+			STAILQ_INSERT_TAIL(&zfs_vdevs, vdev, v_alllink);
+		}
+	}
 
 	return (vdev);
 }
 
-static int
-vdev_init_from_nvlist(const unsigned char *nvlist, vdev_t *pvdev,
-    vdev_t **vdevp, int is_newer)
+static void
+vdev_set_initial_state(vdev_t *vdev, const unsigned char *nvlist)
 {
-	int rc;
-	uint64_t guid, id, ashift, asize, nparity;
-	const char *type;
-	const char *path;
-	vdev_t *vdev, *kid;
-	const unsigned char *kids;
-	int nkids, i, is_new;
 	uint64_t is_offline, is_faulted, is_degraded, is_removed, isnt_present;
 	uint64_t is_log;
 
-	if (nvlist_find(nvlist, ZPOOL_CONFIG_GUID, DATA_TYPE_UINT64,
-	    NULL, &guid) ||
-	    nvlist_find(nvlist, ZPOOL_CONFIG_ID, DATA_TYPE_UINT64, NULL, &id) ||
+	is_offline = is_removed = is_faulted = is_degraded = isnt_present = 0;
+	is_log = 0;
+	(void) nvlist_find(nvlist, ZPOOL_CONFIG_OFFLINE, DATA_TYPE_UINT64, NULL,
+	    &is_offline);
+	(void) nvlist_find(nvlist, ZPOOL_CONFIG_REMOVED, DATA_TYPE_UINT64, NULL,
+	    &is_removed);
+	(void) nvlist_find(nvlist, ZPOOL_CONFIG_FAULTED, DATA_TYPE_UINT64, NULL,
+	    &is_faulted);
+	(void) nvlist_find(nvlist, ZPOOL_CONFIG_DEGRADED, DATA_TYPE_UINT64,
+	    NULL, &is_degraded);
+	(void) nvlist_find(nvlist, ZPOOL_CONFIG_NOT_PRESENT, DATA_TYPE_UINT64,
+	    NULL, &isnt_present);
+	(void) nvlist_find(nvlist, ZPOOL_CONFIG_IS_LOG, DATA_TYPE_UINT64, NULL,
+	    &is_log);
+
+	if (is_offline != 0)
+		vdev->v_state = VDEV_STATE_OFFLINE;
+	else if (is_removed != 0)
+		vdev->v_state = VDEV_STATE_REMOVED;
+	else if (is_faulted != 0)
+		vdev->v_state = VDEV_STATE_FAULTED;
+	else if (is_degraded != 0)
+		vdev->v_state = VDEV_STATE_DEGRADED;
+	else if (isnt_present != 0)
+		vdev->v_state = VDEV_STATE_CANT_OPEN;
+
+	vdev->v_islog = is_log != 0;
+}
+
+static int
+vdev_init(uint64_t guid, const unsigned char *nvlist, vdev_t **vdevp)
+{
+	uint64_t id, ashift, asize, nparity;
+	const char *path;
+	const char *type;
+	vdev_t *vdev;
+
+	if (nvlist_find(nvlist, ZPOOL_CONFIG_ID, DATA_TYPE_UINT64, NULL, &id) ||
 	    nvlist_find(nvlist, ZPOOL_CONFIG_TYPE, DATA_TYPE_STRING,
 	    NULL, &type)) {
-		printf("ZFS: can't find vdev details\n");
 		return (ENOENT);
 	}
 
@@ -1132,153 +1168,238 @@ vdev_init_from_nvlist(const unsigned char *nvlist, vdev_t *pvdev,
 		return (EIO);
 	}
 
-	is_offline = is_removed = is_faulted = is_degraded = isnt_present = 0;
-	is_log = 0;
-
-	nvlist_find(nvlist, ZPOOL_CONFIG_OFFLINE, DATA_TYPE_UINT64, NULL,
-	    &is_offline);
-	nvlist_find(nvlist, ZPOOL_CONFIG_REMOVED, DATA_TYPE_UINT64, NULL,
-	    &is_removed);
-	nvlist_find(nvlist, ZPOOL_CONFIG_FAULTED, DATA_TYPE_UINT64, NULL,
-	    &is_faulted);
-	nvlist_find(nvlist, ZPOOL_CONFIG_DEGRADED, DATA_TYPE_UINT64, NULL,
-	    &is_degraded);
-	nvlist_find(nvlist, ZPOOL_CONFIG_NOT_PRESENT, DATA_TYPE_UINT64, NULL,
-	    &isnt_present);
-	nvlist_find(nvlist, ZPOOL_CONFIG_IS_LOG, DATA_TYPE_UINT64, NULL,
-	    &is_log);
+	if (strcmp(type, VDEV_TYPE_MIRROR) == 0)
+		vdev = vdev_create(guid, vdev_mirror_read);
+	else if (strcmp(type, VDEV_TYPE_RAIDZ) == 0)
+		vdev = vdev_create(guid, vdev_raidz_read);
+	else if (strcmp(type, VDEV_TYPE_REPLACING) == 0)
+		vdev = vdev_create(guid, vdev_replacing_read);
+	else if (strcmp(type, VDEV_TYPE_INDIRECT) == 0) {
+		vdev_indirect_config_t *vic;
 
-	vdev = vdev_find(guid);
-	if (!vdev) {
-		is_new = 1;
-
-		if (strcmp(type, VDEV_TYPE_MIRROR) == 0)
-			vdev = vdev_create(guid, vdev_mirror_read);
-		else if (strcmp(type, VDEV_TYPE_RAIDZ) == 0)
-			vdev = vdev_create(guid, vdev_raidz_read);
-		else if (strcmp(type, VDEV_TYPE_REPLACING) == 0)
-			vdev = vdev_create(guid, vdev_replacing_read);
-		else if (strcmp(type, VDEV_TYPE_INDIRECT) == 0) {
-			vdev_indirect_config_t *vic;
-
-			vdev = vdev_create(guid, vdev_indirect_read);
+		vdev = vdev_create(guid, vdev_indirect_read);
+		if (vdev != NULL) {
 			vdev->v_state = VDEV_STATE_HEALTHY;
 			vic = &vdev->vdev_indirect_config;
 
 			nvlist_find(nvlist,
-			    ZPOOL_CONFIG_INDIRECT_OBJECT, DATA_TYPE_UINT64,
+			    ZPOOL_CONFIG_INDIRECT_OBJECT,
+			    DATA_TYPE_UINT64,
 			    NULL, &vic->vic_mapping_object);
 			nvlist_find(nvlist,
-			    ZPOOL_CONFIG_INDIRECT_BIRTHS, DATA_TYPE_UINT64,
+			    ZPOOL_CONFIG_INDIRECT_BIRTHS,
+			    DATA_TYPE_UINT64,
 			    NULL, &vic->vic_births_object);
 			nvlist_find(nvlist,
-			    ZPOOL_CONFIG_PREV_INDIRECT_VDEV, DATA_TYPE_UINT64,
+			    ZPOOL_CONFIG_PREV_INDIRECT_VDEV,
+			    DATA_TYPE_UINT64,
 			    NULL, &vic->vic_prev_indirect_vdev);
-		} else
-			vdev = vdev_create(guid, vdev_disk_read);
-
-		vdev->v_id = id;
-		vdev->v_top = pvdev != NULL ? pvdev : vdev;
-		if (nvlist_find(nvlist, ZPOOL_CONFIG_ASHIFT,
-		    DATA_TYPE_UINT64, NULL, &ashift) == 0) {
-			vdev->v_ashift = ashift;
-		} else {
-			vdev->v_ashift = 0;
 		}
-		if (nvlist_find(nvlist, ZPOOL_CONFIG_ASIZE,
-		    DATA_TYPE_UINT64, NULL, &asize) == 0) {
-			vdev->v_psize = asize +
-			    VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE;
-		}
-		if (nvlist_find(nvlist, ZPOOL_CONFIG_NPARITY,
-		    DATA_TYPE_UINT64, NULL, &nparity) == 0) {
-			vdev->v_nparity = nparity;
+	} else {
+		vdev = vdev_create(guid, vdev_disk_read);
+	}
+
+	if (vdev == NULL)
+		return (ENOMEM);
+
+	vdev_set_initial_state(vdev, nvlist);
+	vdev->v_id = id;
+	if (nvlist_find(nvlist, ZPOOL_CONFIG_ASHIFT,
+	    DATA_TYPE_UINT64, NULL, &ashift) == 0)
+		vdev->v_ashift = ashift;
+
+	if (nvlist_find(nvlist, ZPOOL_CONFIG_ASIZE,
+	    DATA_TYPE_UINT64, NULL, &asize) == 0) {
+		vdev->v_psize = asize +
+		    VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE;
+	}
+
+	if (nvlist_find(nvlist, ZPOOL_CONFIG_NPARITY,
+	    DATA_TYPE_UINT64, NULL, &nparity) == 0)
+		vdev->v_nparity = nparity;
+
+	if (nvlist_find(nvlist, ZPOOL_CONFIG_PATH,
+	    DATA_TYPE_STRING, NULL, &path) == 0) {
+		if (strncmp(path, "/dev/dsk/", 9) == 0)
+			path += 9;
+		vdev->v_name = strdup(path);
+		if (nvlist_find(nvlist, ZPOOL_CONFIG_PHYS_PATH,
+		    DATA_TYPE_STRING, NULL, &path) == 0) {
+			vdev->v_phys_path = strdup(path);
 		} else {
-			vdev->v_nparity = 0;
+			vdev->v_phys_path = NULL;
 		}
-		if (nvlist_find(nvlist, ZPOOL_CONFIG_PATH,
+		if (nvlist_find(nvlist, ZPOOL_CONFIG_DEVID,
 		    DATA_TYPE_STRING, NULL, &path) == 0) {
-			if (strncmp(path, "/dev/dsk/", 9) == 0)
-				path += 9;
-			vdev->v_name = strdup(path);
-			if (nvlist_find(nvlist, ZPOOL_CONFIG_PHYS_PATH,
-			    DATA_TYPE_STRING, NULL, &path) == 0) {
-				vdev->v_phys_path = strdup(path);
-			} else {
-				vdev->v_phys_path = NULL;
-			}
-			if (nvlist_find(nvlist, ZPOOL_CONFIG_DEVID,
-			    DATA_TYPE_STRING, NULL, &path) == 0) {
-				vdev->v_devid = strdup(path);
-			} else {
-				vdev->v_devid = NULL;
-			}
+			vdev->v_devid = strdup(path);
 		} else {
-			char *name;
-
-			if (strcmp(type, "raidz") == 0) {
-				if (vdev->v_nparity < 1 ||
-				    vdev->v_nparity > 3) {
-					printf("ZFS: can only boot from disk, "
-					    "mirror, raidz1, raidz2 and raidz3 "
-					    "vdevs\n");
-					return (EIO);
-				}
-				rc = asprintf(&name, "%s%d-%" PRIu64, type,
-				    vdev->v_nparity, id);
-			} else {
-				rc = asprintf(&name, "%s-%" PRIu64, type, id);
-			}
-			if (rc < 0)
-				return (ENOMEM);
-			vdev->v_name = name;
+			vdev->v_devid = NULL;
 		}
-		vdev->v_islog = is_log == 1;
 	} else {
-		is_new = 0;
+		char *name;
+
+		name = NULL;
+		if (strcmp(type, "raidz") == 0) {
+			if (vdev->v_nparity < 1 ||
+			    vdev->v_nparity > 3) {
+				printf("ZFS: invalid raidz parity: %d\n",
+				    vdev->v_nparity);
+				return (EIO);
+			}
+			(void) asprintf(&name, "%s%d-%" PRIu64, type,
+			    vdev->v_nparity, id);
+		} else {
+			(void) asprintf(&name, "%s-%" PRIu64, type, id);
+		}
+		vdev->v_name = name;
 	}
+	*vdevp = vdev;
+	return (0);
+}
+
+/*
+ * Find slot for vdev. We return either NULL to signal to use
+ * STAILQ_INSERT_HEAD, or we return link element to be used with
+ * STAILQ_INSERT_AFTER.
+ */
+static vdev_t *
+vdev_find_previous(vdev_t *top_vdev, vdev_t *vdev)
+{
+	vdev_t *v, *previous;
+
+	if (STAILQ_EMPTY(&top_vdev->v_children))
+		return (NULL);
+
+	previous = NULL;
+	STAILQ_FOREACH(v, &top_vdev->v_children, v_childlink) {
+		if (v->v_id > vdev->v_id)
+			return (previous);
+
+		if (v->v_id == vdev->v_id)
+			return (v);
+
+		if (v->v_id < vdev->v_id)
+			previous = v;
+	}
+	return (previous);
+}
+
+static size_t
+vdev_child_count(vdev_t *vdev)
+{
+	vdev_t *v;
+	size_t count;
+
+	count = 0;
+	STAILQ_FOREACH(v, &vdev->v_children, v_childlink) {
+		count++;
+	}
+	return (count);
+}
+
+/*
+ * Insert vdev into top_vdev children list. List is ordered by v_id.
+ */
+static void
+vdev_insert(vdev_t *top_vdev, vdev_t *vdev)
+{
+	vdev_t *previous;
+	size_t count;
 
-	if (is_new || is_newer) {
+	/*
+	 * The top level vdev can appear in random order, depending how
+	 * the firmware is presenting the disk devices.
+	 * However, we will insert vdev to create list ordered by v_id,
+	 * so we can use either STAILQ_INSERT_HEAD or STAILQ_INSERT_AFTER
+	 * as STAILQ does not have insert before.
+	 */
+	previous = vdev_find_previous(top_vdev, vdev);
+
+	if (previous == NULL) {
+		STAILQ_INSERT_HEAD(&top_vdev->v_children, vdev, v_childlink);
+	} else if (previous->v_id == vdev->v_id) {
 		/*
-		 * This is either new vdev or we've already seen this vdev,
-		 * but from an older vdev label, so let's refresh its state
-		 * from the newer label.
+		 * This vdev was configured from label config,
+		 * do not insert duplicate.
 		 */
-		if (is_offline)
-			vdev->v_state = VDEV_STATE_OFFLINE;
-		else if (is_removed)
-			vdev->v_state = VDEV_STATE_REMOVED;
-		else if (is_faulted)
-			vdev->v_state = VDEV_STATE_FAULTED;
-		else if (is_degraded)
-			vdev->v_state = VDEV_STATE_DEGRADED;
-		else if (isnt_present)
-			vdev->v_state = VDEV_STATE_CANT_OPEN;
+		return;
+	} else {
+		STAILQ_INSERT_AFTER(&top_vdev->v_children, previous, vdev,
+		    v_childlink);
+	}
+
+	count = vdev_child_count(top_vdev);
+	if (top_vdev->v_nchildren < count)
+		top_vdev->v_nchildren = count;
+}
+
+static int
+vdev_from_nvlist(spa_t *spa, uint64_t top_guid, const unsigned char *nvlist)
+{
+	vdev_t *top_vdev, *vdev;
+	const unsigned char *kids;
+	int rc, nkids;
+
+	/* Get top vdev. */
+	top_vdev = vdev_find(top_guid);
+	if (top_vdev == NULL) {
+		rc = vdev_init(top_guid, nvlist, &top_vdev);
+		if (rc != 0)
+			return (rc);
+		top_vdev->v_spa = spa;
+		top_vdev->v_top = top_vdev;
+		vdev_insert(spa->spa_root_vdev, top_vdev);
 	}
 
+	/* Add children if there are any. */
 	rc = nvlist_find(nvlist, ZPOOL_CONFIG_CHILDREN, DATA_TYPE_NVLIST_ARRAY,
 	    &nkids, &kids);
-	/*
-	 * Its ok if we don't have any kids.
-	 */
 	if (rc == 0) {
-		vdev->v_nchildren = nkids;
-		for (i = 0; i < nkids; i++) {
-			rc = vdev_init_from_nvlist(kids, vdev, &kid, is_newer);
-			if (rc)
+		for (int i = 0; i < nkids; i++) {
+			uint64_t guid;
+
+			rc = nvlist_find(kids, ZPOOL_CONFIG_GUID,
+			    DATA_TYPE_UINT64, NULL, &guid);
+			if (rc != 0)
+				return (rc);
+			rc = vdev_init(guid, kids, &vdev);
+			if (rc != 0)
 				return (rc);
-			if (is_new)
-				STAILQ_INSERT_TAIL(&vdev->v_children, kid,
-				    v_childlink);
+
+			vdev->v_spa = spa;
+			vdev->v_top = top_vdev;
+			vdev_insert(top_vdev, vdev);
+
 			kids = nvlist_next(kids);
 		}
 	} else {
-		vdev->v_nchildren = 0;
+		/*
+		 * When there are no children, nvlist_find() does return
+		 * error, reset it because leaf devices have no children.
+		 */
+		rc = 0;
 	}
 
-	if (vdevp)
-		*vdevp = vdev;
-	return (0);
+	return (rc);
+}
+
+static int
+vdev_init_from_label(spa_t *spa, const unsigned char *nvlist)
+{
+	uint64_t pool_guid, top_guid;
+	const unsigned char *vdevs;
+
+	if (nvlist_find(nvlist, ZPOOL_CONFIG_POOL_GUID, DATA_TYPE_UINT64,
+	    NULL, &pool_guid) ||
+	    nvlist_find(nvlist, ZPOOL_CONFIG_TOP_GUID, DATA_TYPE_UINT64,
+	    NULL, &top_guid) ||
+	    nvlist_find(nvlist, ZPOOL_CONFIG_VDEV_TREE, DATA_TYPE_NVLIST,
+	    NULL, &vdevs)) {
+		printf("ZFS: can't find vdev details\n");
+		return (ENOENT);
+	}
+
+	return (vdev_from_nvlist(spa, top_guid, vdevs));
 }
 
 static void
@@ -1288,6 +1409,10 @@ vdev_set_state(vdev_t *vdev)
 	int good_kids;
 	int bad_kids;
 
+	STAILQ_FOREACH(kid, &vdev->v_children, v_childlink) {
+		vdev_set_state(kid);
+	}
+
 	/*
 	 * A mirror or raidz is healthy if all its kids are healthy. A
 	 * mirror is degraded if any of its kids is healthy; a raidz
@@ -1322,6 +1447,104 @@ vdev_set_state(vdev_t *vdev)
 	}
 }
 
+static int
+vdev_update_from_nvlist(uint64_t top_guid, const unsigned char *nvlist)
+{
+	vdev_t *vdev;
+	const unsigned char *kids;
+	int rc, nkids;
+
+	/* Update top vdev. */
+	vdev = vdev_find(top_guid);
+	if (vdev != NULL)
+		vdev_set_initial_state(vdev, nvlist);
+
+	/* Update children if there are any. */
+	rc = nvlist_find(nvlist, ZPOOL_CONFIG_CHILDREN, DATA_TYPE_NVLIST_ARRAY,
+	    &nkids, &kids);
+	if (rc == 0) {
+		for (int i = 0; i < nkids; i++) {
+			uint64_t guid;
+
+			rc = nvlist_find(kids, ZPOOL_CONFIG_GUID,
+			    DATA_TYPE_UINT64, NULL, &guid);
+			if (rc != 0)
+				break;
+
+			vdev = vdev_find(guid);
+			if (vdev != NULL)
+				vdev_set_initial_state(vdev, kids);
+
+			kids = nvlist_next(kids);
+		}
+	} else {
+		rc = 0;
+	}
+
+	return (rc);
+}
+
+static int
+vdev_init_from_nvlist(spa_t *spa, const unsigned char *nvlist)
+{
+	uint64_t pool_guid, vdev_children;
+	const unsigned char *vdevs, *kids;
+	int rc, nkids;
+
+	if (nvlist_find(nvlist, ZPOOL_CONFIG_POOL_GUID, DATA_TYPE_UINT64,
+	    NULL, &pool_guid) ||
+	    nvlist_find(nvlist, ZPOOL_CONFIG_VDEV_CHILDREN, DATA_TYPE_UINT64,
+	    NULL, &vdev_children) ||
+	    nvlist_find(nvlist, ZPOOL_CONFIG_VDEV_TREE, DATA_TYPE_NVLIST,
+	    NULL, &vdevs)) {
+		printf("ZFS: can't find vdev details\n");
+		return (ENOENT);
+	}
+
+	/* Wrong guid?! */
+	if (spa->spa_guid != pool_guid)
+		return (EINVAL);
+
+	spa->spa_root_vdev->v_nchildren = vdev_children;
+
+	rc = nvlist_find(vdevs, ZPOOL_CONFIG_CHILDREN, DATA_TYPE_NVLIST_ARRAY,
+	    &nkids, &kids);
+
+	/*
+	 * MOS config has at least one child for root vdev.
+	 */
+	if (rc != 0)
+		return (rc);
+
+	for (int i = 0; i < nkids; i++) {
+		uint64_t guid;
+		vdev_t *vdev;
+
+		rc = nvlist_find(kids, ZPOOL_CONFIG_GUID, DATA_TYPE_UINT64,
+		    NULL, &guid);
+		if (rc != 0)
+			break;
+		vdev = vdev_find(guid);
+		/*
+		 * Top level vdev is missing, create it.
+		 */
+		if (vdev == NULL)
+			rc = vdev_from_nvlist(spa, guid, kids);
+		else
+			rc = vdev_update_from_nvlist(guid, kids);
+		if (rc != 0)
+			break;
+		kids = nvlist_next(kids);
+	}
+
+	/*
+	 * Re-evaluate top-level vdev state.
+	 */
+	vdev_set_state(spa->spa_root_vdev);
+
+	return (rc);
+}
+
 static spa_t *
 spa_find_by_guid(uint64_t guid)
 {
@@ -1362,7 +1585,7 @@ spa_get_primary_vdev(const spa_t *spa)
 		spa = spa_get_primary();
 	if (spa == NULL)
 		return (NULL);
-	vdev = STAILQ_FIRST(&spa->spa_vdevs);
+	vdev = spa->spa_root_vdev;
 	if (vdev == NULL)
 		return (NULL);
 	for (kid = STAILQ_FIRST(&vdev->v_children); kid != NULL;
@@ -1382,8 +1605,14 @@ spa_create(uint64_t guid, const char *name)
 		free(spa);
 		return (NULL);
 	}
-	STAILQ_INIT(&spa->spa_vdevs);
 	spa->spa_guid = guid;
+	spa->spa_root_vdev = vdev_create(guid, NULL);
+	if (spa->spa_root_vdev == NULL) {
+		free(spa->spa_name);
+		free(spa);
+		return (NULL);
+	}
+	spa->spa_root_vdev->v_name = strdup("root");
 	STAILQ_INSERT_TAIL(&zfs_pools, spa, spa_link);
 
 	return (spa);
@@ -1460,6 +1689,7 @@ spa_status(spa_t *spa)
 {
 	static char bootfs[ZFS_MAXNAMELEN];
 	uint64_t rootid;
+	vdev_list_t *vlist;
 	vdev_t *vdev;
 	int good_kids, bad_kids, degraded_kids, ret;
 	vdev_state_t state;
@@ -1488,7 +1718,8 @@ spa_status(spa_t *spa)
 	good_kids = 0;
 	degraded_kids = 0;
 	bad_kids = 0;
-	STAILQ_FOREACH(vdev, &spa->spa_vdevs, v_childlink) {
+	vlist = &spa->spa_root_vdev->v_children;
+	STAILQ_FOREACH(vdev, vlist, v_childlink) {
 		if (vdev->v_state == VDEV_STATE_HEALTHY)
 			good_kids++;
 		else if (vdev->v_state == VDEV_STATE_DEGRADED)
@@ -1506,7 +1737,8 @@ spa_status(spa_t *spa)
 	ret = print_state(0, spa->spa_name, state);
 	if (ret != 0)
 		return (ret);
-	STAILQ_FOREACH(vdev, &spa->spa_vdevs, v_childlink) {
+
+	STAILQ_FOREACH(vdev, vlist, v_childlink) {
 		ret = vdev_status(vdev, 1);
 		if (ret != 0)
 			return (ret);
@@ -1698,15 +1930,14 @@ vdev_probe(vdev_phys_read_t *phys_read, void *read_priv, spa_t **spap)
 {
 	vdev_t vtmp;
 	spa_t *spa;
-	vdev_t *vdev, *top_vdev, *pool_vdev;
+	vdev_t *vdev;
 	unsigned char *nvlist;
 	uint64_t val;
-	uint64_t guid;
+	uint64_t guid, vdev_children;
 	uint64_t pool_txg, pool_guid;
 	const char *pool_name;
-	const unsigned char *vdevs;
 	const unsigned char *features;
-	int rc, is_newer;
+	int rc;
 
 	/*
 	 * Load the vdev label and figure out which
@@ -1778,18 +2009,17 @@ vdev_probe(vdev_phys_read_t *phys_read, void *read_priv, spa_t **spap)
 	 */
 	spa = spa_find_by_guid(pool_guid);
 	if (spa == NULL) {
+		nvlist_find(nvlist, ZPOOL_CONFIG_VDEV_CHILDREN,
+		    DATA_TYPE_UINT64, NULL, &vdev_children);
 		spa = spa_create(pool_guid, pool_name);
 		if (spa == NULL) {
 			free(nvlist);
 			return (ENOMEM);
 		}
+		spa->spa_root_vdev->v_nchildren = vdev_children;
 	}
-	if (pool_txg > spa->spa_txg) {
+	if (pool_txg > spa->spa_txg)
 		spa->spa_txg = pool_txg;
-		is_newer = 1;
-	} else {
-		is_newer = 0;
-	}
 
 	/*
 	 * Get the vdev tree and create our in-core copy of it.
@@ -1809,39 +2039,25 @@ vdev_probe(vdev_phys_read_t *phys_read, void *read_priv, spa_t **spap)
 		return (EIO);
 	}
 
-	if (nvlist_find(nvlist, ZPOOL_CONFIG_VDEV_TREE, DATA_TYPE_NVLIST,
-	    NULL, &vdevs)) {
-		free(nvlist);
-		return (EIO);
-	}
-
-	rc = vdev_init_from_nvlist(vdevs, NULL, &top_vdev, is_newer);
+	rc = vdev_init_from_label(spa, nvlist);
 	free(nvlist);
 	if (rc != 0)
 		return (rc);
 
 	/*
-	 * Add the toplevel vdev to the pool if its not already there.
-	 */
-	STAILQ_FOREACH(pool_vdev, &spa->spa_vdevs, v_childlink)
-		if (top_vdev == pool_vdev)
-			break;
-
-	if (!pool_vdev && top_vdev) {
-		top_vdev->spa = spa;
-		STAILQ_INSERT_TAIL(&spa->spa_vdevs, top_vdev, v_childlink);
-	}
-
-	/*
 	 * We should already have created an incomplete vdev for this
 	 * vdev. Find it and initialise it with our read proc.
 	 */
 	vdev = vdev_find(guid);
-	if (vdev) {
+	if (vdev != NULL) {
 		vdev->v_phys_read = phys_read;
 		vdev->v_read_priv = read_priv;
-		vdev->v_state = VDEV_STATE_HEALTHY;
 		vdev->v_psize = vtmp.v_psize;
+		/*
+		 * If no other state is set, mark vdev healthy.
+		 */
+		if (vdev->v_state == VDEV_STATE_UNKNOWN)
+			vdev->v_state = VDEV_STATE_HEALTHY;
 	} else {
 		printf("ZFS: inconsistent nvlist contents\n");
 		return (EIO);
@@ -1851,13 +2067,13 @@ vdev_probe(vdev_phys_read_t *phys_read, void *read_priv, spa_t **spap)
 		spa->spa_with_log = vdev->v_islog;
 
 	/* Record boot vdev for spa. */
-	if (is_newer == 1)
+	if (spa->spa_boot_vdev == NULL)
 		spa->spa_boot_vdev = vdev;
 
 	/*
 	 * Re-evaluate top-level vdev state.
 	 */
-	vdev_set_state(top_vdev);
+	vdev_set_state(vdev->v_top);
 
 	/*
 	 * Ok, we are happy with the pool so far. Lets find
@@ -1866,7 +2082,6 @@ vdev_probe(vdev_phys_read_t *phys_read, void *read_priv, spa_t **spap)
 	 */
 	vdev_uberblock_load(vdev, &spa->spa_uberblock);
 
-	vdev->spa = spa;
 	if (spap != NULL)
 		*spap = spa;
 	return (0);
@@ -1961,7 +2176,8 @@ zio_read(const spa_t *spa, const blkptr_t *bp, void *buf)
 	for (i = 0; i < SPA_DVAS_PER_BP; i++) {
 		const dva_t *dva = &bp->blk_dva[i];
 		vdev_t *vdev;
-		int vdevid;
+		vdev_list_t *vlist;
+		uint64_t vdevid;
 		off_t offset;
 
 		if (!dva->dva_word[0] && !dva->dva_word[1])
@@ -1969,7 +2185,8 @@ zio_read(const spa_t *spa, const blkptr_t *bp, void *buf)
 
 		vdevid = DVA_GET_VDEV(dva);
 		offset = DVA_GET_OFFSET(dva);
-		STAILQ_FOREACH(vdev, &spa->spa_vdevs, v_childlink) {
+		vlist = &spa->spa_root_vdev->v_children;
+		STAILQ_FOREACH(vdev, vlist, v_childlink) {
 			if (vdev->v_id == vdevid)
 				break;
 		}
@@ -1978,7 +2195,7 @@ zio_read(const spa_t *spa, const blkptr_t *bp, void *buf)
 
 		size = BP_GET_PSIZE(bp);
 		if (vdev->v_read == vdev_raidz_read) {
-			align = 1ULL << vdev->v_top->v_ashift;
+			align = 1ULL << vdev->v_ashift;
 			if (P2PHASE(size, align) != 0)
 				size = P2ROUNDUP(size, align);
 		}
@@ -3026,9 +3243,7 @@ zfs_spa_init(spa_t *spa)
 	dnode_phys_t dir;
 	uint64_t config_object;
 	unsigned char *nvlist;
-	char *type;
-	const unsigned char *nv;
-	int nkids, rc;
+	int rc;
 
 	if (zio_read(spa, &spa->spa_uberblock.ub_rootbp, &spa->spa_mos)) {
 		printf("ZFS: can't read MOS of pool %s\n", spa->spa_name);
@@ -3066,62 +3281,11 @@ zfs_spa_init(spa_t *spa)
 	if (rc != 0)
 		return (rc);
 
-	/* Update vdevs from MOS config. */
-	if (nvlist_find(nvlist + 4, ZPOOL_CONFIG_VDEV_TREE, DATA_TYPE_NVLIST,
-	    NULL, &nv)) {
-		rc = EIO;
-		goto done;
-	}
-
-	if (nvlist_find(nv, ZPOOL_CONFIG_TYPE, DATA_TYPE_STRING, NULL, &type)) {
-		printf("ZFS: can't find vdev details\n");
-		rc = ENOENT;
-		goto done;
-	}
-	if (strcmp(type, VDEV_TYPE_ROOT) != 0) {
-		rc = ENOENT;
-		goto done;
-	}
-
-	rc = nvlist_find(nv, ZPOOL_CONFIG_CHILDREN, DATA_TYPE_NVLIST_ARRAY,
-	    &nkids, &nv);
-	if (rc != 0)
-		goto done;
-
-	for (int i = 0; i < nkids; i++) {
-		vdev_t *vd, *prev, *kid = NULL;
-		rc = vdev_init_from_nvlist(nv, NULL, &kid, 0);
-		if (rc != 0) {
-			printf("vdev_init_from_nvlist: %d\n", rc);
-			break;
-		}
-		kid->spa = spa;
-		prev = NULL;
-		STAILQ_FOREACH(vd, &spa->spa_vdevs, v_childlink) {
-			/* Already present? */
-			if (kid->v_id == vd->v_id) {
-				kid = NULL;
-				break;
-			}
-			if (vd->v_id > kid->v_id) {
-				if (prev == NULL) {
-					STAILQ_INSERT_HEAD(&spa->spa_vdevs,
-					    kid, v_childlink);
-				} else {
-					STAILQ_INSERT_AFTER(&spa->spa_vdevs,
-					    prev, kid, v_childlink);
-				}
-				kid = NULL;
-				break;
-			}
-			prev = vd;
-		}
-		if (kid != NULL)
-			STAILQ_INSERT_TAIL(&spa->spa_vdevs, kid, v_childlink);
-		nv = nvlist_next(nv);
-	}
-	rc = 0;
-done:
+	/*
+	 * Update vdevs from MOS config. Note, we do skip encoding bytes
+	 * here. See also vdev_label_read_config().
+	 */
+	rc = vdev_init_from_nvlist(spa, nvlist + 4);
 	free(nvlist);
 	return (rc);
 }
diff --git a/usr/src/boot/sys/cddl/boot/zfs/zfsimpl.h b/usr/src/boot/sys/cddl/boot/zfs/zfsimpl.h
index 6b629f8fe5..c57181b670 100644
--- a/usr/src/boot/sys/cddl/boot/zfs/zfsimpl.h
+++ b/usr/src/boot/sys/cddl/boot/zfs/zfsimpl.h
@@ -763,6 +763,7 @@ typedef enum {
 #define	ZPOOL_CONFIG_IS_LOG		"is_log"
 #define	ZPOOL_CONFIG_TIMESTAMP		"timestamp" /* not stored on disk */
 #define	ZPOOL_CONFIG_FEATURES_FOR_READ	"features_for_read"
+#define	ZPOOL_CONFIG_VDEV_CHILDREN	"vdev_children"
 
 /*
  * The persistent vdev state is stored as separate values rather than a single
@@ -1765,13 +1766,13 @@ typedef struct vdev {
 	int		v_ashift;	/* offset to block shift */
 	int		v_nparity;	/* # parity for raidz */
 	struct vdev	*v_top;		/* parent vdev */
-	int		v_nchildren;	/* # children */
+	size_t		v_nchildren;	/* # children */
 	vdev_state_t	v_state;	/* current state */
 	vdev_phys_read_t *v_phys_read;	/* read from raw leaf vdev */
 	vdev_read_t	*v_read;	/* read from vdev */
 	void		*v_read_priv;	/* private data for read function */
 	boolean_t	v_islog;
-	struct spa	*spa;		/* link to spa */
+	struct spa	*v_spa;		/* link to spa */
 	/*
 	 * Values stored in the config for an indirect or removing vdev.
 	 */
@@ -1790,11 +1791,10 @@ typedef struct spa {
 	uint64_t	spa_guid;	/* pool guid */
 	uint64_t	spa_txg;	/* most recent transaction */
 	struct uberblock spa_uberblock;	/* best uberblock so far */
-	vdev_list_t	spa_vdevs;	/* list of all toplevel vdevs */
+	vdev_t		*spa_root_vdev;	/* toplevel vdev container */
 	objset_phys_t	spa_mos;	/* MOS for this pool */
 	zio_cksum_salt_t spa_cksum_salt;	/* secret salt for cksum */
 	void		*spa_cksum_tmpls[ZIO_CHECKSUM_FUNCTIONS];
-	int		spa_inited;	/* initialized */
 	vdev_t		*spa_boot_vdev;	/* boot device for kernel */
 	boolean_t	spa_with_log;	/* this pool has log */
 } spa_t;
diff --git a/usr/src/boot/sys/cddl/boot/zfs/zfssubr.c b/usr/src/boot/sys/cddl/boot/zfs/zfssubr.c
index 4cfd337213..0d509ed4a1 100644
--- a/usr/src/boot/sys/cddl/boot/zfs/zfssubr.c
+++ b/usr/src/boot/sys/cddl/boot/zfs/zfssubr.c
@@ -1643,7 +1643,7 @@ reconstruct:
 		int rv;
 
 		if (data_errors == 0) {
-			rv = raidz_checksum_verify(vd->spa, bp, data, bytes);
+			rv = raidz_checksum_verify(vd->v_spa, bp, data, bytes);
 			if (rv == 0) {
 				/*
 				 * If we read parity information (unnecessarily
@@ -1689,7 +1689,7 @@ reconstruct:
 
 			code = vdev_raidz_reconstruct(rm, tgts, n);
 
-			rv = raidz_checksum_verify(vd->spa, bp, data, bytes);
+			rv = raidz_checksum_verify(vd->v_spa, bp, data, bytes);
 			if (rv == 0) {
 				/*
 				 * If we read more parity disks than were used
@@ -1764,7 +1764,7 @@ reconstruct:
 	if (total_errors > rm->rm_firstdatacol) {
 		error = EIO;
 	} else if (total_errors < rm->rm_firstdatacol &&
-	    (code = vdev_raidz_combrec(vd->spa, rm, bp, data, offset, bytes,
+	    (code = vdev_raidz_combrec(vd->v_spa, rm, bp, data, offset, bytes,
 	    total_errors, data_errors)) != 0) {
 		/*
 		 * If we didn't use all the available parity for the
diff --git a/usr/src/cmd/addbadsec/Makefile b/usr/src/cmd/addbadsec/Makefile
index 54fc4fa23b..4a05f046ae 100644
--- a/usr/src/cmd/addbadsec/Makefile
+++ b/usr/src/cmd/addbadsec/Makefile
@@ -25,7 +25,7 @@
 #
 # Copyright (c) 2018, Joyent, Inc.
 
-PROG= 		addbadsec
+PROG=		addbadsec
 OBJECTS=	addbadsec.o ix_altsctr.o
 SRCS=		$(OBJECTS:.o=.c)
 
@@ -43,7 +43,7 @@ SMATCH=off
 all: $(PROG)
 
 $(PROG):        $(OBJECTS)
-	$(CC) -o $@ $(OBJECTS) $(LDFLAGS) $(LDLIBS)
+	$(LINK.c) -o $@ $(OBJECTS) $(LDLIBS)
 	$(POST_PROCESS)
 
 install: all $(ROOTPROG)
diff --git a/usr/src/cmd/auditreduce/Makefile b/usr/src/cmd/auditreduce/Makefile
index 1c39fe869f..f8b5cd7f0e 100644
--- a/usr/src/cmd/auditreduce/Makefile
+++ b/usr/src/cmd/auditreduce/Makefile
@@ -54,7 +54,7 @@ all: $(PROG)
 install: all $(ROOTUSRSBINPROG)
 
 $(PROG): $(OBJS)
-	$(CC) -o $(PROG) $(OBJS) $(LDFLAGS) $(LDLIBS)
+	$(LINK.c) -o $(PROG) $(OBJS) $(LDLIBS)
 	$(POST_PROCESS)
 
 toktable.o:	$(TABLEDIR)/toktable.c
diff --git a/usr/src/cmd/bart/Makefile b/usr/src/cmd/bart/Makefile
index 47938144ab..e5ac75c33b 100644
--- a/usr/src/cmd/bart/Makefile
+++ b/usr/src/cmd/bart/Makefile
@@ -52,7 +52,7 @@ ROOTLIBDIFFH=	$(DIFFH:%=$(ROOTLIB)/%)
 all:	$(PROG)
 
 $(PROG):	$(OBJS)
-	$(CC) -o $(PROG) $(OBJS) $(LDFLAGS) $(LDLIBS)
+	$(LINK.c) -o $(PROG) $(OBJS) $(LDLIBS)
 	$(POST_PROCESS)
 
 clean:
diff --git a/usr/src/cmd/devfsadm/Makefile.com b/usr/src/cmd/devfsadm/Makefile.com
index 4332220e5f..ec9037409e 100644
--- a/usr/src/cmd/devfsadm/Makefile.com
+++ b/usr/src/cmd/devfsadm/Makefile.com
@@ -185,7 +185,7 @@ $(DEVFSADM_MOD): $(DEVFSADM_OBJ)
 	$(POST_PROCESS)
 
 SUNW_%.so: %.o $(MAPFILES)
-	$(CC) -o $@ $(GSHARED) $(DYNFLAGS) -h $@ $< $(LDLIBS) -lc
+	$(LINK.c) -o $@ $(GSHARED) $(DYNFLAGS) -h $@ $< $(LDLIBS) -lc
 	$(POST_PROCESS_SO)
 
 %.o: $(COMMON)/%.c
diff --git a/usr/src/cmd/fs.d/nfs/nfsd/Makefile b/usr/src/cmd/fs.d/nfs/nfsd/Makefile
index 384d6a0c66..d9553ac9a2 100644
--- a/usr/src/cmd/fs.d/nfs/nfsd/Makefile
+++ b/usr/src/cmd/fs.d/nfs/nfsd/Makefile
@@ -46,7 +46,7 @@ CERRWARN +=	-_gcc=-Wno-extra
 SMATCH=off
 
 $(TYPEPROG):	$(OBJS)
-		$(CC) -o $@ $(OBJS) $(LDFLAGS) $(LDLIBS)
+		$(LINK.c) -o $@ $(OBJS) $(LDLIBS)
 		$(POST_PROCESS)
 
 nfs_tbind.o:	../lib/nfs_tbind.c
diff --git a/usr/src/cmd/genmsg/Makefile b/usr/src/cmd/genmsg/Makefile
index aafd07c843..420bfd58d1 100644
--- a/usr/src/cmd/genmsg/Makefile
+++ b/usr/src/cmd/genmsg/Makefile
@@ -69,7 +69,7 @@ all:    $(PROG)
 install: all $(ROOTPROG)
 
 $(PROG): $(OBJS) $(MAPFILES)
-	$(CC) -o $@ $(OBJS) $(LDFLAGS) $(LDLIBS)
+	$(LINK.c) -o $@ $(OBJS) $(LDLIBS)
 	$(POST_PROCESS)
 
 $(LEXINTSRCS): $(LEXSRCS)
diff --git a/usr/src/cmd/power/Makefile b/usr/src/cmd/power/Makefile
index 77c3699bb9..0b77a62513 100644
--- a/usr/src/cmd/power/Makefile
+++ b/usr/src/cmd/power/Makefile
@@ -27,14 +27,14 @@
 DAEMON_SRCS =	powerd.c sysstat.c
 DAEMON_OBJS =	$(DAEMON_SRCS:%.c=%.o)
 DAEMON =	powerd
-PMCFG_SRCS = 	conf.c parse.c handlers.c
+PMCFG_SRCS =	conf.c parse.c handlers.c
 PMCFG_OBJS =	$(PMCFG_SRCS:%.c=%.o)
-PMCFG = 	pmconfig
-SUSPEND_SRCS = 	sys-suspend.c pm_pam_conv.c
+PMCFG =		pmconfig
+SUSPEND_SRCS =	sys-suspend.c pm_pam_conv.c
 SUSPEND_OBJS =	$(SUSPEND_SRCS:%.c=%.o)
-SUSPEND = 	sys-suspend
-SRCS = 		$(DAEMON_SRCS) $(PMCFG_SRCS) $(SUSPEND_SRCS)
-OBJS = 		$(SRCS:%.c=%.o)
+SUSPEND =	sys-suspend
+SRCS =		$(DAEMON_SRCS) $(PMCFG_SRCS) $(SUSPEND_SRCS)
+OBJS =		$(SRCS:%.c=%.o)
 PROG =		$(DAEMON) $(PMCFG) $(SUSPEND)
 POWERCONF=	power.conf
 ETCFILES =	$(POWERCONF)
@@ -99,7 +99,7 @@ $(DAEMON_OBJS): $(DAEMON_SRCS)
 	$(PROCESS_COMMENT) $@
 
 $(DAEMON): $(DAEMON_OBJS)
-	$(CC) -o $@ $(DAEMON_OBJS) $(LDFLAGS) $(DAEMON_LDLIBS)
+	$(LINK.c) -o $@ $(DAEMON_OBJS) $(DAEMON_LDLIBS)
 	$(POST_PROCESS)
 
 $(PMCFG_OBJS): pmconfig.h
diff --git a/usr/src/cmd/praudit/Makefile b/usr/src/cmd/praudit/Makefile
index 9ae3e01a8b..0749cada9a 100644
--- a/usr/src/cmd/praudit/Makefile
+++ b/usr/src/cmd/praudit/Makefile
@@ -56,7 +56,7 @@ all: $(PROG)
 install: all $(ROOTUSRSBINPROG)
 
 $(PROG): $(OBJS)
-	$(CC) -o $(PROG) $(OBJS) $(LDFLAGS) $(LDLIBS)
+	$(LINK.c) -o $(PROG) $(OBJS) $(LDLIBS)
 	$(POST_PROCESS)
 
 adt_xlate.o: $(XLATEDIR)/adt_xlate.c
diff --git a/usr/src/cmd/vgrind/Makefile b/usr/src/cmd/vgrind/Makefile
index 1a1eb08217..b873dedafb 100644
--- a/usr/src/cmd/vgrind/Makefile
+++ b/usr/src/cmd/vgrind/Makefile
@@ -28,7 +28,7 @@
 # These are the objects associated with the overall vgrind command.
 #
 VFONTEDPR=	vfontedpr
-RETEST= 	retest
+RETEST=		retest
 MACROS=		tmac.vgrind
 LANGDEFS=	vgrindefs
 KSHPROG=	vgrind
@@ -41,8 +41,8 @@ KSHPROG=	vgrind
 # installed.  We omit it here, so that the NSE doesn't spend cycles
 # on it when acquiring and reconciling.
 #
-PROG= 		$(KSHPROG)
-LIBPROG= 	$(VFONTEDPR) $(LANGDEFS)
+PROG=		$(KSHPROG)
+LIBPROG=	$(VFONTEDPR) $(LANGDEFS)
 TMACPROG=	$(MACROS)
 
 VFONTEDPROBJS=	vfontedpr.o vgrindefs.o regexp.o
@@ -91,8 +91,8 @@ ROOTTMACPROG= $(TMACPROG:%=$(ROOTTMAC)/%)
 #
 # Conditional assignments pertinent to installation.
 #
-$(ROOTLIB)/$(LANGDEFS) 	:= FILEMODE= $(LIBFILEMODE)
-$(ROOTTMACPROG) 	:= FILEMODE= 0644
+$(ROOTLIB)/$(LANGDEFS)	:= FILEMODE= $(LIBFILEMODE)
+$(ROOTTMACPROG)		:= FILEMODE= 0644
 
 #
 # The standard set of rules doesn't know about installing into
@@ -116,14 +116,14 @@ $(POFILE): $(POFILES) $(POFILE_KSH)
 	cat $(POFILES) $(POFILE_KSH) > $@
 
 $(VFONTEDPR): $(VFONTEDPROBJS)
-	$(CC) -o $@ $(VFONTEDPROBJS) $(LDFLAGS) $(LDLIBS)
+	$(LINK.c) -o $@ $(VFONTEDPROBJS) $(LDLIBS)
 	$(POST_PROCESS)
 
 $(LANGDEFS):	$(LANGDEFS).src
 	$(CP) $? $@
 
 $(RETEST): $(RETESTOBJS)
-	$(CC) -o $@ $(RETESTOBJS) $(LDFLAGS) $(LDLIBS)
+	$(LINK.c) -o $@ $(RETESTOBJS) $(LDLIBS)
 	$(POST_PROCESS)
 
 #
@@ -131,7 +131,7 @@ $(RETEST): $(RETESTOBJS)
 # matching rules see everything they should.  (This is a safety net.)
 #
 # XXX:	ROOTTMAC shouldn't appear as a dependent; it's here as a
-# 	bandaid(TM) until /usr/lib/tmac becomes a symlink to
+#	bandaid(TM) until /usr/lib/tmac becomes a symlink to
 #	/usr/share/lib/tmac.
 #
 install: all $(ROOTTMAC) $(ROOTPROG) $(ROOTLIBPROG) $(ROOTTMACPROG)
diff --git a/usr/src/grub/grub-0.97/Makefile.solaris.defs b/usr/src/grub/grub-0.97/Makefile.solaris.defs
index 86e79c484b..4c9a5a81b0 100644
--- a/usr/src/grub/grub-0.97/Makefile.solaris.defs
+++ b/usr/src/grub/grub-0.97/Makefile.solaris.defs
@@ -44,9 +44,11 @@ OPTION_DOCS	= $(POUND_SIGN)
 OPTION_FS	= $(POUND_SIGN)
 
 
-BASE_CFLAGS	= -B$(GNUC_ROOT)/bin/ -g $(CPPFLAGS) $(OPTFLAGS) -std=gnu89
-BASE_CCASFLAGS	= -B$(GNUC_ROOT)/bin/ -g $(CPPFLAGS) $(OPTFLAGS)
-BASE_LDFLAGS	=
+BASE_CFLAGS	= $($(MACH)_XARCH)
+BASE_CFLAGS	+= -B$(GNUC_ROOT)/bin/ -g $(CPPFLAGS) $(OPTFLAGS) -std=gnu89
+BASE_CCASFLAGS	= $($(MACH)_XARCH)
+BASE_CCASFLAGS	+= -B$(GNUC_ROOT)/bin/ -g $(CPPFLAGS) $(OPTFLAGS)
+BASE_LDFLAGS	= $($(MACH)_XARCH)
 
 CC		= $(GNUC_ROOT)/bin/gcc
 CFLAGS		= $(BASE_CFLAGS)
diff --git a/usr/src/lib/libbe/common/be_create.c b/usr/src/lib/libbe/common/be_create.c
index 791d678302..4c69ac38f6 100644
--- a/usr/src/lib/libbe/common/be_create.c
+++ b/usr/src/lib/libbe/common/be_create.c
@@ -24,6 +24,7 @@
  * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
  * Copyright (c) 2014, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2016 Martin Matuska. All rights reserved.
+ * Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
  */
 
 /*
@@ -611,6 +612,8 @@ be_copy(nvlist_t *be_attrs)
 	uuid_t		parent_uu = { 0 };
 	char		obe_root_ds[MAXPATHLEN];
 	char		nbe_root_ds[MAXPATHLEN];
+	char		obe_root_container[MAXPATHLEN];
+	char		nbe_root_container[MAXPATHLEN];
 	char		ss[MAXPATHLEN];
 	char		*new_mp = NULL;
 	char		*obe_name = NULL;
@@ -1095,8 +1098,14 @@ be_copy(nvlist_t *be_attrs)
 	/*
 	 * Update new BE's vfstab.
 	 */
-	if ((ret = be_update_vfstab(bt.nbe_name, bt.obe_zpool, bt.nbe_zpool,
-	    &fld, new_mp)) != BE_SUCCESS) {
+
+	be_make_root_container_ds(bt.obe_zpool, obe_root_container,
+	    sizeof (obe_root_container));
+	be_make_root_container_ds(bt.nbe_zpool, nbe_root_container,
+	    sizeof (nbe_root_container));
+
+	if ((ret = be_update_vfstab(bt.nbe_name, obe_root_container,
+	    nbe_root_container, &fld, new_mp)) != BE_SUCCESS) {
 		be_print_err(gettext("be_copy: failed to "
 		    "update new BE's vfstab (%s)\n"), bt.nbe_name);
 		goto done;
diff --git a/usr/src/lib/libbe/common/be_rename.c b/usr/src/lib/libbe/common/be_rename.c
index dee131834c..64c26943bf 100644
--- a/usr/src/lib/libbe/common/be_rename.c
+++ b/usr/src/lib/libbe/common/be_rename.c
@@ -69,6 +69,7 @@ be_rename(nvlist_t *be_attrs)
 	be_fs_list_data_t	fld = { 0 };
 	zfs_handle_t	*zhp = NULL;
 	char		root_ds[MAXPATHLEN];
+	char		be_root_container[MAXPATHLEN];
 	char		*mp = NULL;
 	int		zret = 0, ret = BE_SUCCESS;
 
@@ -205,8 +206,16 @@ be_rename(nvlist_t *be_attrs)
 	}
 
 	/* Update BE's vfstab */
-	if ((ret = be_update_vfstab(bt.nbe_name, bt.obe_zpool, bt.nbe_zpool,
-	    &fld, mp)) != BE_SUCCESS) {
+
+	/*
+	 * Since the new and old BEs reside in the same pool (see above),
+	 * the same variable can be used for the container for both.
+	 */
+	be_make_root_container_ds(bt.obe_zpool, be_root_container,
+	    sizeof (be_root_container));
+
+	if ((ret = be_update_vfstab(bt.nbe_name, be_root_container,
+	    be_root_container, &fld, mp)) != BE_SUCCESS) {
 		be_print_err(gettext("be_rename: "
 		    "failed to update new BE's vfstab (%s)\n"), bt.nbe_name);
 		goto done;
diff --git a/usr/src/lib/libbe/common/be_utils.c b/usr/src/lib/libbe/common/be_utils.c
index 72b41b3315..f748ad978f 100644
--- a/usr/src/lib/libbe/common/be_utils.c
+++ b/usr/src/lib/libbe/common/be_utils.c
@@ -24,7 +24,7 @@
  * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
  * Copyright 2016 Toomas Soome <tsoome@me.com>
  * Copyright (c) 2015 by Delphix. All rights reserved.
- * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
+ * Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
  * Copyright (c) 2018, Joyent, Inc.
  */
 
@@ -517,6 +517,35 @@ be_make_container_ds(const char *zpool,  char *container_ds,
 }
 
 /*
+ * Function:	be_make_root_container_ds
+ * Description:	Generate string for the BE root container dataset given a pool
+ *              name.
+ * Parameters:
+ *		zpool - pointer zpool name.
+ *		container_ds - pointer to buffer in which to return result
+ *		container_ds_size - size of container_ds
+ * Returns:
+ *		None
+ * Scope:
+ *		Semi-private (library wide use only)
+ */
+void
+be_make_root_container_ds(const char *zpool, char *container_ds,
+    int container_ds_size)
+{
+	char *root;
+
+	be_make_container_ds(zpool, container_ds, container_ds_size);
+
+	/* If the container DS ends with /ROOT, remove it.  */
+
+	if ((root = strrchr(container_ds, '/')) != NULL &&
+	    strcmp(root + 1, BE_CONTAINER_DS_NAME) == 0) {
+		*root = '\0';
+	}
+}
+
+/*
  * Function:	be_make_name_from_ds
  * Description:	This function takes a dataset name and strips off the
  *		BE container dataset portion from the beginning.  The
diff --git a/usr/src/lib/libbe/common/libbe_priv.h b/usr/src/lib/libbe/common/libbe_priv.h
index cbd382242b..f2960c4f17 100644
--- a/usr/src/lib/libbe/common/libbe_priv.h
+++ b/usr/src/lib/libbe/common/libbe_priv.h
@@ -24,7 +24,7 @@
  * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
  * Copyright 2016 Toomas Soome <tsoome@me.com>
  * Copyright (c) 2015 by Delphix. All rights reserved.
- * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
+ * Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #ifndef	_LIBBE_PRIV_H
@@ -175,6 +175,7 @@ boolean_t be_zfs_init(void);
 void be_zfs_fini(void);
 void be_make_root_ds(const char *, const char *, char *, int);
 void be_make_container_ds(const char *, char *, int);
+void be_make_root_container_ds(const char *, char *, int);
 char *be_make_name_from_ds(const char *, char *);
 int be_append_menu(char *, char *, char *, char *, char *);
 int be_remove_menu(char *, char *, char *);
diff --git a/usr/src/lib/libc/i386/Makefile.com b/usr/src/lib/libc/i386/Makefile.com
index 5b51713a06..9987e2ee87 100644
--- a/usr/src/lib/libc/i386/Makefile.com
+++ b/usr/src/lib/libc/i386/Makefile.com
@@ -1279,8 +1279,8 @@ $(LIB_PIC): pics $$(PICS)
 	$(POST_PROCESS_A)
 
 $(LIBCBASE)/crt/_rtbootld.s: $(LIBCBASE)/crt/_rtboot.s $(LIBCBASE)/crt/_rtld.c
-	$(CC) $(CPPFLAGS) -_smatch=off $(CTF_FLAGS) -O -S $(C_PICFLAGS) \
-	    $(LIBCBASE)/crt/_rtld.c -o $(LIBCBASE)/crt/_rtld.s
+	$(CC) $($(MACH)_XARCH) $(CPPFLAGS) -_smatch=off $(CTF_FLAGS) -O -S \
+	    $(C_PICFLAGS) $(LIBCBASE)/crt/_rtld.c -o $(LIBCBASE)/crt/_rtld.s
 	$(CAT) $(LIBCBASE)/crt/_rtboot.s $(LIBCBASE)/crt/_rtld.s > $@
 	$(RM) $(LIBCBASE)/crt/_rtld.s
 
diff --git a/usr/src/ucblib/libcurses/i386/Makefile b/usr/src/ucblib/libcurses/i386/Makefile
index 75f501557f..aa983b2923 100644
--- a/usr/src/ucblib/libcurses/i386/Makefile
+++ b/usr/src/ucblib/libcurses/i386/Makefile
@@ -26,7 +26,7 @@
 include ../Makefile.com
 
 DYNFLAGS += $(DYNFLAGS32)
-BUILD.SO= $(CC) -o $@ $(GSHARED) $(DYNFLAGS) $(PICS) -L $(ROOT)/usr/ucblib $(LDLIBS)
+BUILD.SO= $(CC) $(CFLAGS) -o $@ $(GSHARED) $(DYNFLAGS) $(PICS) -L $(ROOT)/usr/ucblib $(LDLIBS)
 
 .KEEP_STATE:
 
diff --git a/usr/src/ucblib/libdbm/i386/Makefile b/usr/src/ucblib/libdbm/i386/Makefile
index 85705049e2..2d7d1fdc53 100644
--- a/usr/src/ucblib/libdbm/i386/Makefile
+++ b/usr/src/ucblib/libdbm/i386/Makefile
@@ -25,7 +25,7 @@
 
 include ../Makefile.com
 
-BUILD.SO= $(CC) -o $@ $(GSHARED) $(DYNFLAGS) $(PICS) -L $(ROOT)/usr/ucblib $(LDLIBS)
+BUILD.SO= $(CC) $(CFLAGS) -o $@ $(GSHARED) $(DYNFLAGS) $(PICS) -L $(ROOT)/usr/ucblib $(LDLIBS)
 
 .KEEP_STATE:
 
diff --git a/usr/src/ucblib/librpcsoc/i386/Makefile b/usr/src/ucblib/librpcsoc/i386/Makefile
index 8c2e44b15a..adaefd83fb 100644
--- a/usr/src/ucblib/librpcsoc/i386/Makefile
+++ b/usr/src/ucblib/librpcsoc/i386/Makefile
@@ -26,7 +26,7 @@
 include ../Makefile.com
 
 DYNFLAGS += $(DYNFLAGS32)
-BUILD.SO= $(CC) -o $@ $(GSHARED) $(DYNFLAGS) $(PICS) -L $(ROOT)/usr/ucblib $(LDLIBS)
+BUILD.SO= $(CC) $(CFLAGS) -o $@ $(GSHARED) $(DYNFLAGS) $(PICS) -L $(ROOT)/usr/ucblib $(LDLIBS)
 
 LIBS = $(DYNLIB)
 
diff --git a/usr/src/ucblib/libtermcap/i386/Makefile b/usr/src/ucblib/libtermcap/i386/Makefile
index 85705049e2..2d7d1fdc53 100644
--- a/usr/src/ucblib/libtermcap/i386/Makefile
+++ b/usr/src/ucblib/libtermcap/i386/Makefile
@@ -25,7 +25,7 @@
 
 include ../Makefile.com
 
-BUILD.SO= $(CC) -o $@ $(GSHARED) $(DYNFLAGS) $(PICS) -L $(ROOT)/usr/ucblib $(LDLIBS)
+BUILD.SO= $(CC) $(CFLAGS) -o $@ $(GSHARED) $(DYNFLAGS) $(PICS) -L $(ROOT)/usr/ucblib $(LDLIBS)
 
 .KEEP_STATE:
 
diff --git a/usr/src/uts/common/fs/zfs/btree.c b/usr/src/uts/common/fs/zfs/btree.c
index 72c68fb0f5..0c1013d5b7 100644
--- a/usr/src/uts/common/fs/zfs/btree.c
+++ b/usr/src/uts/common/fs/zfs/btree.c
@@ -980,7 +980,7 @@ zfs_btree_bulk_finish(zfs_btree_t *tree)
  * Insert value into tree at the location specified by where.
  */
 void
-zfs_btree_insert(zfs_btree_t *tree, const void *value,
+zfs_btree_add_idx(zfs_btree_t *tree, const void *value,
     const zfs_btree_index_t *where)
 {
 	zfs_btree_index_t idx = {0};
@@ -1294,7 +1294,7 @@ zfs_btree_add(zfs_btree_t *tree, const void *node)
 {
 	zfs_btree_index_t where = {0};
 	VERIFY3P(zfs_btree_find(tree, node, &where), ==, NULL);
-	zfs_btree_insert(tree, node, &where);
+	zfs_btree_add_idx(tree, node, &where);
 }
 
 /* Helper function to free a tree node. */
@@ -1542,7 +1542,7 @@ zfs_btree_remove_from_node(zfs_btree_t *tree, zfs_btree_core_t *node,
 
 /* Remove the element at the specific location. */
 void
-zfs_btree_remove_from(zfs_btree_t *tree, zfs_btree_index_t *where)
+zfs_btree_remove_idx(zfs_btree_t *tree, zfs_btree_index_t *where)
 {
 	size_t size = tree->bt_elem_size;
 	zfs_btree_hdr_t *hdr = where->bti_node;
@@ -1778,7 +1778,7 @@ zfs_btree_remove(zfs_btree_t *tree, const void *value)
 {
 	zfs_btree_index_t where = {0};
 	VERIFY3P(zfs_btree_find(tree, value, &where), !=, NULL);
-	zfs_btree_remove_from(tree, &where);
+	zfs_btree_remove_idx(tree, &where);
 }
 
 /* Return the number of elements in the tree. */
diff --git a/usr/src/uts/common/fs/zfs/dsl_scan.c b/usr/src/uts/common/fs/zfs/dsl_scan.c
index 81c7d599a4..b619719ba9 100644
--- a/usr/src/uts/common/fs/zfs/dsl_scan.c
+++ b/usr/src/uts/common/fs/zfs/dsl_scan.c
@@ -2848,13 +2848,23 @@ scan_io_queue_fetch_ext(dsl_scan_io_queue_t *queue)
 		if (zfs_scan_issue_strategy == 1) {
 			return (range_tree_first(rt));
 		} else if (zfs_scan_issue_strategy == 2) {
+			/*
+			 * We need to get the original entry in the by_addr
+			 * tree so we can modify it.
+			 */
 			range_seg_t *size_rs =
 			    zfs_btree_first(&queue->q_exts_by_size, NULL);
+			if (size_rs == NULL)
+				return (NULL);
 			uint64_t start = rs_get_start(size_rs, rt);
 			uint64_t size = rs_get_end(size_rs, rt) - start;
 			range_seg_t *addr_rs = range_tree_find(rt, start,
 			    size);
 			ASSERT3P(addr_rs, !=, NULL);
+			ASSERT3U(rs_get_start(size_rs, rt), ==,
+			    rs_get_start(addr_rs, rt));
+			ASSERT3U(rs_get_end(size_rs, rt), ==,
+			    rs_get_end(addr_rs, rt));
 			return (addr_rs);
 		}
 	}
@@ -2871,12 +2881,21 @@ scan_io_queue_fetch_ext(dsl_scan_io_queue_t *queue)
 	if (scn->scn_checkpointing) {
 		return (range_tree_first(rt));
 	} else if (scn->scn_clearing) {
+		/*
+		 * We need to get the original entry in the by_addr
+		 * tree so we can modify it.
+		 */
 		range_seg_t *size_rs = zfs_btree_first(&queue->q_exts_by_size,
 		    NULL);
+		if (size_rs == NULL)
+			return (NULL);
 		uint64_t start = rs_get_start(size_rs, rt);
 		uint64_t size = rs_get_end(size_rs, rt) - start;
 		range_seg_t *addr_rs = range_tree_find(rt, start, size);
 		ASSERT3P(addr_rs, !=, NULL);
+		ASSERT3U(rs_get_start(size_rs, rt), ==, rs_get_start(addr_rs,
+		    rt));
+		ASSERT3U(rs_get_end(size_rs, rt), ==, rs_get_end(addr_rs, rt));
 		return (addr_rs);
 	} else {
 		return (NULL);
diff --git a/usr/src/uts/common/fs/zfs/range_tree.c b/usr/src/uts/common/fs/zfs/range_tree.c
index 8b670c6193..8eb1ba384b 100644
--- a/usr/src/uts/common/fs/zfs/range_tree.c
+++ b/usr/src/uts/common/fs/zfs/range_tree.c
@@ -349,7 +349,7 @@ range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill)
 		uint64_t before_start = rs_get_start_raw(rs_before, rt);
 		uint64_t before_fill = rs_get_fill(rs_before, rt);
 		uint64_t after_fill = rs_get_fill(rs_after, rt);
-		zfs_btree_remove_from(&rt->rt_root, &where_before);
+		zfs_btree_remove_idx(&rt->rt_root, &where_before);
 
 		/*
 		 * We have to re-find the node because our old reference is
@@ -385,7 +385,7 @@ range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill)
 		rs_set_start(rs, rt, start);
 		rs_set_end(rs, rt, end);
 		rs_set_fill(rs, rt, fill);
-		zfs_btree_insert(&rt->rt_root, rs, &where);
+		zfs_btree_add_idx(&rt->rt_root, rs, &where);
 	}
 
 	if (gap != 0) {
@@ -488,7 +488,7 @@ range_tree_remove_impl(range_tree_t *rt, uint64_t start, uint64_t size,
 
 		rs_copy(rs, &rs_tmp, rt);
 		if (zfs_btree_next(&rt->rt_root, &where, &where) != NULL)
-			zfs_btree_insert(&rt->rt_root, &newseg, &where);
+			zfs_btree_add_idx(&rt->rt_root, &newseg, &where);
 		else
 			zfs_btree_add(&rt->rt_root, &newseg);
 
@@ -503,7 +503,7 @@ range_tree_remove_impl(range_tree_t *rt, uint64_t start, uint64_t size,
 		rs_set_start(rs, rt, end);
 		rs_copy(rs, &rs_tmp, rt);
 	} else {
-		zfs_btree_remove_from(&rt->rt_root, &where);
+		zfs_btree_remove_idx(&rt->rt_root, &where);
 		rs = NULL;
 	}
 
diff --git a/usr/src/uts/common/fs/zfs/sys/btree.h b/usr/src/uts/common/fs/zfs/sys/btree.h
index 432a6c6886..e518565c83 100644
--- a/usr/src/uts/common/fs/zfs/sys/btree.h
+++ b/usr/src/uts/common/fs/zfs/sys/btree.h
@@ -129,10 +129,10 @@ void zfs_btree_create(zfs_btree_t *, int (*) (const void *, const void *),
 /*
  * Find a node with a matching value in the tree. Returns the matching node
  * found. If not found, it returns NULL and then if "where" is not NULL it sets
- * "where" for use with zfs_btree_insert() or zfs_btree_nearest().
+ * "where" for use with zfs_btree_add_idx() or zfs_btree_nearest().
  *
  * node   - node that has the value being looked for
- * where  - position for use with zfs_btree_nearest() or zfs_btree_insert(),
+ * where  - position for use with zfs_btree_nearest() or zfs_btree_add_idx(),
  *          may be NULL
  */
 void *zfs_btree_find(zfs_btree_t *, const void *, zfs_btree_index_t *);
@@ -143,17 +143,20 @@ void *zfs_btree_find(zfs_btree_t *, const void *, zfs_btree_index_t *);
  * node   - the node to insert
  * where  - position as returned from zfs_btree_find()
  */
-void zfs_btree_insert(zfs_btree_t *, const void *, const zfs_btree_index_t *);
+void zfs_btree_add_idx(zfs_btree_t *, const void *, const zfs_btree_index_t *);
 
 /*
- * Return the first or last valued node in the tree. Will return NULL
- * if the tree is empty.
+ * Return the first or last valued node in the tree. Will return NULL if the
+ * tree is empty. The index can be NULL if the location of the first or last
+ * element isn't required.
  */
 void *zfs_btree_first(zfs_btree_t *, zfs_btree_index_t *);
 void *zfs_btree_last(zfs_btree_t *, zfs_btree_index_t *);
 
 /*
- * Return the next or previous valued node in the tree.
+ * Return the next or previous valued node in the tree. The second index can
+ * safely be NULL, if the location of the next or previous value isn't
+ * required.
  */
 void *zfs_btree_next(zfs_btree_t *, const zfs_btree_index_t *,
     zfs_btree_index_t *);
@@ -167,7 +170,9 @@ void *zfs_btree_get(zfs_btree_t *, zfs_btree_index_t *);
 
 /*
  * Add a single value to the tree. The value must not compare equal to any
- * other node already in the tree.
+ * other node already in the tree. Note that the value will be copied out, not
+ * inserted directly. It is safe to free or destroy the value once this
+ * function returns.
  */
 void zfs_btree_add(zfs_btree_t *, const void *);
 
@@ -181,7 +186,7 @@ void zfs_btree_remove(zfs_btree_t *, const void *);
 /*
  * Remove the value at the given location from the tree.
  */
-void zfs_btree_remove_from(zfs_btree_t *, zfs_btree_index_t *);
+void zfs_btree_remove_idx(zfs_btree_t *, zfs_btree_index_t *);
 
 /*
  * Return the number of nodes in the tree