13017 Dump unique configurations and Uberblocks in zdb -lu

Portions contributed by: Ryan Moeller <ryan@iXsystems.com>
Portions contributed by: Jason King <jason.king@joyent.com>
Reviewed by: Tim Chase <tim@chase2k.com>
Reviewed by: Don Brady <don.brady@intel.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed by: John Kennedy <john.kennedy@delphix.com>
Reviewed by: Andy Fiddaman <andy@omniosce.org>
Reviewed by: C Fraire <cfraire@me.com>
Approved by: Richard Lowe <richlowe@richlowe.net>

11 files changed, 500 insertions, 73 deletions

diff --git a/usr/src/cmd/zdb/zdb.c b/usr/src/cmd/zdb/zdb.c
index 8272bbf64f..bbd637c49f 100644
--- a/usr/src/cmd/zdb/zdb.c
+++ b/usr/src/cmd/zdb/zdb.c
@@ -73,6 +73,7 @@
 
 #include <libnvpair.h>
 #include <libzutil.h>
+#include <zfs_fletcher.h>
 
 #include "zdb.h"
 
@@ -2550,7 +2551,7 @@ dump_uberblock(uberblock_t *ub, const char *header, const char *footer)
 		    (unsigned int) ub->ub_mmp_config & 0xFF);
 	}
 
-	if (dump_opt['u'] >= 3) {
+	if (dump_opt['u'] >= 4) {
 		char blkbuf[BP_SPRINTF_LEN];
 		snprintf_blkptr(blkbuf, sizeof (blkbuf), &ub->ub_rootbp);
 		(void) printf("\trootbp = %s\n", blkbuf);
@@ -2630,36 +2631,6 @@ dump_cachefile(const char *cachefile)
 	nvlist_free(config);
 }
 
-#define	ZDB_MAX_UB_HEADER_SIZE 32
-
-static void
-dump_label_uberblocks(vdev_label_t *lbl, uint64_t ashift)
-{
-	vdev_t vd;
-	vdev_t *vdp = &vd;
-	char header[ZDB_MAX_UB_HEADER_SIZE];
-
-	vd.vdev_ashift = ashift;
-	vdp->vdev_top = vdp;
-
-	for (int i = 0; i < VDEV_UBERBLOCK_COUNT(vdp); i++) {
-		uint64_t uoff = VDEV_UBERBLOCK_OFFSET(vdp, i);
-		uberblock_t *ub = (void *)((char *)lbl + uoff);
-
-		if (uberblock_verify(ub))
-			continue;
-
-		if ((dump_opt['u'] < 4) &&
-		    (ub->ub_mmp_magic == MMP_MAGIC) && ub->ub_mmp_delay &&
-		    (i >= VDEV_UBERBLOCK_COUNT(&vd) - MMP_BLOCKS_PER_LABEL))
-			continue;
-
-		(void) snprintf(header, ZDB_MAX_UB_HEADER_SIZE,
-		    "Uberblock[%d]\n", i);
-		dump_uberblock(ub, header, "");
-	}
-}
-
 static char curpath[PATH_MAX];
 
 /*
@@ -2762,17 +2733,179 @@ dump_path(char *ds, char *path)
 	return (err);
 }
 
+typedef struct cksum_record {
+	zio_cksum_t cksum;
+	boolean_t labels[VDEV_LABELS];
+	avl_node_t link;
+} cksum_record_t;
+
 static int
-dump_label(const char *dev)
+cksum_record_compare(const void *x1, const void *x2)
 {
-	int fd;
+	const cksum_record_t *l = (cksum_record_t *)x1;
+	const cksum_record_t *r = (cksum_record_t *)x2;
+	int arraysize = ARRAY_SIZE(l->cksum.zc_word);
+	int difference;
+
+	for (int i = 0; i < arraysize; i++) {
+		difference = AVL_CMP(l->cksum.zc_word[i], r->cksum.zc_word[i]);
+		if (difference)
+			break;
+	}
+
+	return (difference);
+}
+
+static cksum_record_t *
+cksum_record_alloc(zio_cksum_t *cksum, int l)
+{
+	cksum_record_t *rec;
+
+	rec = umem_zalloc(sizeof (*rec), UMEM_NOFAIL);
+	rec->cksum = *cksum;
+	rec->labels[l] = B_TRUE;
+
+	return (rec);
+}
+
+static cksum_record_t *
+cksum_record_lookup(avl_tree_t *tree, zio_cksum_t *cksum)
+{
+	cksum_record_t lookup = { .cksum = *cksum };
+	avl_index_t where;
+
+	return (avl_find(tree, &lookup, &where));
+}
+
+static cksum_record_t *
+cksum_record_insert(avl_tree_t *tree, zio_cksum_t *cksum, int l)
+{
+	cksum_record_t *rec;
+
+	rec = cksum_record_lookup(tree, cksum);
+	if (rec) {
+		rec->labels[l] = B_TRUE;
+	} else {
+		rec = cksum_record_alloc(cksum, l);
+		avl_add(tree, rec);
+	}
+
+	return (rec);
+}
+
+static int
+first_label(cksum_record_t *rec)
+{
+	for (int i = 0; i < VDEV_LABELS; i++)
+		if (rec->labels[i])
+			return (i);
+
+	return (-1);
+}
+
+static void
+print_label_numbers(char *prefix, cksum_record_t *rec)
+{
+	printf("%s", prefix);
+	for (int i = 0; i < VDEV_LABELS; i++)
+		if (rec->labels[i] == B_TRUE)
+			printf("%d ", i);
+	printf("\n");
+}
+
+#define	MAX_UBERBLOCK_COUNT (VDEV_UBERBLOCK_RING >> UBERBLOCK_SHIFT)
+
+typedef struct zdb_label {
 	vdev_label_t label;
+	nvlist_t *config_nv;
+	cksum_record_t *config;
+	cksum_record_t *uberblocks[MAX_UBERBLOCK_COUNT];
+	boolean_t header_printed;
+	boolean_t read_failed;
+} zdb_label_t;
+
+static void
+print_label_header(zdb_label_t *label, int l)
+{
+
+	if (dump_opt['q'])
+		return;
+
+	if (label->header_printed == B_TRUE)
+		return;
+
+	(void) printf("------------------------------------\n");
+	(void) printf("LABEL %d\n", l);
+	(void) printf("------------------------------------\n");
+
+	label->header_printed = B_TRUE;
+}
+
+static void
+dump_config_from_label(zdb_label_t *label, size_t buflen, int l)
+{
+	if (dump_opt['q'])
+		return;
+
+	if ((dump_opt['l'] < 3) && (first_label(label->config) != l))
+		return;
+
+	print_label_header(label, l);
+	dump_nvlist(label->config_nv, 4);
+	print_label_numbers("    labels = ", label->config);
+}
+
+#define	ZDB_MAX_UB_HEADER_SIZE 32
+
+static void
+dump_label_uberblocks(zdb_label_t *label, uint64_t ashift, int label_num)
+{
+
+	vdev_t vd;
+	char header[ZDB_MAX_UB_HEADER_SIZE];
+
+	vd.vdev_ashift = ashift;
+	vd.vdev_top = &vd;
+
+	for (int i = 0; i < VDEV_UBERBLOCK_COUNT(&vd); i++) {
+		uint64_t uoff = VDEV_UBERBLOCK_OFFSET(&vd, i);
+		uberblock_t *ub = (void *)((char *)&label->label + uoff);
+		cksum_record_t *rec = label->uberblocks[i];
+
+		if (rec == NULL) {
+			if (dump_opt['u'] >= 2) {
+				print_label_header(label, label_num);
+				(void) printf("    Uberblock[%d] invalid\n", i);
+			}
+			continue;
+		}
+
+		if ((dump_opt['u'] < 3) && (first_label(rec) != label_num))
+			continue;
+
+		print_label_header(label, label_num);
+		(void) snprintf(header, ZDB_MAX_UB_HEADER_SIZE,
+		    "    Uberblock[%d]\n", i);
+		dump_uberblock(ub, header, "");
+		print_label_numbers("        labels = ", rec);
+	}
+}
+
+static int
+dump_label(const char *dev)
+{
 	char path[MAXPATHLEN];
-	char *buf = label.vl_vdev_phys.vp_nvlist;
-	size_t buflen = sizeof (label.vl_vdev_phys.vp_nvlist);
-	struct stat64 statbuf;
+	zdb_label_t labels[VDEV_LABELS];
 	uint64_t psize, ashift;
-	boolean_t label_found = B_FALSE;
+	struct stat64 statbuf;
+	boolean_t config_found = B_FALSE;
+	boolean_t error = B_FALSE;
+	avl_tree_t config_tree;
+	avl_tree_t uberblock_tree;
+	void *node, *cookie;
+	int fd;
+
+	bzero(labels, sizeof (labels));
 
 	(void) strlcpy(path, dev, sizeof (path));
 	if (dev[0] == '/') {
@@ -2812,49 +2945,117 @@ dump_label(const char *dev)
 		exit(1);
 	}
 
+	avl_create(&config_tree, cksum_record_compare,
+	    sizeof (cksum_record_t), offsetof(cksum_record_t, link));
+	avl_create(&uberblock_tree, cksum_record_compare,
+	    sizeof (cksum_record_t), offsetof(cksum_record_t, link));
+
 	psize = statbuf.st_size;
 	psize = P2ALIGN(psize, (uint64_t)sizeof (vdev_label_t));
+	ashift = SPA_MINBLOCKSHIFT;
 
+	/*
+	 * 1. Read the label from disk
+	 * 2. Unpack the configuration and insert in config tree.
+	 * 3. Traverse all uberblocks and insert in uberblock tree.
+	 */
 	for (int l = 0; l < VDEV_LABELS; l++) {
-		nvlist_t *config = NULL;
-
-		if (!dump_opt['q']) {
-			(void) printf("------------------------------------\n");
-			(void) printf("LABEL %d\n", l);
-			(void) printf("------------------------------------\n");
-		}
-
-		if (pread64(fd, &label, sizeof (label),
-		    vdev_label_offset(psize, l, 0)) != sizeof (label)) {
+		zdb_label_t *label = &labels[l];
+		char *buf = label->label.vl_vdev_phys.vp_nvlist;
+		size_t buflen = sizeof (label->label.vl_vdev_phys.vp_nvlist);
+		nvlist_t *config;
+		cksum_record_t *rec;
+		zio_cksum_t cksum;
+		vdev_t vd;
+
+		if (pread64(fd, &label->label, sizeof (label->label),
+		    vdev_label_offset(psize, l, 0)) != sizeof (label->label)) {
 			if (!dump_opt['q'])
 				(void) printf("failed to read label %d\n", l);
+			label->read_failed = B_TRUE;
+			error = B_TRUE;
 			continue;
 		}
 
-		if (nvlist_unpack(buf, buflen, &config, 0) != 0) {
-			if (!dump_opt['q'])
-				(void) printf("failed to unpack label %d\n", l);
-			ashift = SPA_MINBLOCKSHIFT;
-		} else {
+		label->read_failed = B_FALSE;
+
+		if (nvlist_unpack(buf, buflen, &config, 0) == 0) {
 			nvlist_t *vdev_tree = NULL;
+			size_t size;
 
-			if (!dump_opt['q'])
-				dump_nvlist(config, 4);
 			if ((nvlist_lookup_nvlist(config,
 			    ZPOOL_CONFIG_VDEV_TREE, &vdev_tree) != 0) ||
 			    (nvlist_lookup_uint64(vdev_tree,
 			    ZPOOL_CONFIG_ASHIFT, &ashift) != 0))
 				ashift = SPA_MINBLOCKSHIFT;
-			nvlist_free(config);
-			label_found = B_TRUE;
+
+			if (nvlist_size(config, &size, NV_ENCODE_XDR) != 0)
+				size = buflen;
+
+			fletcher_4_native(buf, size, NULL, &cksum);
+			rec = cksum_record_insert(&config_tree, &cksum, l);
+
+			label->config = rec;
+			label->config_nv = config;
+			config_found = B_TRUE;
+		} else {
+			error = B_TRUE;
+		}
+
+		vd.vdev_ashift = ashift;
+		vd.vdev_top = &vd;
+
+		for (int i = 0; i < VDEV_UBERBLOCK_COUNT(&vd); i++) {
+			uint64_t uoff = VDEV_UBERBLOCK_OFFSET(&vd, i);
+			uberblock_t *ub = (void *)((char *)label + uoff);
+
+			if (uberblock_verify(ub))
+				continue;
+
+			fletcher_4_native(ub, sizeof (*ub), NULL, &cksum);
+			rec = cksum_record_insert(&uberblock_tree, &cksum, l);
+
+			label->uberblocks[i] = rec;
+		}
+	}
+
+	/*
+	 * Dump the label and uberblocks.
+	 */
+	for (int l = 0; l < VDEV_LABELS; l++) {
+		zdb_label_t *label = &labels[l];
+		size_t buflen = sizeof (label->label.vl_vdev_phys.vp_nvlist);
+
+		if (label->read_failed == B_TRUE)
+			continue;
+
+		if (label->config_nv) {
+			dump_config_from_label(label, buflen, l);
+		} else {
+			if (!dump_opt['q'])
+				(void) printf("failed to unpack label %d\n", l);
 		}
 		if (dump_opt['u'])
-			dump_label_uberblocks(&label, ashift);
+			dump_label_uberblocks(label, ashift, l);
+
+		nvlist_free(label->config_nv);
 	}
 
+	cookie = NULL;
+	while ((node = avl_destroy_nodes(&config_tree, &cookie)) != NULL)
+		umem_free(node, sizeof (cksum_record_t));
+
+	cookie = NULL;
+	while ((node = avl_destroy_nodes(&uberblock_tree, &cookie)) != NULL)
+		umem_free(node, sizeof (cksum_record_t));
+
+	avl_destroy(&config_tree);
+	avl_destroy(&uberblock_tree);
+
 	(void) close(fd);
 
-	return (label_found ? 0 : 2);
+	return (config_found == B_FALSE ? 2 :
+	    (error == B_TRUE ? 1 : 0));
 }
 
 static uint64_t dataset_feature_count[SPA_FEATURES];
diff --git a/usr/src/man/man1m/zdb.1m b/usr/src/man/man1m/zdb.1m
index 422fba96d9..9720024f90 100644
--- a/usr/src/man/man1m/zdb.1m
+++ b/usr/src/man/man1m/zdb.1m
@@ -174,18 +174,21 @@ If specified multiple times, display counts of each intent log transaction type.
 Examine the checkpointed state of the pool.
 Note, the on disk format of the pool is not reverted to the checkpointed state.
 .It Fl l Ar device
-Read the vdev labels from the specified device.
+Read the vdev labels from the specified device and dump the unique configuration
+nvlists(t).
 .Nm Fl l
-will return 0 if valid label was found, 1 if error occurred, and 2 if no valid
-labels were found.
+will return 1 if an error occurred, 2 if no configuration nvlist could be
+unpacked (errors or not), and 0 otherwise.
+Specify multiple times to increase verbosity.
 .Pp
 If the
 .Fl q
-option is also specified, don't print the labels.
+option is also specified, don't dump the configurations or the uberblocks.
 .Pp
 If the
 .Fl u
 option is also specified, also display the uberblocks on this device.
+Specify multiple times to increase verbosity.
 .It Fl L
 Disable leak detection and the loading of space maps.
 By default,
diff --git a/usr/src/pkg/manifests/system-test-zfstest.mf b/usr/src/pkg/manifests/system-test-zfstest.mf
index 78e459f92c..eca36deda1 100644
--- a/usr/src/pkg/manifests/system-test-zfstest.mf
+++ b/usr/src/pkg/manifests/system-test-zfstest.mf
@@ -717,6 +717,9 @@ file path=opt/zfs-tests/tests/functional/clean_mirror/setup mode=0555
 file path=opt/zfs-tests/tests/functional/cli_root/cli_common.kshlib mode=0444
 file path=opt/zfs-tests/tests/functional/cli_root/zdb/zdb_001_neg mode=0555
 file path=opt/zfs-tests/tests/functional/cli_root/zdb/zdb_002_pos mode=0555
+file path=opt/zfs-tests/tests/functional/cli_root/zdb/zdb_003_pos mode=0555
+file path=opt/zfs-tests/tests/functional/cli_root/zdb/zdb_004_pos mode=0555
+file path=opt/zfs-tests/tests/functional/cli_root/zdb/zdb_005_pos mode=0555
 file path=opt/zfs-tests/tests/functional/cli_root/zfs/cleanup mode=0555
 file path=opt/zfs-tests/tests/functional/cli_root/zfs/setup mode=0555
 file path=opt/zfs-tests/tests/functional/cli_root/zfs/zfs_001_neg mode=0555
diff --git a/usr/src/test/zfs-tests/include/default.cfg b/usr/src/test/zfs-tests/include/default.cfg
index a4e0e28bee..88ca99b50d 100644
--- a/usr/src/test/zfs-tests/include/default.cfg
+++ b/usr/src/test/zfs-tests/include/default.cfg
@@ -27,6 +27,7 @@
 #
 # Copyright (c) 2012, 2016 by Delphix. All rights reserved.
 # Copyright 2019 Joyent, Inc.
+# Copyright (c) 2017 Lawrence Livermore National Security, LLC.
 #
 
 . $STF_SUITE/include/libtest.shlib
@@ -84,6 +85,13 @@ export TESTDIR0=${TEST_BASE_DIR%%/}/testdir0
 export TESTDIR1=${TEST_BASE_DIR%%/}/testdir1
 export TESTDIR2=${TEST_BASE_DIR%%/}/testdir2
 
+# some temp files
+export TEMPFILE=${TEST_BASE_DIR%%/}/tempfile$$
+export TEMPFILE0=${TEST_BASE_DIR%%/}/tempfile0$$
+export TEMPFILE1=${TEST_BASE_DIR%%/}/tempfile1$$
+export TEMPFILE2=${TEST_BASE_DIR%%/}/tempfile2$$
+
+
 # some test sub file system names
 export TESTSUBFS=subfs
 export TESTSUBFS1=subfs1
diff --git a/usr/src/test/zfs-tests/runfiles/omnios.run b/usr/src/test/zfs-tests/runfiles/omnios.run
index 9cad083b13..5693772b65 100644
--- a/usr/src/test/zfs-tests/runfiles/omnios.run
+++ b/usr/src/test/zfs-tests/runfiles/omnios.run
@@ -107,7 +107,8 @@ tests = [ 'clean_mirror_001_pos', 'clean_mirror_002_pos',
     'clean_mirror_003_pos', 'clean_mirror_004_pos']
 
 [/opt/zfs-tests/tests/functional/cli_root/zdb]
-tests = ['zdb_001_neg', 'zdb_002_pos']
+tests = ['zdb_001_neg', 'zdb_002_pos', 'zdb_003_pos', 'zdb_004_pos',
+    'zdb_005_pos']
 pre =
 post =
 
diff --git a/usr/src/test/zfs-tests/runfiles/openindiana.run b/usr/src/test/zfs-tests/runfiles/openindiana.run
index 809cf72855..b004b2e96a 100644
--- a/usr/src/test/zfs-tests/runfiles/openindiana.run
+++ b/usr/src/test/zfs-tests/runfiles/openindiana.run
@@ -107,7 +107,8 @@ tests = [ 'clean_mirror_001_pos', 'clean_mirror_002_pos',
     'clean_mirror_003_pos', 'clean_mirror_004_pos']
 
 [/opt/zfs-tests/tests/functional/cli_root/zdb]
-tests = ['zdb_001_neg', 'zdb_002_pos']
+tests = ['zdb_001_neg', 'zdb_002_pos', 'zdb_003_pos', 'zdb_004_pos',
+    'zdb_005_pos']
 pre =
 post =
 
diff --git a/usr/src/test/zfs-tests/runfiles/smartos.run b/usr/src/test/zfs-tests/runfiles/smartos.run
index e7cf7279dd..e8f2a35b12 100644
--- a/usr/src/test/zfs-tests/runfiles/smartos.run
+++ b/usr/src/test/zfs-tests/runfiles/smartos.run
@@ -69,7 +69,8 @@ tests = [ 'clean_mirror_001_pos', 'clean_mirror_002_pos',
     'clean_mirror_003_pos', 'clean_mirror_004_pos']
 
 [/opt/zfs-tests/tests/functional/cli_root/zdb]
-tests = ['zdb_001_neg', 'zdb_002_pos']
+tests = ['zdb_001_neg', 'zdb_002_pos', 'zdb_003_pos', 'zdb_004_pos',
+    'zdb_005_pos']
 pre =
 post =
 
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zdb/zdb_003_pos.ksh b/usr/src/test/zfs-tests/tests/functional/cli_root/zdb/zdb_003_pos.ksh
new file mode 100755
index 0000000000..7645df08fa
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zdb/zdb_003_pos.ksh
@@ -0,0 +1,58 @@
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# Description:
+# zdb will not produce redundant dumps of configurations
+#
+# Strategy:
+# 1. Create a pool with two vdevs
+# 2. Copy label 1 from the first vdev to the second vdev
+# 3. Collect zdb -l output for both vdevs
+# 4. Verify that the correct number of configs is dumped for each
+#
+
+log_assert "Verify zdb does not produce redundant dumps of configurations"
+log_onexit cleanup
+
+function cleanup
+{
+	datasetexists $TESTPOOL && destroy_pool $TESTPOOL
+}
+
+verify_runnable "global"
+verify_disk_count "$DISKS" 2
+
+config_count=(1 2)
+set -A DISK $DISKS
+
+default_mirror_setup_noexit $DISKS
+log_must dd if=${DEV_DSKDIR}/${DISK[0]}s0 of=${DEV_DSKDIR}/${DISK[1]}s0 bs=1024 count=256 conv=notrunc
+
+for x in 0 1 ; do
+	config_count=$(zdb -l $DEV_RDSKDIR/${DISK[$x]}s0 | grep -c features_for_read)
+	(( $? != 0)) && log_fail "failed to get config_count from DISK[$x]"
+	log_note "vdev $x: message_count $config_count"
+	[ $config_count -ne ${config_count[$x]} ] && \
+		log_fail "zdb produces an incorrect number of configuration dumps."
+done
+
+cleanup
+
+log_pass "zdb produces unique dumps of configurations."
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zdb/zdb_004_pos.ksh b/usr/src/test/zfs-tests/tests/functional/cli_root/zdb/zdb_004_pos.ksh
new file mode 100755
index 0000000000..8957f1a23f
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zdb/zdb_004_pos.ksh
@@ -0,0 +1,78 @@
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# Description:
+# zdb will not produce redundant dumps of uberblocks
+#
+# Strategy:
+# 1. Create a pool with two vdevs, A and B
+# 2. Offline vdev A
+# 3. Do some I/O
+# 4. Export the pool
+# 5. Copy label 1 from vdev A to vdev B
+# 6. Collect zdb -lu output for vdev B
+# 7. Verify labels 0 and 1 have unique Uberblocks, but 2 and 3 have none
+#
+
+log_assert "Verify zdb produces unique dumps of uberblocks"
+log_onexit cleanup
+
+function cleanup
+{
+	datasetexists $TESTPOOL && destroy_pool $TESTPOOL
+	for DISK in $DISKS; do
+		zpool labelclear -f $DEV_RDSKDIR/$DISK
+	done
+}
+
+verify_runnable "global"
+verify_disk_count "$DISKS" 2
+set -A DISK $DISKS
+
+default_mirror_setup_noexit $DISKS
+log_must zpool offline $TESTPOOL ${DISK[0]}
+log_must dd if=/dev/urandom of=$TESTDIR/testfile bs=1024 count=2
+log_must zpool export $TESTPOOL
+log_must dd if=$DEV_DSKDIR/${DISK[0]}s0 of=$DEV_DSKDIR/${DISK[1]}s0 bs=1024 count=256 conv=notrunc
+
+ubs=$(zdb -lu $DEV_RDSKDIR/${DISK[1]}s0 | grep -e LABEL -e Uberblock -e 'labels = ')
+log_note "vdev 1: ubs $ubs"
+
+ub_dump_counts=$(zdb -lu $DEV_RDSKDIR/${DISK[1]}s0 | \
+	awk '	/LABEL/	{label=$NF; blocks[label]=0};
+		/Uberblock/ {blocks[label]++};
+		END {print blocks[0],blocks[1],blocks[2],blocks[3]}')
+(( $? != 0)) && log_fail "failed to get ub_dump_counts from DISK[1]"
+log_note "vdev 1: ub_dump_counts $ub_dump_counts"
+
+set -A dump_count $ub_dump_counts
+for label in 0 1 2 3; do
+	if [[ $label -lt 2 ]]; then
+		[[ ${dump_count[$label]} -eq 0 ]] && \
+		    log_fail "zdb incorrectly dumps duplicate uberblocks"
+	else
+		[[ ${dump_count[$label]} -ne 0 ]] && \
+		    log_fail "zdb incorrectly dumps duplicate uberblocks"
+	fi
+done
+
+cleanup
+
+log_pass "zdb produces unique dumps of uberblocks"
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zdb/zdb_005_pos.ksh b/usr/src/test/zfs-tests/tests/functional/cli_root/zdb/zdb_005_pos.ksh
new file mode 100755
index 0000000000..5b5ab135be
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zdb/zdb_005_pos.ksh
@@ -0,0 +1,65 @@
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# Description:
+# zdb -l exit codes are correct
+#
+# Strategy:
+# 1. Create a pool
+# 2. Overwrite label 0 on vdev[1] with dd
+# 3. Create an empty file
+# 3. Run zdb -l on vdev[0] and verify exit value 0
+# 4. Run zdb -l on vdev[1] and verify exit value 1
+# 5. Run zdb -l on the empty file and verify exit value 2
+#
+
+log_assert "Verify zdb -l exit codes are correct"
+log_onexit cleanup
+
+function cleanup
+{
+	datasetexists $TESTPOOL && destroy_pool $TESTPOOL
+	rm -f $TEMPFILE
+}
+
+verify_runnable "global"
+verify_disk_count "$DISKS" 2
+
+set -A DISK $DISKS
+
+default_mirror_setup_noexit $DISKS
+log_must dd if=/dev/zero of=$DEV_DSKDIR/${DISK[1]}s0 bs=1024 count=256 conv=notrunc
+log_must truncate -s 0 $TEMPFILE
+
+zdb -l $DEV_RDSKDIR/${DISK[0]}s0
+[[ $? -ne 0 ]] &&
+	log_fail "zdb -l exit codes are incorrect."
+
+zdb -l $DEV_RDSKDIR/${DISK[1]}s0
+[[ $? -ne 1 ]] &&
+	log_fail "zdb -l exit codes are incorrect."
+
+zdb -l $TEMPFILE
+[[ $? -ne 2 ]] &&
+	log_fail "zdb -l exit codes are incorrect."
+
+cleanup
+
+log_pass "zdb -l exit codes are correct."
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_create/create-o_ashift.ksh b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_create/create-o_ashift.ksh
index c4375e0eea..d692518754 100644
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_create/create-o_ashift.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_create/create-o_ashift.ksh
@@ -85,14 +85,22 @@ function verify_device_uberblocks
 	typeset device=$1
 	typeset ubcount=$2
 
-	zdb -quuul $device | egrep '^(\s+)?Uberblock' |
-	    egrep -v 'invalid$' | nawk \
-	    -v ubcount=$ubcount '{ uberblocks[$0]++; }
-	    END { for (i in uberblocks) {
-		count++;
-		if (uberblocks[i] != 4) { exit 1; }
-	    }
-	    if (count != ubcount) { exit 1; } }'
+        zdb -quuul $device | awk -v ubcount=$ubcount '
+            /Uberblock/ && ! /invalid/ { uberblocks[$0]++ }
+            END {
+                count = 0
+                for (i in uberblocks) {
+                    if (uberblocks[i] != 4) {
+                        printf "%s count: %s != 4\n", i, uberblocks[i]
+                        exit 1
+                    }
+                    count++;
+                }
+                if (count != ubcount) {
+                    printf "Total uberblock count: %s != %s\n", count, ubcount
+                    exit 1
+                }
+            }'
 
 	return $?
 }