5767 fix several problems with zfs test suite

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Christopher Siden <christopher.siden@delphix.com>
Approved by: Gordon Ross <gwr@nexenta.com>

40 files changed, 1003 insertions, 586 deletions

diff --git a/usr/src/cmd/zpool/zpool_main.c b/usr/src/cmd/zpool/zpool_main.c
index 8b992efcb4..143ee2155d 100644
--- a/usr/src/cmd/zpool/zpool_main.c
+++ b/usr/src/cmd/zpool/zpool_main.c
@@ -2792,6 +2792,9 @@ print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
 	uint_t c, children;
 	char *vname;
 	boolean_t scripted = cb->cb_scripted;
+	uint64_t islog = B_FALSE;
+	boolean_t haslog = B_FALSE;
+	char *dashes = "%-*s      -      -      -         -      -      -\n";
 
 	verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
 	    (uint64_t **)&vs, &c) == 0);
@@ -2842,24 +2845,47 @@ print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
 		    ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
 			continue;
 
+		if (nvlist_lookup_uint64(child[c],
+		    ZPOOL_CONFIG_IS_LOG, &islog) == 0 && islog) {
+			haslog = B_TRUE;
+			continue;
+		}
+
 		vname = zpool_vdev_name(g_zfs, zhp, child[c], B_FALSE);
 		print_list_stats(zhp, vname, child[c], cb, depth + 2);
 		free(vname);
 	}
 
-	/*
-	 * Include level 2 ARC devices in iostat output
-	 */
+	if (haslog == B_TRUE) {
+		/* LINTED E_SEC_PRINTF_VAR_FMT */
+		(void) printf(dashes, cb->cb_namewidth, "log");
+		for (c = 0; c < children; c++) {
+			if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
+			    &islog) != 0 || !islog)
+				continue;
+			vname = zpool_vdev_name(g_zfs, zhp, child[c], B_FALSE);
+			print_list_stats(zhp, vname, child[c], cb, depth + 2);
+			free(vname);
+		}
+	}
+
 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
-	    &child, &children) != 0)
-		return;
+	    &child, &children) == 0 && children > 0) {
+		/* LINTED E_SEC_PRINTF_VAR_FMT */
+		(void) printf(dashes, cb->cb_namewidth, "cache");
+		for (c = 0; c < children; c++) {
+			vname = zpool_vdev_name(g_zfs, zhp, child[c], B_FALSE);
+			print_list_stats(zhp, vname, child[c], cb, depth + 2);
+			free(vname);
+		}
+	}
 
-	if (children > 0) {
-		(void) printf("%-*s      -      -      -      -      -      "
-		    "-\n", cb->cb_namewidth, "cache");
+	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES, &child,
+	    &children) == 0 && children > 0) {
+		/* LINTED E_SEC_PRINTF_VAR_FMT */
+		(void) printf(dashes, cb->cb_namewidth, "spare");
 		for (c = 0; c < children; c++) {
-			vname = zpool_vdev_name(g_zfs, zhp, child[c],
-			    B_FALSE);
+			vname = zpool_vdev_name(g_zfs, zhp, child[c], B_FALSE);
 			print_list_stats(zhp, vname, child[c], cb, depth + 2);
 			free(vname);
 		}
diff --git a/usr/src/pkg/manifests/system-test-zfstest.mf b/usr/src/pkg/manifests/system-test-zfstest.mf
index b3a8c55353..dab3f35c90 100644
--- a/usr/src/pkg/manifests/system-test-zfstest.mf
+++ b/usr/src/pkg/manifests/system-test-zfstest.mf
@@ -96,6 +96,7 @@ dir path=opt/zfs-tests/tests/functional/features/async_destroy
 dir path=opt/zfs-tests/tests/functional/grow_pool
 dir path=opt/zfs-tests/tests/functional/grow_replicas
 dir path=opt/zfs-tests/tests/functional/history
+dir path=opt/zfs-tests/tests/functional/holes
 dir path=opt/zfs-tests/tests/functional/inheritance
 dir path=opt/zfs-tests/tests/functional/interop
 dir path=opt/zfs-tests/tests/functional/inuse
@@ -147,8 +148,10 @@ file path=opt/zfs-tests/bin/dir_rd_update mode=0555
 file path=opt/zfs-tests/bin/file_check mode=0555
 file path=opt/zfs-tests/bin/file_trunc mode=0555
 file path=opt/zfs-tests/bin/file_write mode=0555
+file path=opt/zfs-tests/bin/getholes mode=0555
 file path=opt/zfs-tests/bin/largest_file mode=0555
 file path=opt/zfs-tests/bin/mkbusy mode=0555
+file path=opt/zfs-tests/bin/mkholes mode=0555
 file path=opt/zfs-tests/bin/mktree mode=0555
 file path=opt/zfs-tests/bin/mmapwrite mode=0555
 file path=opt/zfs-tests/bin/randfree_file mode=0555
@@ -1702,6 +1705,10 @@ file path=opt/zfs-tests/tests/functional/history/sparc.migratedpool.DAT.Z \
 file path=opt/zfs-tests/tests/functional/history/sparc.orig_history.txt \
     mode=0444
 file path=opt/zfs-tests/tests/functional/history/zfs-pool-v4.dat.Z mode=0444
+file path=opt/zfs-tests/tests/functional/holes/cleanup mode=0555
+file path=opt/zfs-tests/tests/functional/holes/holes.shlib mode=0555
+file path=opt/zfs-tests/tests/functional/holes/holes_sanity mode=0555
+file path=opt/zfs-tests/tests/functional/holes/setup mode=0555
 file path=opt/zfs-tests/tests/functional/inheritance/cleanup mode=0555
 file path=opt/zfs-tests/tests/functional/inheritance/config001.cfg mode=0555
 file path=opt/zfs-tests/tests/functional/inheritance/config002.cfg mode=0555
@@ -1836,8 +1843,6 @@ file path=opt/zfs-tests/tests/functional/nopwrite/nopwrite_varying_compression \
 file path=opt/zfs-tests/tests/functional/nopwrite/nopwrite_volume mode=0555
 file path=opt/zfs-tests/tests/functional/nopwrite/setup mode=0555
 file path=opt/zfs-tests/tests/functional/online_offline/cleanup mode=0555
-file path=opt/zfs-tests/tests/functional/online_offline/online_offline.cfg \
-    mode=0555
 file path=opt/zfs-tests/tests/functional/online_offline/online_offline_001_pos \
     mode=0555
 file path=opt/zfs-tests/tests/functional/online_offline/online_offline_002_neg \
@@ -1973,6 +1978,7 @@ file path=opt/zfs-tests/tests/functional/rsend/rsend_010_pos mode=0555
 file path=opt/zfs-tests/tests/functional/rsend/rsend_011_pos mode=0555
 file path=opt/zfs-tests/tests/functional/rsend/rsend_012_pos mode=0555
 file path=opt/zfs-tests/tests/functional/rsend/rsend_013_pos mode=0555
+file path=opt/zfs-tests/tests/functional/rsend/rsend_014_pos mode=0555
 file path=opt/zfs-tests/tests/functional/rsend/setup mode=0555
 file path=opt/zfs-tests/tests/functional/scrub_mirror/cleanup mode=0555
 file path=opt/zfs-tests/tests/functional/scrub_mirror/default.cfg mode=0555
diff --git a/usr/src/test/zfs-tests/cmd/Makefile b/usr/src/test/zfs-tests/cmd/Makefile
index d3f9661a29..031b8ff1f6 100644
--- a/usr/src/test/zfs-tests/cmd/Makefile
+++ b/usr/src/test/zfs-tests/cmd/Makefile
@@ -10,7 +10,7 @@
 #
 
 #
-# Copyright (c) 2012 by Delphix. All rights reserved.
+# Copyright (c) 2012, 2014 by Delphix. All rights reserved.
 #
 
 .PARALLEL: $(SUBDIRS)
@@ -21,8 +21,10 @@ SUBDIRS = chg_usr_exec \
 	file_check \
 	file_trunc \
 	file_write \
+	getholes \
 	largest_file \
 	mkbusy \
+	mkholes \
 	mktree \
 	mmapwrite \
 	randfree_file \
diff --git a/usr/src/test/zfs-tests/cmd/file_trunc/file_trunc.c b/usr/src/test/zfs-tests/cmd/file_trunc/file_trunc.c
index 818492714c..a6e78d153c 100644
--- a/usr/src/test/zfs-tests/cmd/file_trunc/file_trunc.c
+++ b/usr/src/test/zfs-tests/cmd/file_trunc/file_trunc.c
@@ -25,7 +25,7 @@
  */
 
 /*
- * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #include <stdio.h>
@@ -76,7 +76,7 @@ usage(char *execname)
 int
 main(int argc, char *argv[])
 {
-	int i = 0;
+	int i;
 	int fd = -1;
 
 	parse_options(argc, argv);
@@ -87,11 +87,9 @@ main(int argc, char *argv[])
 		exit(3);
 	}
 
-	while (i < count) {
+	for (i = 0; count == 0 || i < count; i++) {
 		(void) do_write(fd);
 		(void) do_trunc(fd);
-
-		i++;
 	}
 
 	(void) close(fd);
@@ -188,7 +186,7 @@ do_write(int fd)
 		exit(5);
 	}
 
-	buf = "ZFS Test Suite Truncation Test";
+	(void) strcpy(buf, "ZFS Test Suite Truncation Test");
 	if (write(fd, buf, bsize) < bsize) {
 		perror("write");
 		exit(6);
diff --git a/usr/src/test/zfs-tests/cmd/getholes/Makefile b/usr/src/test/zfs-tests/cmd/getholes/Makefile
new file mode 100644
index 0000000000..c032dafcf3
--- /dev/null
+++ b/usr/src/test/zfs-tests/cmd/getholes/Makefile
@@ -0,0 +1,22 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
+
+PROG = getholes
+
+include $(SRC)/cmd/Makefile.cmd
+
+LDLIBS += -lcmdutils -lumem -lzfs
+
+include ../Makefile.subdirs
diff --git a/usr/src/test/zfs-tests/cmd/getholes/getholes.c b/usr/src/test/zfs-tests/cmd/getholes/getholes.c
new file mode 100644
index 0000000000..ff54b50704
--- /dev/null
+++ b/usr/src/test/zfs-tests/cmd/getholes/getholes.c
@@ -0,0 +1,194 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright (c) 2014 by Delphix. All rights reserved.
+ */
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <libzfs.h>
+#include <umem.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <sys/types.h>
+#include <sys/list.h>
+#include <sys/stat.h>
+#include <sys/errno.h>
+
+#define	PRINT_HOLE 0x1
+#define	PRINT_DATA 0x2
+#define	PRINT_VERBOSE 0x4
+
+extern int errno;
+
+static void
+usage(char *msg, int exit_value)
+{
+	(void) fprintf(stderr, "getholes [-dhv] filename\n");
+	(void) fprintf(stderr, "%s\n", msg);
+	exit(exit_value);
+}
+
+typedef struct segment {
+	list_node_t	seg_node;
+	int		seg_type;
+	off_t		seg_offset;
+	off_t		seg_len;
+} seg_t;
+
+/*
+ * Return an appropriate whence value, depending on whether the file begins
+ * with a holes or data.
+ */
+static int
+starts_with_hole(int fd)
+{
+	off_t	off;
+
+	if ((off = lseek(fd, 0, SEEK_HOLE)) == -1) {
+		/* ENXIO means no holes were found */
+		if (errno == ENXIO)
+			return (SEEK_DATA);
+		perror("lseek failed");
+		exit(1);
+	}
+
+	return (off == 0 ? SEEK_HOLE : SEEK_DATA);
+}
+
+static void
+print_list(list_t *seg_list, char *fname, int options)
+{
+	uint64_t	lz_holes, bs = 0;
+	uint64_t	hole_blks_seen = 0, data_blks_seen = 0;
+	seg_t		*seg;
+
+	if (0 == bs)
+		if (zfs_get_hole_count(fname, &lz_holes, &bs) != 0) {
+			perror("zfs_get_hole_count");
+			exit(1);
+		}
+
+	while ((seg = list_remove_head(seg_list)) != NULL) {
+		if (options & PRINT_VERBOSE)
+			(void) fprintf(stdout, "%c %llu:%llu\n",
+			    seg->seg_type == SEEK_HOLE ? 'h' : 'd',
+			    seg->seg_offset, seg->seg_len);
+
+		if (seg->seg_type == SEEK_HOLE) {
+			hole_blks_seen += seg->seg_len / bs;
+		} else {
+			data_blks_seen += seg->seg_len / bs;
+		}
+		umem_free(seg, sizeof (seg_t));
+	}
+
+	/* Verify libzfs sees the same number of hole blocks found manually. */
+	if (lz_holes != hole_blks_seen) {
+		(void) fprintf(stderr, "Counted %llu holes, but libzfs found "
+		    "%llu\n", hole_blks_seen, lz_holes);
+		exit(1);
+	}
+
+	if (options & PRINT_HOLE && options & PRINT_DATA) {
+		(void) fprintf(stdout, "datablks: %llu\n", data_blks_seen);
+		(void) fprintf(stdout, "holeblks: %llu\n", hole_blks_seen);
+		return;
+	}
+
+	if (options & PRINT_DATA)
+		(void) fprintf(stdout, "%llu\n", data_blks_seen);
+	if (options & PRINT_HOLE)
+		(void) fprintf(stdout, "%llu\n", hole_blks_seen);
+}
+
+int
+main(int argc, char *argv[])
+{
+	off_t		len, off = 0;
+	int		c, fd, options = 0, whence = SEEK_DATA;
+	struct stat	statbuf;
+	char		*fname;
+	list_t		seg_list;
+	seg_t		*seg = NULL;
+
+	list_create(&seg_list, sizeof (seg_t), offsetof(seg_t, seg_node));
+
+	while ((c = getopt(argc, argv, "dhv")) != -1) {
+		switch (c) {
+		case 'd':
+			options |= PRINT_DATA;
+			break;
+		case 'h':
+			options |= PRINT_HOLE;
+			break;
+		case 'v':
+			options |= PRINT_VERBOSE;
+			break;
+		}
+	}
+	argc -= optind;
+	argv += optind;
+
+	if (argc != 1)
+		usage("Incorrect number of arguments.", 1);
+
+	if ((fname = argv[0]) == NULL)
+		usage("No filename provided.", 1);
+
+	if ((fd = open(fname, O_LARGEFILE | O_RDONLY)) < 0) {
+		perror("open failed");
+		exit(1);
+	}
+
+	if (fstat(fd, &statbuf) != 0) {
+		perror("fstat failed");
+		exit(1);
+	}
+	len = statbuf.st_size;
+
+	whence = starts_with_hole(fd);
+	while ((off = lseek(fd, off, whence)) != -1) {
+		seg_t	*s;
+
+		seg = umem_alloc(sizeof (seg_t), UMEM_DEFAULT);
+		seg->seg_type = whence;
+		seg->seg_offset = off;
+
+		list_insert_tail(&seg_list, seg);
+		if ((s = list_prev(&seg_list, seg)) != NULL)
+			s->seg_len = seg->seg_offset - s->seg_offset;
+
+		whence = whence == SEEK_HOLE ? SEEK_DATA : SEEK_HOLE;
+	}
+	if (errno != ENXIO) {
+		perror("lseek failed");
+		exit(1);
+	}
+	(void) close(fd);
+
+	/*
+	 * If this file ends with a hole block, then populate the length of
+	 * the last segment, otherwise this is the end of the file, so
+	 * discard the remaining zero length segment.
+	 */
+	if (seg && seg->seg_offset != len) {
+		seg->seg_len = len - seg->seg_offset;
+	} else {
+		(void) list_remove_tail(&seg_list);
+	}
+
+	print_list(&seg_list, fname, options);
+	list_destroy(&seg_list);
+	return (0);
+}
diff --git a/usr/src/test/zfs-tests/cmd/mkholes/Makefile b/usr/src/test/zfs-tests/cmd/mkholes/Makefile
new file mode 100644
index 0000000000..6c53ec60ee
--- /dev/null
+++ b/usr/src/test/zfs-tests/cmd/mkholes/Makefile
@@ -0,0 +1,22 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
+
+PROG = mkholes
+
+include $(SRC)/cmd/Makefile.cmd
+
+LDLIBS += -lumem -lcmdutils
+
+include ../Makefile.subdirs
diff --git a/usr/src/test/zfs-tests/cmd/mkholes/mkholes.c b/usr/src/test/zfs-tests/cmd/mkholes/mkholes.c
new file mode 100644
index 0000000000..18fd42aa8c
--- /dev/null
+++ b/usr/src/test/zfs-tests/cmd/mkholes/mkholes.c
@@ -0,0 +1,212 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright (c) 2014 by Delphix. All rights reserved.
+ */
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <umem.h>
+#include <stddef.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/errno.h>
+#include <sys/list.h>
+
+extern int errno;
+
+typedef enum {
+	SEG_HOLE,
+	SEG_DATA,
+	SEG_TYPES
+} seg_type_t;
+
+typedef struct segment {
+	list_node_t	seg_node;
+	seg_type_t	seg_type;
+	off_t		seg_offset;
+	off_t		seg_len;
+} seg_t;
+
+static int
+no_memory(void) {
+	(void) fprintf(stderr, "malloc failed\n");
+	return (UMEM_CALLBACK_EXIT(255));
+}
+
+static void
+usage(char *msg, int exit_value)
+{
+	(void) fprintf(stderr, "mkholes [-d|h offset:length] ... filename\n");
+	(void) fprintf(stderr, "%s\n", msg);
+	exit(exit_value);
+}
+
+static char *
+get_random_buffer(size_t len)
+{
+	int	rand_fd;
+	char	*buf;
+
+	buf = umem_alloc(len, UMEM_NOFAIL);
+
+	/*
+	 * Fill the buffer from /dev/urandom to counteract the
+	 * effects of compression.
+	 */
+	if ((rand_fd = open("/dev/urandom", O_RDONLY)) < 0) {
+		perror("open /dev/urandom failed");
+		exit(1);
+	}
+
+	if (read(rand_fd, buf, len) < 0) {
+		perror("read /dev/urandom failed");
+		exit(1);
+	}
+
+	(void) close(rand_fd);
+
+	return (buf);
+}
+
+static void
+push_segment(list_t *seg_list, seg_type_t seg_type, char *optarg)
+{
+	char		*off_str, *len_str;
+	static off_t	file_size = 0;
+	off_t		off, len;
+	seg_t		*seg;
+
+	off_str = strtok(optarg, ":");
+	len_str = strtok(NULL, ":");
+
+	if (off_str == NULL || len_str == NULL)
+		usage("Bad offset or length", 1);
+
+	off = strtoull(off_str, NULL, 0);
+	len = strtoull(len_str, NULL, 0);
+
+	if (file_size >= off + len)
+		usage("Ranges must ascend and may not overlap.", 1);
+	file_size = off + len;
+
+	seg = umem_alloc(sizeof (seg_t), UMEM_NOFAIL);
+	seg->seg_type = seg_type;
+	seg->seg_offset = off;
+	seg->seg_len = len;
+
+	list_insert_tail(seg_list, seg);
+}
+
+int
+main(int argc, char *argv[])
+{
+	int	c, fd;
+	char	*fname;
+	list_t	seg_list;
+	seg_t	*seg;
+
+	umem_nofail_callback(no_memory);
+	list_create(&seg_list, sizeof (seg_t), offsetof(seg_t, seg_node));
+
+	while ((c = getopt(argc, argv, "d:h:")) != -1) {
+		switch (c) {
+		case 'd':
+			push_segment(&seg_list, SEG_DATA, optarg);
+			break;
+		case 'h':
+			push_segment(&seg_list, SEG_HOLE, optarg);
+			break;
+		}
+	}
+	argc -= optind;
+	argv += optind;
+
+	if ((fname = argv[0]) == NULL)
+		usage("No filename specified", 1);
+	fname = argv[0];
+
+	if ((fd = open(fname, O_LARGEFILE | O_RDWR | O_CREAT | O_SYNC,
+	    00666)) < 0) {
+		perror("open failed");
+		exit(1);
+	}
+
+	while ((seg = list_remove_head(&seg_list)) != NULL) {
+		char	*buf, *vbuf;
+		off_t	off = seg->seg_offset;
+		off_t	len = seg->seg_len;
+
+		if (seg->seg_type == SEG_HOLE) {
+			struct flock	fl;
+			off_t bytes_read = 0;
+			ssize_t readlen = 1024 * 1024 * 16;
+
+			fl.l_whence = SEEK_SET;
+			fl.l_start = off;
+			fl.l_len = len;
+			if (fcntl(fd, F_FREESP, &fl) != 0) {
+				perror("freesp failed");
+				exit(1);
+			}
+
+			buf = (char *)umem_alloc(readlen, UMEM_NOFAIL);
+			vbuf = (char *)umem_zalloc(readlen, UMEM_NOFAIL);
+			while (bytes_read < len) {
+				ssize_t bytes = pread(fd, buf, readlen, off);
+				if (bytes < 0) {
+					perror("pread hole failed");
+					exit(1);
+				}
+
+				if (memcmp(buf, vbuf, bytes) != 0) {
+					(void) fprintf(stderr, "Read back hole "
+					    "didn't match.\n");
+					exit(1);
+				}
+				bytes_read += bytes;
+				off += bytes;
+			}
+
+			umem_free(buf, readlen);
+			umem_free(vbuf, readlen);
+			umem_free(seg, sizeof (seg_t));
+		} else if (seg->seg_type == SEG_DATA) {
+			buf = get_random_buffer(len);
+			vbuf = (char *)umem_alloc(len, UMEM_NOFAIL);
+			if ((pwrite(fd, buf, len, off)) < 0) {
+				perror("pwrite failed");
+				exit(1);
+			}
+
+			if ((pread(fd, vbuf, len, off)) != len) {
+				perror("pread failed");
+				exit(1);
+			}
+
+			if (memcmp(buf, vbuf, len) != 0) {
+				(void) fprintf(stderr, "Read back buf didn't "
+				    "match.\n");
+				exit(1);
+			}
+
+			umem_free(buf, len);
+			umem_free(vbuf, len);
+			umem_free(seg, sizeof (seg_t));
+		}
+	}
+
+	(void) close(fd);
+	return (0);
+}
diff --git a/usr/src/test/zfs-tests/include/commands.cfg b/usr/src/test/zfs-tests/include/commands.cfg
index 1d19d9d473..36f9457197 100644
--- a/usr/src/test/zfs-tests/include/commands.cfg
+++ b/usr/src/test/zfs-tests/include/commands.cfg
@@ -10,7 +10,7 @@
 #
 
 #
-# Copyright (c) 2013 by Delphix. All rights reserved.
+# Copyright (c) 2012, 2014 by Delphix. All rights reserved.
 #
 
 export ARP="/usr/sbin/arp"
@@ -88,6 +88,7 @@ export METAINIT="/usr/sbin/metainit"
 export METASTAT="/usr/sbin/metastat"
 export MKDIR="/usr/bin/mkdir"
 export MKFILE="/usr/sbin/mkfile"
+export MKTEMP="/usr/bin/mktemp"
 export MKNOD="/usr/sbin/mknod"
 export MODINFO="/usr/sbin/modinfo"
 export MODUNLOAD="/usr/sbin/modunload"
diff --git a/usr/src/test/zfs-tests/include/default.cfg b/usr/src/test/zfs-tests/include/default.cfg
index 72079889cc..9580001eb1 100644
--- a/usr/src/test/zfs-tests/include/default.cfg
+++ b/usr/src/test/zfs-tests/include/default.cfg
@@ -25,7 +25,7 @@
 #
 
 #
-# Copyright (c) 2012 by Delphix. All rights reserved.
+# Copyright (c) 2012, 2014 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/include/commands.cfg
@@ -47,8 +47,10 @@ export DIR_RD_UPDATE="/opt/zfs-tests/bin/dir_rd_update"
 export FILE_CHECK="/opt/zfs-tests/bin/file_check"
 export FILE_TRUNC="/opt/zfs-tests/bin/file_trunc"
 export FILE_WRITE="/opt/zfs-tests/bin/file_write"
+export GETHOLES="/opt/zfs-tests/bin/getholes"
 export LARGEST_FILE="/opt/zfs-tests/bin/largest_file"
 export MKBUSY="/opt/zfs-tests/bin/mkbusy"
+export MKHOLES="/opt/zfs-tests/bin/mkholes"
 export MKTREE="/opt/zfs-tests/bin/mktree"
 export MMAPWRITE="/opt/zfs-tests/bin/mmapwrite"
 export RANDFREE_FILE="/opt/zfs-tests/bin/randfree_file"
diff --git a/usr/src/test/zfs-tests/include/libtest.shlib b/usr/src/test/zfs-tests/include/libtest.shlib
index f077e8dc31..7c25516d51 100644
--- a/usr/src/test/zfs-tests/include/libtest.shlib
+++ b/usr/src/test/zfs-tests/include/libtest.shlib
@@ -25,7 +25,7 @@
 #
 
 #
-# Copyright (c) 2013 by Delphix. All rights reserved.
+# Copyright (c) 2012, 2014 by Delphix. All rights reserved.
 #
 
 . ${STF_TOOLS}/contrib/include/logapi.shlib
@@ -2573,7 +2573,7 @@ function is_te_enabled
 # Utility function to determine if a system has multiple cpus.
 function is_mp
 {
-	(($(psrinfo -p) > 1))
+	(($($PSRINFO | $WC -l) > 1))
 }
 
 # Run the given command as the user provided.
@@ -2585,3 +2585,36 @@ function user_run
 	eval \$SU \$user -c \"$@\" > /tmp/out 2>/tmp/err
 	return $?
 }
+
+#
+# Check if the pool contains the specified vdevs
+#
+# $1 pool
+# $2..n <vdev> ...
+#
+# Return 0 if the vdevs are contained in the pool, 1 if any of the specified
+# vdevs is not in the pool, and 2 if pool name is missing.
+#
+function vdevs_in_pool
+{
+	typeset pool=$1
+	typeset vdev
+
+        if [[ -z $pool ]]; then
+                log_note "Missing pool name."
+                return 2
+        fi
+
+	shift
+
+	typeset tmpfile=$($MKTEMP)
+	$ZPOOL list -Hv "$pool" >$tmpfile
+	for vdev in $@; do
+		$GREP -w ${vdev##*/} $tmpfile >/dev/null 2>&1
+		[[ $? -ne 0 ]] && return 1
+	done
+
+	$RM -f $tmpfile
+
+	return 0;
+}
diff --git a/usr/src/test/zfs-tests/runfiles/delphix.run b/usr/src/test/zfs-tests/runfiles/delphix.run
index 61dd469c66..c98ed9f888 100644
--- a/usr/src/test/zfs-tests/runfiles/delphix.run
+++ b/usr/src/test/zfs-tests/runfiles/delphix.run
@@ -351,6 +351,9 @@ tests = ['history_001_pos', 'history_002_pos', 'history_003_pos',
     'history_004_pos', 'history_005_neg', 'history_006_neg', 'history_007_pos',
     'history_008_pos', 'history_009_pos', 'history_010_pos']
 
+[/opt/zfs-tests/tests/functional/holes]
+tests = ['holes_sanity']
+
 [/opt/zfs-tests/tests/functional/inheritance]
 tests = ['inherit_001_pos']
 pre =
@@ -455,7 +458,7 @@ tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_neg']
 tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos',
     'rsend_005_pos', 'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos',
     'rsend_009_pos', 'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos',
-    'rsend_013_pos']
+    'rsend_013_pos', 'rsend_014_pos']
 
 [/opt/zfs-tests/tests/functional/scrub_mirror]
 tests = ['scrub_mirror_001_pos', 'scrub_mirror_002_pos',
diff --git a/usr/src/test/zfs-tests/runfiles/omnios.run b/usr/src/test/zfs-tests/runfiles/omnios.run
index 474365fcd0..8908515500 100644
--- a/usr/src/test/zfs-tests/runfiles/omnios.run
+++ b/usr/src/test/zfs-tests/runfiles/omnios.run
@@ -351,6 +351,9 @@ tests = ['history_001_pos', 'history_002_pos', 'history_003_pos',
     'history_004_pos', 'history_005_neg', 'history_006_neg', 'history_007_pos',
     'history_008_pos', 'history_009_pos', 'history_010_pos']
 
+[/opt/zfs-tests/tests/functional/holes]
+tests = ['holes_sanity']
+
 [/opt/zfs-tests/tests/functional/inheritance]
 tests = ['inherit_001_pos']
 pre =
@@ -455,7 +458,7 @@ tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_neg']
 tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos',
     'rsend_005_pos', 'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos',
     'rsend_009_pos', 'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos',
-    'rsend_013_pos']
+    'rsend_013_pos', 'rsend_014_pos']
 
 [/opt/zfs-tests/tests/functional/scrub_mirror]
 tests = ['scrub_mirror_001_pos', 'scrub_mirror_002_pos',
diff --git a/usr/src/test/zfs-tests/runfiles/openindiana.run b/usr/src/test/zfs-tests/runfiles/openindiana.run
index 474365fcd0..8908515500 100644
--- a/usr/src/test/zfs-tests/runfiles/openindiana.run
+++ b/usr/src/test/zfs-tests/runfiles/openindiana.run
@@ -351,6 +351,9 @@ tests = ['history_001_pos', 'history_002_pos', 'history_003_pos',
     'history_004_pos', 'history_005_neg', 'history_006_neg', 'history_007_pos',
     'history_008_pos', 'history_009_pos', 'history_010_pos']
 
+[/opt/zfs-tests/tests/functional/holes]
+tests = ['holes_sanity']
+
 [/opt/zfs-tests/tests/functional/inheritance]
 tests = ['inherit_001_pos']
 pre =
@@ -455,7 +458,7 @@ tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_neg']
 tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos',
     'rsend_005_pos', 'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos',
     'rsend_009_pos', 'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos',
-    'rsend_013_pos']
+    'rsend_013_pos', 'rsend_014_pos']
 
 [/opt/zfs-tests/tests/functional/scrub_mirror]
 tests = ['scrub_mirror_001_pos', 'scrub_mirror_002_pos',
diff --git a/usr/src/test/zfs-tests/tests/functional/Makefile b/usr/src/test/zfs-tests/tests/functional/Makefile
index 6248d0d42e..65838aba43 100644
--- a/usr/src/test/zfs-tests/tests/functional/Makefile
+++ b/usr/src/test/zfs-tests/tests/functional/Makefile
@@ -11,7 +11,7 @@
 #
 
 #
-# Copyright (c) 2013 by Delphix. All rights reserved.
+# Copyright (c) 2013, 2014 by Delphix. All rights reserved.
 #
 
 .PARALLEL: $(SUBDIRS)
@@ -33,6 +33,7 @@ SUBDIRS = acl \
 	grow_pool \
 	grow_replicas \
 	history \
+	holes \
 	inheritance \
 	interop \
 	inuse \
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zfs_set/readonly_001_pos.ksh b/usr/src/test/zfs-tests/tests/functional/cli_root/zfs_set/readonly_001_pos.ksh
index b59333b775..fd8a9e4daf 100644
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zfs_set/readonly_001_pos.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zfs_set/readonly_001_pos.ksh
@@ -25,6 +25,10 @@
 # Use is subject to license terms.
 #
 
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
+
 . $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
 
 #
@@ -81,7 +85,7 @@ function verify_readonly # $1 dataset, $2 on|off
 	typeset value=$2
 
 	if datasetnonexists $dataset ; then
-		log_note "$dataset not exist!"
+		log_note "$dataset does not exist!"
 		return 1
 	fi
 
@@ -98,18 +102,19 @@ function verify_readonly # $1 dataset, $2 on|off
 			typeset mtpt=$(get_prop mountpoint $dataset)
 			$expect $TOUCH $mtpt/$TESTFILE1
 			$expect $MKDIR -p $mtpt/$TESTDIR1
-			$expect $ECHO 'y' | $RM $mtpt/$TESTFILE0
+			$expect eval "$ECHO 'y' | $RM $mtpt/$TESTFILE0"
 			$expect $RMDIR $mtpt/$TESTDIR0
 
 			if [[ $expect == "log_must" ]] ; then
-				log_must $ECHO 'y' | $RM $mtpt/$TESTFILE1
+				log_must eval "$ECHO 'y' | $RM $mtpt/$TESTFILE1"
 				log_must $RMDIR $mtpt/$TESTDIR1
 				log_must $TOUCH $mtpt/$TESTFILE0
 				log_must $MKDIR -p $mtpt/$TESTDIR0
 			fi
 			;;
 		volume)
-			$expect eval "$ECHO 'y' | $NEWFS /dev/zvol/dsk/$dataset > /dev/null 2>&1"
+			$expect eval "$ECHO 'y' | $NEWFS \
+			    /dev/zvol/dsk/$dataset > /dev/null 2>&1"
 			;;
 		*)
 			;;
@@ -132,7 +137,8 @@ log_must $ZFS clone $TESTPOOL/$TESTFS@$TESTSNAP $TESTPOOL/$TESTCLONE
 if is_global_zone ; then
 	log_must $ZFS snapshot $TESTPOOL/$TESTVOL@$TESTSNAP
 	log_must $ZFS clone $TESTPOOL/$TESTVOL@$TESTSNAP $TESTPOOL/$TESTCLONE1
-	all_datasets="$TESTPOOL $TESTPOOL/$TESTFS $TESTPOOL/$TESTVOL $TESTPOOL/$TESTCLONE $TESTPOOL/$TESTCLONE1"
+	all_datasets="$TESTPOOL $TESTPOOL/$TESTFS $TESTPOOL/$TESTVOL "
+	all_datasets+="$TESTPOOL/$TESTCLONE $TESTPOOL/$TESTCLONE1"
 else
 	all_datasets="$TESTPOOL $TESTPOOL/$TESTFS $TESTPOOL/$TESTCLONE"
 fi
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib b/usr/src/test/zfs-tests/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
index 17c954cd51..5c4363b19e 100644
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
@@ -24,6 +24,10 @@
 # Use is subject to license terms.
 #
 
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
+
 . $STF_SUITE/include/libtest.shlib
 
 set -A VALID_NAME_CHAR a b c d e f g h i j k l m n o p q r s t u v w x y z \
@@ -133,6 +137,9 @@ function random_string
 
 	eval typeset -i count=\${#$char_set[@]}
 
+	# No consumers want an empty string.
+	((len == 0)) && len=3
+
 	typeset str
 	typeset -i i=0
 	while ((i < len)); do
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.cfg b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.cfg
index e8578830fa..05da4cb795 100644
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.cfg
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.cfg
@@ -25,7 +25,7 @@
 #
 
 #
-# Copyright (c) 2012 by Delphix. All rights reserved.
+# Copyright (c) 2012, 2014 by Delphix. All rights reserved.
 #
 
 export DISK_ARRAY_NUM=0
@@ -35,9 +35,7 @@ export DISKSARRAY=""
 #
 # Variables for zpool_add_006
 #
-export STF_TIMEOUT=2400
-export VDEVS_NUM=300
-export FILE_SIZE=100 #100mb
+export VDEVS_NUM=32
 
 function set_disks
 {
@@ -74,27 +72,9 @@ export SIZE="150m"
 export SIZE1="250m"
 export SLICE0=0
 export SLICE1=1
-export SLICE2=2
 export SLICE3=3
 export SLICE4=4
 export SLICE5=5
 export SLICE6=6
-export SLICE7=7
-
-export FILEDISK=filedisk.zpool_add
-export FILEDISK0=filedisk0.zpool_add
-export FILEDISK1=filedisk1.zpool_add
-export FILEDISK2=filedisk2.zpool_add
-export FILEDISK3=filedisk3.zpool_add
 
 export VOLSIZE=64mb
-
-export BYND_MAX_NAME="byondmaxnamelength\
-012345678901234567890123456789\
-012345678901234567890123456789\
-012345678901234567890123456789\
-012345678901234567890123456789\
-012345678901234567890123456789\
-012345678901234567890123456789\
-012345678901234567890123456789\
-012345678901234567890123456789"
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.kshlib b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.kshlib
index 3a3498be87..2ea78fde6d 100644
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.kshlib
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.kshlib
@@ -25,51 +25,13 @@
 #
 
 #
-# Copyright (c) 2012 by Delphix. All rights reserved.
+# Copyright (c) 2012, 2014 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.cfg
 
 #
-# check if the <pool> contains <vdev> ...
-#
-# $1 pool
-# $2..n <vdev> ...
-#
-# Return 0 if <vdev> are contained in the <pool>; 1 if not used; 2 if pool
-# name is missing
-#
-function iscontained
-{
-	typeset pool=$1
-	typeset vdev
-
-        if [[ -z $pool ]]; then
-                log_note "Missing pool name."
-                return 2
-        fi
-
-	shift
-
-	for vdev in $@; do
-
-# remove /dev/dsk in vdev if there is
-		$ECHO $vdev | $GREP "^/dev/dsk" >/dev/null 2>&1
-		(( $? == 0 )) && \
-			vdev=${vdev##*/}
-
-		$ZPOOL status "$pool" | $AWK '$1 == vdevname {exit 1}' \
-			vdevname=$vdev >/dev/null 2>&1
-		(( $? != 1 )) && \
-			return 1
-	done
-
-	return 0;
-
-}
-
-#
 # Find the storage device in /etc/vfstab
 #
 function find_vfstab_dev
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_001_pos.ksh b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_001_pos.ksh
index f0ea12641b..f57eb348bf 100644
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_001_pos.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_001_pos.ksh
@@ -24,6 +24,11 @@
 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
+
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
+
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib
 
@@ -88,7 +93,7 @@ while (( $i < ${#keywords[*]} )); do
 			create_pool "$TESTPOOL" "${disk}s${SLICE6}"
 			log_must poolexists "$TESTPOOL"
 			log_must $ZPOOL add -f "$TESTPOOL" ${keywords[i]} $vdev
-			log_must iscontained "$TESTPOOL" "$vdev"
+			log_must vdevs_in_pool "$TESTPOOL" "$vdev"
 			destroy_pool "$TESTPOOL"
 		done
 
@@ -99,7 +104,7 @@ while (( $i < ${#keywords[*]} )); do
 				"${disk}s${SLICE4}" "${disk}s${SLICE5}"
 			log_must poolexists "$TESTPOOL"
 			log_must $ZPOOL add "$TESTPOOL" ${keywords[i]} $vdev
-			log_must iscontained "$TESTPOOL" "$vdev"
+			log_must vdevs_in_pool "$TESTPOOL" "$vdev"
 			destroy_pool "$TESTPOOL"
 		done
 
@@ -110,7 +115,7 @@ while (( $i < ${#keywords[*]} )); do
 				"${disk}s${SLICE4}" "${disk}s${SLICE5}"
 			log_must poolexists "$TESTPOOL"
 			log_must $ZPOOL add "$TESTPOOL" ${keywords[i]} $vdev
-			log_must iscontained "$TESTPOOL" "$vdev"
+			log_must vdevs_in_pool "$TESTPOOL" "$vdev"
 			destroy_pool "$TESTPOOL"
 		done
 
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_002_pos.ksh b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_002_pos.ksh
index 7e245c43cc..8008669bf8 100644
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_002_pos.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_002_pos.ksh
@@ -24,6 +24,10 @@
 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
+
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib
 
@@ -59,9 +63,9 @@ create_pool "$TESTPOOL" mirror "${disk}s${SLICE0}" "${disk}s${SLICE1}"
 log_must poolexists "$TESTPOOL"
 
 log_mustnot $ZPOOL add "$TESTPOOL" ${disk}s${SLICE3}
-log_mustnot iscontained "$TESTPOOL" "${disk}s${SLICE3}"
+log_mustnot vdevs_in_pool "$TESTPOOL" "${disk}s${SLICE3}"
 
 log_must $ZPOOL add -f "$TESTPOOL" ${disk}s${SLICE3}
-log_must iscontained "$TESTPOOL" "${disk}s${SLICE3}"
+log_must vdevs_in_pool "$TESTPOOL" "${disk}s${SLICE3}"
 
 log_pass "'zpool add -f <pool> <vdev> ...' executes successfully."
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_003_pos.ksh b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_003_pos.ksh
index e4704dbd2e..a0ef7e5e12 100644
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_003_pos.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_003_pos.ksh
@@ -24,6 +24,11 @@
 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
+
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
+
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib
 
@@ -63,7 +68,7 @@ log_must poolexists "$TESTPOOL"
 
 $ZPOOL add -n "$TESTPOOL" ${disk}s${SLICE1} > $tmpfile
 
-log_mustnot iscontained "$TESTPOOL" "${disk}s${SLICE1}"
+log_mustnot vdevs_in_pool "$TESTPOOL" "${disk}s${SLICE1}"
 
 str="would update '$TESTPOOL' to the following configuration:"
 $CAT $tmpfile | $GREP "$str" >/dev/null 2>&1
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_004_pos.ksh b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_004_pos.ksh
index 6f52126874..ced64a4260 100644
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_004_pos.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_004_pos.ksh
@@ -24,6 +24,11 @@
 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
+
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
+
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib
 
@@ -67,6 +72,6 @@ log_must $ZFS create -V $VOLSIZE $TESTPOOL1/$TESTVOL
 
 log_must $ZPOOL add "$TESTPOOL" /dev/zvol/dsk/$TESTPOOL1/$TESTVOL
 
-log_must iscontained "$TESTPOOL" "/dev/zvol/dsk/$TESTPOOL1/$TESTVOL"
+log_must vdevs_in_pool "$TESTPOOL" "/dev/zvol/dsk/$TESTPOOL1/$TESTVOL"
 
 log_pass "'zpool add <pool> <vdev> ...' adds zfs volume to the pool successfully"
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_006_pos.ksh b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_006_pos.ksh
index df263112db..c60814d40d 100644
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_006_pos.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_006_pos.ksh
@@ -24,21 +24,22 @@
 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
+
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
+
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib
 
 #
 # DESCRIPTION:
-# 'zpool add [-f]' can add large numbers of file-in-zfs-filesystem-based vdevs
-# to the specified pool without any errors.
+# Adding a large number of file based vdevs to a zpool works.
 #
 # STRATEGY:
-# 1. Create assigned number of files in ZFS filesystem as vdevs and use the first
-# file to create a pool
-# 2. Add other vdevs to the pool should get success
-# 3  Fill in the filesystem and create a partially written file
-# as vdev
-# 4. Add the new file into the pool should be failed.
+# 1. Create a file based pool.
+# 2. Add 32 file based vdevs to it.
+# 3. Attempt to add a file based vdev that's too small; verify failure.
 #
 
 verify_runnable "global"
@@ -48,106 +49,30 @@ function cleanup
 	poolexists $TESTPOOL1 && \
 		destroy_pool $TESTPOOL1
 
-	datasetexists $TESTPOOL/$TESTFS && \
-		log_must $ZFS destroy -f $TESTPOOL/$TESTFS
 	poolexists $TESTPOOL && \
 		destroy_pool $TESTPOOL
 
-	if [[ -d $TESTDIR ]]; then
-		log_must $RM -rf $TESTDIR
-	fi
-
+	[[ -d $TESTDIR ]] && log_must $RM -rf $TESTDIR
 	partition_cleanup
 }
 
-
-#
-# Create a pool and fs on the assigned disk, and dynamically create large
-# numbers of files as vdevs.(the default value is <VDEVS_NUM>)
-# the first file will be used to create a pool for other vdevs to be added into
-#
-
-function setup_vdevs #<disk>
-{
-	typeset disk=$1
-	typeset -l count=0
-	typeset -l largest_num=0
-	typeset -l slicesize=0
-	typeset vdev=""
-
-
-	#
-	# Get disk size for zfs filesystem
-	#
-	create_pool foo $disk
-	log_must $ZFS create foo/fs
-	typeset -l fs_size=$(get_prop "available" foo/fs)
-	destroy_pool foo
-
-	#64m is the minmum size for pool
-	(( largest_num = fs_size / (1024 * 1024 * 64) ))
-	if (( largest_num < $VDEVS_NUM )); then
-		# minus $largest_num/40 to leave 2.5% space for metadata.
-		(( vdevs_num=largest_num - largest_num/40 ))
-		file_size=64
-		vdev=$disk
-	else
-		vdevs_num=$VDEVS_NUM
-		file_size=$((fs_size / (1024 * 1024 * \
-		    (vdevs_num + vdevs_num / 40))))
-		if (( file_size > FILE_SIZE )); then
-			file_size=$FILE_SIZE
-		fi
-		# plus $vdevs_num/40 to provide enough space for metadata.
-		(( slice_size = file_size * (vdevs_num + vdevs_num/40) ))
-		set_partition 0 "" ${slice_size}m $disk
-		vdev=${disk}s0
-        fi
-
-	create_pool $TESTPOOL $vdev
-	[[ -d $TESTDIR ]] && \
-		log_must $RM -rf $TESTDIR
-        log_must $MKDIR -p $TESTDIR
-        log_must $ZFS create $TESTPOOL/$TESTFS
-        log_must $ZFS set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
-
-	# Create a pool first using the first file, and make subsequent files
-	# ready as vdevs to add to the pool
-
-	log_must $MKFILE ${file_size}m ${TESTDIR}/file.$count
-	create_pool "$TESTPOOL1" "${TESTDIR}/file.$count"
-	log_must poolexists "$TESTPOOL1"
-
-	while (( count < vdevs_num )); do # minus 1 to avoid space non-enough
-		(( count = count + 1 ))
-		log_must $MKFILE ${file_size}m ${TESTDIR}/file.$count
-		vdevs_list="$vdevs_list ${TESTDIR}/file.$count"
-	done
-}
-
-log_assert " 'zpool add [-f]' can add large numbers of vdevs to the specified" \
-	   " pool without any errors."
+log_assert "Adding a large number of file based vdevs to a zpool works."
 log_onexit cleanup
 
-if [[ $DISK_ARRAY_NUM == 0 ]]; then
-        disk=$DISK
-else
-        disk=$DISK0
-fi
+create_pool $TESTPOOL ${DISKS%% *}
+log_must $ZFS create -o mountpoint=$TESTDIR $TESTPOOL/$TESTFS
+log_must $MKFILE 64m $TESTDIR/file.00
+create_pool "$TESTPOOL1" "$TESTDIR/file.00"
 
-vdevs_list=""
-vdevs_num=$VDEVS_NUM
-file_size=$FILE_SIZE
+vdevs_list=$($ECHO $TESTDIR/file.{01..32})
+log_must $MKFILE 64m $vdevs_list
 
-setup_vdevs $disk
 log_must $ZPOOL add -f "$TESTPOOL1" $vdevs_list
-log_must iscontained "$TESTPOOL1" "$vdevs_list"
-
-(( file_size = file_size * (vdevs_num/40 + 1 ) ))
-log_mustnot $MKFILE ${file_size}m ${TESTDIR}/broken_file
+log_must vdevs_in_pool "$TESTPOOL1" "$vdevs_list"
 
+# Attempt to add a file based vdev that's too small.
+log_must $MKFILE 32m $TESTDIR/broken_file
 log_mustnot $ZPOOL add -f "$TESTPOOL1" ${TESTDIR}/broken_file
-log_mustnot iscontained "$TESTPOOL1" "${TESTDIR}/broken_file"
+log_mustnot vdevs_in_pool "$TESTPOOL1" "${TESTDIR}/broken_file"
 
-log_pass "'zpool successfully add [-f]' can add large numbers of vdevs to the" \
-	 "specified pool without any errors."
+log_pass "Adding a large number of file based vdevs to a zpool works."
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create.cfg b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create.cfg
index 1a8a3d876a..3798b929f1 100644
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create.cfg
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create.cfg
@@ -25,7 +25,7 @@
 #
 
 #
-# Copyright (c) 2012 by Delphix. All rights reserved.
+# Copyright (c) 2012, 2014 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
@@ -33,14 +33,7 @@
 export DISK_ARRAY_NUM=0
 export DISK_ARRAY_LIMIT=4
 export DISKSARRAY=""
-
-#
-# Variables for zpool_create_005
-#
-export VDEVS_NUM=300
-export FILE_SIZE=100 #100mb
-export MD_OVERHEAD=10 # 10%
-export POOL_MINSIZE=64  # the minimum size(64m) to create a storage pool
+export VDEVS_NUM=32
 
 function set_disks
 {
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create_004_pos.ksh b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create_004_pos.ksh
index 568da695c1..ee604fce13 100644
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create_004_pos.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create_004_pos.ksh
@@ -26,22 +26,19 @@
 #
 
 #
-# Copyright (c) 2012 by Delphix. All rights reserved.
+# Copyright (c) 2012, 2014 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/tests/functional/cli_root/zpool_create/zpool_create.shlib
 
 #
 # DESCRIPTION:
-# 'zpool create [-f]' can create a storage pool with large number of
-# file-in-zfs-filesystem-based vdevs without any errors.
+# Create a storage pool with many file based vdevs.
 #
 # STRATEGY:
-# 1. Create assigned number of files in ZFS filesystem as vdevs
-# 2. Creating a new pool based on the vdevs should get success
-# 3. Fill in the filesystem and create a partially writen file as vdev
-# 4. Add the new file into vdevs list and create a pool
-# 5. Creating a storage pool with the new vdevs list should be failed.
+# 1. Create assigned number of files in ZFS filesystem as vdevs.
+# 2. Creating a new pool based on the vdevs should work.
+# 3. Creating a pool with a file based vdev that is too small should fail.
 #
 
 verify_runnable "global"
@@ -50,132 +47,34 @@ function cleanup
 {
 	typeset pool=""
 
-	for pool in foo $TESTPOOL2 $TESTPOOL1; do
-		poolexists $pool && \
-			destroy_pool $pool
-	done
-
-	if datasetexists $TESTPOOL/$TESTFS; then
-		log_must $ZFS destroy -f $TESTPOOL/$TESTFS
-	fi
-
-	if poolexists $TESTPOOL; then
-		destroy_pool $TESTPOOL
-	fi
-
-	if [[ -d $TESTDIR ]]; then
-		log_must $RM -rf $TESTDIR
-	fi
+	poolexists $TESTPOOL1 && destroy_pool $TESTPOOL1
+	poolexists $TESTPOOL && destroy_pool $TESTPOOL
 
+	[[ -d $TESTDIR ]] && log_must $RM -rf $TESTDIR
 	partition_disk $SIZE $disk 6
 }
 
-#
-# Create a pool and fs on the assigned disk, and dynamically create large
-# numbers of files as vdevs.(the default value is <VDEVS_NUM>)
-#
-
-function setup_vdevs #<disk>
-{
-	typeset disk=$1
-	typeset -l largest_num=0
-	typeset -l slice_size=0
-	typeset vdev=""
-
-
-	#
-	# Get disk size for zfs filesystem
-	#
-	create_pool foo $disk
-	log_must $ZFS create foo/fs
-	typeset -l fs_size=$(get_prop "available" foo/fs)
-	destroy_pool foo
-
-	(( largest_num = fs_size / (1024 * 1024 * ${POOL_MINSIZE}) ))
-	if (( largest_num < $VDEVS_NUM )); then
-		#minus $largest_num/$MD_OVERHEAD to leave space for metadata
-		(( vdevs_num=largest_num - largest_num/$MD_OVERHEAD ))
-		file_size=$POOL_MINSIZE
-		vdev=$disk
-	else
-		vdevs_num=$VDEVS_NUM
-		(( file_size = fs_size / \
-		 (1024 * 1024 * (vdevs_num + vdevs_num/$MD_OVERHEAD)) ))
-		if (( file_size > FILE_SIZE )); then
-			# If file_size too large, the time cost will increase so
-                        # we limit the file_size to $FILE_SIZE, and thus the
-			# total time spent on creating file can be controlled
-			# within the timeout.
-			file_size=$FILE_SIZE
-		fi
-		# Add $vdevs_num/$MD_OVERHEAD to provide enough space for
-		# metadata
-		(( slice_size = file_size * (vdevs_num + \
-		    vdevs_num/$MD_OVERHEAD) ))
-		set_partition 0 "" ${slice_size}m $disk
-		vdev=${disk}s0
-        fi
-
-	create_pool $TESTPOOL $vdev
-	[[ -d $TESTDIR ]] && \
-		log_must $RM -rf $TESTDIR
-        log_must $MKDIR -p $TESTDIR
-        log_must $ZFS create $TESTPOOL/$TESTFS
-        log_must $ZFS set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
-
-	typeset -l count=0
-	typeset PIDLIST=""
-	while (( count < vdevs_num )); do
-		$MKFILE ${file_size}m ${TESTDIR}/file.$count &
-		PIDLIST="$PIDLIST $!"
-		vdevs_list="$vdevs_list ${TESTDIR}/file.$count"
-		(( count = count + 1 ))
-	done
-
-	# wait all mkfiles to finish
-        wait $PIDLIST
-        if (( $? != 0 )); then
-                log_fail "create vdevs failed."
-        fi
-
-        return 0
-}
-
-log_assert " 'zpool create [-f]' can create a storage pool with large " \
-    "numbers of vdevs without any errors."
+log_assert "Storage pools with $VDEVS_NUM file based vdevs can be created."
 log_onexit cleanup
 
-if [[ -n $DISK ]]; then
-        disk=$DISK
-else
-        disk=$DISK0
-fi
+disk=${DISKS%% *}
+create_pool $TESTPOOL $disk
+log_must $ZFS create -o mountpoint=$TESTDIR $TESTPOOL/$TESTFS
 
-log_note "Create storage pool with number of $VDEVS_NUM file vdevs should " \
-    "succeed."
-vdevs_list=""
-vdevs_num=$VDEVS_NUM
-file_size=$FILE_SIZE
+vdevs_list=$($ECHO $TESTDIR/file.{01..32})
+log_must $MKFILE 64m $vdevs_list
 
-setup_vdevs $disk
-create_pool $TESTPOOL1 $vdevs_list
+create_pool "$TESTPOOL1" $vdevs_list
+log_must vdevs_in_pool "$TESTPOOL1" "$vdevs_list"
 
 if poolexists $TESTPOOL1; then
 	destroy_pool $TESTPOOL1
 else
-	log_fail " Creating pool with large numbers of file-vdevs fail."
+	log_fail "Creating pool with large numbers of file-vdevs failed."
 fi
 
-log_note "Creating storage pool with partially written file as vdev should " \
-    "fail."
-
-left_space=$(get_prop "available" $TESTPOOL/$TESTFS)
-# Count the broken file size. make sure it should be greater than $left_space
-# so, here, we plus a number -- $file_size, this number can be any other number.
-(( file_size = left_space / (1024 * 1024) + file_size ))
-log_mustnot $MKFILE ${file_size}m ${TESTDIR}/broken_file
+log_must $MKFILE 32m $TESTDIR/broken_file
 vdevs_list="$vdevs_list ${TESTDIR}/broken_file"
+log_mustnot $ZPOOL create -f $TESTPOOL1 $vdevs_list
 
-log_mustnot $ZPOOL create -f $TESTPOOL2 $vdevs_list
-
-log_pass "'zpool create [-f]' with $vdevs_num vdevs success."
+log_pass "Storage pools with many file based vdevs can be created."
diff --git a/usr/src/test/zfs-tests/tests/functional/holes/Makefile b/usr/src/test/zfs-tests/tests/functional/holes/Makefile
new file mode 100644
index 0000000000..2ee3940779
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/holes/Makefile
@@ -0,0 +1,43 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
+
+include $(SRC)/Makefile.master
+
+ROOTOPTPKG = $(ROOT)/opt/zfs-tests
+TESTDIR = $(ROOTOPTPKG)/tests/functional/holes
+
+PROGS = cleanup \
+	holes_sanity \
+	setup
+
+FILES = holes.shlib
+
+CMDS = $(PROGS:%=$(TESTDIR)/%) $(FILES:%=$(TESTDIR)/%)
+$(CMDS) := FILEMODE = 0555
+
+all lint clean clobber:
+
+install: $(CMDS)
+
+$(CMDS): $(TESTDIR)
+
+$(TESTDIR):
+	$(INS.dir)
+
+$(TESTDIR)/%: %.ksh
+	$(INS.rename)
+
+$(TESTDIR)/%: %
+	$(INS.file)
diff --git a/usr/src/test/zfs-tests/tests/functional/holes/cleanup.ksh b/usr/src/test/zfs-tests/tests/functional/holes/cleanup.ksh
new file mode 100644
index 0000000000..2ee524ecf1
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/holes/cleanup.ksh
@@ -0,0 +1,20 @@
+#!/usr/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+default_cleanup
diff --git a/usr/src/test/zfs-tests/tests/functional/holes/holes.shlib b/usr/src/test/zfs-tests/tests/functional/holes/holes.shlib
new file mode 100644
index 0000000000..9420c82120
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/holes/holes.shlib
@@ -0,0 +1,39 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
+
+# testfile The file to examine.
+# hole_blks The expected number of holes.
+# data_blks The expected number of data blocks.
+function verify_holes_and_data_blocks
+{
+	typeset testfile=$1
+	typeset -i hole_blks=$2
+	typeset -i data_blks=$3
+	typeset -i failures=0
+
+	found_hole_blks=$($GETHOLES -h $testfile)
+	found_data_blks=$($GETHOLES -d $testfile)
+	if [[ $found_hole_blks -ne $hole_blks ]] then;
+		log_note "Found $found_hole_blks, not $hole_blks hole blocks."
+		((failures++))
+	fi
+
+	if [[ $found_data_blks -ne $data_blks ]] then;
+		log_note "Found $found_data_blks, not $data_blks data blocks."
+		((failures++))
+	fi
+
+	[[ $failures -eq 0 ]] || log_fail "Wrong number of data/hole blocks."
+}
diff --git a/usr/src/test/zfs-tests/tests/functional/holes/holes_sanity.ksh b/usr/src/test/zfs-tests/tests/functional/holes/holes_sanity.ksh
new file mode 100644
index 0000000000..f00bba59de
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/holes/holes_sanity.ksh
@@ -0,0 +1,85 @@
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
+
+#
+# Description:
+# Verify that holes can be written and read back correctly in ZFS.
+#
+# Strategy:
+# 1. Create a testfile with varying holes and data throughout the file.
+# 2. Verify that each created file has the correct number of holes and
+# data blocks as seen by both lseek and libzfs.
+# 3. Do the same verification for a largefile.
+# 4. Repeat for each recsize.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/holes/holes.shlib
+
+verify_runnable "both"
+testfile="$TESTDIR/testfile"
+
+for bs in 512 1024 2048 4096 8192 16384 32768 65536 131072; do
+	log_must $ZFS set recsize=$bs $TESTPOOL/$TESTFS
+
+	#
+	# Create combinations of holes and data to verify holes ending files
+	# and the like. (hhh, hhd, hdh...)
+	#
+	log_must $MKHOLES -h 0:$((bs * 6)) $testfile
+	verify_holes_and_data_blocks $testfile 6 0
+	log_must $RM $testfile
+
+	log_must $MKHOLES -h 0:$((bs * 4)) -d $((bs * 4)):$((bs * 2)) $testfile
+	verify_holes_and_data_blocks $testfile 4 2
+	log_must $RM $testfile
+
+	log_must $MKHOLES -h 0:$((bs * 2)) -d $((bs * 2)):$((bs * 2)) \
+	    -h $((bs * 4)):$((bs * 2)) $testfile
+	verify_holes_and_data_blocks $testfile 4 2
+	log_must $RM $testfile
+
+	log_must $MKHOLES -h 0:$((bs * 2)) -d $((bs * 2)):$((bs * 4)) $testfile
+	verify_holes_and_data_blocks $testfile 2 4
+	log_must $RM $testfile
+
+	log_must $MKHOLES -d 0:$((bs * 2)) -h $((bs * 2)):$((bs * 4)) $testfile
+	verify_holes_and_data_blocks $testfile 4 2
+	log_must $RM $testfile
+
+	log_must $MKHOLES -d 0:$((bs * 2)) -h $((bs * 2)):$((bs * 2)) \
+	    -d $((bs * 4)):$((bs * 2)) $testfile
+	verify_holes_and_data_blocks $testfile 2 4
+	log_must $RM $testfile
+
+	log_must $MKHOLES -d 0:$((bs * 4)) -h $((bs * 4)):$((bs * 2)) $testfile
+	verify_holes_and_data_blocks $testfile 2 4
+	log_must $RM $testfile
+
+	log_must $MKHOLES -d 0:$((bs * 6)) $testfile
+	verify_holes_and_data_blocks $testfile 0 6
+	log_must $RM $testfile
+
+	# Verify holes are correctly seen past the largefile limit.
+	len=$((1024**3 * 5))
+	nblks=$((len / bs))
+	log_must $MKHOLES -h 0:$len -d $len:$bs $testfile
+	verify_holes_and_data_blocks $testfile $nblks 1
+	log_must $RM $testfile
+done
+
+log_pass "Basic hole tests pass."
diff --git a/usr/src/test/zfs-tests/tests/functional/holes/setup.ksh b/usr/src/test/zfs-tests/tests/functional/holes/setup.ksh
new file mode 100644
index 0000000000..9b2d9ff181
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/holes/setup.ksh
@@ -0,0 +1,21 @@
+#!/usr/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+DISK=${DISKS%% *}
+default_setup $DISK
diff --git a/usr/src/test/zfs-tests/tests/functional/online_offline/Makefile b/usr/src/test/zfs-tests/tests/functional/online_offline/Makefile
index 53211b61a5..98135a4a31 100644
--- a/usr/src/test/zfs-tests/tests/functional/online_offline/Makefile
+++ b/usr/src/test/zfs-tests/tests/functional/online_offline/Makefile
@@ -10,7 +10,7 @@
 #
 
 #
-# Copyright (c) 2013 by Delphix. All rights reserved.
+# Copyright (c) 2013, 2014 by Delphix. All rights reserved.
 #
 
 include $(SRC)/Makefile.master
@@ -24,9 +24,7 @@ PROGS = cleanup \
 	online_offline_003_neg \
 	setup
 
-FILES = online_offline.cfg
-
-CMDS = $(PROGS:%=$(TESTDIR)/%) $(FILES:%=$(TESTDIR)/%)
+CMDS = $(PROGS:%=$(TESTDIR)/%)
 $(CMDS) := FILEMODE = 0555
 
 all lint clean clobber:
diff --git a/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline.cfg b/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline.cfg
deleted file mode 100644
index e68e8363f6..0000000000
--- a/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline.cfg
+++ /dev/null
@@ -1,38 +0,0 @@
-#
-# CDDL HEADER START
-#
-# The contents of this file are subject to the terms of the
-# Common Development and Distribution License (the "License").
-# You may not use this file except in compliance with the License.
-#
-# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
-# or http://www.opensolaris.org/os/licensing.
-# See the License for the specific language governing permissions
-# and limitations under the License.
-#
-# When distributing Covered Code, include this CDDL HEADER in each
-# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
-# If applicable, add the following below this CDDL HEADER, with the
-# fields enclosed by brackets "[]" replaced with your own identifying
-# information: Portions Copyright [yyyy] [name of copyright owner]
-#
-# CDDL HEADER END
-#
-
-#
-# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
-# Use is subject to license terms.
-#
-
-#
-# Copyright (c) 2013 by Delphix. All rights reserved.
-#
-
-export TESTFILE=testfile.$$
-export TESTFILE1=testfile1.$$
-export HOLES_FILESIZE=${HOLES_FILESIZE-"67108864"} # 64 Mb
-export HOLES_BLKSIZE=${HOLES_BLKSIZE-"512"}
-export HOLES_SEED=${HOLES_SEED-""}
-export HOLES_FILEOFFSET=${HOLES_FILEOFFSET-""}
-export HOLES_COUNT=${HOLES_COUNT-"16384"}	   # FILESIZE/BLKSIZE/8
-export STF_TIMEOUT=3600
diff --git a/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline_001_pos.ksh b/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline_001_pos.ksh
index 7c45718a2b..b1b9b1383c 100644
--- a/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline_001_pos.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline_001_pos.ksh
@@ -26,23 +26,23 @@
 #
 
 #
-# Copyright (c) 2013 by Delphix. All rights reserved.
+# Copyright (c) 2013, 2014 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
-. $STF_SUITE/tests/functional/online_offline/online_offline.cfg
 
 #
 # DESCRIPTION:
-# 	Turning a disk offline and back online during I/O completes.
+# Turning a disk offline and back online during I/O completes.
 #
 # STRATEGY:
-#	1. Create a mirror and start some random I/O
-#	2. For each disk in the mirror, set it offline and online
-#	3. Verify the integrity of the file system and the resilvering.
+# 1. Create a mirror and start some random I/O
+# 2. For each disk in the mirror, set it offline and online
+# 3. Verify the integrity of the file system and the resilvering.
 #
 
 verify_runnable "global"
+log_onexit cleanup
 
 DISKLIST=$(get_disklist $TESTPOOL)
 
@@ -60,72 +60,24 @@ function cleanup
 
 	done
 
+	$KILL $killpid >/dev/null 2>&1
 	[[ -e $TESTDIR ]] && log_must $RM -rf $TESTDIR/*
 }
 
 log_assert "Turning a disk offline and back online during I/O completes."
 
-options=""
-options_display="default options"
-
-log_onexit cleanup
-
-[[ -n "$HOLES_FILESIZE" ]] && options=" $options -f $HOLES_FILESIZE "
-
-[[ -n "$HOLES_BLKSIZE" ]] && options="$options -b $HOLES_BLKSIZE "
-
-[[ -n "$HOLES_COUNT" ]] && options="$options -c $HOLES_COUNT "
-
-[[ -n "$HOLES_SEED" ]] && options="$options -s $HOLES_SEED "
-
-[[ -n "$HOLES_FILEOFFSET" ]] && options="$options -o $HOLES_FILEOFFSET "
-
-options="$options -r "
-
-[[ -n "$options" ]] && options_display=$options
-
-typeset child_pid=""
-
-typeset -i iters=2
-typeset -i index=0
-
-i=0
-while [[ $i -lt $iters ]]; do
-	log_note "Invoking $FILE_TRUNC with: $options_display"
-	$FILE_TRUNC $options $TESTDIR/$TESTFILE.$i &
-	typeset pid=$!
-
-	$SLEEP 1
-	if ! $PS -p $pid > /dev/null 2>&1; then
-		log_fail "$FILE_TRUNC $options $TESTDIR/$TESTFILE.$i"
-	fi
-
-	child_pids="$child_pids $pid"
-	((i = i + 1))
-done
+$FILE_TRUNC -f $((64 * 1024 * 1024)) -b 8192 -c 0 -r $TESTDIR/$TESTFILE1 &
+typeset killpid="$! "
 
 for disk in $DISKLIST; do
-	#
-	# Allow some common data to reach each side of the mirror.
-	#
-	$SLEEP 30
-
-        i=0
-        while [[ $i -lt $iters ]]; do
+        for i in 'do_offline' 'do_offline_while_already_offline'; do
 		log_must $ZPOOL offline $TESTPOOL $disk
 		check_state $TESTPOOL $disk "offline"
                 if [[ $? != 0 ]]; then
                         log_fail "$disk of $TESTPOOL is not offline."
                 fi
-
-                (( i = i + 1 ))
         done
 
-	#
-	# Sleep to allow the two sides to get out of sync
-	#
-	$SLEEP 60
-
         log_must $ZPOOL online $TESTPOOL $disk
         check_state $TESTPOOL $disk "online"
         if [[ $? != 0 ]]; then
@@ -133,10 +85,8 @@ for disk in $DISKLIST; do
         fi
 done
 
-for wait_pid in $child_pids
-do
-	$KILL $wait_pid
-done
+log_must $KILL $killpid
+$SYNC
 
 typeset dir=$(get_device_dir $DISKS)
 verify_filesys "$TESTPOOL" "$TESTPOOL/$TESTFS" "$dir"
diff --git a/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline_002_neg.ksh b/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline_002_neg.ksh
index 23014011a3..6bd2411b11 100644
--- a/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline_002_neg.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline_002_neg.ksh
@@ -26,21 +26,21 @@
 #
 
 #
-# Copyright (c) 2013 by Delphix. All rights reserved.
+# Copyright (c) 2013, 2014 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
-. $STF_SUITE/tests/functional/online_offline/online_offline.cfg
 
 #
 # DESCRIPTION:
-# 	Turning both disks offline should fail.
+# Turning disks in a pool offline should fail when there is no longer
+# sufficient redundancy.
 #
 # STRATEGY:
-#	1. Create a mirror and start some random I/O
-#	2. For each disk in the mirror, set them offline sequentially.
-#	3. Only one disk can be offline at any one time.
-#	4. Verify the integrity of the file system and the resilvering.
+# 1. Start some random I/O on the mirror or raidz.
+# 2. Verify that we can offline as many disks as the redundancy level
+# will support, but not more.
+# 3. Verify the integrity of the file system and the resilvering.
 #
 
 verify_runnable "global"
@@ -49,13 +49,6 @@ DISKLIST=$(get_disklist $TESTPOOL)
 
 function cleanup
 {
-	if [[ -n "$child_pids" ]]; then
-		for wait_pid in $child_pids
-		do
-		        $KILL $wait_pid
-		done
-	fi
-
 	#
 	# Ensure we don't leave disks in the offline state
 	#
@@ -68,72 +61,70 @@ function cleanup
 
 	done
 
+	$KILL $killpid >/dev/null 2>&1
 	[[ -e $TESTDIR ]] && log_must $RM -rf $TESTDIR/*
 }
 
 log_assert "Turning both disks offline should fail."
 
-options=""
-options_display="default options"
-
 log_onexit cleanup
 
-[[ -n "$HOLES_FILESIZE" ]] && options=" $options -f $HOLES_FILESIZE "
-
-[[ -n "$HOLES_BLKSIZE" ]] && options="$options -b $HOLES_BLKSIZE "
-
-[[ -n "$HOLES_COUNT" ]] && options="$options -c $HOLES_COUNT "
-
-[[ -n "$HOLES_SEED" ]] && options="$options -s $HOLES_SEED "
-
-[[ -n "$HOLES_FILEOFFSET" ]] && options="$options -o $HOLES_FILEOFFSET "
-
-options="$options -r "
-
-[[ -n "$options" ]] && options_display=$options
-
-child_pid=""
-
-typeset -i iters=2
-typeset -i index=0
+$FILE_TRUNC -f $((64 * 1024 * 1024)) -b 8192 -c 0 -r $TESTDIR/$TESTFILE1 &
+typeset killpid="$! "
 
-i=0
-while [[ $i -lt $iters ]]; do
-	log_note "Invoking $FILE_TRUNC with: $options_display"
-	$FILE_TRUNC $options $TESTDIR/$TESTFILE.$i &
-	typeset pid=$!
+disks=($DISKLIST)
 
-	$SLEEP 1
-	if ! $PS -p $pid > /dev/null 2>&1; then
-		log_fail "$FILE_TRUNC $options $TESTDIR/$TESTFILE.$i"
+#
+# The setup script will give us either a mirror or a raidz. The former can have
+# all but one vdev offlined, whereas with raidz there can be only one.
+#
+pooltype='mirror'
+$ZPOOL list -v $TESTPOOL | $GREP raidz >/dev/null 2>&1 && pooltype='raidz'
+
+typeset -i i=0
+while [[ $i -lt ${#disks[*]} ]]; do
+	typeset -i j=0
+	if [[ $pooltype = 'mirror' ]]; then
+		# Hold one disk online, verify the others can be offlined.
+		log_must $ZPOOL online $TESTPOOL ${disks[$i]}
+		check_state $TESTPOOL ${disks[$i]} "online" || \
+		    log_fail "Failed to set ${disks[$i]} online"
+		while [[ $j -lt ${#disks[*]} ]]; do
+			if [[ $j -eq $i ]]; then
+				((j++))
+				continue
+			fi
+			log_must $ZPOOL offline $TESTPOOL ${disks[$j]}
+			check_state $TESTPOOL ${disks[$j]} "offline" || \
+			    log_fail "Failed to set ${disks[$j]} offline"
+			((j++))
+		done
+	elif [[ $pooltype = 'raidz' ]]; then
+		# Hold one disk offline, verify the others can't be offlined.
+		log_must $ZPOOL offline $TESTPOOL ${disks[$i]}
+		check_state $TESTPOOL ${disks[$i]} "offline" || \
+		    log_fail "Failed to set ${disks[$i]} offline"
+		while [[ $j -lt ${#disks[*]} ]]; do
+			if [[ $j -eq $i ]]; then
+				((j++))
+				continue
+			fi
+			log_mustnot $ZPOOL offline $TESTPOOL ${disks[$j]}
+			check_state $TESTPOOL ${disks[$j]} "online" || \
+			    log_fail "Failed to set ${disks[$j]} online"
+			check_state $TESTPOOL ${disks[$i]} "offline" || \
+			    log_fail "Failed to set ${disks[$i]} offline"
+			((j++))
+		done
+		log_must $ZPOOL online $TESTPOOL ${disks[$i]}
+		check_state $TESTPOOL ${disks[$i]} "online" || \
+		    log_fail "Failed to set ${disks[$i]} online"
 	fi
-
-	child_pids="$child_pids $pid"
-	((i = i + 1))
-done
-
-set -A disk "" $DISKLIST
-
-log_must $ZPOOL offline $TESTPOOL ${disk[1]}
-
-$SLEEP 60
-
-for wait_pid in $child_pids
-do
-	$KILL $wait_pid
+	((i++))
 done
-child_pids=""
-
-i=1
-while [[ $i != ${#disk[*]} ]]; do
-	log_must $ZPOOL online $TESTPOOL ${disk[$i]}
-	check_state $TESTPOOL ${disk[$i]} "online"
-	if [[ $? != 0 ]]; then
-		log_fail "${disk[$i]} of $TESTPOOL did not match online state"
-	fi
 
-	((i = i + 1))
-done
+log_must $KILL $killpid
+$SYNC
 
 typeset dir=$(get_device_dir $DISKS)
 verify_filesys "$TESTPOOL" "$TESTPOOL/$TESTFS" "$dir"
diff --git a/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline_003_neg.ksh b/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline_003_neg.ksh
index ef789a8c0a..95d423c73e 100644
--- a/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline_003_neg.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/online_offline/online_offline_003_neg.ksh
@@ -26,108 +26,56 @@
 #
 
 #
-# Copyright (c) 2013 by Delphix. All rights reserved.
+# Copyright (c) 2013, 2014 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
-. $STF_SUITE/tests/functional/online_offline/online_offline.cfg
 
 #
 # DESCRIPTION:
-# 	Turning both disks offline should fail.
+# Offlining disks in a non-redundant pool should fail.
 #
 # STRATEGY:
-#	1. Create a multidisk stripe and start some random I/O
-#	2. For two disks in the stripe, set them offline sequentially.
-#	3. Zpool offline should fail in both cases.
-#	4. Verify the integrity of the file system and the resilvering.
+# 1. Create a multidisk stripe and start some random I/O
+# 2. zpool offline should fail on each disk.
 #
 
 verify_runnable "global"
 
-DISKLIST=$(get_disklist $TESTPOOL)
-
 function cleanup
 {
-	if [[ -n "$child_pids" ]]; then
-		for wait_pid in $child_pids
-		do
-		        $KILL $wait_pid
-		done
-	fi
-
 	if poolexists $TESTPOOL1; then
 		destroy_pool $TESTPOOL1
 	fi
 
+	$KILL $killpid >/dev/null 2>&1
 	[[ -e $TESTDIR ]] && log_must $RM -rf $TESTDIR/*
 }
 
-log_assert "Turning a disk offline and back online during I/O completes."
-
-options=""
-options_display="default options"
+log_assert "Offlining disks in a non-redundant pool should fail."
 
 log_onexit cleanup
 
-[[ -n "$HOLES_FILESIZE" ]] && options=" $options -f $HOLES_FILESIZE "
-
-[[ -n "$HOLES_BLKSIZE" ]] && options="$options -b $HOLES_BLKSIZE "
-
-[[ -n "$HOLES_COUNT" ]] && options="$options -c $HOLES_COUNT "
-
-[[ -n "$HOLES_SEED" ]] && options="$options -s $HOLES_SEED "
-
-[[ -n "$HOLES_FILEOFFSET" ]] && options="$options -o $HOLES_FILEOFFSET "
-
-options="$options -r "
-
-[[ -n "$options" ]] && options_display=$options
-
-child_pid=""
-
-typeset -i iters=2
-typeset -i index=0
-
 specials_list=""
-i=0
-while [[ $i != 3 ]]; do
-	$MKFILE 100m $TESTDIR/$TESTFILE1.$i
+for i in 0 1 2; do
+	$MKFILE 64m $TESTDIR/$TESTFILE1.$i
 	specials_list="$specials_list $TESTDIR/$TESTFILE1.$i"
-
-	((i = i + 1))
 done
+disk=($specials_list)
 
 create_pool $TESTPOOL1 $specials_list
 log_must $ZFS create $TESTPOOL1/$TESTFS1
 log_must $ZFS set mountpoint=$TESTDIR1 $TESTPOOL1/$TESTFS1
 
-i=0
-while [[ $i -lt $iters ]]; do
-	log_note "Invoking $FILE_TRUNC with: $options_display"
-	$FILE_TRUNC $options $TESTDIR/$TESTFILE.$i &
-	typeset pid=$!
-
-	$SLEEP 1
-	if ! $PS -p $pid > /dev/null 2>&1; then
-		log_fail "$FILE_TRUNC $options $TESTDIR/$TESTFILE.$i"
-	fi
+$FILE_TRUNC -f $((64 * 1024 * 1024)) -b 8192 -c 0 -r $TESTDIR/$TESTFILE1 &
+typeset killpid="$! "
 
-	child_pids="$child_pids $pid"
-	((i = i + 1))
+for i in 0 1 2; do
+	log_mustnot $ZPOOL offline $TESTPOOL1 ${disk[$i]}
+	check_state $TESTPOOL1 ${disk[$i]} "online"
 done
 
-set -A disk "" $specials_list
-
-log_mustnot $ZPOOL offline $TESTPOOL1 ${disk[1]}
-log_mustnot $ZPOOL offline $TESTPOOL1 ${disk[2]}
-
-$SLEEP 60
-
-for wait_pid in $child_pids
-do
-	$KILL $wait_pid
-done
-child_pids=""
+log_must $KILL $killpid
+$SYNC
 
 log_pass
diff --git a/usr/src/test/zfs-tests/tests/functional/rsend/Makefile b/usr/src/test/zfs-tests/tests/functional/rsend/Makefile
index 1d238a0d55..fb757b29a3 100644
--- a/usr/src/test/zfs-tests/tests/functional/rsend/Makefile
+++ b/usr/src/test/zfs-tests/tests/functional/rsend/Makefile
@@ -10,7 +10,7 @@
 #
 
 #
-# Copyright (c) 2013 by Delphix. All rights reserved.
+# Copyright (c) 2013, 2014 by Delphix. All rights reserved.
 #
 
 include $(SRC)/Makefile.master
@@ -32,6 +32,7 @@ PROGS = cleanup \
 	rsend_011_pos \
 	rsend_012_pos \
 	rsend_013_pos \
+	rsend_014_pos \
 	setup
 
 FILES = rsend.cfg \
diff --git a/usr/src/test/zfs-tests/tests/functional/rsend/cleanup.ksh b/usr/src/test/zfs-tests/tests/functional/rsend/cleanup.ksh
index a13ea7e814..b835b54cbc 100644
--- a/usr/src/test/zfs-tests/tests/functional/rsend/cleanup.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/rsend/cleanup.ksh
@@ -26,21 +26,13 @@
 #
 
 #
-# Copyright (c) 2013 by Delphix. All rights reserved.
+# Copyright (c) 2013, 2014 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/tests/functional/rsend/rsend.kshlib
 
 verify_runnable "both"
 
-#
-# Check if the system support 'send -R'
-#
-$ZFS send 2>&1 | $GREP "\[-[a-zA-Z]*R[a-zA-Z]*\]" >/dev/null 2>&1
-if (($? != 0)); then
-	log_unsupported
-fi
-
 if is_global_zone ; then
 	destroy_pool $POOL
 	destroy_pool $POOL2
diff --git a/usr/src/test/zfs-tests/tests/functional/rsend/rsend_014_pos.ksh b/usr/src/test/zfs-tests/tests/functional/rsend/rsend_014_pos.ksh
new file mode 100644
index 0000000000..cf8fc423f0
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/rsend/rsend_014_pos.ksh
@@ -0,0 +1,56 @@
+#!/usr/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2014 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify that a pool imported readonly can be sent and received.
+#
+# Strategy:
+# 1. Make the source pool readonly, and receive it into pool2.
+# 2. Reset pool2, and repeat the send from a non-root fs of the source pool.
+# 3. Make the source pool read-write again.
+#
+
+verify_runnable "both"
+
+log_assert "zfs send will work on filesystems and volumes in a read-only pool."
+log_onexit cleanup_pool $POOL2
+
+log_must eval "$ZPOOL export $POOL"
+log_must eval "$ZPOOL import -o readonly=on $POOL"
+log_must eval "$ZFS send -R $POOL@final > $BACKDIR/pool-final-R"
+log_must eval "$ZFS receive -d -F $POOL2 < $BACKDIR/pool-final-R"
+
+dstds=$(get_dst_ds $POOL $POOL2)
+log_must cmp_ds_subs $POOL $dstds
+log_must cmp_ds_cont $POOL $dstds
+
+log_must cleanup_pool $POOL2
+
+log_must eval "$ZFS send -R $POOL/$FS@final > $BACKDIR/fs-final-R"
+log_must eval "$ZFS receive -d $POOL2 < $BACKDIR/fs-final-R"
+log_must eval "$ZPOOL export $POOL"
+log_must eval "$ZPOOL import $POOL"
+
+dstds=$(get_dst_ds $POOL/$FS $POOL2)
+log_must cmp_ds_subs $POOL/$FS $dstds
+log_must cmp_ds_cont $POOL/$FS $dstds
+
+log_pass "zfs send will work on filesystems and volumes in a read-only pool."
diff --git a/usr/src/test/zfs-tests/tests/functional/rsend/setup.ksh b/usr/src/test/zfs-tests/tests/functional/rsend/setup.ksh
index fbecf9b828..939356ad7e 100644
--- a/usr/src/test/zfs-tests/tests/functional/rsend/setup.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/rsend/setup.ksh
@@ -26,7 +26,7 @@
 #
 
 #
-# Copyright (c) 2013 by Delphix. All rights reserved.
+# Copyright (c) 2013, 2014 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/tests/functional/rsend/rsend.kshlib
@@ -34,14 +34,6 @@
 verify_runnable "both"
 verify_disk_count "$DISKS" 2
 
-#
-# Check if the system support 'send -R'
-#
-$ZFS send 2>&1 | $GREP "\[-[a-zA-Z]*R[a-zA-Z]*\]" >/dev/null 2>&1
-if (($? != 0)); then
-	log_unsupported
-fi
-
 if is_global_zone ; then
 	log_must $ZPOOL create $POOL $DISK1
 	log_must $ZPOOL create $POOL2 $DISK2