12002 async unlinked drain

Portions contributed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Portions contributed by: Roman Strashkin <roman.strashkin@nexenta.com>
Portions contributed by: Saso Kiselkov <saso.kiselkov@nexenta.com>
Reviewed by: Jorgen Lundman <lundman@lundman.net>
Reviewed by: Tom Caputi <tcaputi@datto.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed by: Matt Ahrens <mahrens@delphix.com>
Reviewed by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Sanjay Nadkarni <sanjay.nadkarni@nexenta.com>
Reviewed by: Josef 'Jeff' Sipek <josef.sipek@nexenta.com>
Reviewed by: C Fraire <cfraire@me.com>
Reviewed by: Kody Kantor <kody.kantor@joyent.com>
Approved by: Dan McDonald <danmcd@joyent.com>

14 files changed, 270 insertions, 11 deletions

diff --git a/usr/src/pkg/manifests/system-test-zfstest.mf b/usr/src/pkg/manifests/system-test-zfstest.mf
index b90f2836a5..3c5cbfbd42 100644
--- a/usr/src/pkg/manifests/system-test-zfstest.mf
+++ b/usr/src/pkg/manifests/system-test-zfstest.mf
@@ -2617,6 +2617,8 @@ file path=opt/zfs-tests/tests/functional/mmp/setup mode=0555
 file path=opt/zfs-tests/tests/functional/mount/cleanup mode=0555
 file path=opt/zfs-tests/tests/functional/mount/setup mode=0555
 file path=opt/zfs-tests/tests/functional/mount/umount_001 mode=0555
+file path=opt/zfs-tests/tests/functional/mount/umount_002 mode=0555
+file path=opt/zfs-tests/tests/functional/mount/umount_unlinked_drain mode=0555
 file path=opt/zfs-tests/tests/functional/mount/umountall_001 mode=0555
 file path=opt/zfs-tests/tests/functional/mv_files/cleanup mode=0555
 file path=opt/zfs-tests/tests/functional/mv_files/mv_files.cfg mode=0444
diff --git a/usr/src/test/zfs-tests/runfiles/delphix.run b/usr/src/test/zfs-tests/runfiles/delphix.run
index 8fea4e3ca4..e97d0e6e0b 100644
--- a/usr/src/test/zfs-tests/runfiles/delphix.run
+++ b/usr/src/test/zfs-tests/runfiles/delphix.run
@@ -521,7 +521,7 @@ tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval',
     'mmp_on_zdb']
 
 [/opt/zfs-tests/tests/functional/mount]
-tests = ['umount_001', 'umountall_001']
+tests = ['umount_001', 'umount_002', 'umountall_001']
 
 [/opt/zfs-tests/tests/functional/mv_files]
 tests = ['mv_files_001_pos', 'mv_files_002_pos']
diff --git a/usr/src/test/zfs-tests/runfiles/omnios.run b/usr/src/test/zfs-tests/runfiles/omnios.run
index a89ed48b8e..ff7a697d5f 100644
--- a/usr/src/test/zfs-tests/runfiles/omnios.run
+++ b/usr/src/test/zfs-tests/runfiles/omnios.run
@@ -520,7 +520,7 @@ tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval',
     'mmp_on_zdb']
 
 [/opt/zfs-tests/tests/functional/mount]
-tests = ['umount_001', 'umountall_001']
+tests = ['umount_001', 'umount_002', 'umountall_001']
 
 [/opt/zfs-tests/tests/functional/mv_files]
 tests = ['mv_files_001_pos', 'mv_files_002_pos']
diff --git a/usr/src/test/zfs-tests/runfiles/openindiana.run b/usr/src/test/zfs-tests/runfiles/openindiana.run
index 1364265ac6..9ab1d42293 100644
--- a/usr/src/test/zfs-tests/runfiles/openindiana.run
+++ b/usr/src/test/zfs-tests/runfiles/openindiana.run
@@ -520,7 +520,7 @@ tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval',
     'mmp_on_zdb']
 
 [/opt/zfs-tests/tests/functional/mount]
-tests = ['umount_001', 'umountall_001']
+tests = ['umount_001', 'umount_002', 'umountall_001']
 
 [/opt/zfs-tests/tests/functional/mv_files]
 tests = ['mv_files_001_pos', 'mv_files_002_pos']
diff --git a/usr/src/test/zfs-tests/runfiles/smartos.run b/usr/src/test/zfs-tests/runfiles/smartos.run
index f9bfcb71f7..52fbf045dc 100644
--- a/usr/src/test/zfs-tests/runfiles/smartos.run
+++ b/usr/src/test/zfs-tests/runfiles/smartos.run
@@ -452,7 +452,7 @@ tests = ['mmp_on_thread', 'mmp_on_off', 'mmp_interval',
     'mmp_on_zdb']
 
 [/opt/zfs-tests/tests/functional/mount]
-tests = ['umount_001', 'umountall_001']
+tests = ['umount_001', 'umount_002', 'umountall_001']
 
 [/opt/zfs-tests/tests/functional/mv_files]
 tests = ['mv_files_001_pos', 'mv_files_002_pos']
diff --git a/usr/src/test/zfs-tests/tests/functional/mount/umount_002.ksh b/usr/src/test/zfs-tests/tests/functional/mount/umount_002.ksh
new file mode 100644
index 0000000000..c57d787e4c
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/mount/umount_002.ksh
@@ -0,0 +1,52 @@
+#! /usr/bin/ksh -p
+#
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright 2016 Nexenta Systems, Inc. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+# zfs umount should not fail because flushing of DNLC
+# uses async implementation of zfs_inactive
+#
+# STRATEGY:
+# 1. Call zfs unmount/mount to be sure DNLC is empty
+# 2. Create a lot of files
+# 3. Call zfs unmount command
+# 4. Make sure the file systems were unmounted
+# 5. Mount them back
+#
+
+for fs in 1 2 3; do
+	log_must mounted $TESTDIR.$fs
+	log_must zfs umount $TESTPOOL/$TESTFS.$fs
+	log_must unmounted $TESTDIR.$fs
+	log_must zfs mount $TESTPOOL/$TESTFS.$fs
+	log_must mounted $TESTDIR.$fs
+
+	for fn in $(seq 1 8096); do
+		log_must dd if=/dev/urandom of=$TESTDIR.$fs/file$fn bs=1024 \
+		    count=1
+	done
+
+	log_must zfs umount $TESTPOOL/$TESTFS.$fs
+	log_must unmounted $TESTDIR.$fs
+	log_must zfs mount $TESTPOOL/$TESTFS.$fs
+	log_must mounted $TESTDIR.$fs
+	log_must rm -f $TESTDIR.$fs/file*
+done
+
+log_pass "All file systems are unmounted"
diff --git a/usr/src/test/zfs-tests/tests/functional/mount/umount_unlinked_drain.ksh b/usr/src/test/zfs-tests/tests/functional/mount/umount_unlinked_drain.ksh
new file mode 100755
index 0000000000..0d2628079c
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/mount/umount_unlinked_drain.ksh
@@ -0,0 +1,119 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright 2018 Datto Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+# Test async unlinked drain to ensure mounting is not held up when there are
+# entries in the unlinked set. We also try to test that the list is able to be
+# filled up and drained at the same time.
+#
+# STRATEGY:
+# 1. Use zfs_unlink_suspend_progress tunable to disable freeing to build up
+#    the unlinked set
+# 2. Make sure mount happens even when there are entries in the unlinked set
+# 3. Drain and build up the unlinked list at the same time to test for races
+#
+
+function cleanup
+{
+	log_must set_tunable32 zfs_unlink_suspend_progress $default_unlink_sp
+	for fs in $(seq 1 3); do
+		mounted $TESTDIR.$fs || zfs mount $TESTPOOL/$TESTFS.$fs
+		rm -f $TESTDIR.$fs/file-*
+		zfs set xattr=on $TESTPOOL/$TESTFS.$fs
+	done
+}
+
+function unlinked_size_is
+{
+	MAX_ITERS=5 # iteration to do before we consider reported number stable
+	iters=0
+	last_usize=0
+	while [[ $iters -le $MAX_ITERS ]]; do
+		kstat_file=$(grep -nrwl /proc/spl/kstat/zfs/$2/objset-0x* -e $3)
+		nunlinks=`cat $kstat_file | grep nunlinks | awk '{print $3}'`
+		nunlinked=`cat $kstat_file | grep nunlinked | awk '{print $3}'`
+		usize=$(($nunlinks - $nunlinked))
+		if [[ $iters == $MAX_ITERS && $usize == $1 ]]; then
+			return 0
+		fi
+		if [[ $usize == $last_usize ]]; then
+			(( iters++ ))
+		else
+			iters=0
+		fi
+		last_usize=$usize
+	done
+
+	log_note "Unexpected unlinked set size: $last_usize, expected $1"
+	return 1
+}
+
+
+UNLINK_SP_PARAM=/sys/module/zfs/parameters/zfs_unlink_suspend_progress
+default_unlink_sp=$(get_tunable zfs_unlink_suspend_progress)
+
+log_onexit cleanup
+
+log_assert "Unlinked list drain does not hold up mounting of fs"
+
+for fs in 1 2 3; do
+	set -A xattrs on sa off
+	for xa in ${xattrs[@]}; do
+		# setup fs and ensure all deleted files got into unliked set
+		log_must mounted $TESTDIR.$fs
+
+		log_must zfs set xattr=$xa $TESTPOOL/$TESTFS.$fs
+
+		if [[ $xa == off ]]; then
+			for fn in $(seq 1 175); do
+				log_must mkfile 128k $TESTDIR.$fs/file-$fn
+			done
+		else
+			log_must xattrtest -f 175 -x 3 -r -k -p $TESTDIR.$fs
+		fi
+
+		log_must set_tunable32 zfs_unlink_suspend_progress 1
+		log_must unlinked_size_is 0 $TESTPOOL $TESTPOOL/$TESTFS.$fs
+
+		# build up unlinked set
+		for fn in $(seq 1 100); do
+			log_must eval "rm $TESTDIR.$fs/file-$fn &"
+		done
+		log_must unlinked_size_is 100 $TESTPOOL $TESTPOOL/$TESTFS.$fs
+
+		# test that we can mount fs without emptying the unlinked list
+		log_must zfs umount $TESTPOOL/$TESTFS.$fs
+		log_must unmounted $TESTDIR.$fs
+		log_must zfs mount $TESTPOOL/$TESTFS.$fs
+		log_must mounted $TESTDIR.$fs
+		log_must unlinked_size_is 100 $TESTPOOL $TESTPOOL/$TESTFS.$fs
+
+		# confirm we can drain and add to unlinked set at the same time
+		log_must set_tunable32 zfs_unlink_suspend_progress 0
+		log_must zfs umount $TESTPOOL/$TESTFS.$fs
+		log_must zfs mount $TESTPOOL/$TESTFS.$fs
+		for fn in $(seq 101 175); do
+			log_must eval "rm $TESTDIR.$fs/file-$fn &"
+		done
+		log_must unlinked_size_is 0 $TESTPOOL $TESTPOOL/$TESTFS.$fs
+	done
+done
+
+log_pass "Confirmed unlinked list drain does not hold up mounting of fs"
diff --git a/usr/src/uts/common/fs/zfs/dsl_pool.c b/usr/src/uts/common/fs/zfs/dsl_pool.c
index abc69ec57c..bc6f9aff77 100644
--- a/usr/src/uts/common/fs/zfs/dsl_pool.c
+++ b/usr/src/uts/common/fs/zfs/dsl_pool.c
@@ -221,6 +221,9 @@ dsl_pool_open_impl(spa_t *spa, uint64_t txg)
 
 	dp->dp_vnrele_taskq = taskq_create("zfs_vn_rele_taskq", 1, minclsyspri,
 	    1, 4, 0);
+	dp->dp_unlinked_drain_taskq = taskq_create("z_unlinked_drain",
+	    max_ncpus, minclsyspri, max_ncpus, INT_MAX,
+	    TASKQ_PREPOPULATE | TASKQ_DYNAMIC);
 
 	return (dp);
 }
@@ -402,6 +405,7 @@ dsl_pool_close(dsl_pool_t *dp)
 
 	rrw_destroy(&dp->dp_config_rwlock);
 	mutex_destroy(&dp->dp_lock);
+	taskq_destroy(dp->dp_unlinked_drain_taskq);
 	taskq_destroy(dp->dp_vnrele_taskq);
 	if (dp->dp_blkstats != NULL)
 		kmem_free(dp->dp_blkstats, sizeof (zfs_all_blkstats_t));
@@ -1077,6 +1081,12 @@ dsl_pool_vnrele_taskq(dsl_pool_t *dp)
 	return (dp->dp_vnrele_taskq);
 }
 
+taskq_t *
+dsl_pool_unlinked_drain_taskq(dsl_pool_t *dp)
+{
+	return (dp->dp_unlinked_drain_taskq);
+}
+
 /*
  * Walk through the pool-wide zap object of temporary snapshot user holds
  * and release them.
diff --git a/usr/src/uts/common/fs/zfs/sys/dsl_pool.h b/usr/src/uts/common/fs/zfs/sys/dsl_pool.h
index de13fa8bfa..cda8324da6 100644
--- a/usr/src/uts/common/fs/zfs/sys/dsl_pool.h
+++ b/usr/src/uts/common/fs/zfs/sys/dsl_pool.h
@@ -93,6 +93,7 @@ typedef struct dsl_pool {
 	struct dsl_dataset *dp_origin_snap;
 	uint64_t dp_root_dir_obj;
 	taskq_t *dp_vnrele_taskq;
+	struct taskq *dp_unlinked_drain_taskq;
 
 	/* No lock needed - sync context only */
 	blkptr_t dp_meta_rootbp;
@@ -173,6 +174,7 @@ boolean_t dsl_pool_config_held_writer(dsl_pool_t *dp);
 boolean_t dsl_pool_need_dirty_delay(dsl_pool_t *dp);
 
 taskq_t *dsl_pool_vnrele_taskq(dsl_pool_t *dp);
+taskq_t *dsl_pool_unlinked_drain_taskq(dsl_pool_t *dp);
 
 int dsl_pool_user_hold(dsl_pool_t *dp, uint64_t dsobj,
     const char *tag, uint64_t now, dmu_tx_t *tx);
diff --git a/usr/src/uts/common/fs/zfs/sys/zfs_dir.h b/usr/src/uts/common/fs/zfs/sys/zfs_dir.h
index 349f8ef373..d8edb79286 100644
--- a/usr/src/uts/common/fs/zfs/sys/zfs_dir.h
+++ b/usr/src/uts/common/fs/zfs/sys/zfs_dir.h
@@ -63,6 +63,7 @@ extern void zfs_dl_name_switch(zfs_dirlock_t *dl, char *new, char **old);
 extern boolean_t zfs_dirempty(znode_t *);
 extern void zfs_unlinked_add(znode_t *, dmu_tx_t *);
 extern void zfs_unlinked_drain(zfsvfs_t *zfsvfs);
+extern void zfs_unlinked_drain_stop_wait(zfsvfs_t *zfsvfs);
 extern int zfs_sticky_remove_access(znode_t *, znode_t *, cred_t *cr);
 extern int zfs_get_xattrdir(znode_t *, vnode_t **, cred_t *, int);
 extern int zfs_make_xattrdir(znode_t *, vattr_t *, vnode_t **, cred_t *);
diff --git a/usr/src/uts/common/fs/zfs/sys/zfs_vfsops.h b/usr/src/uts/common/fs/zfs/sys/zfs_vfsops.h
index 199c8446ca..0d1611b66c 100644
--- a/usr/src/uts/common/fs/zfs/sys/zfs_vfsops.h
+++ b/usr/src/uts/common/fs/zfs/sys/zfs_vfsops.h
@@ -75,6 +75,8 @@ struct zfsvfs {
 	boolean_t	z_use_fuids;	/* version allows fuids */
 	boolean_t	z_replay;	/* set during ZIL replay */
 	boolean_t	z_use_sa;	/* version allow system attributes */
+	boolean_t	z_draining;	/* is true when drain is active */
+	boolean_t	z_drain_cancel; /* signal the unlinked drain to stop */
 	uint64_t	z_version;	/* ZPL version */
 	uint64_t	z_shares_dir;	/* hidden shares dir */
 	kmutex_t	z_lock;
@@ -88,6 +90,7 @@ struct zfsvfs {
 	sa_attr_type_t	*z_attr_table;	/* SA attr mapping->id */
 #define	ZFS_OBJ_MTX_SZ	64
 	kmutex_t	z_hold_mtx[ZFS_OBJ_MTX_SZ];	/* znode hold locks */
+	taskqid_t	z_drain_task;	/* task id for the unlink drain task */
 };
 
 /*
diff --git a/usr/src/uts/common/fs/zfs/zfs_dir.c b/usr/src/uts/common/fs/zfs/zfs_dir.c
index 990abb784f..3841c11d11 100644
--- a/usr/src/uts/common/fs/zfs/zfs_dir.c
+++ b/usr/src/uts/common/fs/zfs/zfs_dir.c
@@ -480,20 +480,23 @@ zfs_unlinked_add(znode_t *zp, dmu_tx_t *tx)
  * Clean up any znodes that had no links when we either crashed or
  * (force) umounted the file system.
  */
-void
-zfs_unlinked_drain(zfsvfs_t *zfsvfs)
+static void
+zfs_unlinked_drain_task(void *arg)
 {
+	zfsvfs_t *zfsvfs = arg;
 	zap_cursor_t	zc;
 	zap_attribute_t zap;
 	dmu_object_info_t doi;
 	znode_t		*zp;
 	int		error;
 
+	ASSERT3B(zfsvfs->z_draining, ==, B_TRUE);
+
 	/*
 	 * Interate over the contents of the unlinked set.
 	 */
 	for (zap_cursor_init(&zc, zfsvfs->z_os, zfsvfs->z_unlinkedobj);
-	    zap_cursor_retrieve(&zc, &zap) == 0;
+	    zap_cursor_retrieve(&zc, &zap) == 0 && !zfsvfs->z_drain_cancel;
 	    zap_cursor_advance(&zc)) {
 
 		/*
@@ -523,9 +526,52 @@ zfs_unlinked_drain(zfsvfs_t *zfsvfs)
 			continue;
 
 		zp->z_unlinked = B_TRUE;
+
 		VN_RELE(ZTOV(zp));
+		ASSERT3B(zfsvfs->z_unmounted, ==, B_FALSE);
 	}
 	zap_cursor_fini(&zc);
+
+	zfsvfs->z_draining = B_FALSE;
+	zfsvfs->z_drain_task = TASKQID_INVALID;
+}
+
+/*
+ * Sets z_draining then tries to dispatch async unlinked drain.
+ * If that fails executes synchronous unlinked drain.
+ */
+void
+zfs_unlinked_drain(zfsvfs_t *zfsvfs)
+{
+	ASSERT3B(zfsvfs->z_unmounted, ==, B_FALSE);
+	ASSERT3B(zfsvfs->z_draining, ==, B_FALSE);
+
+	zfsvfs->z_draining = B_TRUE;
+	zfsvfs->z_drain_cancel = B_FALSE;
+
+	zfsvfs->z_drain_task = taskq_dispatch(
+	    dsl_pool_unlinked_drain_taskq(dmu_objset_pool(zfsvfs->z_os)),
+	    zfs_unlinked_drain_task, zfsvfs, TQ_SLEEP);
+	if (zfsvfs->z_drain_task == TASKQID_INVALID) {
+		zfs_dbgmsg("async zfs_unlinked_drain dispatch failed");
+		zfs_unlinked_drain_task(zfsvfs);
+	}
+}
+
+/*
+ * Wait for the unlinked drain taskq task to stop. This will interrupt the
+ * unlinked set processing if it is in progress.
+ */
+void
+zfs_unlinked_drain_stop_wait(zfsvfs_t *zfsvfs)
+{
+	ASSERT3B(zfsvfs->z_unmounted, ==, B_FALSE);
+
+	while (zfsvfs->z_draining) {
+		zfsvfs->z_drain_cancel = B_TRUE;
+		taskq_wait(dsl_pool_unlinked_drain_taskq(
+		    dmu_objset_pool(zfsvfs->z_os)));
+	}
 }
 
 /*
@@ -1109,7 +1155,7 @@ top:
 int
 zfs_sticky_remove_access(znode_t *zdp, znode_t *zp, cred_t *cr)
 {
-	uid_t  		uid;
+	uid_t		uid;
 	uid_t		downer;
 	uid_t		fowner;
 	zfsvfs_t	*zfsvfs = zdp->z_zfsvfs;
diff --git a/usr/src/uts/common/fs/zfs/zfs_vfsops.c b/usr/src/uts/common/fs/zfs/zfs_vfsops.c
index aad3855d98..1314204765 100644
--- a/usr/src/uts/common/fs/zfs/zfs_vfsops.c
+++ b/usr/src/uts/common/fs/zfs/zfs_vfsops.c
@@ -1200,6 +1200,10 @@ zfsvfs_create_impl(zfsvfs_t **zfvp, zfsvfs_t *zfsvfs, objset_t *os)
 		return (error);
 	}
 
+	zfsvfs->z_drain_task = TASKQID_INVALID;
+	zfsvfs->z_draining = B_FALSE;
+	zfsvfs->z_drain_cancel = B_TRUE;
+
 	*zfvp = zfsvfs;
 	return (0);
 }
@@ -1228,10 +1232,11 @@ zfsvfs_setup(zfsvfs_t *zfsvfs, boolean_t mounting)
 		 * allow replays to succeed.
 		 */
 		readonly = zfsvfs->z_vfs->vfs_flag & VFS_RDONLY;
-		if (readonly != 0)
+		if (readonly != 0) {
 			zfsvfs->z_vfs->vfs_flag &= ~VFS_RDONLY;
-		else
+		} else {
 			zfs_unlinked_drain(zfsvfs);
+		}
 
 		/*
 		 * Parse and replay the intent log.
@@ -2038,6 +2043,8 @@ zfsvfs_teardown(zfsvfs_t *zfsvfs, boolean_t unmounting)
 {
 	znode_t	*zp;
 
+	zfs_unlinked_drain_stop_wait(zfsvfs);
+
 	rrm_enter(&zfsvfs->z_teardown_lock, RW_WRITER, FTAG);
 
 	if (!unmounting) {
@@ -2359,6 +2366,16 @@ zfs_resume_fs(zfsvfs_t *zfsvfs, dsl_dataset_t *ds)
 	}
 	mutex_exit(&zfsvfs->z_znodes_lock);
 
+	if (((zfsvfs->z_vfs->vfs_flag & VFS_RDONLY) == 0) &&
+	    !zfsvfs->z_unmounted) {
+		/*
+		 * zfs_suspend_fs() could have interrupted freeing
+		 * of dnodes. We need to restart this freeing so
+		 * that we don't "leak" the space.
+		 */
+		zfs_unlinked_drain(zfsvfs);
+	}
+
 bail:
 	/* release the VOPs */
 	rw_exit(&zfsvfs->z_teardown_inactive_lock);
diff --git a/usr/src/uts/common/fs/zfs/zfs_znode.c b/usr/src/uts/common/fs/zfs/zfs_znode.c
index 9abfc025d5..257d5b2a35 100644
--- a/usr/src/uts/common/fs/zfs/zfs_znode.c
+++ b/usr/src/uts/common/fs/zfs/zfs_znode.c
@@ -100,6 +100,12 @@ krwlock_t zfsvfs_lock;
 
 static kmem_cache_t *znode_cache = NULL;
 
+/*
+ * This is used by the test suite so that it can delay znodes from being
+ * freed in order to inspect the unlinked set.
+ */
+int zfs_unlink_suspend_progress = 0;
+
 /*ARGSUSED*/
 static void
 znode_evict_error(dmu_buf_t *dbuf, void *user_ptr)
@@ -1416,7 +1422,8 @@ zfs_zinactive(znode_t *zp)
 	 */
 	if (zp->z_unlinked) {
 		ASSERT(!zfsvfs->z_issnap);
-		if ((zfsvfs->z_vfs->vfs_flag & VFS_RDONLY) == 0) {
+		if ((zfsvfs->z_vfs->vfs_flag & VFS_RDONLY) == 0 &&
+		    !zfs_unlink_suspend_progress) {
 			mutex_exit(&zp->z_lock);
 			ZFS_OBJ_HOLD_EXIT(zfsvfs, z_id);
 			zfs_rmnode(zp);