OS-8054 inotify watches lead to EBUSY during zfs mountOS-8054

10 files changed, 353 insertions, 7 deletions

diff --git a/usr/src/test/zfs-tests/cmd/scripts/zfstest.ksh b/usr/src/test/zfs-tests/cmd/scripts/zfstest.ksh
index e38b5a34a1..dd93e6e833 100644
--- a/usr/src/test/zfs-tests/cmd/scripts/zfstest.ksh
+++ b/usr/src/test/zfs-tests/cmd/scripts/zfstest.ksh
@@ -135,6 +135,9 @@ function constrain_path
 	pkgsrc_bin=/opt/local/bin
 	pkgsrc_packages="sudo truncate python base64 shuf sha256sum"
 	for pkg in $pkgsrc_packages; do
+		if [[ ! -x $pkgsrc_bin/$pkg ]]; then
+			fail "$pkg not found in $pkgsrc_bin"
+		fi
 		if [[ ! -x $PATHDIR/$pkg ]]; then
 			rm $PATHDIR/$pkg &&
 			    ln -s $pkgsrc_bin/$pkg $PATHDIR/$pkg ||
diff --git a/usr/src/test/zfs-tests/cmd/watch_dir/Makefile b/usr/src/test/zfs-tests/cmd/watch_dir/Makefile
new file mode 100644
index 0000000000..e178ec4b34
--- /dev/null
+++ b/usr/src/test/zfs-tests/cmd/watch_dir/Makefile
@@ -0,0 +1,20 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright 2019 Joyent, Inc.
+#
+
+PROG = watch_dir
+
+include $(SRC)/cmd/Makefile.cmd
+
+include ../Makefile.subdirs
diff --git a/usr/src/test/zfs-tests/cmd/watch_dir/watch_dir.c b/usr/src/test/zfs-tests/cmd/watch_dir/watch_dir.c
new file mode 100644
index 0000000000..9f68df1159
--- /dev/null
+++ b/usr/src/test/zfs-tests/cmd/watch_dir/watch_dir.c
@@ -0,0 +1,144 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2019 Joyent, Inc.
+ */
+
+/*
+ * This program watches a directory with portfs or inotify, exiting when the
+ * directory is removed.  It is useful in tests that ensure that watching a
+ * directory does not prevent it from being used as a mount point.
+ */
+#include <limits.h>
+#include <port.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/inotify.h>
+#include <unistd.h>
+
+void
+fail_usage(void)
+{
+	(void) fprintf(stderr, "Usage: watch <portfs|inotify> directory\n");
+	exit(1);
+}
+
+void
+watch_port(char *path)
+{
+	int port;
+	struct file_obj fobj = {0};
+
+	if ((port = port_create()) == -1) {
+		perror("port_create");
+		exit(1);
+	}
+
+	fobj.fo_name = path;
+	for (;;) {
+		timespec_t ts = {300, 0};
+		port_event_t pe;
+
+		if (port_associate(port, PORT_SOURCE_FILE, (uintptr_t)&fobj, 0,
+		    path) != 0) {
+			perror("port_associate");
+			exit(1);
+		}
+
+		if (port_get(port, &pe, &ts) != 0) {
+			perror("port_get");
+			exit(1);
+		}
+
+		if (pe.portev_events & FILE_DELETE) {
+			(void) printf("DELETE\t%s\n", path);
+			exit(0);
+		}
+		if (pe.portev_events & MOUNTEDOVER) {
+			(void) printf("MOUNTEDOVER\t%s\n", path);
+		}
+	}
+}
+
+void
+watch_inotify(char *path)
+{
+	int in, wd;
+	struct inotify_event ev;
+
+	if ((in = inotify_init()) == -1) {
+		perror("inotify_init");
+		exit(1);
+	}
+	if ((wd = inotify_add_watch(in, path, IN_DELETE_SELF)) == -1) {
+		perror("inotify_add_watch");
+		exit(1);
+	}
+
+	for (;;) {
+		ssize_t cnt;
+		char evpath[PATH_MAX];
+
+		cnt = read(in, &ev, sizeof (ev));
+		if (cnt != sizeof (ev)) {
+			(void) fprintf(stderr,
+			    "read: expected %ld bytes got %ld\n",
+			    sizeof (ev), cnt);
+			exit(1);
+		}
+		if (ev.len != 0) {
+			if (ev.len > sizeof (evpath)) {
+				(void) fprintf(stderr, "read: oversize "
+				    "path (%u bytes)\n", ev.len);
+				exit(1);
+			}
+			cnt = read(in, evpath, ev.len);
+			if (cnt != ev.len) {
+				(void) fprintf(stderr, "read: expected %ld "
+				    "bytes for path, got %ld\n", ev.len, cnt);
+				exit(1);
+			}
+			evpath[ev.len - 1] = '\0';
+		} else {
+			evpath[0] = '\0';
+		}
+		if (ev.mask & IN_DELETE_SELF) {
+			(void) printf("DELETE_SELF\t%s\n", evpath);
+			exit(0);
+		} else {
+			(void) printf("EVENT_%08x\t%s\n", ev.mask, evpath);
+		}
+	}
+}
+
+int
+main(int argc, char **argv)
+{
+	char *watcher, *path;
+
+	if (argc != 3) {
+		fail_usage();
+	}
+	watcher = argv[1];
+	path = argv[2];
+
+	if (strcmp(watcher, "portfs") == 0) {
+		watch_port(path);
+	} else if (strcmp(watcher, "inotify") == 0) {
+		watch_inotify(path);
+	} else {
+		fail_usage();
+	}
+
+	return (0);
+}
diff --git a/usr/src/test/zfs-tests/include/commands.cfg b/usr/src/test/zfs-tests/include/commands.cfg
index 050d6caba7..8b023a16ba 100644
--- a/usr/src/test/zfs-tests/include/commands.cfg
+++ b/usr/src/test/zfs-tests/include/commands.cfg
@@ -195,4 +195,5 @@ export ZFSTEST_FILES='chg_usr_exec
     randwritecomp
     readmmap
     rename_dir
-    rm_lnkcnt_zero_file'
+    rm_lnkcnt_zero_file
+    watch_dir'
diff --git a/usr/src/test/zfs-tests/runfiles/smartos.run b/usr/src/test/zfs-tests/runfiles/smartos.run
index a9ce4c13d1..187f6694ff 100644
--- a/usr/src/test/zfs-tests/runfiles/smartos.run
+++ b/usr/src/test/zfs-tests/runfiles/smartos.run
@@ -130,7 +130,8 @@ tests = ['zfs_mount_001_pos', 'zfs_mount_002_pos', 'zfs_mount_003_pos',
     'zfs_mount_007_pos', 'zfs_mount_008_pos', 'zfs_mount_009_neg',
     'zfs_mount_010_neg', 'zfs_mount_011_neg', 'zfs_mount_012_neg',
     'zfs_mount_all_001_pos', 'zfs_mount_all_fail', 'zfs_mount_all_mountpoints',
-    'zfs_mount_encrypted']
+    'zfs_mount_encrypted', 'zfs_mount_watched_inotify',
+    'zfs_mount_watched_portfs' ]
 
 [/opt/zfs-tests/tests/functional/cli_root/zfs_program]
 tests = ['zfs_program_json']
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zfs_mount/zfs_mount_watched_inotify.ksh b/usr/src/test/zfs-tests/tests/functional/cli_root/zfs_mount/zfs_mount_watched_inotify.ksh
new file mode 100644
index 0000000000..7330bb6265
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zfs_mount/zfs_mount_watched_inotify.ksh
@@ -0,0 +1,74 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2019 Joyent, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+# 'zfs mount' should not get EBUSY due to inotify(5) watching a directory
+#
+# STRATEGY:
+# 1. Create a directory
+# 2. Start watching the directory with inotify(5).
+# 3. Create a filesystem
+# 4. Mount the filesystem at the directory created in step 1
+# 5. Destroy the filesystem
+# 6. Remove the directory
+# 7. Verify the watcher saw the directory removal
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+	datasetexists $TESTPOOL/$TESTFS1 && \
+		log_must zfs destroy -f $TESTPOOL/$TESTFS1
+	log_must rm -rf "$TESTDIR/mntpt"
+}
+
+log_onexit cleanup
+
+log_assert "'zfs mount' should not get EBUSY due to inotify(5) watching a directory"
+
+# 1. Create a directory.
+log_must mkdir -p "$TESTDIR/mntpt"
+
+# 2. Start watching the directory with inotify(5).
+watch_dir inotify $TESTDIR/mntpt > $TESTDIR/watch_dir.log &
+
+# 3. Create a filesystem
+log_must zfs create $TESTPOOL/$TESTFS1
+
+# 4. Mount the file system at the directory created in step 1
+log_must zfs set mountpoint=$TESTDIR/mntpt $TESTPOOL/$TESTFS1
+
+# 5. Destroy the filesystem
+log_must zfs destroy $TESTPOOL/$TESTFS1
+
+# 6. Remove the directory.  The corresponding inotify event will cause the
+# watcher to exit.
+log_must rmdir $TESTDIR/mntpt
+
+# 7. Verify the watcher saw the directory removal. This ensures that the watcher
+# was watching the directory we are interested in.
+wait
+log_must grep -q DELETE_SELF $TESTDIR/watch_dir.log
+
+log_pass "'zfs mount' should not get EBUSY due to inotify(5) watching a directory"
diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zfs_mount/zfs_mount_watched_portfs.ksh b/usr/src/test/zfs-tests/tests/functional/cli_root/zfs_mount/zfs_mount_watched_portfs.ksh
new file mode 100644
index 0000000000..a84c9bb8e7
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zfs_mount/zfs_mount_watched_portfs.ksh
@@ -0,0 +1,74 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2019 Joyent, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+# 'zfs mount' should not get EBUSY due to portfs watching a directory
+#
+# STRATEGY:
+# 1. Create a directory
+# 2. Start watching the directory with port_associate
+# 3. Create a filesystem
+# 4. Mount the filesystem at the directory created in step 1
+# 5. Destroy the filesystem
+# 6. Remove the directory
+# 7. Verify the watcher saw the directory removal
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+	datasetexists $TESTPOOL/$TESTFS1 && \
+		log_must zfs destroy -f $TESTPOOL/$TESTFS1
+	log_must rm -rf "$TESTDIR/mntpt"
+}
+
+log_onexit cleanup
+
+log_assert "'zfs mount' should not get EBUSY due to portfs watching a directory"
+
+# 1. Create a directory.
+log_must mkdir -p "$TESTDIR/mntpt"
+
+# 2. Start watching the directory with port_associate
+watch_dir portfs $TESTDIR/mntpt > $TESTDIR/watch_dir.log &
+
+# 3. Create a filesystem
+log_must zfs create $TESTPOOL/$TESTFS1
+
+# 4. Mount the file system at the directory created in step 1
+log_must zfs set mountpoint=$TESTDIR/mntpt $TESTPOOL/$TESTFS1
+
+# 5. Destroy the filesystem
+log_must zfs destroy $TESTPOOL/$TESTFS1
+
+# 6. Remove the directory.  The corresponding portfs event will cause the
+# watcher to exit.
+log_must rmdir $TESTDIR/mntpt
+
+# 7. Verify the watcher saw the directory removal. This ensures that the watcher
+# was watching the directory we are interested in.
+wait
+log_must grep -q DELETE $TESTDIR/watch_dir.log
+
+log_pass "'zfs mount' should not get EBUSY due to portfs watching a directory"
diff --git a/usr/src/uts/common/fs/fem.c b/usr/src/uts/common/fs/fem.c
index 50633859ce..e5c318c6d7 100644
--- a/usr/src/uts/common/fs/fem.c
+++ b/usr/src/uts/common/fs/fem.c
@@ -24,7 +24,7 @@
  */
 
 /*
- * Copyright (c) 2015, Joyent, Inc.  All rights reserved.
+ * Copyright 2019 Joyent, Inc.
  */
 
 #include <sys/types.h>
@@ -3977,6 +3977,35 @@ fem_getvnops(vnode_t *v)
 	return (r);
 }
 
+/*
+ * Returns the number of vnode refs that are not associated with fem.
+ */
+int
+fem_getvnrefs(vnode_t *v)
+{
+	int val;
+	struct fem_list *fl;
+
+	ASSERT(MUTEX_HELD(&v->v_lock));
+	if (v->v_femhead == NULL) {
+		return (v->v_count);
+	}
+
+	if ((fl = fem_lock(v->v_femhead)) == NULL) {
+		val = v->v_count;
+	} else {
+		ASSERT3S(fl->feml_tos, >, 0);
+		ASSERT3S(fl->feml_tos, <=, v->v_count);
+		/*
+		 * The first item on the list is a guard entry that does not
+		 * have a vnode ref.
+		 */
+		val = fl->feml_tos - v->v_count + 1;
+	}
+	fem_unlock(v->v_femhead);
+
+	return (val);
+}
 
 /*
  * VFS interposition
diff --git a/usr/src/uts/common/fs/zfs/zfs_vfsops.c b/usr/src/uts/common/fs/zfs/zfs_vfsops.c
index 6b61cd7a84..cb542f1037 100644
--- a/usr/src/uts/common/fs/zfs/zfs_vfsops.c
+++ b/usr/src/uts/common/fs/zfs/zfs_vfsops.c
@@ -20,11 +20,10 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, Joyent, Inc. All rights reserved.
+ * Copyright 2019 Joyent, Inc.
  * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
  * Copyright (c) 2014 Integros [integros.com]
  * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
- * Copyright 2019 Joyent, Inc.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -1832,7 +1831,7 @@ zfs_mount(vfs_t *vfsp, vnode_t *mvp, struct mounta *uap, cred_t *cr)
 	mutex_enter(&mvp->v_lock);
 	if ((uap->flags & MS_REMOUNT) == 0 &&
 	    (uap->flags & MS_OVERLAY) == 0 &&
-	    (mvp->v_count != 1 || (mvp->v_flag & VROOT))) {
+	    (fem_getvnrefs(mvp) != 1 || (mvp->v_flag & VROOT))) {
 		mutex_exit(&mvp->v_lock);
 		return (SET_ERROR(EBUSY));
 	}
diff --git a/usr/src/uts/common/sys/fem.h b/usr/src/uts/common/sys/fem.h
index beb838fdfa..f606c47e3b 100644
--- a/usr/src/uts/common/sys/fem.h
+++ b/usr/src/uts/common/sys/fem.h
@@ -23,6 +23,7 @@
  * Use is subject to license terms.
  *
  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright 2019 Joyent, Inc.
  */
 
 #ifndef _SYS_FEM_H
@@ -427,7 +428,7 @@ extern int fem_is_installed(struct vnode *v, fem_t *mon, void *arg);
 extern int fem_uninstall(struct vnode *v, fem_t *mon, void *arg);
 extern vnodeops_t *fem_getvnops(struct vnode *v);
 extern void fem_setvnops(struct vnode *v, struct vnodeops *nops);
-
+extern int fem_getvnrefs(vnode_t *);
 
 extern int fsem_create(char *name, const struct fs_operation_def *templ,
 			fsem_t **actual);