12554 want sdev plugin framework

Portions contributed by: John Levon <john.levon@joyent.com>
Portions contributed by: Mike Gerdts <mike.gerdts@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Toomas Soome <tsoome@me.com>
Approved by: Dan McDonald <danmcd@joyent.com>

8 files changed, 1153 insertions, 236 deletions

diff --git a/usr/src/uts/common/Makefile.files b/usr/src/uts/common/Makefile.files
index 36b2c49d67..dbed5ea9cc 100644
--- a/usr/src/uts/common/Makefile.files
+++ b/usr/src/uts/common/Makefile.files
@@ -1067,8 +1067,7 @@ DEVFS_OBJS +=	devfs_subr.o	devfs_vfsops.o	devfs_vnops.o
 DEV_OBJS  +=	sdev_subr.o	sdev_vfsops.o	sdev_vnops.o	\
 		sdev_ptsops.o	sdev_zvolops.o	sdev_comm.o	\
 		sdev_profile.o	sdev_ncache.o	sdev_netops.o	\
-		sdev_ipnetops.o	\
-		sdev_vtops.o
+		sdev_ipnetops.o	sdev_vtops.o	sdev_plugin.o
 
 CTFS_OBJS +=	ctfs_all.o ctfs_cdir.o ctfs_ctl.o ctfs_event.o \
 		ctfs_latest.o ctfs_root.o ctfs_sym.o ctfs_tdir.o ctfs_tmpl.o
diff --git a/usr/src/uts/common/fs/dev/sdev_plugin.c b/usr/src/uts/common/fs/dev/sdev_plugin.c
new file mode 100644
index 0000000000..8f825eccff
--- /dev/null
+++ b/usr/src/uts/common/fs/dev/sdev_plugin.c
@@ -0,0 +1,948 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2019 Joyent, Inc.
+ */
+
+/*
+ * Dynamic directory plugin interface for sdev.
+ *
+ * The sdev plugin interfaces provides a means for a dynamic directory based on
+ * in-kernel state to be simply created. Traditionally, dynamic directories were
+ * built into sdev itself. While these legacy plugins are useful, it makes more
+ * sense for these pieces of functionality to live with the individual drivers.
+ *
+ * The plugin interface requires folks to implement three interfaces and
+ * provides a series of callbacks that can be made in the context of those
+ * interfaces to interrogate the sdev_node_t without having to leak
+ * implementation details of the sdev_node_t. These interfaces are:
+ *
+ *   o spo_validate
+ *
+ *   Given a particular node, answer the question as to whether or not this
+ *   entry is still valid. Here, plugins should use the name and the dev_t
+ *   associated with the node to verify that it matches something that still
+ *   exists.
+ *
+ *   o spo_filldir
+ *
+ *   Fill all the entries inside of a directory. Note that some of these entries
+ *   may already exist.
+ *
+ *   o spo_inactive
+ *
+ *   The given node is no longer being used. This allows the consumer to
+ *   potentially tear down anything that was being held open related to this.
+ *   Note that this only fires when the given sdev_node_t becomes a zombie.
+ *
+ * During these callbacks a consumer is not allowed to register or unregister a
+ * plugin, especially their own. They may call the sdev_ctx style functions. All
+ * callbacks fire in a context where blocking is allowed (eg. the spl is below
+ * LOCK_LEVEL).
+ *
+ * When a plugin is added, we create its directory in the global zone. By doing
+ * that, we ensure that something isn't already there and that nothing else can
+ * come along and try and create something without our knowledge. We only have
+ * to create it in the GZ and not for all other instances of sdev because an
+ * instance of sdev that isn't at /dev does not have dynamic directories, and
+ * second, any instance of sdev present in a non-global zone cannot create
+ * anything, therefore we know that by it not being in the global zone's
+ * instance of sdev that we're good to go.
+ *
+ * Lock Ordering
+ * -------------
+ *
+ * The global sdev_plugin_lock must be held before any of the individual
+ * sdev_plugin_t`sp_lock. Further, once any plugin related lock has been held,
+ * it is not legal to take any holds on any sdev_node_t or to grab the
+ * sdev_node_t`contents_lock in any way.
+ */
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/fs/sdev_impl.h>
+#include <sys/fs/sdev_plugin.h>
+#include <fs/fs_subr.h>
+#include <sys/ddi.h>
+#include <sys/sunddi.h>
+#include <sys/ksynch.h>
+#include <sys/sysmacros.h>
+#include <sys/list.h>
+#include <sys/ctype.h>
+
+kmutex_t sdev_plugin_lock;
+list_t sdev_plugin_list;
+kmem_cache_t *sdev_plugin_cache;
+struct vnodeops *sdev_plugin_vnops;
+
+#define	SDEV_PLUGIN_NAMELEN	64
+
+typedef struct sdev_plugin {
+	list_node_t sp_link;
+	char sp_name[SDEV_PLUGIN_NAMELEN];	/* E */
+	int sp_nflags;				/* E */
+	struct vnodeops *sp_vnops;		/* E */
+	sdev_plugin_ops_t *sp_pops;		/* E */
+	boolean_t sp_islegacy;			/* E */
+	int (*sp_lvtor)(sdev_node_t *);		/* E */
+	kmutex_t sp_lock;			/* Protects everything below */
+	kcondvar_t sp_nodecv;
+	size_t sp_nnodes;
+} sdev_plugin_t;
+
+/* ARGSUSED */
+static int
+sdev_plugin_cache_constructor(void *buf, void *arg, int tags)
+{
+	sdev_plugin_t *spp = buf;
+	mutex_init(&spp->sp_lock, NULL, MUTEX_DRIVER, 0);
+	cv_init(&spp->sp_nodecv, NULL, CV_DRIVER, NULL);
+	return (0);
+}
+
+/* ARGSUSED */
+static void
+sdev_plugin_cache_destructor(void *buf, void *arg)
+{
+	sdev_plugin_t *spp = buf;
+	cv_destroy(&spp->sp_nodecv);
+	mutex_destroy(&spp->sp_lock);
+}
+
+enum vtype
+sdev_ctx_vtype(sdev_ctx_t ctx)
+{
+	sdev_node_t *sdp = (sdev_node_t *)ctx;
+
+	ASSERT(RW_LOCK_HELD(&sdp->sdev_contents));
+	return (sdp->sdev_vnode->v_type);
+}
+
+const char *
+sdev_ctx_path(sdev_ctx_t ctx)
+{
+	sdev_node_t *sdp = (sdev_node_t *)ctx;
+
+	ASSERT(RW_LOCK_HELD(&sdp->sdev_contents));
+	return (sdp->sdev_path);
+}
+
+const char *
+sdev_ctx_name(sdev_ctx_t ctx)
+{
+	sdev_node_t *sdp = (sdev_node_t *)ctx;
+
+	ASSERT(RW_LOCK_HELD(&sdp->sdev_contents));
+	return (sdp->sdev_name);
+}
+
+int
+sdev_ctx_minor(sdev_ctx_t ctx, minor_t *minorp)
+{
+	sdev_node_t *sdp = (sdev_node_t *)ctx;
+
+	ASSERT(RW_LOCK_HELD(&sdp->sdev_contents));
+	ASSERT(minorp != NULL);
+	if (sdp->sdev_vnode->v_type == VCHR ||
+	    sdp->sdev_vnode->v_type == VBLK) {
+		*minorp = getminor(sdp->sdev_vnode->v_rdev);
+		return (0);
+	}
+
+	return (ENODEV);
+}
+
+/*
+ * Currently we only support psasing through a single flag -- SDEV_IS_GLOBAL.
+ */
+sdev_ctx_flags_t
+sdev_ctx_flags(sdev_ctx_t ctx)
+{
+	sdev_node_t *sdp = (sdev_node_t *)ctx;
+
+	ASSERT(RW_LOCK_HELD(&sdp->sdev_contents));
+	return (sdp->sdev_flags & SDEV_GLOBAL);
+}
+
+/*
+ * Use the same rules as zones for a name. isalphanum + '-', '_', and '.'.
+ */
+static int
+sdev_plugin_name_isvalid(const char *c, int buflen)
+{
+	int i;
+
+	for (i = 0; i < buflen; i++, c++) {
+		if (*c == '\0')
+			return (1);
+
+		if (!isalnum(*c) && *c != '-' && *c != '_' && *c != '.')
+			return (0);
+	}
+	/* Never found a null terminator */
+	return (0);
+}
+
+static int
+sdev_plugin_mknode(sdev_plugin_t *spp, sdev_node_t *sdvp, char *name,
+    vattr_t *vap)
+{
+	int ret;
+	sdev_node_t *svp;
+
+	ASSERT(RW_WRITE_HELD(&sdvp->sdev_contents));
+	ASSERT(spp != NULL);
+	svp = sdev_cache_lookup(sdvp, name);
+	if (svp != NULL) {
+		SDEV_SIMPLE_RELE(svp);
+		return (EEXIST);
+	}
+
+	ret = sdev_mknode(sdvp, name, &svp, vap, NULL, NULL, kcred,
+	    SDEV_READY);
+	if (ret != 0)
+		return (ret);
+	SDEV_SIMPLE_RELE(svp);
+
+	return (0);
+}
+
+/*
+ * Plugin node creation callbacks
+ */
+int
+sdev_plugin_mkdir(sdev_ctx_t ctx, char *name)
+{
+	sdev_node_t *sdvp;
+	timestruc_t now;
+	struct vattr vap;
+
+	if (sdev_plugin_name_isvalid(name, SDEV_PLUGIN_NAMELEN) == 0)
+		return (EINVAL);
+
+	sdvp = (sdev_node_t *)ctx;
+	ASSERT(sdvp->sdev_private != NULL);
+	ASSERT(RW_WRITE_HELD(&sdvp->sdev_contents));
+
+	vap = *sdev_getdefault_attr(VDIR);
+	gethrestime(&now);
+	vap.va_atime = now;
+	vap.va_mtime = now;
+	vap.va_ctime = now;
+
+	return (sdev_plugin_mknode(sdvp->sdev_private, sdvp, name, &vap));
+}
+
+int
+sdev_plugin_mknod(sdev_ctx_t ctx, char *name, mode_t mode, dev_t dev)
+{
+	sdev_node_t *sdvp;
+	timestruc_t now;
+	struct vattr vap;
+	mode_t type = mode & S_IFMT;
+	mode_t access = mode & S_IAMB;
+
+	if (sdev_plugin_name_isvalid(name, SDEV_PLUGIN_NAMELEN) == 0)
+		return (EINVAL);
+
+	sdvp = (sdev_node_t *)ctx;
+	ASSERT(RW_WRITE_HELD(&sdvp->sdev_contents));
+
+	/*
+	 * Ensure only type and user/group/other permission bits are present.
+	 * Do not allow setuid, setgid, etc.
+	 */
+	if ((mode & ~(S_IFMT | S_IAMB)) != 0)
+		return (EINVAL);
+
+	/* Disallow types other than character and block devices */
+	if (type != S_IFCHR && type != S_IFBLK)
+		return (EINVAL);
+
+	/* Disallow execute bits */
+	if ((access & (S_IXUSR | S_IXGRP | S_IXOTH)) != 0)
+		return (EINVAL);
+
+	/* No bits other than 0666 in access */
+	ASSERT((access &
+	    ~(S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) == 0);
+
+	/* Default to relatively safe access bits if none specified. */
+	if (access == 0)
+		access = 0600;
+
+	ASSERT(sdvp->sdev_private != NULL);
+
+	vap = *sdev_getdefault_attr(type == S_IFCHR ? VCHR : VBLK);
+	gethrestime(&now);
+	vap.va_atime = now;
+	vap.va_mtime = now;
+	vap.va_ctime = now;
+	vap.va_rdev = dev;
+	vap.va_mode = type | access;
+
+	/* Despite the similar name, this is in fact a different function */
+	return (sdev_plugin_mknode(sdvp->sdev_private, sdvp, name, &vap));
+}
+
+static int
+sdev_plugin_validate(sdev_node_t *sdp)
+{
+	int ret;
+	sdev_plugin_t *spp;
+
+	ASSERT(sdp->sdev_private != NULL);
+	spp = sdp->sdev_private;
+	ASSERT(spp->sp_islegacy == B_FALSE);
+	ASSERT(spp->sp_pops != NULL);
+	rw_enter(&sdp->sdev_contents, RW_READER);
+	ret = spp->sp_pops->spo_validate((uintptr_t)sdp);
+	rw_exit(&sdp->sdev_contents);
+	return (ret);
+}
+
+static void
+sdev_plugin_validate_dir(sdev_node_t *sdvp)
+{
+	int ret;
+	sdev_node_t *svp, *next;
+
+	ASSERT(RW_WRITE_HELD(&sdvp->sdev_contents));
+
+	for (svp = SDEV_FIRST_ENTRY(sdvp); svp != NULL; svp = next) {
+
+		next = SDEV_NEXT_ENTRY(sdvp, svp);
+		ASSERT(svp->sdev_state != SDEV_ZOMBIE);
+		/* skip nodes that aren't ready */
+		if (svp->sdev_state == SDEV_INIT)
+			continue;
+
+		switch (sdev_plugin_validate(svp)) {
+		case SDEV_VTOR_VALID:
+		case SDEV_VTOR_SKIP:
+			continue;
+		case SDEV_VTOR_INVALID:
+		case SDEV_VTOR_STALE:
+			break;
+		}
+
+		SDEV_HOLD(svp);
+
+		/*
+		 * Clean out everything underneath this node before we
+		 * remove it.
+		 */
+		if (svp->sdev_vnode->v_type == VDIR) {
+			ret = sdev_cleandir(svp, NULL, 0);
+			ASSERT(ret == 0);
+		}
+		/* remove the cache node */
+		(void) sdev_cache_update(sdvp, &svp, svp->sdev_name,
+		    SDEV_CACHE_DELETE);
+		SDEV_RELE(svp);
+	}
+}
+
+/* ARGSUSED */
+static int
+sdev_plugin_vop_readdir(struct vnode *dvp, struct uio *uiop, struct cred *cred,
+    int *eofp, caller_context_t *ct_unused, int flags_unused)
+{
+	int ret;
+	sdev_node_t *sdvp = VTOSDEV(dvp);
+	sdev_plugin_t *spp;
+
+	ASSERT(RW_READ_HELD(&sdvp->sdev_contents));
+
+	/* Sanity check we're not a zombie before we do anyting else */
+	if (sdvp->sdev_state == SDEV_ZOMBIE)
+		return (ENOENT);
+
+	spp = sdvp->sdev_private;
+	ASSERT(spp != NULL);
+	ASSERT(spp->sp_islegacy == B_FALSE);
+	ASSERT(spp->sp_pops != NULL);
+
+	if (crgetzoneid(cred) == GLOBAL_ZONEID && !SDEV_IS_GLOBAL(sdvp))
+		return (EPERM);
+
+	if (uiop->uio_offset == 0) {
+		/*
+		 * We upgrade to a write lock and grab the plugin's lock along
+		 * the way. We're almost certainly going to get creation
+		 * callbacks, so this is the only safe way to go.
+		 */
+		if (rw_tryupgrade(&sdvp->sdev_contents) == 0) {
+			rw_exit(&sdvp->sdev_contents);
+			rw_enter(&sdvp->sdev_contents, RW_WRITER);
+			if (sdvp->sdev_state == SDEV_ZOMBIE) {
+				rw_downgrade(&sdvp->sdev_contents);
+				return (ENOENT);
+			}
+		}
+
+		sdev_plugin_validate_dir(sdvp);
+		ret = spp->sp_pops->spo_filldir((uintptr_t)sdvp);
+		rw_downgrade(&sdvp->sdev_contents);
+		if (ret != 0)
+			return (ret);
+	}
+
+	return (devname_readdir_func(dvp, uiop, cred, eofp, 0));
+}
+
+/*
+ * If we don't have a callback function that returns a failure, then sdev will
+ * try to create a node for us which violates all of our basic assertions. To
+ * work around that we create our own callback for devname_lookup_func which
+ * always returns ENOENT as at this point either it was created with the filldir
+ * callback or it was not.
+ */
+/*ARGSUSED*/
+static int
+sdev_plugin_vop_lookup_cb(sdev_node_t *ddv, char *nm, void **arg, cred_t *cred,
+    void *unused, char *unused2)
+{
+	return (ENOENT);
+}
+
+/* ARGSUSED */
+static int
+sdev_plugin_vop_lookup(struct vnode *dvp, char *nm, struct vnode **vpp,
+    struct pathname *pnp, int flags, struct vnode *rdir, struct cred *cred,
+    caller_context_t *ct, int *direntflags, pathname_t *realpnp)
+{
+	int ret;
+	sdev_node_t *sdvp;
+	sdev_plugin_t *spp;
+
+	/* execute access is required to search the directory */
+	if ((ret = VOP_ACCESS(dvp, VEXEC, 0, cred, ct)) != 0)
+		return (ret);
+
+	sdvp = VTOSDEV(dvp);
+	spp = sdvp->sdev_private;
+	ASSERT(spp != NULL);
+	ASSERT(spp->sp_islegacy == B_FALSE);
+	ASSERT(spp->sp_pops != NULL);
+
+	if (crgetzoneid(cred) == GLOBAL_ZONEID && !SDEV_IS_GLOBAL(sdvp))
+		return (EPERM);
+
+	/*
+	 * Go straight for the write lock.
+	 */
+	rw_enter(&sdvp->sdev_contents, RW_WRITER);
+	if (sdvp->sdev_state == SDEV_ZOMBIE) {
+		rw_exit(&sdvp->sdev_contents);
+		return (ENOENT);
+	}
+	sdev_plugin_validate_dir(sdvp);
+	ret = spp->sp_pops->spo_filldir((uintptr_t)sdvp);
+	rw_exit(&sdvp->sdev_contents);
+	if (ret != 0)
+		return (ret);
+
+	return (devname_lookup_func(sdvp, nm, vpp, cred,
+	    sdev_plugin_vop_lookup_cb, SDEV_VATTR));
+}
+
+/*
+ * sdev is not a good citizen. We get inactive callbacks whenever a vnode goes
+ * to zero, but isn't necessairily a zombie yet. As such, to make things easier
+ * for users, we only fire the inactive callback when the node becomes a zombie
+ * and thus will be torn down here.
+ */
+static void
+sdev_plugin_vop_inactive_cb(struct vnode *dvp)
+{
+	sdev_node_t *sdp = VTOSDEV(dvp);
+	sdev_plugin_t *spp = sdp->sdev_private;
+
+	rw_enter(&sdp->sdev_contents, RW_READER);
+	if (sdp->sdev_state != SDEV_ZOMBIE) {
+		rw_exit(&sdp->sdev_contents);
+		return;
+	}
+	spp->sp_pops->spo_inactive((uintptr_t)sdp);
+	mutex_enter(&spp->sp_lock);
+	VERIFY(spp->sp_nnodes > 0);
+	spp->sp_nnodes--;
+	cv_signal(&spp->sp_nodecv);
+	mutex_exit(&spp->sp_lock);
+	rw_exit(&sdp->sdev_contents);
+}
+
+/*ARGSUSED*/
+static void
+sdev_plugin_vop_inactive(struct vnode *dvp, struct cred *cred,
+    caller_context_t *ct)
+{
+	sdev_node_t *sdp = VTOSDEV(dvp);
+	sdev_plugin_t *spp = sdp->sdev_private;
+	ASSERT(sdp->sdev_private != NULL);
+	ASSERT(spp->sp_islegacy == B_FALSE);
+	devname_inactive_func(dvp, cred, sdev_plugin_vop_inactive_cb);
+}
+
+const fs_operation_def_t sdev_plugin_vnodeops_tbl[] = {
+	VOPNAME_READDIR,	{ .vop_readdir = sdev_plugin_vop_readdir },
+	VOPNAME_LOOKUP,		{ .vop_lookup = sdev_plugin_vop_lookup },
+	VOPNAME_INACTIVE,	{ .vop_inactive = sdev_plugin_vop_inactive },
+	VOPNAME_CREATE,		{ .error = fs_nosys },
+	VOPNAME_REMOVE,		{ .error = fs_nosys },
+	VOPNAME_MKDIR,		{ .error = fs_nosys },
+	VOPNAME_RMDIR,		{ .error = fs_nosys },
+	VOPNAME_SYMLINK,	{ .error = fs_nosys },
+	VOPNAME_SETSECATTR,	{ .error = fs_nosys },
+	NULL,			NULL
+};
+
+/*
+ * construct a new template with overrides from vtab
+ */
+static fs_operation_def_t *
+sdev_merge_vtab(const fs_operation_def_t tab[])
+{
+	fs_operation_def_t *new;
+	const fs_operation_def_t *tab_entry;
+
+	/* make a copy of standard vnode ops table */
+	new = kmem_alloc(sdev_vnodeops_tbl_size, KM_SLEEP);
+	bcopy((void *)sdev_vnodeops_tbl, new, sdev_vnodeops_tbl_size);
+
+	/* replace the overrides from tab */
+	for (tab_entry = tab; tab_entry->name != NULL; tab_entry++) {
+		fs_operation_def_t *std_entry = new;
+		while (std_entry->name) {
+			if (strcmp(tab_entry->name, std_entry->name) == 0) {
+				std_entry->func = tab_entry->func;
+				break;
+			}
+			std_entry++;
+		}
+	}
+
+	return (new);
+}
+
+/* free memory allocated by sdev_merge_vtab */
+static void
+sdev_free_vtab(fs_operation_def_t *new)
+{
+	kmem_free(new, sdev_vnodeops_tbl_size);
+}
+
+/*
+ * Register a new plugin.
+ */
+sdev_plugin_hdl_t
+sdev_plugin_register(const char *name, sdev_plugin_ops_t *ops, int *errp)
+{
+	char buf[sizeof ("dev")] = "";
+	struct pathname pn = { 0 };
+	sdev_plugin_t *spp, *iter;
+	vnode_t *vp, *nvp;
+	sdev_node_t *sdp, *slp;
+	timestruc_t now;
+	struct vattr vap;
+	int ret, err;
+
+	/*
+	 * Some consumers don't care about why they failed. To keep the code
+	 * simple, we'll just pretend they gave us something.
+	 */
+	if (errp == NULL)
+		errp = &err;
+
+	if (sdev_plugin_name_isvalid(name, SDEV_PLUGIN_NAMELEN) == 0) {
+		*errp = EINVAL;
+		return ((sdev_plugin_hdl_t)NULL);
+	}
+
+	if (ops->spo_version != 1) {
+		*errp = EINVAL;
+		return ((sdev_plugin_hdl_t)NULL);
+	}
+
+	if (ops->spo_validate == NULL || ops->spo_filldir == NULL ||
+	    ops->spo_inactive == NULL) {
+		*errp = EINVAL;
+		return ((sdev_plugin_hdl_t)NULL);
+	}
+
+	if ((ops->spo_flags & ~SDEV_PLUGIN_FLAGS_MASK) != 0) {
+		*errp = EINVAL;
+		return ((sdev_plugin_hdl_t)NULL);
+	}
+
+	spp = kmem_cache_alloc(sdev_plugin_cache, KM_SLEEP);
+	(void) strlcpy(spp->sp_name, name, SDEV_PLUGIN_NAMELEN);
+
+	spp->sp_pops = ops;
+	spp->sp_nflags = SDEV_DYNAMIC | SDEV_VTOR;
+	if (ops->spo_flags & SDEV_PLUGIN_NO_NCACHE)
+		spp->sp_nflags |= SDEV_NO_NCACHE;
+	if (ops->spo_flags & SDEV_PLUGIN_SUBDIR)
+		spp->sp_nflags |= SDEV_SUBDIR;
+	spp->sp_vnops = sdev_plugin_vnops;
+	spp->sp_islegacy = B_FALSE;
+	spp->sp_lvtor = NULL;
+	spp->sp_nnodes = 0;
+
+	/*
+	 * Make sure our /dev entry is unique and install it.  We also need to
+	 * go through and grab the sdev root node as we cannot grab any sdev
+	 * node locks once we've grabbed the sdev_plugin_lock. We effectively
+	 * assert that if a directory is not present in the GZ's /dev, then it
+	 * doesn't exist in any of the local zones.
+	 *
+	 * Note that we may be in NGZ context: during a prof_filldir(".../dev/")
+	 * enumeration, for example. So we have to dig as deep as lookuppnvp()
+	 * to make sure we really get to the global /dev (i.e.  escape both
+	 * CRED() and ->u_rdir).
+	 */
+	(void) pn_get_buf("dev", UIO_SYSSPACE, &pn, buf, sizeof (buf));
+	VN_HOLD(rootdir);
+	ret = lookuppnvp(&pn, NULL, NO_FOLLOW, NULLVPP,
+	    &vp, rootdir, rootdir, kcred);
+
+	if (ret != 0) {
+		*errp = ret;
+		kmem_cache_free(sdev_plugin_cache, spp);
+		return ((sdev_plugin_hdl_t)NULL);
+	}
+	/* Make sure we have the real vnode */
+	if (VOP_REALVP(vp, &nvp, NULL) == 0) {
+		VN_HOLD(nvp);
+		VN_RELE(vp);
+		vp = nvp;
+		nvp = NULL;
+	}
+	VERIFY(vp->v_op == sdev_vnodeops);
+	sdp = VTOSDEV(vp);
+	rw_enter(&sdp->sdev_contents, RW_WRITER);
+	slp = sdev_cache_lookup(sdp, spp->sp_name);
+	if (slp != NULL) {
+		SDEV_RELE(slp);
+		rw_exit(&sdp->sdev_contents);
+		VN_RELE(vp);
+		*errp = EEXIST;
+		kmem_cache_free(sdev_plugin_cache, spp);
+		return ((sdev_plugin_hdl_t)NULL);
+	}
+
+	mutex_enter(&sdev_plugin_lock);
+	for (iter = list_head(&sdev_plugin_list); iter != NULL;
+	    iter = list_next(&sdev_plugin_list, iter)) {
+		if (strcmp(spp->sp_name, iter->sp_name) == 0) {
+			mutex_exit(&sdev_plugin_lock);
+			rw_exit(&sdp->sdev_contents);
+			VN_RELE(vp);
+			*errp = EEXIST;
+			kmem_cache_free(sdev_plugin_cache, spp);
+			return ((sdev_plugin_hdl_t)NULL);
+		}
+	}
+
+	list_insert_tail(&sdev_plugin_list, spp);
+	mutex_exit(&sdev_plugin_lock);
+
+	/*
+	 * Now go ahead and create the top level directory for the global zone.
+	 */
+	vap = *sdev_getdefault_attr(VDIR);
+	gethrestime(&now);
+	vap.va_atime = now;
+	vap.va_mtime = now;
+	vap.va_ctime = now;
+
+	(void) sdev_plugin_mknode(spp, sdp, spp->sp_name, &vap);
+
+	rw_exit(&sdp->sdev_contents);
+	VN_RELE(vp);
+
+	*errp = 0;
+
+	return ((sdev_plugin_hdl_t)spp);
+}
+
+static void
+sdev_plugin_unregister_cb(sdev_node_t *rdp, void *arg)
+{
+	sdev_plugin_t *spp = arg;
+	sdev_node_t *sdp;
+
+	rw_enter(&rdp->sdev_contents, RW_WRITER);
+	sdp = sdev_cache_lookup(rdp, spp->sp_name);
+	/* If it doesn't exist, we're done here */
+	if (sdp == NULL) {
+		rw_exit(&rdp->sdev_contents);
+		return;
+	}
+
+	/*
+	 * We first delete the directory before recursively marking everything
+	 * else stale. This ordering should ensure that we don't accidentally
+	 * miss anything.
+	 */
+	sdev_cache_update(rdp, &sdp, spp->sp_name, SDEV_CACHE_DELETE);
+	sdev_stale(sdp);
+	SDEV_RELE(sdp);
+	rw_exit(&rdp->sdev_contents);
+}
+
+int sdev_plugin_unregister_allowed;
+
+/*
+ * Remove a plugin. This will block until everything has become a zombie, thus
+ * guaranteeing the caller that nothing will call into them again once this call
+ * returns. While the call is ongoing, it could be called into. Note that while
+ * this is ongoing, it will block other mounts.
+ *
+ * NB: this is not safe when used from detach() context - we will be DEVI_BUSY,
+ * and other sdev threads may be waiting for this.  Only use the over-ride if
+ * willing to risk it.
+ */
+int
+sdev_plugin_unregister(sdev_plugin_hdl_t hdl)
+{
+	sdev_plugin_t *spp = (sdev_plugin_t *)hdl;
+	if (spp->sp_islegacy)
+		return (EINVAL);
+
+	if (!sdev_plugin_unregister_allowed)
+		return (EBUSY);
+
+	mutex_enter(&sdev_plugin_lock);
+	list_remove(&sdev_plugin_list, spp);
+	mutex_exit(&sdev_plugin_lock);
+
+	sdev_mnt_walk(sdev_plugin_unregister_cb, spp);
+	mutex_enter(&spp->sp_lock);
+	while (spp->sp_nnodes > 0)
+		cv_wait(&spp->sp_nodecv, &spp->sp_lock);
+	mutex_exit(&spp->sp_lock);
+	kmem_cache_free(sdev_plugin_cache, spp);
+	return (0);
+}
+
+/*
+ * Register an old sdev style plugin to deal with what used to be in the vtab.
+ */
+static int
+sdev_plugin_register_legacy(struct sdev_vop_table *vtp)
+{
+	sdev_plugin_t *spp;
+
+	spp = kmem_cache_alloc(sdev_plugin_cache, KM_SLEEP);
+	(void) strlcpy(spp->sp_name, vtp->vt_name, SDEV_PLUGIN_NAMELEN);
+	spp->sp_islegacy = B_TRUE;
+	spp->sp_pops = NULL;
+	spp->sp_nflags = vtp->vt_flags;
+	spp->sp_lvtor = vtp->vt_vtor;
+	spp->sp_nnodes = 0;
+
+	if (vtp->vt_service != NULL) {
+		fs_operation_def_t *templ;
+		templ = sdev_merge_vtab(vtp->vt_service);
+		if (vn_make_ops(vtp->vt_name,
+		    (const fs_operation_def_t *)templ,
+		    &spp->sp_vnops) != 0) {
+			cmn_err(CE_WARN, "%s: malformed vnode ops\n",
+			    vtp->vt_name);
+			sdev_free_vtab(templ);
+			kmem_cache_free(sdev_plugin_cache, spp);
+			return (1);
+		}
+
+		if (vtp->vt_global_vops) {
+			*(vtp->vt_global_vops) = spp->sp_vnops;
+		}
+
+		sdev_free_vtab(templ);
+	} else {
+		spp->sp_vnops = sdev_vnodeops;
+	}
+
+	/*
+	 * No need to check for EEXIST here. These are loaded as a part of the
+	 * sdev's initialization function. Further, we don't have to create them
+	 * as that's taken care of in sdev's mount for the GZ.
+	 */
+	mutex_enter(&sdev_plugin_lock);
+	list_insert_tail(&sdev_plugin_list, spp);
+	mutex_exit(&sdev_plugin_lock);
+
+	return (0);
+}
+
+/*
+ * We need to match off of the sdev_path, not the sdev_name. We are only allowed
+ * to exist directly under /dev.
+ */
+static sdev_plugin_t *
+sdev_match(sdev_node_t *dv)
+{
+	int vlen;
+	const char *path;
+	sdev_plugin_t *spp;
+
+	if (strlen(dv->sdev_path) <= 5)
+		return (NULL);
+
+	if (strncmp(dv->sdev_path, "/dev/", 5) != 0)
+		return (NULL);
+	path = dv->sdev_path + 5;
+
+	mutex_enter(&sdev_plugin_lock);
+
+	for (spp = list_head(&sdev_plugin_list); spp != NULL;
+	    spp = list_next(&sdev_plugin_list, spp)) {
+		if (strcmp(spp->sp_name, path) == 0) {
+			mutex_exit(&sdev_plugin_lock);
+			return (spp);
+		}
+
+		if (spp->sp_nflags & SDEV_SUBDIR) {
+			vlen = strlen(spp->sp_name);
+			if ((strncmp(spp->sp_name, path,
+			    vlen - 1) == 0) && path[vlen] == '/') {
+				mutex_exit(&sdev_plugin_lock);
+				return (spp);
+			}
+
+		}
+	}
+
+	mutex_exit(&sdev_plugin_lock);
+	return (NULL);
+}
+
+void
+sdev_set_no_negcache(sdev_node_t *dv)
+{
+	char *path;
+	sdev_plugin_t *spp;
+
+	ASSERT(dv->sdev_path);
+	path = dv->sdev_path + strlen("/dev/");
+
+	mutex_enter(&sdev_plugin_lock);
+	for (spp = list_head(&sdev_plugin_list); spp != NULL;
+	    spp = list_next(&sdev_plugin_list, spp)) {
+		if (strcmp(spp->sp_name, path) == 0) {
+			if (spp->sp_nflags & SDEV_NO_NCACHE)
+				dv->sdev_flags |= SDEV_NO_NCACHE;
+			break;
+		}
+	}
+	mutex_exit(&sdev_plugin_lock);
+}
+
+struct vnodeops *
+sdev_get_vop(sdev_node_t *dv)
+{
+	char *path;
+	sdev_plugin_t *spp;
+
+	path = dv->sdev_path;
+	ASSERT(path);
+
+	/* gets the relative path to /dev/ */
+	path += 5;
+
+	if ((spp = sdev_match(dv)) != NULL) {
+		dv->sdev_flags |= spp->sp_nflags;
+		if (SDEV_IS_PERSIST(dv->sdev_dotdot) &&
+		    (SDEV_IS_PERSIST(dv) || !SDEV_IS_DYNAMIC(dv)))
+			dv->sdev_flags |= SDEV_PERSIST;
+		return (spp->sp_vnops);
+	}
+
+	/* child inherits the persistence of the parent */
+	if (SDEV_IS_PERSIST(dv->sdev_dotdot))
+		dv->sdev_flags |= SDEV_PERSIST;
+	return (sdev_vnodeops);
+}
+
+void *
+sdev_get_vtor(sdev_node_t *dv)
+{
+	sdev_plugin_t *spp;
+
+	if (dv->sdev_private == NULL) {
+		spp = sdev_match(dv);
+		if (spp == NULL)
+			return (NULL);
+	} else {
+		spp = dv->sdev_private;
+	}
+
+	if (spp->sp_islegacy)
+		return ((void *)spp->sp_lvtor);
+	else
+		return ((void *)sdev_plugin_validate);
+}
+
+void
+sdev_plugin_nodeready(sdev_node_t *sdp)
+{
+	sdev_plugin_t *spp;
+
+	ASSERT(RW_WRITE_HELD(&sdp->sdev_contents));
+	ASSERT(sdp->sdev_private == NULL);
+
+	spp = sdev_match(sdp);
+	if (spp == NULL)
+		return;
+	if (spp->sp_islegacy)
+		return;
+	sdp->sdev_private = spp;
+	mutex_enter(&spp->sp_lock);
+	spp->sp_nnodes++;
+	mutex_exit(&spp->sp_lock);
+}
+
+int
+sdev_plugin_init(void)
+{
+	sdev_vop_table_t *vtp;
+	fs_operation_def_t *templ;
+
+	sdev_plugin_cache = kmem_cache_create("sdev_plugin",
+	    sizeof (sdev_plugin_t), 0, sdev_plugin_cache_constructor,
+	    sdev_plugin_cache_destructor, NULL, NULL, NULL, 0);
+	if (sdev_plugin_cache == NULL)
+		return (1);
+	mutex_init(&sdev_plugin_lock, NULL, MUTEX_DRIVER, NULL);
+	list_create(&sdev_plugin_list, sizeof (sdev_plugin_t),
+	    offsetof(sdev_plugin_t, sp_link));
+
+	/*
+	 * Register all of the legacy vnops
+	 */
+	for (vtp = &vtab[0]; vtp->vt_name != NULL; vtp++)
+		if (sdev_plugin_register_legacy(vtp) != 0)
+			return (1);
+
+	templ = sdev_merge_vtab(sdev_plugin_vnodeops_tbl);
+	if (vn_make_ops("sdev_plugin",
+	    (const fs_operation_def_t *)templ,
+	    &sdev_plugin_vnops) != 0) {
+		sdev_free_vtab(templ);
+		return (1);
+	}
+
+	sdev_free_vtab(templ);
+	return (0);
+}
diff --git a/usr/src/uts/common/fs/dev/sdev_subr.c b/usr/src/uts/common/fs/dev/sdev_subr.c
index d810dd9a31..42a3874b95 100644
--- a/usr/src/uts/common/fs/dev/sdev_subr.c
+++ b/usr/src/uts/common/fs/dev/sdev_subr.c
@@ -151,12 +151,6 @@ vattr_t sdev_vattr_chr = {
 kmem_cache_t	*sdev_node_cache;	/* sdev_node cache */
 int		devtype;		/* fstype */
 
-/* static */
-static struct vnodeops *sdev_get_vop(struct sdev_node *);
-static void sdev_set_no_negcache(struct sdev_node *);
-static fs_operation_def_t *sdev_merge_vtab(const fs_operation_def_t []);
-static void sdev_free_vtab(fs_operation_def_t *);
-
 static void
 sdev_prof_free(struct sdev_node *dv)
 {
@@ -314,6 +308,7 @@ sdev_nodeinit(struct sdev_node *ddv, char *nm, struct sdev_node **newdv,
 	(void) snprintf(dv->sdev_path, len, "%s/%s", ddv->sdev_path, nm);
 	/* overwritten for VLNK nodes */
 	dv->sdev_symlink = NULL;
+	list_link_init(&dv->sdev_plist);
 
 	vp = SDEVTOV(dv);
 	vn_reinit(vp);
@@ -402,6 +397,7 @@ sdev_nodeready(struct sdev_node *dv, struct vattr *vap, struct vnode *avp,
 	} else {
 		dv->sdev_nlink = 1;
 	}
+	sdev_plugin_nodeready(dv);
 
 	if (!(SDEV_IS_GLOBAL(dv))) {
 		dv->sdev_origin = (struct sdev_node *)args;
@@ -498,37 +494,22 @@ sdev_mkroot(struct vfs *vfsp, dev_t devdev, struct vnode *mvp,
 	return (dv);
 }
 
-/* directory dependent vop table */
-struct sdev_vop_table {
-	char *vt_name;				/* subdirectory name */
-	const fs_operation_def_t *vt_service;	/* vnodeops table */
-	struct vnodeops *vt_vops;		/* constructed vop */
-	struct vnodeops **vt_global_vops;	/* global container for vop */
-	int (*vt_vtor)(struct sdev_node *);	/* validate sdev_node */
-	int vt_flags;
-};
-
-/*
- * A nice improvement would be to provide a plug-in mechanism
- * for this table instead of a const table.
- */
-static struct sdev_vop_table vtab[] =
-{
-	{ "pts", devpts_vnodeops_tbl, NULL, &devpts_vnodeops, devpts_validate,
+struct sdev_vop_table vtab[] = {
+	{ "pts", devpts_vnodeops_tbl, &devpts_vnodeops, devpts_validate,
 	SDEV_DYNAMIC | SDEV_VTOR },
 
-	{ "vt", devvt_vnodeops_tbl, NULL, &devvt_vnodeops, devvt_validate,
+	{ "vt", devvt_vnodeops_tbl, &devvt_vnodeops, devvt_validate,
 	SDEV_DYNAMIC | SDEV_VTOR },
 
-	{ "zvol", devzvol_vnodeops_tbl, NULL, &devzvol_vnodeops,
+	{ "zvol", devzvol_vnodeops_tbl, &devzvol_vnodeops,
 	devzvol_validate, SDEV_ZONED | SDEV_DYNAMIC | SDEV_VTOR | SDEV_SUBDIR },
 
-	{ "zcons", NULL, NULL, NULL, NULL, SDEV_NO_NCACHE },
+	{ "zcons", NULL, NULL, NULL, SDEV_NO_NCACHE },
 
-	{ "net", devnet_vnodeops_tbl, NULL, &devnet_vnodeops, devnet_validate,
-	SDEV_DYNAMIC | SDEV_VTOR },
+	{ "net", devnet_vnodeops_tbl, &devnet_vnodeops, devnet_validate,
+	SDEV_DYNAMIC | SDEV_VTOR | SDEV_SUBDIR },
 
-	{ "ipnet", devipnet_vnodeops_tbl, NULL, &devipnet_vnodeops,
+	{ "ipnet", devipnet_vnodeops_tbl, &devipnet_vnodeops,
 	devipnet_validate, SDEV_DYNAMIC | SDEV_VTOR | SDEV_NO_NCACHE },
 
 	/*
@@ -543,132 +524,14 @@ static struct sdev_vop_table vtab[] =
 	 * preventing a mkdir.
 	 */
 
-	{ "lofi", NULL, NULL, NULL, NULL,
+	{ "lofi", NULL, NULL, NULL,
 	    SDEV_ZONED | SDEV_DYNAMIC | SDEV_PERSIST },
-	{ "rlofi", NULL, NULL, NULL, NULL,
+	{ "rlofi", NULL, NULL, NULL,
 	    SDEV_ZONED | SDEV_DYNAMIC | SDEV_PERSIST },
 
-	{ NULL, NULL, NULL, NULL, NULL, 0}
+	{ NULL, NULL, NULL, NULL, 0}
 };
 
-/*
- * We need to match off of the sdev_path, not the sdev_name. We are only allowed
- * to exist directly under /dev.
- */
-struct sdev_vop_table *
-sdev_match(struct sdev_node *dv)
-{
-	int vlen;
-	int i;
-	const char *path;
-
-	if (strlen(dv->sdev_path) <= 5)
-		return (NULL);
-
-	if (strncmp(dv->sdev_path, "/dev/", 5) != 0)
-		return (NULL);
-	path = dv->sdev_path + 5;
-
-	for (i = 0; vtab[i].vt_name; i++) {
-		if (strcmp(vtab[i].vt_name, path) == 0)
-			return (&vtab[i]);
-		if (vtab[i].vt_flags & SDEV_SUBDIR) {
-			vlen = strlen(vtab[i].vt_name);
-			if ((strncmp(vtab[i].vt_name, path,
-			    vlen - 1) == 0) && path[vlen] == '/')
-				return (&vtab[i]);
-		}
-
-	}
-	return (NULL);
-}
-
-/*
- *  sets a directory's vnodeops if the directory is in the vtab;
- */
-static struct vnodeops *
-sdev_get_vop(struct sdev_node *dv)
-{
-	struct sdev_vop_table *vtp;
-	char *path;
-
-	path = dv->sdev_path;
-	ASSERT(path);
-
-	/* gets the relative path to /dev/ */
-	path += 5;
-
-	/* gets the vtab entry it matches */
-	if ((vtp = sdev_match(dv)) != NULL) {
-		dv->sdev_flags |= vtp->vt_flags;
-		if (SDEV_IS_PERSIST(dv->sdev_dotdot) &&
-		    (SDEV_IS_PERSIST(dv) || !SDEV_IS_DYNAMIC(dv)))
-			dv->sdev_flags |= SDEV_PERSIST;
-
-		if (vtp->vt_vops) {
-			if (vtp->vt_global_vops)
-				*(vtp->vt_global_vops) = vtp->vt_vops;
-
-			return (vtp->vt_vops);
-		}
-
-		if (vtp->vt_service) {
-			fs_operation_def_t *templ;
-			templ = sdev_merge_vtab(vtp->vt_service);
-			if (vn_make_ops(vtp->vt_name,
-			    (const fs_operation_def_t *)templ,
-			    &vtp->vt_vops) != 0) {
-				cmn_err(CE_PANIC, "%s: malformed vnode ops\n",
-				    vtp->vt_name);
-				/*NOTREACHED*/
-			}
-			if (vtp->vt_global_vops) {
-				*(vtp->vt_global_vops) = vtp->vt_vops;
-			}
-			sdev_free_vtab(templ);
-
-			return (vtp->vt_vops);
-		}
-
-		return (sdev_vnodeops);
-	}
-
-	/* child inherits the persistence of the parent */
-	if (SDEV_IS_PERSIST(dv->sdev_dotdot))
-		dv->sdev_flags |= SDEV_PERSIST;
-
-	return (sdev_vnodeops);
-}
-
-static void
-sdev_set_no_negcache(struct sdev_node *dv)
-{
-	int i;
-	char *path;
-
-	ASSERT(dv->sdev_path);
-	path = dv->sdev_path + strlen("/dev/");
-
-	for (i = 0; vtab[i].vt_name; i++) {
-		if (strcmp(vtab[i].vt_name, path) == 0) {
-			if (vtab[i].vt_flags & SDEV_NO_NCACHE)
-				dv->sdev_flags |= SDEV_NO_NCACHE;
-			break;
-		}
-	}
-}
-
-void *
-sdev_get_vtor(struct sdev_node *dv)
-{
-	struct sdev_vop_table *vtp;
-
-	vtp = sdev_match(dv);
-	if (vtp)
-		return ((void *)vtp->vt_vtor);
-	else
-		return (NULL);
-}
 
 /*
  * Build the base root inode
@@ -948,8 +811,11 @@ sdev_nodedestroy(struct sdev_node *dv, uint_t flags)
 		dv->sdev_path = NULL;
 	}
 
-	if (!SDEV_IS_GLOBAL(dv))
+	if (!SDEV_IS_GLOBAL(dv)) {
 		sdev_prof_free(dv);
+		if (dv->sdev_vnode->v_type != VLNK && dv->sdev_origin != NULL)
+			SDEV_RELE(dv->sdev_origin);
+	}
 
 	if (SDEVTOV(dv)->v_type == VDIR) {
 		ASSERT(SDEV_FIRST_ENTRY(dv) == NULL);
@@ -963,6 +829,7 @@ sdev_nodedestroy(struct sdev_node *dv, uint_t flags)
 	(void) memset((void *)&dv->sdev_instance_data, 0,
 	    sizeof (dv->sdev_instance_data));
 	vn_invalid(SDEVTOV(dv));
+	dv->sdev_private = NULL;
 	kmem_cache_free(sdev_node_cache, dv);
 }
 
@@ -2945,46 +2812,6 @@ sdev_modctl_devexists(const char *path)
 	return (error);
 }
 
-extern int sdev_vnodeops_tbl_size;
-
-/*
- * construct a new template with overrides from vtab
- */
-static fs_operation_def_t *
-sdev_merge_vtab(const fs_operation_def_t tab[])
-{
-	fs_operation_def_t *new;
-	const fs_operation_def_t *tab_entry;
-
-	/* make a copy of standard vnode ops table */
-	new = kmem_alloc(sdev_vnodeops_tbl_size, KM_SLEEP);
-	bcopy((void *)sdev_vnodeops_tbl, new, sdev_vnodeops_tbl_size);
-
-	/* replace the overrides from tab */
-	for (tab_entry = tab; tab_entry->name != NULL; tab_entry++) {
-		fs_operation_def_t *std_entry = new;
-		while (std_entry->name) {
-			if (strcmp(tab_entry->name, std_entry->name) == 0) {
-				std_entry->func = tab_entry->func;
-				break;
-			}
-			std_entry++;
-		}
-		if (std_entry->name == NULL)
-			cmn_err(CE_NOTE, "sdev_merge_vtab: entry %s unused.",
-			    tab_entry->name);
-	}
-
-	return (new);
-}
-
-/* free memory allocated by sdev_merge_vtab */
-static void
-sdev_free_vtab(fs_operation_def_t *new)
-{
-	kmem_free(new, sdev_vnodeops_tbl_size);
-}
-
 /*
  * a generic setattr() function
  *
diff --git a/usr/src/uts/common/fs/dev/sdev_vfsops.c b/usr/src/uts/common/fs/dev/sdev_vfsops.c
index d81702185e..55b388c2d4 100644
--- a/usr/src/uts/common/fs/dev/sdev_vfsops.c
+++ b/usr/src/uts/common/fs/dev/sdev_vfsops.c
@@ -173,7 +173,13 @@ devinit(int fstype, char *name)
 
 	if ((devmajor = getudev()) == (major_t)-1) {
 		cmn_err(CE_WARN, "%s: can't get unique dev", sdev_vfssw.name);
-		return (1);
+		return (ENXIO);
+	}
+
+	if (sdev_plugin_init() != 0) {
+		cmn_err(CE_WARN, "%s: failed to set init plugin subsystem",
+		    sdev_vfssw.name);
+		return (EIO);
 	}
 
 	/* initialize negative cache */
@@ -350,6 +356,7 @@ sdev_mount(struct vfs *vfsp, struct vnode *mvp, struct mounta *uap,
 		ASSERT(sdev_origins);
 		dv->sdev_flags &= ~SDEV_GLOBAL;
 		dv->sdev_origin = sdev_origins->sdev_root;
+		SDEV_HOLD(dv->sdev_origin);
 	} else {
 		sdev_ncache_setup();
 		rw_enter(&dv->sdev_contents, RW_WRITER);
@@ -527,3 +534,17 @@ sdev_mntinfo_rele(struct sdev_data *mntinfo)
 	mutex_exit(&vp->v_lock);
 	mutex_exit(&sdev_lock);
 }
+
+void
+sdev_mnt_walk(void (*func)(struct sdev_node *, void *), void *arg)
+{
+	struct sdev_data *mntinfo;
+
+	mutex_enter(&sdev_lock);
+	mntinfo = sdev_mntinfo;
+	while (mntinfo != NULL) {
+		func(mntinfo->sdev_root, arg);
+		mntinfo = mntinfo->sdev_next;
+	}
+	mutex_exit(&sdev_lock);
+}
diff --git a/usr/src/uts/common/fs/dev/sdev_vnops.c b/usr/src/uts/common/fs/dev/sdev_vnops.c
index 79ebd8b2e5..8fe926f6fb 100644
--- a/usr/src/uts/common/fs/dev/sdev_vnops.c
+++ b/usr/src/uts/common/fs/dev/sdev_vnops.c
@@ -22,7 +22,7 @@
  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 /*
- * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
+ * Copyright 2018, Joyent, Inc.
  */
 
 /*
@@ -372,7 +372,7 @@ sdev_close(struct vnode *vp, int flag, int count,
 /*ARGSUSED*/
 static int
 sdev_read(struct vnode *vp, struct uio *uio, int ioflag, struct cred *cred,
-	struct caller_context *ct)
+    struct caller_context *ct)
 {
 	struct sdev_node *dv = (struct sdev_node *)VTOSDEV(vp);
 	int	error;
@@ -399,7 +399,7 @@ sdev_read(struct vnode *vp, struct uio *uio, int ioflag, struct cred *cred,
 /*ARGSUSED*/
 static int
 sdev_write(struct vnode *vp, struct uio *uio, int ioflag, struct cred *cred,
-	struct caller_context *ct)
+    struct caller_context *ct)
 {
 	struct sdev_node *dv = VTOSDEV(vp);
 	int	error = 0;
@@ -582,7 +582,9 @@ sdev_self_access(sdev_node_t *dv, int mode, int flags, struct cred *cr,
 {
 	int ret;
 
+	ASSERT(RW_READ_HELD(&dv->sdev_contents));
 	ASSERT(dv->sdev_attr || dv->sdev_attrvp);
+
 	if (dv->sdev_attrvp) {
 		ret = VOP_ACCESS(dv->sdev_attrvp, mode, flags, cr, ct);
 	} else if (dv->sdev_attr) {
@@ -1438,32 +1440,24 @@ sdev_readlink(struct vnode *vp, struct uio *uiop, struct cred *cred,
 
 /*ARGSUSED4*/
 static int
-sdev_readdir(struct vnode *dvp, struct uio *uiop, struct cred *cred, int *eofp,
+sdev_readdir(struct vnode *vp, struct uio *uiop, struct cred *cred, int *eofp,
     caller_context_t *ct, int flags)
 {
-	struct sdev_node *parent = VTOSDEV(dvp);
+	struct sdev_node *dv = VTOSDEV(vp);
 	int error;
 
+	VERIFY(RW_READ_HELD(&dv->sdev_contents));
+
 	/*
-	 * We must check that we have execute access to search the directory --
-	 * but because our sdev_contents lock is already held as a reader (the
-	 * caller must have done a VOP_RWLOCK()), we call directly into the
-	 * underlying access routine if sdev_attr is non-NULL.
+	 * We can't recursively take ->sdev_contents via an indirect
+	 * VOP_ACCESS(), but we don't need to use that anyway.
 	 */
-	if (parent->sdev_attr != NULL) {
-		VERIFY(RW_READ_HELD(&parent->sdev_contents));
-
-		if (sdev_unlocked_access(parent, VEXEC, cred) != 0)
-			return (EACCES);
-	} else {
-		if ((error = VOP_ACCESS(dvp, VEXEC, 0, cred, ct)) != 0)
-			return (error);
-	}
+	if ((error = sdev_self_access(dv, VEXEC, 0, cred, ct)) != 0)
+		return (error);
 
-	ASSERT(parent);
-	if (!SDEV_IS_GLOBAL(parent))
-		prof_filldir(parent);
-	return (devname_readdir_func(dvp, uiop, cred, eofp, SDEV_BROWSE));
+	if (!SDEV_IS_GLOBAL(dv))
+		prof_filldir(dv);
+	return (devname_readdir_func(vp, uiop, cred, eofp, SDEV_BROWSE));
 }
 
 /*ARGSUSED1*/
diff --git a/usr/src/uts/common/sys/fs/sdev_impl.h b/usr/src/uts/common/sys/fs/sdev_impl.h
index 9f9ce5c8c1..dc6601bb43 100644
--- a/usr/src/uts/common/sys/fs/sdev_impl.h
+++ b/usr/src/uts/common/sys/fs/sdev_impl.h
@@ -38,6 +38,7 @@ extern "C" {
 #include <sys/list.h>
 #include <sys/nvpair.h>
 #include <sys/sunddi.h>
+#include <sys/fs/sdev_plugin.h>
 
 /*
  * sdev_nodes are the file-system specific part of the
@@ -129,6 +130,21 @@ typedef struct sdev_local_data {
 	struct sdev_dprof sdev_lprof;	/* profile for multi-inst */
 } sdev_local_data_t;
 
+/* sdev_flags */
+typedef enum sdev_flags {
+	SDEV_BUILD =		0x0001,	/* directory cache out-of-date */
+	SDEV_GLOBAL =		0x0002,	/* global /dev nodes */
+	SDEV_PERSIST =		0x0004,	/* backing store persisted node */
+	SDEV_NO_NCACHE = 	0x0008,	/* do not include in neg. cache */
+	SDEV_DYNAMIC =		0x0010,	/* special-purpose vnode ops */
+					/* (ex: pts) */
+	SDEV_VTOR =		0x0020,	/* validate sdev_nodes during search */
+	SDEV_ATTR_INVALID =	0x0040,	/* invalid node attributes, */
+					/* need update */
+	SDEV_SUBDIR =		0x0080,	/* match all subdirs under here */
+	SDEV_ZONED =		0x0100	/* zoned subdir */
+} sdev_flags_t;
+
 /*
  * /dev filesystem sdev_node defines
  */
@@ -151,7 +167,7 @@ typedef struct sdev_node {
 	ino64_t		sdev_ino;	/* inode */
 	uint_t		sdev_nlink;	/* link count */
 	int		sdev_state;	/* state of this node */
-	int		sdev_flags;	/* flags bit */
+	sdev_flags_t	sdev_flags;	/* flags bit */
 
 	kmutex_t	sdev_lookup_lock; /* node creation synch lock */
 	kcondvar_t	sdev_lookup_cv;	/* node creation sync cv */
@@ -162,7 +178,7 @@ typedef struct sdev_node {
 		struct sdev_global_data	sdev_globaldata;
 		struct sdev_local_data	sdev_localdata;
 	} sdev_instance_data;
-
+	list_node_t	sdev_plist;	/* link on plugin list */
 	void		*sdev_private;
 } sdev_node_t;
 
@@ -193,29 +209,11 @@ typedef enum {
 	SDEV_READY
 } sdev_node_state_t;
 
-/* sdev_flags */
-#define	SDEV_BUILD		0x0001	/* directory cache out-of-date */
-#define	SDEV_GLOBAL		0x0002	/* global /dev nodes */
-#define	SDEV_PERSIST		0x0004	/* backing store persisted node */
-#define	SDEV_NO_NCACHE		0x0008	/* do not include in neg. cache */
-#define	SDEV_DYNAMIC		0x0010	/* special-purpose vnode ops */
-					/* (ex: pts) */
-#define	SDEV_VTOR		0x0020	/* validate sdev_nodes during search */
-#define	SDEV_ATTR_INVALID	0x0040	/* invalid node attributes, */
-					/* need update */
-#define	SDEV_SUBDIR		0x0080	/* match all subdirs under here */
-#define	SDEV_ZONED		0x0100  /* zoned subdir */
-
 /* sdev_lookup_flags */
 #define	SDEV_LOOKUP	0x0001	/* node creation in progress */
 #define	SDEV_READDIR	0x0002	/* VDIR readdir in progress */
 #define	SDEV_LGWAITING	0x0004	/* waiting for devfsadm completion */
 
-#define	SDEV_VTOR_INVALID	-1
-#define	SDEV_VTOR_SKIP		0
-#define	SDEV_VTOR_VALID		1
-#define	SDEV_VTOR_STALE		2
-
 /* convenient macros */
 #define	SDEV_IS_GLOBAL(dv)	\
 	(dv->sdev_flags & SDEV_GLOBAL)
@@ -368,8 +366,13 @@ extern void sdev_devfsadmd_thread(struct sdev_node *, struct sdev_node *,
 extern int devname_profile_update(char *, size_t);
 extern struct sdev_data *sdev_find_mntinfo(char *);
 void sdev_mntinfo_rele(struct sdev_data *);
+typedef void (*sdev_mnt_walk_f)(struct sdev_node *, void *);
+void sdev_mnt_walk(sdev_mnt_walk_f, void *);
 extern struct vnodeops *devpts_getvnodeops(void);
 extern struct vnodeops *devvt_getvnodeops(void);
+extern void sdev_plugin_nodeready(struct sdev_node *);
+extern int sdev_plugin_init(void);
+extern int sdev_plugin_fini(void);
 
 /*
  * boot states - warning, the ordering here is significant
@@ -515,6 +518,23 @@ extern void sdev_nc_path_exists(sdev_nc_list_t *, char *);
 extern void sdev_modctl_dump_files(void);
 
 /*
+ * plugin and legacy vtab stuff
+ */
+/* directory dependent vop table */
+typedef struct sdev_vop_table {
+	char *vt_name;				/* subdirectory name */
+	const fs_operation_def_t *vt_service;	/* vnodeops table */
+	struct vnodeops **vt_global_vops;	/* global container for vop */
+	int (*vt_vtor)(struct sdev_node *);	/* validate sdev_node */
+	int vt_flags;
+} sdev_vop_table_t;
+
+extern struct sdev_vop_table vtab[];
+extern struct vnodeops *sdev_get_vop(struct sdev_node *);
+extern void sdev_set_no_negcache(struct sdev_node *);
+extern void *sdev_get_vtor(struct sdev_node *dv);
+
+/*
  * globals
  */
 extern kmutex_t sdev_lock;
@@ -527,6 +547,7 @@ extern struct vnodeops		*devipnet_vnodeops;
 extern struct vnodeops		*devvt_vnodeops;
 extern struct sdev_data *sdev_origins; /* mount info for global /dev instance */
 extern struct vnodeops		*devzvol_vnodeops;
+extern int			sdev_vnodeops_tbl_size;
 
 extern const fs_operation_def_t	sdev_vnodeops_tbl[];
 extern const fs_operation_def_t	devpts_vnodeops_tbl[];
diff --git a/usr/src/uts/common/sys/fs/sdev_plugin.h b/usr/src/uts/common/sys/fs/sdev_plugin.h
new file mode 100644
index 0000000000..f4ed813c1e
--- /dev/null
+++ b/usr/src/uts/common/sys/fs/sdev_plugin.h
@@ -0,0 +1,106 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright (c) 2018, Joyent, Inc.
+ */
+
+#ifndef _SYS_SDEV_PLUGIN_H
+#define	_SYS_SDEV_PLUGIN_H
+
+/*
+ * Kernel sdev plugin interface
+ */
+
+#ifdef _KERNEL
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/vnode.h>
+
+#endif	/* _KERNEL */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _KERNEL
+
+typedef uintptr_t sdev_plugin_hdl_t;
+typedef uintptr_t sdev_ctx_t;
+
+/*
+ * Valid return values for sdev_plugin_validate_t.
+ */
+typedef enum sdev_plugin_validate {
+	SDEV_VTOR_INVALID = -1,
+	SDEV_VTOR_SKIP = 0,
+	SDEV_VTOR_VALID	= 1,
+	SDEV_VTOR_STALE	= 2
+} sdev_plugin_validate_t;
+
+/*
+ * Valid flags
+ */
+typedef enum sdev_plugin_flags {
+	SDEV_PLUGIN_NO_NCACHE = 0x1,
+	SDEV_PLUGIN_SUBDIR = 0x2
+} sdev_plugin_flags_t;
+
+#define	SDEV_PLUGIN_FLAGS_MASK	0x3
+
+/*
+ * Functions a module must implement
+ */
+typedef sdev_plugin_validate_t (*sp_valid_f)(sdev_ctx_t);
+typedef int (*sp_filldir_f)(sdev_ctx_t);
+typedef void (*sp_inactive_f)(sdev_ctx_t);
+
+#define	SDEV_PLUGIN_VERSION	1
+
+typedef struct sdev_plugin_ops {
+	int spo_version;
+	sdev_plugin_flags_t spo_flags;
+	sp_valid_f spo_validate;
+	sp_filldir_f spo_filldir;
+	sp_inactive_f spo_inactive;
+} sdev_plugin_ops_t;
+
+extern sdev_plugin_hdl_t sdev_plugin_register(const char *, sdev_plugin_ops_t *,
+    int *);
+extern int sdev_plugin_unregister(sdev_plugin_hdl_t);
+
+typedef enum sdev_ctx_flags {
+	SDEV_CTX_GLOBAL = 0x2	/* node belongs to the GZ */
+} sdev_ctx_flags_t;
+
+/*
+ * Context helper functions
+ */
+extern sdev_ctx_flags_t sdev_ctx_flags(sdev_ctx_t);
+extern const char *sdev_ctx_name(sdev_ctx_t);
+extern const char *sdev_ctx_path(sdev_ctx_t);
+extern int sdev_ctx_minor(sdev_ctx_t, minor_t *);
+extern enum vtype sdev_ctx_vtype(sdev_ctx_t);
+
+/*
+ * Callbacks to manipulate nodes
+ */
+extern int sdev_plugin_mkdir(sdev_ctx_t, char *);
+extern int sdev_plugin_mknod(sdev_ctx_t, char *, mode_t, dev_t);
+
+#endif	/* _KERNEL */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_SDEV_PLUGIN_H */
diff --git a/usr/src/uts/intel/dev/Makefile b/usr/src/uts/intel/dev/Makefile
index 6cc3fda6cd..cc100922a0 100644
--- a/usr/src/uts/intel/dev/Makefile
+++ b/usr/src/uts/intel/dev/Makefile
@@ -71,6 +71,7 @@ LINTTAGS	+= -erroff=E_STATIC_UNUSED
 CERRWARN	+= -_gcc=-Wno-parentheses
 CERRWARN	+= -_gcc=-Wno-unused-label
 CERRWARN	+= $(CNOWARN_UNINIT)
+CERRWARN	+= -_gcc=-Wno-unused-function
 
 #
 #	Default build targets.