6792062 FileBench needs work to allow plug-in file system clients

18 files changed, 1251 insertions, 561 deletions

diff --git a/usr/src/cmd/filebench/Makefile.com b/usr/src/cmd/filebench/Makefile.com
index e583cbd054..173fdd5030 100644
--- a/usr/src/cmd/filebench/Makefile.com
+++ b/usr/src/cmd/filebench/Makefile.com
@@ -19,7 +19,7 @@
 # CDDL HEADER END
 #
 #
-# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
 
@@ -31,8 +31,9 @@ include ../../Makefile.targ
 SRCS = \
 	auto_comp.c \
         eventgen.c \
-	fb_random.c \
 	fb_avl.c \
+	fb_localfs.c \
+	fb_random.c \
         fileset.c \
         flowop.c \
         flowop_library.c \
diff --git a/usr/src/cmd/filebench/common/fb_localfs.c b/usr/src/cmd/filebench/common/fb_localfs.c
new file mode 100644
index 0000000000..ff62bb1114
--- /dev/null
+++ b/usr/src/cmd/filebench/common/fb_localfs.c
@@ -0,0 +1,682 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ *
+ * Portions Copyright 2008 Denis Cheng
+ */
+
+#include "config.h"
+#include "filebench.h"
+#include "flowop.h"
+#include "threadflow.h" /* For aiolist definition */
+
+#ifndef HAVE_OFF64_T
+/*
+ * We are probably on linux.
+ * According to http://www.suse.de/~aj/linux_lfs.html, defining the
+ * above, automatically changes type of off_t to off64_t. so let
+ * us use only off_t as off64_t is not defined
+ */
+#defineoff64_t off_t
+#endif /* HAVE_OFF64_T */
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <libgen.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/resource.h>
+
+#include "filebench.h"
+#include "fsplug.h"
+
+#ifdef HAVE_AIO
+#include <aio.h>
+#endif /* HAVE_AIO */
+
+#ifdef HAVE_LIBAIO_H
+#include <libaio.h>
+#endif /* HAVE_LIBAIO_H */
+
+#ifndef HAVE_AIOCB64_T
+#define	aiocb64 aiocb
+#endif /* HAVE_AIOCB64_T */
+
+/*
+ * These routines implement local file access. They are placed into a
+ * vector of functions that are called by all I/O operations in fileset.c
+ * and flowop_library.c. This represents the default file system plug-in,
+ * and may be replaced by vectors for other file system plug-ins.
+ */
+
+static int fb_lfs_freemem(fb_fdesc_t *fd, off64_t size);
+static int fb_lfs_open(fb_fdesc_t *, char *, int, int);
+static int fb_lfs_pread(fb_fdesc_t *, caddr_t, fbint_t, off64_t);
+static int fb_lfs_read(fb_fdesc_t *, caddr_t, fbint_t);
+static int fb_lfs_pwrite(fb_fdesc_t *, caddr_t, fbint_t, off64_t);
+static int fb_lfs_write(fb_fdesc_t *, caddr_t, fbint_t);
+static int fb_lfs_lseek(fb_fdesc_t *, off64_t, int);
+static int fb_lfs_truncate(fb_fdesc_t *, off64_t);
+static int fb_lfs_rename(const char *, const char *);
+static int fb_lfs_close(fb_fdesc_t *);
+static int fb_lfs_link(const char *, const char *);
+static int fb_lfs_symlink(const char *, const char *);
+static int fb_lfs_unlink(char *);
+static ssize_t fb_lfs_readlink(const char *, char *, size_t);
+static int fb_lfs_mkdir(char *, int);
+static int fb_lfs_rmdir(char *);
+static DIR *fb_lfs_opendir(char *);
+static struct dirent *fb_lfs_readdir(DIR *);
+static int fb_lfs_closedir(DIR *);
+static int fb_lfs_fsync(fb_fdesc_t *);
+static int fb_lfs_stat(char *, struct stat64 *);
+static int fb_lfs_fstat(fb_fdesc_t *, struct stat64 *);
+static int fb_lfs_access(const char *, int);
+
+static fsplug_func_t fb_lfs_funcs =
+{
+	"locfs",
+	fb_lfs_freemem,		/* flush page cache */
+	fb_lfs_open,		/* open */
+	fb_lfs_pread,		/* pread */
+	fb_lfs_read,		/* read */
+	fb_lfs_pwrite,		/* pwrite */
+	fb_lfs_write,		/* write */
+	fb_lfs_lseek,		/* lseek */
+	fb_lfs_truncate,	/* ftruncate */
+	fb_lfs_rename,		/* rename */
+	fb_lfs_close,		/* close */
+	fb_lfs_link,		/* link */
+	fb_lfs_symlink,		/* symlink */
+	fb_lfs_unlink,		/* unlink */
+	fb_lfs_readlink,	/* readlink */
+	fb_lfs_mkdir,		/* mkdir */
+	fb_lfs_rmdir,		/* rmdir */
+	fb_lfs_opendir,		/* opendir */
+	fb_lfs_readdir,		/* readdir */
+	fb_lfs_closedir,	/* closedir */
+	fb_lfs_fsync,		/* fsync */
+	fb_lfs_stat,		/* stat */
+	fb_lfs_fstat,		/* fstat */
+	fb_lfs_access		/* access */
+};
+
+#ifdef HAVE_AIO
+/*
+ * Local file system asynchronous IO flowops are in this module, as
+ * they have a number of local file system specific features.
+ */
+static int fb_lfsflow_aiowrite(threadflow_t *threadflow, flowop_t *flowop);
+static int fb_lfsflow_aiowait(threadflow_t *threadflow, flowop_t *flowop);
+
+static flowop_proto_t fb_lfsflow_funcs[] = {
+	FLOW_TYPE_AIO, FLOW_ATTR_WRITE, "aiowrite", flowop_init_generic,
+	fb_lfsflow_aiowrite, flowop_destruct_generic,
+	FLOW_TYPE_AIO, 0, "aiowait", flowop_init_generic,
+	fb_lfsflow_aiowait, flowop_destruct_generic
+};
+
+#endif /* HAVE_AIO */
+
+/*
+ * Initialize this processes I/O functions vector to point to
+ * the vector of local file system I/O functions
+ */
+void
+fb_lfs_funcvecinit(void)
+{
+	fs_functions_vec = &fb_lfs_funcs;
+}
+
+/*
+ * Initialize those flowops whose implementation is file system
+ * specific.
+ */
+void
+fb_lfs_flowinit(void)
+{
+	int nops;
+
+	/*
+	 * re-initialize the I/O functions vector while we are at
+	 * it as it may have been redefined since the process was
+	 * created, at least if this is the master processes
+	 */
+	fb_lfs_funcvecinit();
+
+#ifdef HAVE_AIO
+	nops = sizeof (fb_lfsflow_funcs) / sizeof (flowop_proto_t);
+	flowop_flow_init(fb_lfsflow_funcs, nops);
+#endif /* HAVE_AIO */
+}
+
+/*
+ * Frees up memory mapped file region of supplied size. The
+ * file descriptor "fd" indicates which memory mapped file.
+ * If successful, returns 0. Otherwise returns -1 if "size"
+ * is zero, or -1 times the number of times msync() failed.
+ */
+static int
+fb_lfs_freemem(fb_fdesc_t *fd, off64_t size)
+{
+	off64_t left;
+	int ret = 0;
+
+	for (left = size; left > 0; left -= MMAP_SIZE) {
+		off64_t thismapsize;
+		caddr_t addr;
+
+		thismapsize = MIN(MMAP_SIZE, left);
+		addr = mmap64(0, thismapsize, PROT_READ|PROT_WRITE,
+		    MAP_SHARED, fd->fd_num, size - left);
+		ret += msync(addr, thismapsize, MS_INVALIDATE);
+		(void) munmap(addr, thismapsize);
+	}
+	return (ret);
+}
+
+/*
+ * Does a posix pread. Returns what the pread() returns.
+ */
+static int
+fb_lfs_pread(fb_fdesc_t *fd, caddr_t iobuf, fbint_t iosize, off64_t fileoffset)
+{
+	return (pread64(fd->fd_num, iobuf, iosize, fileoffset));
+}
+
+/*
+ * Does a posix read. Returns what the read() returns.
+ */
+static int
+fb_lfs_read(fb_fdesc_t *fd, caddr_t iobuf, fbint_t iosize)
+{
+	return (read(fd->fd_num, iobuf, iosize));
+}
+
+#ifdef HAVE_AIO
+
+/*
+ * Asynchronous write section. An Asynchronous IO element
+ * (aiolist_t) is used to associate the asynchronous write request with
+ * its subsequent completion. This element includes a aiocb64 struct
+ * that is used by posix aio_xxx calls to track the asynchronous writes.
+ * The flowops aiowrite and aiowait result in calls to these posix
+ * aio_xxx system routines to do the actual asynchronous write IO
+ * operations.
+ */
+
+
+/*
+ * Allocates an asynchronous I/O list (aio, of type
+ * aiolist_t) element. Adds it to the flowop thread's
+ * threadflow aio list. Returns a pointer to the element.
+ */
+static aiolist_t *
+aio_allocate(flowop_t *flowop)
+{
+	aiolist_t *aiolist;
+
+	if ((aiolist = malloc(sizeof (aiolist_t))) == NULL) {
+		filebench_log(LOG_ERROR, "malloc aiolist failed");
+		filebench_shutdown(1);
+	}
+
+	/* Add to list */
+	if (flowop->fo_thread->tf_aiolist == NULL) {
+		flowop->fo_thread->tf_aiolist = aiolist;
+		aiolist->al_next = NULL;
+	} else {
+		aiolist->al_next = flowop->fo_thread->tf_aiolist;
+		flowop->fo_thread->tf_aiolist = aiolist;
+	}
+	return (aiolist);
+}
+
+/*
+ * Searches for the aiolist element that has a matching
+ * completion block, aiocb. If none found returns FILEBENCH_ERROR. If
+ * found, removes the aiolist element from flowop thread's
+ * list and returns FILEBENCH_OK.
+ */
+static int
+aio_deallocate(flowop_t *flowop, struct aiocb64 *aiocb)
+{
+	aiolist_t *aiolist = flowop->fo_thread->tf_aiolist;
+	aiolist_t *previous = NULL;
+	aiolist_t *match = NULL;
+
+	if (aiocb == NULL) {
+		filebench_log(LOG_ERROR, "null aiocb deallocate");
+		return (FILEBENCH_OK);
+	}
+
+	while (aiolist) {
+		if (aiocb == &(aiolist->al_aiocb)) {
+			match = aiolist;
+			break;
+		}
+		previous = aiolist;
+		aiolist = aiolist->al_next;
+	}
+
+	if (match == NULL)
+		return (FILEBENCH_ERROR);
+
+	/* Remove from the list */
+	if (previous)
+		previous->al_next = match->al_next;
+	else
+		flowop->fo_thread->tf_aiolist = match->al_next;
+
+	return (FILEBENCH_OK);
+}
+
+/*
+ * Emulate posix aiowrite(). Determines which file to use,
+ * either one file of a fileset, or the file associated
+ * with a fileobj, allocates and fills an aiolist_t element
+ * for the write, and issues the asynchronous write. This
+ * operation is only valid for random IO, and returns an
+ * error if the flowop is set for sequential IO. Returns
+ * FILEBENCH_OK on success, FILEBENCH_NORSC if iosetup can't
+ * obtain a file to open, and FILEBENCH_ERROR on any
+ * encountered error.
+ */
+static int
+fb_lfsflow_aiowrite(threadflow_t *threadflow, flowop_t *flowop)
+{
+	caddr_t iobuf;
+	fbint_t wss;
+	fbint_t iosize;
+	fb_fdesc_t *fdesc;
+	int ret;
+
+	iosize = avd_get_int(flowop->fo_iosize);
+
+	if ((ret = flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
+	    &fdesc, iosize)) != FILEBENCH_OK)
+		return (ret);
+
+	if (avd_get_bool(flowop->fo_random)) {
+		uint64_t fileoffset;
+		struct aiocb64 *aiocb;
+		aiolist_t *aiolist;
+
+		if (filebench_randomno64(&fileoffset,
+		    wss, iosize, NULL) == -1) {
+			filebench_log(LOG_ERROR,
+			    "file size smaller than IO size for thread %s",
+			    flowop->fo_name);
+			return (FILEBENCH_ERROR);
+		}
+
+		aiolist = aio_allocate(flowop);
+		aiolist->al_type = AL_WRITE;
+		aiocb = &aiolist->al_aiocb;
+
+		aiocb->aio_fildes = fdesc->fd_num;
+		aiocb->aio_buf = iobuf;
+		aiocb->aio_nbytes = (size_t)iosize;
+		aiocb->aio_offset = (off64_t)fileoffset;
+		aiocb->aio_reqprio = 0;
+
+		filebench_log(LOG_DEBUG_IMPL,
+		    "aio fd=%d, bytes=%llu, offset=%llu",
+		    fdesc->fd_num, (u_longlong_t)iosize,
+		    (u_longlong_t)fileoffset);
+
+		flowop_beginop(threadflow, flowop);
+		if (aio_write64(aiocb) < 0) {
+			filebench_log(LOG_ERROR, "aiowrite failed: %s",
+			    strerror(errno));
+			filebench_shutdown(1);
+		}
+		flowop_endop(threadflow, flowop, iosize);
+	} else {
+		return (FILEBENCH_ERROR);
+	}
+
+	return (FILEBENCH_OK);
+}
+
+
+
+#define	MAXREAP 4096
+
+/*
+ * Emulate posix aiowait(). Waits for the completion of half the
+ * outstanding asynchronous IOs, or a single IO, which ever is
+ * larger. The routine will return after a sufficient number of
+ * completed calls issued by any thread in the procflow have
+ * completed, or a 1 second timout elapses. All completed
+ * IO operations are deleted from the thread's aiolist.
+ */
+static int
+fb_lfsflow_aiowait(threadflow_t *threadflow, flowop_t *flowop)
+{
+	struct aiocb64 **worklist;
+	aiolist_t *aio = flowop->fo_thread->tf_aiolist;
+	int uncompleted = 0;
+
+	worklist = calloc(MAXREAP, sizeof (struct aiocb64 *));
+
+	/* Count the list of pending aios */
+	while (aio) {
+		uncompleted++;
+		aio = aio->al_next;
+	}
+
+	do {
+		uint_t ncompleted = 0;
+		uint_t todo;
+		struct timespec timeout;
+		int inprogress;
+		int i;
+
+		/* Wait for half of the outstanding requests */
+		timeout.tv_sec = 1;
+		timeout.tv_nsec = 0;
+
+		if (uncompleted > MAXREAP)
+			todo = MAXREAP;
+		else
+			todo = uncompleted / 2;
+
+		if (todo == 0)
+			todo = 1;
+
+		flowop_beginop(threadflow, flowop);
+
+#ifdef HAVE_AIOWAITN
+		if (((aio_waitn64((struct aiocb64 **)worklist,
+		    MAXREAP, &todo, &timeout)) == -1) &&
+		    errno && (errno != ETIME)) {
+			filebench_log(LOG_ERROR,
+			    "aiowait failed: %s, outstanding = %d, "
+			    "ncompleted = %d ",
+			    strerror(errno), uncompleted, todo);
+		}
+
+		ncompleted = todo;
+		/* Take the  completed I/Os from the list */
+		inprogress = 0;
+		for (i = 0; i < ncompleted; i++) {
+			if ((aio_return64(worklist[i]) == -1) &&
+			    (errno == EINPROGRESS)) {
+				inprogress++;
+				continue;
+			}
+			if (aio_deallocate(flowop, worklist[i])
+			    == FILEBENCH_ERROR) {
+				filebench_log(LOG_ERROR, "Could not remove "
+				    "aio from list ");
+				flowop_endop(threadflow, flowop, 0);
+				return (FILEBENCH_ERROR);
+			}
+		}
+
+		uncompleted -= ncompleted;
+		uncompleted += inprogress;
+
+#else
+
+		for (ncompleted = 0, inprogress = 0,
+		    aio = flowop->fo_thread->tf_aiolist;
+		    ncompleted < todo, aio != NULL; aio = aio->al_next) {
+			int result = aio_error64(aio->al_aiocb);
+
+			if (result == EINPROGRESS) {
+				inprogress++;
+				continue;
+			}
+
+			if ((aio_return64(aio->al_aiocb) == -1) || result) {
+				filebench_log(LOG_ERROR, "aio failed: %s",
+				    strerror(result));
+				continue;
+			}
+
+			ncompleted++;
+
+			if (aio_deallocate(flowop, &aio->al_aiocb) < 0) {
+				filebench_log(LOG_ERROR, "Could not remove "
+				    aio from list ");
+				flowop_endop(threadflow, flowop, 0);
+				return (FILEBENCH_ERROR);
+			}
+		}
+
+		uncompleted -= ncompleted;
+
+#endif
+		filebench_log(LOG_DEBUG_SCRIPT,
+		    "aio2 completed %d ios, uncompleted = %d, inprogress = %d",
+		    ncompleted, uncompleted, inprogress);
+
+	} while (uncompleted > MAXREAP);
+
+	flowop_endop(threadflow, flowop, 0);
+
+	free(worklist);
+
+	return (FILEBENCH_OK);
+}
+
+#endif /* HAVE_AIO */
+
+/*
+ * Does an open64 of a file. Inserts the file descriptor number returned
+ * by open() into the supplied filebench fd. Returns FILEBENCH_OK on
+ * successs, and FILEBENCH_ERROR on failure.
+ */
+
+static int
+fb_lfs_open(fb_fdesc_t *fd, char *path, int flags, int perms)
+{
+	if ((fd->fd_num = open64(path, flags, perms)) < 0)
+		return (FILEBENCH_ERROR);
+	else
+		return (FILEBENCH_OK);
+}
+
+/*
+ * Does an unlink (delete) of a file.
+ */
+static int
+fb_lfs_unlink(char *path)
+{
+	return (unlink(path));
+}
+
+/*
+ * Does a readlink of a symbolic link.
+ */
+static ssize_t
+fb_lfs_readlink(const char *path, char *buf, size_t buf_size)
+{
+	return (readlink(path, buf, buf_size));
+}
+
+/*
+ * Does fsync of a file. Returns with fsync return info.
+ */
+static int
+fb_lfs_fsync(fb_fdesc_t *fd)
+{
+	return (fsync(fd->fd_num));
+}
+
+/*
+ * Do a posix lseek of a file. Return what lseek() returns.
+ */
+static int
+fb_lfs_lseek(fb_fdesc_t *fd, off64_t offset, int whence)
+{
+	return (lseek64(fd->fd_num, offset, whence));
+}
+
+/*
+ * Do a posix rename of a file. Return what rename() returns.
+ */
+static int
+fb_lfs_rename(const char *old, const char *new)
+{
+	return (rename(old, new));
+}
+
+
+/*
+ * Do a posix close of a file. Return what close() returns.
+ */
+static int
+fb_lfs_close(fb_fdesc_t *fd)
+{
+	return (close(fd->fd_num));
+}
+
+/*
+ * Use mkdir to create a directory.
+ */
+static int
+fb_lfs_mkdir(char *path, int perm)
+{
+	return (mkdir(path, perm));
+}
+
+/*
+ * Use rmdir to delete a directory. Returns what rmdir() returns.
+ */
+static int
+fb_lfs_rmdir(char *path)
+{
+	return (rmdir(path));
+}
+
+/*
+ * Does a posix opendir(), Returns a directory handle on success,
+ * NULL on failure.
+ */
+static DIR *
+fb_lfs_opendir(char *path)
+{
+	return (opendir(path));
+}
+
+/*
+ * Does a readdir() call. Returns a pointer to a table of directory
+ * information on success, NULL on failure.
+ */
+static struct dirent *
+fb_lfs_readdir(DIR *dirp)
+{
+	return (readdir(dirp));
+}
+
+/*
+ * Does a closedir() call.
+ */
+static int
+fb_lfs_closedir(DIR *dirp)
+{
+	return (closedir(dirp));
+}
+
+/*
+ * Does an fstat of a file.
+ */
+static int
+fb_lfs_fstat(fb_fdesc_t *fd, struct stat64 *statbufp)
+{
+	return (fstat64(fd->fd_num, statbufp));
+}
+
+/*
+ * Does a stat of a file.
+ */
+static int
+fb_lfs_stat(char *path, struct stat64 *statbufp)
+{
+	return (stat64(path, statbufp));
+}
+
+/*
+ * Do a pwrite64 to a file.
+ */
+static int
+fb_lfs_pwrite(fb_fdesc_t *fd, caddr_t iobuf, fbint_t iosize, off64_t offset)
+{
+	return (pwrite64(fd->fd_num, iobuf, iosize, offset));
+}
+
+/*
+ * Do a write to a file.
+ */
+static int
+fb_lfs_write(fb_fdesc_t *fd, caddr_t iobuf, fbint_t iosize)
+{
+	return (write(fd->fd_num, iobuf, iosize));
+}
+
+/*
+ * Does a truncate operation and returns the result
+ */
+static int
+fb_lfs_truncate(fb_fdesc_t *fd, off64_t fse_size)
+{
+#ifdef HAVE_FTRUNCATE64
+	return (ftruncate64(fd->fd_num, fse_size));
+#else
+	return (ftruncate(fd->fd_num, (off_t)fse_size));
+#endif
+}
+
+/*
+ * Does a link operation and returns the result
+ */
+static int
+fb_lfs_link(const char *existing, const char *new)
+{
+	return (link(existing, new));
+}
+
+/*
+ * Does a symlink operation and returns the result
+ */
+static int
+fb_lfs_symlink(const char *existing, const char *new)
+{
+	return (symlink(existing, new));
+}
+
+/*
+ * Does an access() check on a file.
+ */
+static int
+fb_lfs_access(const char *path, int amode)
+{
+	return (access(path, amode));
+}
diff --git a/usr/src/cmd/filebench/common/filebench.h b/usr/src/cmd/filebench/common/filebench.h
index 91da9418e3..8a6b785679 100644
--- a/usr/src/cmd/filebench/common/filebench.h
+++ b/usr/src/cmd/filebench/common/filebench.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
  * Portions Copyright 2008 Denis Cheng
@@ -91,6 +91,7 @@ extern int noproc;
 void filebench_init();
 void filebench_log __V((int level, const char *fmt, ...));
 void filebench_shutdown(int error);
+void filebench_plugin_funcvecinit(void);
 
 #ifndef HAVE_UINT64_MAX
 #define	UINT64_MAX (((off64_t)1UL<<63UL) - 1UL)
@@ -117,7 +118,7 @@ void filebench_shutdown(int error);
 #define	MIN(x, y) ((x) < (y) ? (x) : (y))
 #endif
 
-#define	FILEBENCH_VERSION	"1.4.2"
+#define	FILEBENCH_VERSION	"1.4.3"
 #define	FILEBENCHDIR	"/usr/benchmarks/filebench"
 #define	FILEBENCH_PROMPT	"filebench> "
 #define	MAX_LINE_LEN	1024
diff --git a/usr/src/cmd/filebench/common/fileset.c b/usr/src/cmd/filebench/common/fileset.c
index 86d1e3efd5..64333a561f 100644
--- a/usr/src/cmd/filebench/common/fileset.c
+++ b/usr/src/cmd/filebench/common/fileset.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
  * Portions Copyright 2008 Denis Cheng
@@ -37,6 +37,7 @@
 #include "fileset.h"
 #include "gamma_dist.h"
 #include "utils.h"
+#include "fsplug.h"
 
 /*
  * File sets, of type fileset_t, are entities which contain
@@ -130,31 +131,6 @@ fileset_usage(void)
 }
 
 /*
- * Frees up memory mapped file region of supplied size. The
- * file descriptor "fd" indicates which memory mapped file.
- * If successful, returns 0. Otherwise returns -1 if "size"
- * is zero, or -1 times the number of times msync() failed.
- */
-static int
-fileset_freemem(int fd, off64_t size)
-{
-	off64_t left;
-	int ret = 0;
-
-	for (left = size; left > 0; left -= MMAP_SIZE) {
-		off64_t thismapsize;
-		caddr_t addr;
-
-		thismapsize = MIN(MMAP_SIZE, left);
-		addr = mmap64(0, thismapsize, PROT_READ|PROT_WRITE,
-		    MAP_SHARED, fd, size - left);
-		ret += msync(addr, thismapsize, MS_INVALIDATE);
-		(void) munmap(addr, thismapsize);
-	}
-	return (ret);
-}
-
-/*
  * Creates a path string from the filesetentry_t "*entry"
  * and all of its parent's path names. The resulting path
  * is a concatination of all the individual parent paths.
@@ -222,7 +198,7 @@ fileset_mkdir(char *path, int mode)
 
 	/* Make the directories, from closest to root downwards. */
 	for (--i; i >= 0; i--) {
-		(void) mkdir(dirs[i], mode);
+		(void) FB_MKDIR(dirs[i], mode);
 		free(dirs[i]);
 	}
 
@@ -304,7 +280,7 @@ fileset_alloc_leafdir(filesetentry_t *entry)
 	if (!((entry->fse_flags & FSE_REUSING) && (stat64(path, &sb) == 0))) {
 
 		/* No file or not reusing, so create */
-		if (mkdir(path, 0755) < 0) {
+		if (FB_MKDIR(path, 0755) < 0) {
 			filebench_log(LOG_ERROR,
 			    "Failed to pre-allocate leaf directory %s: %s",
 			    path, strerror(errno));
@@ -331,7 +307,7 @@ fileset_alloc_file(filesetentry_t *entry)
 	struct stat64 sb;
 	char *pathtmp;
 	off64_t seek;
-	int fd;
+	fb_fdesc_t fdesc;
 
 	fileset = entry->fse_fileset;
 	(void) fb_strlcpy(path, avd_get_str(fileset->fs_path), MAXPATHLEN);
@@ -344,8 +320,8 @@ fileset_alloc_file(filesetentry_t *entry)
 	filebench_log(LOG_DEBUG_IMPL, "Populated %s", entry->fse_path);
 
 	/* see if reusing and this file exists */
-	if ((entry->fse_flags & FSE_REUSING) && (stat64(path, &sb) == 0)) {
-		if ((fd = open64(path, O_RDWR)) < 0) {
+	if ((entry->fse_flags & FSE_REUSING) && (FB_STAT(path, &sb) == 0)) {
+		if (FB_OPEN(&fdesc, path, O_RDWR, 0) == FILEBENCH_ERROR) {
 			filebench_log(LOG_INFO,
 			    "Attempted but failed to Re-use file %s",
 			    path);
@@ -358,10 +334,9 @@ fileset_alloc_file(filesetentry_t *entry)
 			    "Re-using file %s", path);
 
 			if (!avd_get_bool(fileset->fs_cached))
-				(void) fileset_freemem(fd,
-				    entry->fse_size);
+				(void) FB_FREEMEM(&fdesc, entry->fse_size);
 
-			(void) close(fd);
+			(void) FB_CLOSE(&fdesc);
 
 			/* unbusy the allocated entry */
 			fileset_unbusy(entry, TRUE, TRUE, 0);
@@ -372,17 +347,12 @@ fileset_alloc_file(filesetentry_t *entry)
 			filebench_log(LOG_DEBUG_IMPL,
 			    "Truncating & re-using file %s", path);
 
-#ifdef HAVE_FTRUNCATE64
-			(void) ftruncate64(fd, (off64_t)entry->fse_size);
-#else
-			(void) ftruncate(fd, (off_t)entry->fse_size);
-#endif
+			(void) FB_FTRUNC(&fdesc, (off64_t)entry->fse_size);
 
 			if (!avd_get_bool(fileset->fs_cached))
-				(void) fileset_freemem(fd,
-				    entry->fse_size);
+				(void) FB_FREEMEM(&fdesc, entry->fse_size);
 
-			(void) close(fd);
+			(void) FB_CLOSE(&fdesc);
 
 			/* unbusy the allocated entry */
 			fileset_unbusy(entry, TRUE, TRUE, 0);
@@ -391,7 +361,8 @@ fileset_alloc_file(filesetentry_t *entry)
 	} else {
 
 		/* No file or not reusing, so create */
-		if ((fd = open64(path, O_RDWR | O_CREAT, 0644)) < 0) {
+		if (FB_OPEN(&fdesc, path, O_RDWR | O_CREAT, 0644) ==
+		    FILEBENCH_ERROR) {
 			filebench_log(LOG_ERROR,
 			    "Failed to pre-allocate file %s: %s",
 			    path, strerror(errno));
@@ -418,12 +389,12 @@ fileset_alloc_file(filesetentry_t *entry)
 		 */
 		wsize = MIN(entry->fse_size - seek, FILE_ALLOC_BLOCK);
 
-		ret = write(fd, buf, wsize);
+		ret = FB_WRITE(&fdesc, buf, wsize);
 		if (ret != wsize) {
 			filebench_log(LOG_ERROR,
 			    "Failed to pre-allocate file %s: %s",
 			    path, strerror(errno));
-			(void) close(fd);
+			(void) FB_CLOSE(&fdesc);
 			free(buf);
 			fileset_unbusy(entry, TRUE, FALSE, 0);
 			return (FILEBENCH_ERROR);
@@ -432,9 +403,9 @@ fileset_alloc_file(filesetentry_t *entry)
 	}
 
 	if (!avd_get_bool(fileset->fs_cached))
-		(void) fileset_freemem(fd, entry->fse_size);
+		(void) FB_FREEMEM(&fdesc, entry->fse_size);
 
-	(void) close(fd);
+	(void) FB_CLOSE(&fdesc);
 
 	free(buf);
 
@@ -478,17 +449,17 @@ fileset_alloc_thread(filesetentry_t *entry)
  * and opens the file with open64(). It unlocks the fileset
  * entry lock, sets the DIRECTIO_ON or DIRECTIO_OFF flags
  * as requested, and returns the file descriptor integer
- * for the opened file.
+ * for the opened file in the supplied filebench file descriptor.
+ * Returns FILEBENCH_ERROR on error, and FILEBENCH_OK on success.
  */
 int
-fileset_openfile(fileset_t *fileset,
+fileset_openfile(fb_fdesc_t *fdesc, fileset_t *fileset,
     filesetentry_t *entry, int flag, int filemode, int attrs)
 {
 	char path[MAXPATHLEN];
 	char dir[MAXPATHLEN];
 	char *pathtmp;
 	struct stat64 sb;
-	int fd;
 	int open_attrs = 0;
 
 	(void) fb_strlcpy(path, avd_get_str(fileset->fs_path), MAXPATHLEN);
@@ -514,7 +485,8 @@ fileset_openfile(fileset_t *fileset,
 #endif
 	}
 
-	if ((fd = open64(path, flag | open_attrs, filemode)) < 0) {
+	if (FB_OPEN(fdesc, path, flag | open_attrs, filemode)
+	    == FILEBENCH_ERROR) {
 		filebench_log(LOG_ERROR,
 		    "Failed to open file %d, %s, with status %x: %s",
 		    entry->fse_index, path, entry->fse_flags, strerror(errno));
@@ -530,12 +502,12 @@ fileset_openfile(fileset_t *fileset,
 
 #ifdef sun
 	if (attrs & FLOW_ATTR_DIRECTIO)
-		(void) directio(fd, DIRECTIO_ON);
+		(void) directio(fdesc->fd_num, DIRECTIO_ON);
 	else
-		(void) directio(fd, DIRECTIO_OFF);
+		(void) directio(fdesc->fd_num, DIRECTIO_OFF);
 #endif
 
-	return (fd);
+	return (FILEBENCH_OK);
 }
 
 /*
@@ -1071,7 +1043,7 @@ fileset_create(fileset_t *fileset)
 		filebench_log(LOG_VERBOSE,
 		    "making tree for filset %s", path);
 
-		(void) mkdir(path, 0755);
+		(void) FB_MKDIR(path, 0755);
 
 		if (fileset_create_subdirs(fileset, path) == FILEBENCH_ERROR)
 			return (FILEBENCH_ERROR);
@@ -1120,7 +1092,7 @@ fileset_create(fileset_t *fileset)
 				    &filebench_shm->shm_fsparalloc_lock);
 			}
 
-			/* quit if any allocation thread reports and error */
+			/* quit if any allocation thread reports an error */
 			if (filebench_shm->shm_fsparalloc_count < 0) {
 				(void) pthread_mutex_unlock(
 				    &filebench_shm->shm_fsparalloc_lock);
diff --git a/usr/src/cmd/filebench/common/fileset.h b/usr/src/cmd/filebench/common/fileset.h
index 1250d50c32..80537d174a 100644
--- a/usr/src/cmd/filebench/common/fileset.h
+++ b/usr/src/cmd/filebench/common/fileset.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -50,6 +50,7 @@
 
 #include "vars.h"
 #include "fb_avl.h"
+
 #define	FILE_ALLOC_BLOCK (off64_t)(1024 * 1024)
 
 #ifdef	__cplusplus
@@ -173,8 +174,8 @@ typedef struct fileset {
 } fileset_t;
 
 int fileset_createset(fileset_t *);
-int fileset_openfile(fileset_t *fileset, filesetentry_t *entry,
-    int flag, int mode, int attrs);
+int fileset_openfile(fb_fdesc_t *fd, fileset_t *fileset,
+    filesetentry_t *entry, int flag, int mode, int attrs);
 fileset_t *fileset_define(avd_t);
 fileset_t *fileset_find(char *name);
 filesetentry_t *fileset_pick(fileset_t *fileset, int flags, int tid,
diff --git a/usr/src/cmd/filebench/common/flowop.c b/usr/src/cmd/filebench/common/flowop.c
index 6a7773b795..a6b3617e2b 100644
--- a/usr/src/cmd/filebench/common/flowop.c
+++ b/usr/src/cmd/filebench/common/flowop.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -584,14 +584,44 @@ flowop_start(threadflow_t *threadflow)
 	pthread_exit(&threadflow->tf_abort);
 }
 
+void flowoplib_flowinit(void);
+void fb_lfs_flowinit(void);
+
 void
 flowop_init(void)
 {
 	(void) pthread_mutex_init(&controlstats_lock,
 	    ipc_mutexattr(IPC_MUTEX_NORMAL));
-	flowoplib_init();
+	flowoplib_flowinit();
+}
+
+static int plugin_flowinit_done = FALSE;
+
+/*
+ * Initialize any "plug-in" flowops. Called when the first "create fileset"
+ * command is encountered.
+ */
+void
+flowop_plugin_flowinit(void)
+{
+	if (plugin_flowinit_done)
+		return;
+
+	plugin_flowinit_done = TRUE;
+
+	switch (filebench_shm->shm_filesys_type) {
+	case LOCAL_FS_PLUG:
+		fb_lfs_flowinit();
+		break;
+
+	case NFS3_PLUG:
+	case NFS4_PLUG:
+	case CIFS_PLUG:
+		break;
+	}
 }
 
+
 /*
  * Delete the designated flowop from the thread's flowop list.
  */
@@ -1113,3 +1143,62 @@ flowop_composite_destruct(flowop_t *flowop)
 		inner_flowop = inner_flowop->fo_exec_next;
 	}
 }
+
+/*
+ * Support routines for libraries of flowops
+ */
+
+int
+flowop_init_generic(flowop_t *flowop)
+{
+	(void) ipc_mutex_unlock(&flowop->fo_lock);
+	return (FILEBENCH_OK);
+}
+
+void
+flowop_destruct_generic(flowop_t *flowop)
+{
+	char *buf;
+
+	/* release any local resources held by the flowop */
+	(void) ipc_mutex_lock(&flowop->fo_lock);
+	buf = flowop->fo_buf;
+	flowop->fo_buf = NULL;
+	(void) ipc_mutex_unlock(&flowop->fo_lock);
+
+	if (buf)
+		free(buf);
+}
+
+
+/*
+ * Loops through the supplied list of flowops and creates and initializes
+ * a flowop for each one by calling flowop_define. As a side effect of
+ * calling flowop define, the created flowops are placed on the
+ * master flowop list. All created flowops are set to instance "0".
+ */
+void
+flowop_flow_init(flowop_proto_t *list, int nops)
+{
+	int i;
+
+	for (i = 0; i < nops; i++) {
+		flowop_t *flowop;
+		flowop_proto_t *fl;
+
+		fl = &(list[i]);
+
+		if ((flowop = flowop_define(NULL,
+		    fl->fl_name, NULL, NULL, 0, fl->fl_type)) == 0) {
+			filebench_log(LOG_ERROR,
+			    "failed to create flowop %s\n",
+			    fl->fl_name);
+			filebench_shutdown(1);
+		}
+
+		flowop->fo_func = fl->fl_func;
+		flowop->fo_init = fl->fl_init;
+		flowop->fo_destruct = fl->fl_destruct;
+		flowop->fo_attrs = fl->fl_attrs;
+	}
+}
diff --git a/usr/src/cmd/filebench/common/flowop.h b/usr/src/cmd/filebench/common/flowop.h
index 976c3d0505..d08367e7b1 100644
--- a/usr/src/cmd/filebench/common/flowop.h
+++ b/usr/src/cmd/filebench/common/flowop.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -44,6 +44,7 @@
 #include "vars.h"
 #include "fileset.h"
 #include "filebench.h"
+#include "fsplug.h"
 
 #ifdef	__cplusplus
 extern "C" {
@@ -135,17 +136,31 @@ typedef struct flowop {
 #define	FLOW_TYPE_COMPOSITE	4  /* Op is a composite flowop */
 #define	FLOW_TYPE_OTHER		5  /* Op is a something else */
 
+typedef struct flowop_proto {
+	int	fl_type;
+	int	fl_attrs;
+	char	*fl_name;
+	int	(*fl_init)();
+	int	(*fl_func)();
+	void	(*fl_destruct)();
+} flowop_proto_t;
+
 extern flowstat_t controlstats;
 extern pthread_mutex_t controlstats_lock;
 
 void flowop_init(void);
+void flowop_plugin_flowinit(void);
 flowop_t *flowop_define(threadflow_t *, char *name, flowop_t *inherit,
     flowop_t **flowoplist_hdp, int instance, int type);
 flowop_t *flowop_find(char *name);
 flowop_t *flowop_find_one(char *name, int instance);
 flowop_t *flowop_find_from_list(char *name, flowop_t *list);
+int flowop_init_generic(flowop_t *flowop);
+void flowop_destruct_generic(flowop_t *flowop);
+void flowop_flow_init(flowop_proto_t *list, int nops);
 void flowoplib_usage(void);
-void flowoplib_init(void);
+int flowoplib_iosetup(threadflow_t *threadflow, flowop_t *flowop,
+    fbint_t *wssp, caddr_t *iobufp, fb_fdesc_t **filedescp, fbint_t iosize);
 void flowop_delete_all(flowop_t **threadlist);
 void flowop_endop(threadflow_t *threadflow, flowop_t *flowop, int64_t bytes);
 void flowop_beginop(threadflow_t *threadflow, flowop_t *flowop);
diff --git a/usr/src/cmd/filebench/common/flowop_library.c b/usr/src/cmd/filebench/common/flowop_library.c
index b2ad1cf28f..26e5f1a3d5 100644
--- a/usr/src/cmd/filebench/common/flowop_library.c
+++ b/usr/src/cmd/filebench/common/flowop_library.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
  * Portions Copyright 2008 Denis Cheng
@@ -31,6 +31,7 @@
 #ifdef HAVE_SYS_ASYNCH_H
 #include <sys/asynch.h>
 #endif
+#include <stddef.h>
 #include <sys/ipc.h>
 #include <sys/sem.h>
 #include <sys/errno.h>
@@ -44,30 +45,14 @@
 #include <utility.h>
 #endif /* HAVE_UTILITY_H */
 
-#ifdef HAVE_AIO
-#include <aio.h>
-#endif /* HAVE_AIO */
-
-#ifdef HAVE_LIBAIO_H
-#include <libaio.h>
-#endif /* HAVE_LIBAIO_H */
-
 #ifdef HAVE_SYS_ASYNC_H
 #include <sys/asynch.h>
 #endif /* HAVE_SYS_ASYNC_H */
 
-#ifdef HAVE_AIO_H
-#include <aio.h>
-#endif /* HAVE_AIO_H */
-
 #ifndef HAVE_UINT_T
 #define	uint_t unsigned int
 #endif /* HAVE_UINT_T */
 
-#ifndef HAVE_AIOCB64_T
-#define	aiocb64 aiocb
-#endif /* HAVE_AIOCB64_T */
-
 #ifndef HAVE_SYSV_SEM
 #include <semaphore.h>
 #endif /* HAVE_SYSV_SEM */
@@ -77,6 +62,8 @@
 #include "fileset.h"
 #include "fb_random.h"
 #include "utils.h"
+#include "fsplug.h"
+
 /*
  * These routines implement the flowops from the f language. Each
  * flowop has has a name such as "read", and a set of function pointers
@@ -100,16 +87,10 @@
  * by the "name=" attribute.
  */
 
-static int flowoplib_init_generic(flowop_t *flowop);
-static void flowoplib_destruct_generic(flowop_t *flowop);
 static void flowoplib_destruct_noop(flowop_t *flowop);
 static int flowoplib_fdnum(threadflow_t *threadflow, flowop_t *flowop);
 static int flowoplib_print(threadflow_t *threadflow, flowop_t *flowop);
 static int flowoplib_write(threadflow_t *threadflow, flowop_t *flowop);
-#ifdef HAVE_AIO
-static int flowoplib_aiowrite(threadflow_t *threadflow, flowop_t *flowop);
-static int flowoplib_aiowait(threadflow_t *threadflow, flowop_t *flowop);
-#endif
 static int flowoplib_read(threadflow_t *threadflow, flowop_t *flowop);
 static int flowoplib_block_init(flowop_t *flowop);
 static int flowoplib_block(threadflow_t *threadflow, flowop_t *flowop);
@@ -146,140 +127,84 @@ static int flowoplib_testrandvar(threadflow_t *threadflow, flowop_t *flowop);
 static int flowoplib_testrandvar_init(flowop_t *flowop);
 static void flowoplib_testrandvar_destruct(flowop_t *flowop);
 
-typedef struct flowoplib {
-	int	fl_type;
-	int	fl_attrs;
-	char	*fl_name;
-	int	(*fl_init)();
-	int	(*fl_func)();
-	void	(*fl_destruct)();
-} flowoplib_t;
-
-static flowoplib_t flowoplib_funcs[] = {
-	FLOW_TYPE_IO, FLOW_ATTR_WRITE, "write", flowoplib_init_generic,
-	flowoplib_write, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, FLOW_ATTR_READ, "read", flowoplib_init_generic,
-	flowoplib_read, flowoplib_destruct_generic,
-#ifdef HAVE_AIO
-	FLOW_TYPE_AIO, FLOW_ATTR_WRITE, "aiowrite", flowoplib_init_generic,
-	flowoplib_aiowrite, flowoplib_destruct_generic,
-	FLOW_TYPE_AIO, 0, "aiowait", flowoplib_init_generic,
-	flowoplib_aiowait, flowoplib_destruct_generic,
-#endif
+static flowop_proto_t flowoplib_funcs[] = {
+	FLOW_TYPE_IO, FLOW_ATTR_WRITE, "write", flowop_init_generic,
+	flowoplib_write, flowop_destruct_generic,
+	FLOW_TYPE_IO, FLOW_ATTR_READ, "read", flowop_init_generic,
+	flowoplib_read, flowop_destruct_generic,
 	FLOW_TYPE_SYNC, 0, "block", flowoplib_block_init,
-	flowoplib_block, flowoplib_destruct_generic,
-	FLOW_TYPE_SYNC, 0, "wakeup", flowoplib_init_generic,
-	flowoplib_wakeup, flowoplib_destruct_generic,
+	flowoplib_block, flowop_destruct_generic,
+	FLOW_TYPE_SYNC, 0, "wakeup", flowop_init_generic,
+	flowoplib_wakeup, flowop_destruct_generic,
 	FLOW_TYPE_SYNC, 0, "semblock", flowoplib_semblock_init,
 	flowoplib_semblock, flowoplib_semblock_destruct,
 	FLOW_TYPE_SYNC, 0, "sempost", flowoplib_sempost_init,
 	flowoplib_sempost, flowoplib_destruct_noop,
-	FLOW_TYPE_OTHER, 0, "hog", flowoplib_init_generic,
-	flowoplib_hog, flowoplib_destruct_generic,
-	FLOW_TYPE_OTHER, 0, "delay", flowoplib_init_generic,
-	flowoplib_delay, flowoplib_destruct_generic,
-	FLOW_TYPE_OTHER, 0, "eventlimit", flowoplib_init_generic,
-	flowoplib_eventlimit, flowoplib_destruct_generic,
-	FLOW_TYPE_OTHER, 0, "bwlimit", flowoplib_init_generic,
-	flowoplib_bwlimit, flowoplib_destruct_generic,
-	FLOW_TYPE_OTHER, 0, "iopslimit", flowoplib_init_generic,
-	flowoplib_iopslimit, flowoplib_destruct_generic,
-	FLOW_TYPE_OTHER, 0, "opslimit", flowoplib_init_generic,
-	flowoplib_opslimit, flowoplib_destruct_generic,
-	FLOW_TYPE_OTHER, 0, "finishoncount", flowoplib_init_generic,
-	flowoplib_finishoncount, flowoplib_destruct_generic,
-	FLOW_TYPE_OTHER, 0, "finishonbytes", flowoplib_init_generic,
-	flowoplib_finishonbytes, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, 0, "openfile", flowoplib_init_generic,
-	flowoplib_openfile, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, 0, "createfile", flowoplib_init_generic,
-	flowoplib_createfile, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, 0, "closefile", flowoplib_init_generic,
-	flowoplib_closefile, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, 0, "makedir", flowoplib_init_generic,
-	flowoplib_makedir, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, 0, "removedir", flowoplib_init_generic,
-	flowoplib_removedir, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, 0, "listdir", flowoplib_init_generic,
-	flowoplib_listdir, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, 0, "fsync", flowoplib_init_generic,
-	flowoplib_fsync, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, 0, "fsyncset", flowoplib_init_generic,
-	flowoplib_fsyncset, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, 0, "statfile", flowoplib_init_generic,
-	flowoplib_statfile, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, FLOW_ATTR_READ, "readwholefile", flowoplib_init_generic,
-	flowoplib_readwholefile, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, FLOW_ATTR_WRITE, "appendfile", flowoplib_init_generic,
-	flowoplib_appendfile, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, FLOW_ATTR_WRITE, "appendfilerand", flowoplib_init_generic,
-	flowoplib_appendfilerand, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, 0, "deletefile", flowoplib_init_generic,
-	flowoplib_deletefile, flowoplib_destruct_generic,
-	FLOW_TYPE_IO, FLOW_ATTR_WRITE, "writewholefile", flowoplib_init_generic,
-	flowoplib_writewholefile, flowoplib_destruct_generic,
-	FLOW_TYPE_OTHER, 0, "print", flowoplib_init_generic,
-	flowoplib_print, flowoplib_destruct_generic,
+	FLOW_TYPE_OTHER, 0, "hog", flowop_init_generic,
+	flowoplib_hog, flowop_destruct_generic,
+	FLOW_TYPE_OTHER, 0, "delay", flowop_init_generic,
+	flowoplib_delay, flowop_destruct_generic,
+	FLOW_TYPE_OTHER, 0, "eventlimit", flowop_init_generic,
+	flowoplib_eventlimit, flowop_destruct_generic,
+	FLOW_TYPE_OTHER, 0, "bwlimit", flowop_init_generic,
+	flowoplib_bwlimit, flowop_destruct_generic,
+	FLOW_TYPE_OTHER, 0, "iopslimit", flowop_init_generic,
+	flowoplib_iopslimit, flowop_destruct_generic,
+	FLOW_TYPE_OTHER, 0, "opslimit", flowop_init_generic,
+	flowoplib_opslimit, flowop_destruct_generic,
+	FLOW_TYPE_OTHER, 0, "finishoncount", flowop_init_generic,
+	flowoplib_finishoncount, flowop_destruct_generic,
+	FLOW_TYPE_OTHER, 0, "finishonbytes", flowop_init_generic,
+	flowoplib_finishonbytes, flowop_destruct_generic,
+	FLOW_TYPE_IO, 0, "openfile", flowop_init_generic,
+	flowoplib_openfile, flowop_destruct_generic,
+	FLOW_TYPE_IO, 0, "createfile", flowop_init_generic,
+	flowoplib_createfile, flowop_destruct_generic,
+	FLOW_TYPE_IO, 0, "closefile", flowop_init_generic,
+	flowoplib_closefile, flowop_destruct_generic,
+	FLOW_TYPE_IO, 0, "makedir", flowop_init_generic,
+	flowoplib_makedir, flowop_destruct_generic,
+	FLOW_TYPE_IO, 0, "removedir", flowop_init_generic,
+	flowoplib_removedir, flowop_destruct_generic,
+	FLOW_TYPE_IO, 0, "listdir", flowop_init_generic,
+	flowoplib_listdir, flowop_destruct_generic,
+	FLOW_TYPE_IO, 0, "fsync", flowop_init_generic,
+	flowoplib_fsync, flowop_destruct_generic,
+	FLOW_TYPE_IO, 0, "fsyncset", flowop_init_generic,
+	flowoplib_fsyncset, flowop_destruct_generic,
+	FLOW_TYPE_IO, 0, "statfile", flowop_init_generic,
+	flowoplib_statfile, flowop_destruct_generic,
+	FLOW_TYPE_IO, FLOW_ATTR_READ, "readwholefile", flowop_init_generic,
+	flowoplib_readwholefile, flowop_destruct_generic,
+	FLOW_TYPE_IO, FLOW_ATTR_WRITE, "appendfile", flowop_init_generic,
+	flowoplib_appendfile, flowop_destruct_generic,
+	FLOW_TYPE_IO, FLOW_ATTR_WRITE, "appendfilerand", flowop_init_generic,
+	flowoplib_appendfilerand, flowop_destruct_generic,
+	FLOW_TYPE_IO, 0, "deletefile", flowop_init_generic,
+	flowoplib_deletefile, flowop_destruct_generic,
+	FLOW_TYPE_IO, FLOW_ATTR_WRITE, "writewholefile", flowop_init_generic,
+	flowoplib_writewholefile, flowop_destruct_generic,
+	FLOW_TYPE_OTHER, 0, "print", flowop_init_generic,
+	flowoplib_print, flowop_destruct_generic,
 	/* routine to calculate mean and stddev for output from a randvar */
 	FLOW_TYPE_OTHER, 0, "testrandvar", flowoplib_testrandvar_init,
 	flowoplib_testrandvar, flowoplib_testrandvar_destruct
 };
 
 /*
- * Loops through the master list of flowops defined in this
+ * Loops through the list of flowops defined in this
  * module, and creates and initializes a flowop for each one
- * by calling flowop_define. As a side effect of calling
- * flowop define, the created flowops are placed on the
+ * by calling flowop_flow_init. As a side effect of calling
+ * flowop_flow_init, the created flowops are placed on the
  * master flowop list. All created flowops are set to
  * instance "0".
  */
 void
-flowoplib_init()
+flowoplib_flowinit()
 {
-	int nops = sizeof (flowoplib_funcs) / sizeof (flowoplib_t);
-	int i;
-
-	for (i = 0; i < nops; i++) {
-		flowop_t *flowop;
-		flowoplib_t *fl;
+	int nops = sizeof (flowoplib_funcs) / sizeof (flowop_proto_t);
 
-		fl = &flowoplib_funcs[i];
-
-		if ((flowop = flowop_define(NULL,
-		    fl->fl_name, NULL, NULL, 0, fl->fl_type)) == 0) {
-			filebench_log(LOG_ERROR,
-			    "failed to create flowop %s\n",
-			    fl->fl_name);
-			filebench_shutdown(1);
-		}
-
-		flowop->fo_func = fl->fl_func;
-		flowop->fo_init = fl->fl_init;
-		flowop->fo_destruct = fl->fl_destruct;
-		flowop->fo_attrs = fl->fl_attrs;
-	}
-}
-
-static int
-flowoplib_init_generic(flowop_t *flowop)
-{
-	(void) ipc_mutex_unlock(&flowop->fo_lock);
-	return (FILEBENCH_OK);
-}
-
-static void
-flowoplib_destruct_generic(flowop_t *flowop)
-{
-	char *buf;
-
-	/* release any local resources held by the flowop */
-	(void) ipc_mutex_lock(&flowop->fo_lock);
-	buf = flowop->fo_buf;
-	flowop->fo_buf = NULL;
-	(void) ipc_mutex_unlock(&flowop->fo_lock);
-
-	if (buf)
-		free(buf);
+	flowop_flow_init(flowoplib_funcs, nops);
 }
 
 /*
@@ -461,14 +386,14 @@ flowoplib_fdnum(threadflow_t *threadflow, flowop_t *flowop)
  */
 static int
 flowoplib_filesetup(threadflow_t *threadflow, flowop_t *flowop,
-    fbint_t *wssp, int *filedescp)
+    fbint_t *wssp, fb_fdesc_t **fdescp)
 {
 	int fd = flowoplib_fdnum(threadflow, flowop);
 
 	if (fd == -1)
 		return (FILEBENCH_ERROR);
 
-	if (threadflow->tf_fd[fd] == 0) {
+	if (threadflow->tf_fd[fd].fd_ptr == NULL) {
 		int ret;
 
 		if ((ret = flowoplib_openfile_common(
@@ -486,7 +411,7 @@ flowoplib_filesetup(threadflow_t *threadflow, flowop_t *flowop,
 		}
 	}
 
-	*filedescp = threadflow->tf_fd[fd];
+	*fdescp = &(threadflow->tf_fd[fd]);
 
 	if ((*wssp = flowop->fo_constwss) == 0) {
 		if (threadflow->tf_fse[fd])
@@ -556,9 +481,9 @@ flowoplib_iobufsetup(threadflow_t *threadflow, flowop_t *flowop,
  * io buffer or random offset into tf_mem for IO operation and the wss
  * value. Returns FILEBENCH_ERROR on errors, FILEBENCH_OK otherwise.
  */
-static int
+int
 flowoplib_iosetup(threadflow_t *threadflow, flowop_t *flowop,
-    fbint_t *wssp, caddr_t *iobufp, int *filedescp, fbint_t iosize)
+    fbint_t *wssp, caddr_t *iobufp, fb_fdesc_t **filedescp, fbint_t iosize)
 {
 	int ret;
 
@@ -595,13 +520,13 @@ flowoplib_read(threadflow_t *threadflow, flowop_t *flowop)
 	caddr_t iobuf;
 	fbint_t wss;
 	fbint_t iosize;
-	int filedesc;
+	fb_fdesc_t *fdesc;
 	int ret;
 
 
 	iosize = avd_get_int(flowop->fo_iosize);
 	if ((ret = flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
-	    &filedesc, iosize)) != FILEBENCH_OK)
+	    &fdesc, iosize)) != FILEBENCH_OK)
 		return (ret);
 
 	if (avd_get_bool(flowop->fo_random)) {
@@ -616,7 +541,7 @@ flowoplib_read(threadflow_t *threadflow, flowop_t *flowop)
 		}
 
 		(void) flowop_beginop(threadflow, flowop);
-		if ((ret = pread64(filedesc, iobuf,
+		if ((ret = FB_PREAD(fdesc, iobuf,
 		    iosize, (off64_t)fileoffset)) == -1) {
 			(void) flowop_endop(threadflow, flowop, 0);
 			filebench_log(LOG_ERROR,
@@ -630,11 +555,11 @@ flowoplib_read(threadflow_t *threadflow, flowop_t *flowop)
 		(void) flowop_endop(threadflow, flowop, ret);
 
 		if ((ret == 0))
-			(void) lseek64(filedesc, 0, SEEK_SET);
+			(void) FB_LSEEK(fdesc, 0, SEEK_SET);
 
 	} else {
 		(void) flowop_beginop(threadflow, flowop);
-		if ((ret = read(filedesc, iobuf, iosize)) == -1) {
+		if ((ret = FB_READ(fdesc, iobuf, iosize)) == -1) {
 			(void) flowop_endop(threadflow, flowop, 0);
 			filebench_log(LOG_ERROR,
 			    "read file %s failed, io buffer %zd: %s",
@@ -646,281 +571,12 @@ flowoplib_read(threadflow_t *threadflow, flowop_t *flowop)
 		(void) flowop_endop(threadflow, flowop, ret);
 
 		if ((ret == 0))
-			(void) lseek64(filedesc, 0, SEEK_SET);
-	}
-
-	return (FILEBENCH_OK);
-}
-
-#ifdef HAVE_AIO
-
-/*
- * Asynchronous write section. An Asynchronous IO element
- * (aiolist_t) is used to associate the asynchronous write request with
- * its subsequent completion. This element includes a aiocb64 struct
- * that is used by posix aio_xxx calls to track the asynchronous writes.
- * The flowops aiowrite and aiowait result in calls to these posix
- * aio_xxx system routines to do the actual asynchronous write IO
- * operations.
- */
-
-
-/*
- * Allocates an asynchronous I/O list (aio, of type
- * aiolist_t) element. Adds it to the flowop thread's
- * threadflow aio list. Returns a pointer to the element.
- */
-static aiolist_t *
-aio_allocate(flowop_t *flowop)
-{
-	aiolist_t *aiolist;
-
-	if ((aiolist = malloc(sizeof (aiolist_t))) == NULL) {
-		filebench_log(LOG_ERROR, "malloc aiolist failed");
-		filebench_shutdown(1);
-	}
-
-	/* Add to list */
-	if (flowop->fo_thread->tf_aiolist == NULL) {
-		flowop->fo_thread->tf_aiolist = aiolist;
-		aiolist->al_next = NULL;
-	} else {
-		aiolist->al_next = flowop->fo_thread->tf_aiolist;
-		flowop->fo_thread->tf_aiolist = aiolist;
-	}
-	return (aiolist);
-}
-
-/*
- * Searches for the aiolist element that has a matching
- * completion block, aiocb. If none found returns FILEBENCH_ERROR. If
- * found, removes the aiolist element from flowop thread's
- * list and returns FILEBENCH_OK.
- */
-static int
-aio_deallocate(flowop_t *flowop, struct aiocb64 *aiocb)
-{
-	aiolist_t *aiolist = flowop->fo_thread->tf_aiolist;
-	aiolist_t *previous = NULL;
-	aiolist_t *match = NULL;
-
-	if (aiocb == NULL) {
-		filebench_log(LOG_ERROR, "null aiocb deallocate");
-		return (FILEBENCH_OK);
-	}
-
-	while (aiolist) {
-		if (aiocb == &(aiolist->al_aiocb)) {
-			match = aiolist;
-			break;
-		}
-		previous = aiolist;
-		aiolist = aiolist->al_next;
-	}
-
-	if (match == NULL)
-		return (FILEBENCH_ERROR);
-
-	/* Remove from the list */
-	if (previous)
-		previous->al_next = match->al_next;
-	else
-		flowop->fo_thread->tf_aiolist = match->al_next;
-
-	return (FILEBENCH_OK);
-}
-
-/*
- * Emulate posix aiowrite(). Determines which file to use,
- * either one file of a fileset, or the file associated
- * with a fileobj, allocates and fills an aiolist_t element
- * for the write, and issues the asynchronous write. This
- * operation is only valid for random IO, and returns an
- * error if the flowop is set for sequential IO. Returns
- * FILEBENCH_OK on success, FILEBENCH_NORSC if iosetup can't
- * obtain a file to open, and FILEBENCH_ERROR on any
- * encountered error.
- */
-static int
-flowoplib_aiowrite(threadflow_t *threadflow, flowop_t *flowop)
-{
-	caddr_t iobuf;
-	fbint_t wss;
-	fbint_t iosize;
-	int filedesc;
-	int ret;
-
-	iosize = avd_get_int(flowop->fo_iosize);
-
-	if ((ret = flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
-	    &filedesc, iosize)) != FILEBENCH_OK)
-		return (ret);
-
-	if (avd_get_bool(flowop->fo_random)) {
-		uint64_t fileoffset;
-		struct aiocb64 *aiocb;
-		aiolist_t *aiolist;
-
-		if (filebench_randomno64(&fileoffset,
-		    wss, iosize, NULL) == -1) {
-			filebench_log(LOG_ERROR,
-			    "file size smaller than IO size for thread %s",
-			    flowop->fo_name);
-			return (FILEBENCH_ERROR);
-		}
-
-		aiolist = aio_allocate(flowop);
-		aiolist->al_type = AL_WRITE;
-		aiocb = &aiolist->al_aiocb;
-
-		aiocb->aio_fildes = filedesc;
-		aiocb->aio_buf = iobuf;
-		aiocb->aio_nbytes = (size_t)iosize;
-		aiocb->aio_offset = (off64_t)fileoffset;
-		aiocb->aio_reqprio = 0;
-
-		filebench_log(LOG_DEBUG_IMPL,
-		    "aio fd=%d, bytes=%llu, offset=%llu",
-		    filedesc, (u_longlong_t)iosize, (u_longlong_t)fileoffset);
-
-		flowop_beginop(threadflow, flowop);
-		if (aio_write64(aiocb) < 0) {
-			filebench_log(LOG_ERROR, "aiowrite failed: %s",
-			    strerror(errno));
-			filebench_shutdown(1);
-		}
-		flowop_endop(threadflow, flowop, iosize);
-	} else {
-		return (FILEBENCH_ERROR);
-	}
-
-	return (FILEBENCH_OK);
-}
-
-
-
-#define	MAXREAP 4096
-
-/*
- * Emulate posix aiowait(). Waits for the completion of half the
- * outstanding asynchronous IOs, or a single IO, which ever is
- * larger. The routine will return after a sufficient number of
- * completed calls issued by any thread in the procflow have
- * completed, or a 1 second timout elapses. All completed
- * IO operations are deleted from the thread's aiolist.
- */
-static int
-flowoplib_aiowait(threadflow_t *threadflow, flowop_t *flowop)
-{
-	struct aiocb64 **worklist;
-	aiolist_t *aio = flowop->fo_thread->tf_aiolist;
-	int uncompleted = 0;
-
-	worklist = calloc(MAXREAP, sizeof (struct aiocb64 *));
-
-	/* Count the list of pending aios */
-	while (aio) {
-		uncompleted++;
-		aio = aio->al_next;
+			(void) FB_LSEEK(fdesc, 0, SEEK_SET);
 	}
 
-	do {
-		uint_t ncompleted = 0;
-		uint_t todo;
-		struct timespec timeout;
-		int inprogress;
-		int i;
-
-		/* Wait for half of the outstanding requests */
-		timeout.tv_sec = 1;
-		timeout.tv_nsec = 0;
-
-		if (uncompleted > MAXREAP)
-			todo = MAXREAP;
-		else
-			todo = uncompleted / 2;
-
-		if (todo == 0)
-			todo = 1;
-
-		flowop_beginop(threadflow, flowop);
-
-#ifdef HAVE_AIOWAITN
-		if ((aio_waitn64((struct aiocb64 **)worklist,
-		    MAXREAP, &todo, &timeout) == -1) &&
-		    errno && (errno != ETIME)) {
-			filebench_log(LOG_ERROR,
-			    "aiowait failed: %s, outstanding = %d, "
-			    "ncompleted = %d ",
-			    strerror(errno), uncompleted, todo);
-		}
-
-		ncompleted = todo;
-		/* Take the  completed I/Os from the list */
-		inprogress = 0;
-		for (i = 0; i < ncompleted; i++) {
-			if ((aio_return64(worklist[i]) == -1) &&
-			    (errno == EINPROGRESS)) {
-				inprogress++;
-				continue;
-			}
-			if (aio_deallocate(flowop, worklist[i]) < 0) {
-				filebench_log(LOG_ERROR, "Could not remove "
-				    "aio from list ");
-				flowop_endop(threadflow, flowop, 0);
-				return (FILEBENCH_ERROR);
-			}
-		}
-
-		uncompleted -= ncompleted;
-		uncompleted += inprogress;
-
-#else
-
-		for (ncompleted = 0, inprogress = 0,
-		    aio = flowop->fo_thread->tf_aiolist;
-		    ncompleted < todo, aio != NULL; aio = aio->al_next) {
-			int result = aio_error64(&aio->al_aiocb);
-
-			if (result == EINPROGRESS) {
-				inprogress++;
-				continue;
-			}
-
-			if ((aio_return64(&aio->al_aiocb) == -1) || result) {
-				filebench_log(LOG_ERROR, "aio failed: %s",
-				    strerror(result));
-				continue;
-			}
-
-			ncompleted++;
-
-			if (aio_deallocate(flowop, &aio->al_aiocb) < 0) {
-				filebench_log(LOG_ERROR, "Could not remove aio "
-				    "from list ");
-				flowop_endop(threadflow, flowop, 0);
-				return (FILEBENCH_ERROR);
-			}
-		}
-
-		uncompleted -= ncompleted;
-
-#endif
-		filebench_log(LOG_DEBUG_SCRIPT,
-		    "aio2 completed %d ios, uncompleted = %d, inprogress = %d",
-		    ncompleted, uncompleted, inprogress);
-
-	} while (uncompleted > MAXREAP);
-
-	flowop_endop(threadflow, flowop, 0);
-
-	free(worklist);
-
 	return (FILEBENCH_OK);
 }
 
-#endif /* HAVE_AIO */
-
 /*
  * Initializes a "flowop_block" flowop. Specifically, it
  * initializes the flowop's fo_cv and unlocks the fo_lock.
@@ -1791,11 +1447,6 @@ flowoplib_sempost(threadflow_t *threadflow, flowop_t *flowop)
  */
 
 /*
- * XXX Making file selection more consistent among the flowops might good
- */
-
-
-/*
  * Emulates (and actually does) file open. Obtains a file descriptor
  * index, then calls flowoplib_openfile_common() to open. Returns
  * FILEBENCH_ERROR if no file descriptor is found, and returns the
@@ -1852,7 +1503,7 @@ flowoplib_openfile_common(threadflow_t *threadflow, flowop_t *flowop, int fd)
 	if (avd_get_bool(flowop->fo_rotatefd))
 		tid = threadflow->tf_utid;
 
-	if (threadflow->tf_fd[fd] != 0) {
+	if (threadflow->tf_fd[fd].fd_ptr != NULL) {
 		filebench_log(LOG_ERROR,
 		    "flowop %s attempted to open without closing on fd %d",
 		    flowop->fo_name, fd);
@@ -1880,10 +1531,8 @@ flowoplib_openfile_common(threadflow_t *threadflow, flowop_t *flowop, int fd)
 		filebench_log(LOG_DEBUG_SCRIPT,
 		    "open raw device %s flags %d = %d", name, open_attrs, fd);
 
-		threadflow->tf_fd[fd] = open64(name,
-		    O_RDWR | open_attrs, 0666);
-
-		if (threadflow->tf_fd[fd] < 0) {
+		if (FB_OPEN(&(threadflow->tf_fd[fd]), name,
+		    O_RDWR | open_attrs, 0666) == FILEBENCH_ERROR) {
 			filebench_log(LOG_ERROR,
 			    "Failed to open raw device %s: %s",
 			    name, strerror(errno));
@@ -1892,7 +1541,7 @@ flowoplib_openfile_common(threadflow_t *threadflow, flowop_t *flowop, int fd)
 
 		/* if running on Solaris, use un-buffered io */
 #ifdef sun
-		(void) directio(threadflow->tf_fd[fd], DIRECTIO_ON);
+		(void) directio(threadflow->tf_fd[fd].fd_num, DIRECTIO_ON);
 #endif
 
 		threadflow->tf_fse[fd] = NULL;
@@ -1912,11 +1561,11 @@ flowoplib_openfile_common(threadflow_t *threadflow, flowop_t *flowop, int fd)
 	threadflow->tf_fse[fd] = file;
 
 	flowop_beginop(threadflow, flowop);
-	threadflow->tf_fd[fd] = fileset_openfile(flowop->fo_fileset,
+	err = fileset_openfile(&threadflow->tf_fd[fd], flowop->fo_fileset,
 	    file, O_RDWR, 0666, flowoplib_fileattrs(flowop));
 	flowop_endop(threadflow, flowop, 0);
 
-	if (threadflow->tf_fd[fd] < 0) {
+	if (err == FILEBENCH_ERROR) {
 		filebench_log(LOG_ERROR, "flowop %s failed to open file %s",
 		    flowop->fo_name, file->fse_path);
 		return (FILEBENCH_ERROR);
@@ -1948,7 +1597,7 @@ flowoplib_createfile(threadflow_t *threadflow, flowop_t *flowop)
 	int fd = flowop->fo_fdnumber;
 	int err;
 
-	if (threadflow->tf_fd[fd] != 0) {
+	if (threadflow->tf_fd[fd].fd_ptr != NULL) {
 		filebench_log(LOG_ERROR,
 		    "flowop %s attempted to create without closing on fd %d",
 		    flowop->fo_name, fd);
@@ -1982,11 +1631,11 @@ flowoplib_createfile(threadflow_t *threadflow, flowop_t *flowop)
 	threadflow->tf_fse[fd] = file;
 
 	flowop_beginop(threadflow, flowop);
-	threadflow->tf_fd[fd] = fileset_openfile(flowop->fo_fileset,
+	err = fileset_openfile(&threadflow->tf_fd[fd], flowop->fo_fileset,
 	    file, O_RDWR | O_CREAT, 0666, flowoplib_fileattrs(flowop));
 	flowop_endop(threadflow, flowop, 0);
 
-	if (threadflow->tf_fd[fd] < 0) {
+	if (err == FILEBENCH_ERROR) {
 		filebench_log(LOG_ERROR, "failed to create file %s",
 		    flowop->fo_name);
 		return (FILEBENCH_ERROR);
@@ -2074,7 +1723,7 @@ flowoplib_deletefile(threadflow_t *threadflow, flowop_t *flowop)
 	}
 
 	/* don't delete if anyone (other than me) has file open */
-	if ((fd > 0) && (threadflow->tf_fd[fd] > 0)) {
+	if ((fd > 0) && (threadflow->tf_fd[fd].fd_num > 0)) {
 		if (file->fse_open_cnt > 1) {
 			filebench_log(LOG_DEBUG_SCRIPT,
 			    "flowop %s can't delete file opened by other"
@@ -2104,7 +1753,7 @@ flowoplib_deletefile(threadflow_t *threadflow, flowop_t *flowop)
 
 	/* delete the selected file */
 	flowop_beginop(threadflow, flowop);
-	(void) unlink(path);
+	(void) FB_UNLINK(path);
 	flowop_endop(threadflow, flowop, 0);
 
 	/* indicate that it is no longer busy and no longer exists */
@@ -2128,7 +1777,7 @@ flowoplib_fsync(threadflow_t *threadflow, flowop_t *flowop)
 	filesetentry_t *file;
 	int fd = flowop->fo_fdnumber;
 
-	if (threadflow->tf_fd[fd] == 0) {
+	if (threadflow->tf_fd[fd].fd_ptr == NULL) {
 		filebench_log(LOG_ERROR,
 		    "flowop %s attempted to fsync a closed fd %d",
 		    flowop->fo_name, fd);
@@ -2147,7 +1796,7 @@ flowoplib_fsync(threadflow_t *threadflow, flowop_t *flowop)
 
 	/* Measure time to fsync */
 	flowop_beginop(threadflow, flowop);
-	(void) fsync(threadflow->tf_fd[fd]);
+	(void) FB_FSYNC(&threadflow->tf_fd[fd]);
 	flowop_endop(threadflow, flowop, 0);
 
 	filebench_log(LOG_DEBUG_SCRIPT, "fsync file %s", file->fse_path);
@@ -2176,7 +1825,7 @@ flowoplib_fsyncset(threadflow_t *threadflow, flowop_t *flowop)
 
 		/* Measure time to fsync */
 		flowop_beginop(threadflow, flowop);
-		(void) fsync(threadflow->tf_fd[fd]);
+		(void) FB_FSYNC(&threadflow->tf_fd[fd]);
 		flowop_endop(threadflow, flowop, 0);
 
 		file = threadflow->tf_fse[fd];
@@ -2204,7 +1853,7 @@ flowoplib_closefile(threadflow_t *threadflow, flowop_t *flowop)
 	fileset_t *fileset;
 	int fd = flowop->fo_fdnumber;
 
-	if (threadflow->tf_fd[fd] == 0) {
+	if (threadflow->tf_fd[fd].fd_ptr == NULL) {
 		filebench_log(LOG_ERROR,
 		    "flowop %s attempted to close an already closed fd %d",
 		    flowop->fo_name, fd);
@@ -2233,12 +1882,12 @@ flowoplib_closefile(threadflow_t *threadflow, flowop_t *flowop)
 
 	/* Measure time to close */
 	flowop_beginop(threadflow, flowop);
-	(void) close(threadflow->tf_fd[fd]);
+	(void) FB_CLOSE(&threadflow->tf_fd[fd]);
 	flowop_endop(threadflow, flowop, 0);
 
 	fileset_unbusy(file, FALSE, FALSE, -1);
 
-	threadflow->tf_fd[fd] = 0;
+	threadflow->tf_fd[fd].fd_ptr = NULL;
 
 	filebench_log(LOG_DEBUG_SCRIPT, "closed file %s", file->fse_path);
 
@@ -2303,7 +1952,7 @@ flowoplib_makedir(threadflow_t *threadflow, flowop_t *flowop)
 		return (ret);
 
 	flowop_beginop(threadflow, flowop);
-	(void) mkdir(full_path, 0755);
+	(void) FB_MKDIR(full_path, 0755);
 	flowop_endop(threadflow, flowop, 0);
 
 	/* indicate that it is no longer busy and now exists */
@@ -2335,7 +1984,7 @@ flowoplib_removedir(threadflow_t *threadflow, flowop_t *flowop)
 		return (ret);
 
 	flowop_beginop(threadflow, flowop);
-	(void) rmdir(full_path);
+	(void) FB_RMDIR(full_path);
 	flowop_endop(threadflow, flowop, 0);
 
 	/* indicate that it is no longer busy and no longer exists */
@@ -2360,7 +2009,7 @@ flowoplib_listdir(threadflow_t *threadflow, flowop_t *flowop)
 {
 	fileset_t	*fileset;
 	filesetentry_t	*dir;
-	DIR		*dir_handlep;
+	DIR		*dir_handle;
 	struct dirent	*direntp;
 	int		dir_bytes = 0;
 	int		ret;
@@ -2385,7 +2034,7 @@ flowoplib_listdir(threadflow_t *threadflow, flowop_t *flowop)
 	flowop_beginop(threadflow, flowop);
 
 	/* open the directory */
-	if ((dir_handlep = opendir(full_path)) == NULL) {
+	if ((dir_handle = FB_OPENDIR(full_path)) == NULL) {
 		filebench_log(LOG_ERROR,
 		    "flowop %s failed to open directory in fileset %s\n",
 		    flowop->fo_name, avd_get_str(fileset->fs_name));
@@ -2393,13 +2042,13 @@ flowoplib_listdir(threadflow_t *threadflow, flowop_t *flowop)
 	}
 
 	/* read through the directory entries */
-	while ((direntp = readdir(dir_handlep)) != NULL) {
+	while ((direntp = FB_READDIR(dir_handle)) != NULL) {
 		dir_bytes += (strlen(direntp->d_name) +
 		    sizeof (struct dirent) - 1);
 	}
 
 	/* close the directory */
-	(void) closedir(dir_handlep);
+	(void) FB_CLOSEDIR(dir_handle);
 
 	flowop_endop(threadflow, flowop, dir_bytes);
 
@@ -2421,11 +2070,11 @@ flowoplib_statfile(threadflow_t *threadflow, flowop_t *flowop)
 {
 	filesetentry_t *file;
 	fileset_t *fileset;
-	struct stat statbuf;
+	struct stat64 statbuf;
 	int fd = flowop->fo_fdnumber;
 
 	/* if fd specified and the file is open, use it to access file */
-	if ((fd > 0) && ((threadflow->tf_fd[fd]) > 0)) {
+	if ((fd > 0) && (threadflow->tf_fd[fd].fd_num > 0)) {
 
 		/* check whether file handle still valid */
 		if ((file = threadflow->tf_fse[fd]) == NULL) {
@@ -2485,7 +2134,7 @@ flowoplib_statfile(threadflow_t *threadflow, flowop_t *flowop)
 
 		/* stat the file */
 		flowop_beginop(threadflow, flowop);
-		if (stat(path, &statbuf) == -1)
+		if (FB_STAT(path, &statbuf) == -1)
 			filebench_log(LOG_ERROR,
 			    "statfile flowop %s failed", flowop->fo_name);
 		flowop_endop(threadflow, flowop, 0);
@@ -2494,7 +2143,7 @@ flowoplib_statfile(threadflow_t *threadflow, flowop_t *flowop)
 	} else {
 		/* stat specific file */
 		flowop_beginop(threadflow, flowop);
-		if (fstat(threadflow->tf_fd[fd], &statbuf) == -1)
+		if (FB_FSTAT(&threadflow->tf_fd[fd], &statbuf) == -1)
 			filebench_log(LOG_ERROR,
 			    "statfile flowop %s failed", flowop->fo_name);
 		flowop_endop(threadflow, flowop, 0);
@@ -2528,7 +2177,7 @@ flowoplib_readwholefile(threadflow_t *threadflow, flowop_t *flowop)
 {
 	caddr_t iobuf;
 	off64_t bytes = 0;
-	int filedesc;
+	fb_fdesc_t *fdesc;
 	uint64_t wss;
 	fbint_t iosize;
 	int ret;
@@ -2536,7 +2185,7 @@ flowoplib_readwholefile(threadflow_t *threadflow, flowop_t *flowop)
 
 	/* get the file to use */
 	if ((ret = flowoplib_filesetup(threadflow, flowop, &wss,
-	    &filedesc)) != FILEBENCH_OK)
+	    &fdesc)) != FILEBENCH_OK)
 		return (ret);
 
 	/* an I/O size of zero means read entire working set with one I/O */
@@ -2560,8 +2209,8 @@ flowoplib_readwholefile(threadflow_t *threadflow, flowop_t *flowop)
 
 	/* Measure time to read bytes */
 	flowop_beginop(threadflow, flowop);
-	(void) lseek64(filedesc, 0, SEEK_SET);
-	while ((ret = read(filedesc, iobuf, iosize)) > 0)
+	(void) FB_LSEEK(fdesc, 0, SEEK_SET);
+	while ((ret = FB_READ(fdesc, iobuf, iosize)) > 0)
 		bytes += ret;
 
 	flowop_endop(threadflow, flowop, bytes);
@@ -2597,12 +2246,12 @@ flowoplib_write(threadflow_t *threadflow, flowop_t *flowop)
 	caddr_t iobuf;
 	fbint_t wss;
 	fbint_t iosize;
-	int filedesc;
+	fb_fdesc_t *fdesc;
 	int ret;
 
 	iosize = avd_get_int(flowop->fo_iosize);
 	if ((ret = flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
-	    &filedesc, iosize)) != FILEBENCH_OK)
+	    &fdesc, iosize)) != FILEBENCH_OK)
 		return (ret);
 
 	if (avd_get_bool(flowop->fo_random)) {
@@ -2616,7 +2265,7 @@ flowoplib_write(threadflow_t *threadflow, flowop_t *flowop)
 			return (FILEBENCH_ERROR);
 		}
 		flowop_beginop(threadflow, flowop);
-		if (pwrite64(filedesc, iobuf,
+		if (FB_PWRITE(fdesc, iobuf,
 		    iosize, (off64_t)fileoffset) == -1) {
 			filebench_log(LOG_ERROR, "write failed, "
 			    "offset %llu io buffer %zd: %s",
@@ -2627,7 +2276,7 @@ flowoplib_write(threadflow_t *threadflow, flowop_t *flowop)
 		flowop_endop(threadflow, flowop, iosize);
 	} else {
 		flowop_beginop(threadflow, flowop);
-		if (write(filedesc, iobuf, iosize) == -1) {
+		if (FB_WRITE(fdesc, iobuf, iosize) == -1) {
 			filebench_log(LOG_ERROR,
 			    "write failed, io buffer %zd: %s",
 			    iobuf, strerror(errno));
@@ -2658,14 +2307,14 @@ flowoplib_writewholefile(threadflow_t *threadflow, flowop_t *flowop)
 	off64_t bytes = 0;
 	uint64_t wss;
 	fbint_t iosize;
-	int filedesc;
+	fb_fdesc_t *fdesc;
 	int srcfd = flowop->fo_srcfdnumber;
 	int ret;
 	char zerowrtbuf;
 
 	/* get the file to use */
 	if ((ret = flowoplib_filesetup(threadflow, flowop, &wss,
-	    &filedesc)) != FILEBENCH_OK)
+	    &fdesc)) != FILEBENCH_OK)
 		return (ret);
 
 	/* an I/O size of zero means write entire working set with one I/O */
@@ -2702,11 +2351,11 @@ flowoplib_writewholefile(threadflow_t *threadflow, flowop_t *flowop)
 	/* Measure time to write bytes */
 	flowop_beginop(threadflow, flowop);
 	for (seek = 0; seek < wss; seek += wsize) {
-		ret = write(filedesc, iobuf, wsize);
+		ret = FB_WRITE(fdesc, iobuf, wsize);
 		if (ret != wsize) {
 			filebench_log(LOG_ERROR,
 			    "Failed to write %d bytes on fd %d: %s",
-			    wsize, filedesc, strerror(errno));
+			    wsize, fdesc->fd_num, strerror(errno));
 			flowop_endop(threadflow, flowop, 0);
 			return (FILEBENCH_ERROR);
 		}
@@ -2739,26 +2388,26 @@ static int
 flowoplib_appendfile(threadflow_t *threadflow, flowop_t *flowop)
 {
 	caddr_t iobuf;
-	int filedesc;
+	fb_fdesc_t *fdesc;
 	fbint_t wss;
 	fbint_t iosize;
 	int ret;
 
 	iosize = avd_get_int(flowop->fo_iosize);
 	if ((ret = flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
-	    &filedesc, iosize)) != FILEBENCH_OK)
+	    &fdesc, iosize)) != FILEBENCH_OK)
 		return (ret);
 
 	/* XXX wss is not being used */
 
 	/* Measure time to write bytes */
 	flowop_beginop(threadflow, flowop);
-	(void) lseek64(filedesc, 0, SEEK_END);
-	ret = write(filedesc, iobuf, iosize);
+	(void) FB_LSEEK(fdesc, 0, SEEK_END);
+	ret = FB_WRITE(fdesc, iobuf, iosize);
 	if (ret != iosize) {
 		filebench_log(LOG_ERROR,
 		    "Failed to write %llu bytes on fd %d: %s",
-		    (u_longlong_t)iosize, filedesc, strerror(errno));
+		    (u_longlong_t)iosize, fdesc->fd_num, strerror(errno));
 		flowop_endop(threadflow, flowop, ret);
 		return (FILEBENCH_ERROR);
 	}
@@ -2788,7 +2437,7 @@ flowoplib_appendfilerand(threadflow_t *threadflow, flowop_t *flowop)
 {
 	caddr_t iobuf;
 	uint64_t appendsize;
-	int filedesc;
+	fb_fdesc_t *fdesc;
 	fbint_t wss;
 	fbint_t iosize;
 	int ret = 0;
@@ -2810,7 +2459,7 @@ flowoplib_appendfilerand(threadflow_t *threadflow, flowop_t *flowop)
 	}
 
 	if ((ret = flowoplib_iosetup(threadflow, flowop, &wss, &iobuf,
-	    &filedesc, appendsize)) != FILEBENCH_OK)
+	    &fdesc, appendsize)) != FILEBENCH_OK)
 		return (ret);
 
 	/* XXX wss is not being used */
@@ -2818,12 +2467,12 @@ flowoplib_appendfilerand(threadflow_t *threadflow, flowop_t *flowop)
 	/* Measure time to write bytes */
 	flowop_beginop(threadflow, flowop);
 
-	(void) lseek64(filedesc, 0, SEEK_END);
-	ret = write(filedesc, iobuf, appendsize);
+	(void) FB_LSEEK(fdesc, 0, SEEK_END);
+	ret = FB_WRITE(fdesc, iobuf, appendsize);
 	if (ret != appendsize) {
 		filebench_log(LOG_ERROR,
 		    "Failed to write %llu bytes on fd %d: %s",
-		    (u_longlong_t)appendsize, filedesc, strerror(errno));
+		    (u_longlong_t)appendsize, fdesc->fd_num, strerror(errno));
 		flowop_endop(threadflow, flowop, 0);
 		return (FILEBENCH_ERROR);
 	}
diff --git a/usr/src/cmd/filebench/common/fsplug.h b/usr/src/cmd/filebench/common/fsplug.h
new file mode 100644
index 0000000000..4158f9f33a
--- /dev/null
+++ b/usr/src/cmd/filebench/common/fsplug.h
@@ -0,0 +1,144 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ *
+ */
+
+
+#ifndef _FSPLUG_H
+#define	_FSPLUG_H
+
+#include <dirent.h>
+#include "config.h"
+#include <sys/stat.h>
+
+/*
+ * Type of file system client plug-in desired.
+ */
+typedef enum fb_plugin_type {
+	LOCAL_FS_PLUG = 0,
+	NFS3_PLUG,
+	NFS4_PLUG,
+	CIFS_PLUG
+} fb_plugin_type_t;
+
+/* universal file descriptor for both local and nfs file systems */
+typedef union fb_fdesc {
+	int		fd_num;		/* OS file descriptor number */
+	void		*fd_ptr;	/* Pointer to nfs information block */
+} fb_fdesc_t;
+
+typedef struct aiolist aiol_t;
+
+/* Functions vector for file system plug-ins */
+typedef struct fsplug_func_s {
+	char fs_name[16];
+	int (*fsp_freemem)(fb_fdesc_t *, off64_t);
+	int (*fsp_open)(fb_fdesc_t *, char *, int, int);
+	int (*fsp_pread)(fb_fdesc_t *, caddr_t, fbint_t, off64_t);
+	int (*fsp_read)(fb_fdesc_t *, caddr_t, fbint_t);
+	int (*fsp_pwrite)(fb_fdesc_t *, caddr_t, fbint_t, off64_t);
+	int (*fsp_write)(fb_fdesc_t *, caddr_t, fbint_t);
+	int (*fsp_lseek)(fb_fdesc_t *, off64_t, int);
+	int (*fsp_ftrunc)(fb_fdesc_t *, off64_t);
+	int (*fsp_rename)(const char *, const char *);
+	int (*fsp_close)(fb_fdesc_t *);
+	int (*fsp_link)(const char *, const char *);
+	int (*fsp_symlink)(const char *, const char *);
+	int (*fsp_unlink)(char *);
+	ssize_t (*fsp_readlink)(const char *, char *, size_t);
+	int (*fsp_mkdir)(char *, int);
+	int (*fsp_rmdir)(char *);
+	DIR *(*fsp_opendir)(char *);
+	struct dirent *(*fsp_readdir)(DIR *);
+	int (*fsp_closedir)(DIR *);
+	int (*fsp_fsync)(fb_fdesc_t *);
+	int (*fsp_stat)(char *, struct stat64 *);
+	int (*fsp_fstat)(fb_fdesc_t *, struct stat64 *);
+	int (*fsp_access)(const char *, int);
+} fsplug_func_t;
+
+extern fsplug_func_t *fs_functions_vec;
+
+/* Macros for calling functions */
+#define	FB_FREEMEM(fd, sz) \
+	(*fs_functions_vec->fsp_freemem)(fd, sz)
+
+#define	FB_OPEN(fd, path, flags, perms) \
+	(*fs_functions_vec->fsp_open)(fd, path, flags, perms)
+
+#define	FB_PREAD(fdesc, iobuf, iosize, offset) \
+	(*fs_functions_vec->fsp_pread)(fdesc, iobuf, iosize, offset)
+
+#define	FB_READ(fdesc, iobuf, iosize) \
+	(*fs_functions_vec->fsp_read)(fdesc, iobuf, iosize)
+
+#define	FB_PWRITE(fdesc, iobuf, iosize, offset) \
+	(*fs_functions_vec->fsp_pwrite)(fdesc, iobuf, iosize, offset)
+
+#define	FB_WRITE(fdesc, iobuf, iosize) \
+	(*fs_functions_vec->fsp_write)(fdesc, iobuf, iosize)
+
+#define	FB_LSEEK(fdesc, amnt, whence) \
+	(*fs_functions_vec->fsp_lseek)(fdesc, amnt, whence)
+
+#define	FB_CLOSE(fdesc) \
+	(*fs_functions_vec->fsp_close)(fdesc)
+
+#define	FB_UNLINK(path) \
+	(*fs_functions_vec->fsp_unlink)(path)
+
+#define	FB_MKDIR(path, perm) \
+	(*fs_functions_vec->fsp_mkdir)(path, perm)
+
+#define	FB_RMDIR(path) \
+	(*fs_functions_vec->fsp_rmdir)(path)
+
+#define	FB_OPENDIR(path) \
+	(*fs_functions_vec->fsp_opendir)(path)
+
+#define	FB_READDIR(dir) \
+	(*fs_functions_vec->fsp_readdir)(dir)
+
+#define	FB_CLOSEDIR(dir) \
+	(*fs_functions_vec->fsp_closedir)(dir)
+
+#define	FB_FSYNC(fdesc) \
+	(*fs_functions_vec->fsp_fsync)(fdesc)
+
+#define	FB_STAT(path, statp) \
+	(*fs_functions_vec->fsp_stat)(path, statp)
+
+#define	FB_FSTAT(fdesc, statp) \
+	(*fs_functions_vec->fsp_fstat)(fdesc, statp)
+
+#define	FB_FTRUNC(fdesc, size) \
+	(*fs_functions_vec->fsp_ftrunc)(fdesc, size)
+
+#define	FB_LINK(existing, new) \
+	(*fs_functions_vec->fsp_link)(existing, new)
+
+#define	FB_SYMLINK(name1, name2) \
+	(*fs_functions_vec->fsp_symlink)(name1, name2)
+
+#endif /* _FSPLUG_H */
diff --git a/usr/src/cmd/filebench/common/ipc.h b/usr/src/cmd/filebench/common/ipc.h
index 8416a32ca7..cc3372cf9c 100644
--- a/usr/src/cmd/filebench/common/ipc.h
+++ b/usr/src/cmd/filebench/common/ipc.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -34,6 +34,7 @@
 #include "fileset.h"
 #include "flowop.h"
 #include "fb_random.h"
+#include "fsplug.h"
 #include "filebench.h"
 
 #ifdef	__cplusplus
@@ -186,7 +187,16 @@ typedef struct filebench_shm {
 	caddr_t		shm_addr;
 	char		*shm_ptr;
 
-	int		shm_marker;	/* end of pre-zeroed data */
+	/*
+	 * Type of plug-in file system client to use. Defaults to
+	 * local file system, which is type "0".
+	 */
+	fb_plugin_type_t shm_filesys_type;
+
+	/*
+	 * end of pre-zeroed data
+	 */
+	int		shm_marker;
 
 	/*
 	 * actual storage for shared entities.
diff --git a/usr/src/cmd/filebench/common/misc.c b/usr/src/cmd/filebench/common/misc.c
index 553d28bcee..6f2eb4db2f 100644
--- a/usr/src/cmd/filebench/common/misc.c
+++ b/usr/src/cmd/filebench/common/misc.c
@@ -19,12 +19,10 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <stdio.h>
 #include <fcntl.h>
 #include <limits.h>
@@ -36,6 +34,10 @@
 #include "ipc.h"
 #include "eventgen.h"
 #include "utils.h"
+#include "fsplug.h"
+
+/* File System functions vector */
+fsplug_func_t *fs_functions_vec;
 
 /*
  * Routines to access high resolution system time, initialize and
@@ -43,7 +45,6 @@
  * access system information strings.
  */
 
-
 #if !defined(sun) && defined(USE_RDTSC)
 /*
  * Lets us use the rdtsc instruction to get highres time.
@@ -159,7 +160,6 @@ filebench_init(void)
 		filebench_shutdown(1);
 	}
 #endif /* USE_RDTSC */
-
 }
 
 extern int lex_lineno;
@@ -454,3 +454,30 @@ script_var(var_t *var)
 
 	return (var);
 }
+
+void fb_lfs_funcvecinit(void);
+
+/*
+ * Initialize any "plug-in" I/O function vectors. Called by each
+ * filebench process that is forked, as the vector is relative to
+ * its image.
+ */
+void
+filebench_plugin_funcvecinit(void)
+{
+
+	switch (filebench_shm->shm_filesys_type) {
+	case LOCAL_FS_PLUG:
+		fb_lfs_funcvecinit();
+		break;
+
+	case NFS3_PLUG:
+	case NFS4_PLUG:
+	case CIFS_PLUG:
+		break;
+	default:
+		filebench_log(LOG_ERROR,
+		    "filebench_plugin_funcvecinit: unknown file system");
+		break;
+	}
+}
diff --git a/usr/src/cmd/filebench/common/parser_gram.y b/usr/src/cmd/filebench/common/parser_gram.y
index e3a3594c5d..bfce53b9c8 100644
--- a/usr/src/cmd/filebench/common/parser_gram.y
+++ b/usr/src/cmd/filebench/common/parser_gram.y
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
  * Portions Copyright 2008 Denis Cheng
@@ -1786,6 +1786,9 @@ main(int argc, char *argv[])
 			exit(1);
 		}
 
+		/* get correct function pointer for each child process */
+		filebench_plugin_funcvecinit();
+
 		if (procflow_exec(procname, instance) < 0) {
 			filebench_log(LOG_ERROR, "Cannot startup process %s",
 			    procname);
@@ -1802,6 +1805,7 @@ main(int argc, char *argv[])
 	if (fscriptname)
 		(void) strcpy(filebench_shm->shm_fscriptname, fscriptname);
 
+	filebench_plugin_funcvecinit();
 	flowop_init();
 	stats_init();
 	eventgen_init();
@@ -2600,6 +2604,12 @@ parser_fileset_define_common(cmd_t *cmd)
 	attr_t *attr;
 	avd_t pathname;
 
+	/*
+	 * Make sure all plugin flowops are initialized.
+	 * Defaults to local fs for now
+	 */
+	flowop_plugin_flowinit();
+
 	/* Get the name of the file */
 	if (attr = get_attr_fileset(cmd, FSA_NAME)) {
 		name = attr->attr_avd;
diff --git a/usr/src/cmd/filebench/common/procflow.c b/usr/src/cmd/filebench/common/procflow.c
index 4fe5495363..9d124912cc 100644
--- a/usr/src/cmd/filebench/common/procflow.c
+++ b/usr/src/cmd/filebench/common/procflow.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
  * Portions Copyright 2008 Denis Cheng
@@ -628,7 +628,7 @@ procflow_allstarted()
 void
 procflow_shutdown(void)
 {
-	procflow_t *procflow = filebench_shm->shm_proclist;
+	procflow_t *procflow;
 	int wait_cnt;
 
 	(void) ipc_mutex_lock(&filebench_shm->shm_procs_running_lock);
@@ -640,6 +640,7 @@ procflow_shutdown(void)
 	(void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
 
 	(void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
+	procflow = filebench_shm->shm_proclist;
 	filebench_shm->shm_f_abort = 1;
 	wait_cnt = SHUTDOWN_WAIT_SECONDS;
 
diff --git a/usr/src/cmd/filebench/common/threadflow.c b/usr/src/cmd/filebench/common/threadflow.c
index 0d903cb367..f723cc23af 100644
--- a/usr/src/cmd/filebench/common/threadflow.c
+++ b/usr/src/cmd/filebench/common/threadflow.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
  * Portions Copyright 2008 Denis Cheng
@@ -314,10 +314,11 @@ threadflow_delete(threadflow_t **threadlist, threadflow_t *threadflow,
 void
 threadflow_delete_all(threadflow_t **threadlist, int wait_cnt)
 {
-	threadflow_t *threadflow = *threadlist;
+	threadflow_t *threadflow;
 
 	(void) ipc_mutex_lock(&filebench_shm->shm_threadflow_lock);
 
+	threadflow = *threadlist;
 	filebench_log(LOG_DEBUG_IMPL, "Deleting all threads");
 
 	while (threadflow) {
diff --git a/usr/src/cmd/filebench/common/threadflow.h b/usr/src/cmd/filebench/common/threadflow.h
index 125280e816..ee08afbc2f 100644
--- a/usr/src/cmd/filebench/common/threadflow.h
+++ b/usr/src/cmd/filebench/common/threadflow.h
@@ -19,16 +19,15 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef _FB_THREADFLOW_H
 #define	_FB_THREADFLOW_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include "config.h"
+#include "fsplug.h"
 #include <pthread.h>
 
 #ifndef HAVE_CADDR_T1
@@ -86,7 +85,7 @@ typedef struct threadflow {
 	caddr_t		tf_mem;		/* Private Memory */
 	avd_t		tf_memsize;	/* Private Memory size attribute */
 	fbint_t		tf_constmemsize; /* constant copy of memory size */
-	int		tf_fd[THREADFLOW_MAXFD + 1]; /* Thread local fd's */
+	fb_fdesc_t	tf_fd[THREADFLOW_MAXFD + 1]; /* Thread local fd's */
 	filesetentry_t	*tf_fse[THREADFLOW_MAXFD + 1]; /* Thread local files */
 	int		tf_fdrotor;	/* Rotating fd within set */
 	flowstat_t	tf_stats;	/* Thread statistics */
diff --git a/usr/src/cmd/filebench/config/Makefile b/usr/src/cmd/filebench/config/Makefile
index ef6541041e..fe8e43b211 100644
--- a/usr/src/cmd/filebench/config/Makefile
+++ b/usr/src/cmd/filebench/config/Makefile
@@ -19,7 +19,7 @@
 # CDDL HEADER END
 #
 #
-# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
 
@@ -28,7 +28,8 @@
 include ../../Makefile.cmd
 
 CONFIGS = fileio.prof filemacro.prof filemicro.prof generic.func \
-seqread.prof randomread.prof multi_fileserver.prof videoserver.prof
+seqread.prof randomread.prof multi_fileserver.prof newfeatures.prof \
+videoserver.prof
 ROOTUSRBENCHDIR = $(ROOT)/usr/benchmarks
 ROOTUSRBENCHFBCONFIGDIR = $(ROOTUSRBENCHDIR)/filebench/config
 FBCONFIGS = $(CONFIGS:%=$(ROOTUSRBENCHFBCONFIGDIR)/%)
diff --git a/usr/src/cmd/filebench/config/newfeatures.prof b/usr/src/cmd/filebench/config/newfeatures.prof
new file mode 100644
index 0000000000..6040fc92db
--- /dev/null
+++ b/usr/src/cmd/filebench/config/newfeatures.prof
@@ -0,0 +1,86 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+# Excersizes the latest features of FileBench by running their
+# respective test workloads.
+
+DEFAULTS {
+        runtime = 120;
+        dir = /tmp;
+        stats = /tmp;
+        filesystem = tmpfs;
+        description = "newfeatures tmpfs";
+}
+
+CONFIG seqwrite_randvar_gamma {
+        function = generic;
+        personality = filemicro_seqwriterandvargam;
+}
+
+CONFIG seqwrite_randvar_tabular {
+        function = generic;
+        personality = filemicro_seqwriterandvartab;
+}
+
+CONFIG list_dirs_test {
+	function = generic;
+	personality = listdirs;
+}
+
+CONFIG make_dirs_test {
+	function = generic;
+	personality = makedirs;
+}
+
+CONFIG remove_dirs_test {
+	function = generic;
+	personality = removedirs;
+}
+
+CONFIG open_files_test {
+	function = generic;
+	personality = openfiles;
+}
+
+CONFIG video_server_test {
+	function = generic;
+	personality = videoserver;
+	filesize = 2g;
+	numactivevids = 4;
+	numpassivevids = 20;
+	nthreads = 6;
+	srvbwrate = 12;
+	repintval = 20;
+	passvidsname = bigfileset;
+	actvidsname = u2fileset;
+}
+
+CONFIG file_indexing_test {
+        function = generic;
+        personality = randomfileaccess;
+}
+
+CONFIG composite_flowop_test {
+        function = generic;
+        personality = compflow_demo;
+}
diff --git a/usr/src/pkgdefs/SUNWfilebench/prototype_com b/usr/src/pkgdefs/SUNWfilebench/prototype_com
index 2cb8680c29..553ff7e1ea 100644
--- a/usr/src/pkgdefs/SUNWfilebench/prototype_com
+++ b/usr/src/pkgdefs/SUNWfilebench/prototype_com
@@ -20,7 +20,7 @@
 #
 
 #
-# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
 
@@ -44,6 +44,7 @@ f none usr/benchmarks/filebench/config/filemacro.prof 444 root bin
 f none usr/benchmarks/filebench/config/filemicro.prof 444 root bin
 f none usr/benchmarks/filebench/config/generic.func 444 root bin
 f none usr/benchmarks/filebench/config/multi_fileserver.prof 444 root bin
+f none usr/benchmarks/filebench/config/newfeatures.prof 444 root bin
 f none usr/benchmarks/filebench/config/seqread.prof 444 root bin
 f none usr/benchmarks/filebench/config/randomread.prof 444 root bin
 f none usr/benchmarks/filebench/config/videoserver.prof 444 root bin