6680475 Request for per flowop byte limit

6698807 Filebench Shutdown still takes too long with multiple threads
6702350 Filebench Hangs while shutting down on x86 when running oltp.f

19 files changed, 420 insertions, 62 deletions

diff --git a/usr/src/cmd/filebench/common/filebench.h b/usr/src/cmd/filebench/common/filebench.h
index f6c3e63b34..512d713d6c 100644
--- a/usr/src/cmd/filebench/common/filebench.h
+++ b/usr/src/cmd/filebench/common/filebench.h
@@ -119,7 +119,7 @@ void filebench_shutdown(int error);
 #define	MIN(x, y) ((x) < (y) ? (x) : (y))
 #endif
 
-#define	FILEBENCH_VERSION	"1.3.1"
+#define	FILEBENCH_VERSION	"1.3.2"
 #define	FILEBENCHDIR	"/usr/benchmarks/filebench"
 #define	FILEBENCH_PROMPT	"filebench> "
 #define	MAX_LINE_LEN	1024
diff --git a/usr/src/cmd/filebench/common/fileset.c b/usr/src/cmd/filebench/common/fileset.c
index 57d97ad2e6..d402c49431 100644
--- a/usr/src/cmd/filebench/common/fileset.c
+++ b/usr/src/cmd/filebench/common/fileset.c
@@ -47,6 +47,13 @@
  * is rooted in a directory specified by fileset_path, and once the populated
  * fileset has been created, has a tree of directories and files
  * corresponding to the fileset's filesetentry tree.
+ *
+ * This routine is called from fileset_createset(), which is in turn
+ * called from parser_gram.y: parser_create_fileset() when a
+ * "create fileset" or "run" command is encountered.
+ * When the "create fileset" command is used, it is generally paired with
+ * a "create processes" command, and must appear first, in order to
+ * instantiate all the files in the fileset before trying to use them.
  */
 
 static int fileset_checkraw(fileset_t *fileset);
@@ -298,6 +305,12 @@ fileset_alloc_file(filesetentry_t *entry)
 				    entry->fse_size);
 
 			entry->fse_flags |= FSE_EXISTS;
+			(void) ipc_mutex_lock(
+			    &entry->fse_fileset->fs_num_files_lock);
+			entry->fse_fileset->fs_num_act_files++;
+			(void) ipc_mutex_unlock(
+			    &entry->fse_fileset->fs_num_files_lock);
+
 			(void) close(fd);
 			return (0);
 
@@ -317,6 +330,13 @@ fileset_alloc_file(filesetentry_t *entry)
 				    entry->fse_size);
 
 			entry->fse_flags |= FSE_EXISTS;
+
+			(void) ipc_mutex_lock(
+			    &entry->fse_fileset->fs_num_files_lock);
+			entry->fse_fileset->fs_num_act_files++;
+			(void) ipc_mutex_unlock(
+			    &entry->fse_fileset->fs_num_files_lock);
+
 			(void) close(fd);
 			return (0);
 		}
@@ -337,6 +357,10 @@ fileset_alloc_file(filesetentry_t *entry)
 
 	entry->fse_flags |= FSE_EXISTS;
 
+	(void) ipc_mutex_lock(&entry->fse_fileset->fs_num_files_lock);
+	entry->fse_fileset->fs_num_act_files++;
+	(void) ipc_mutex_unlock(&entry->fse_fileset->fs_num_files_lock);
+
 	for (seek = 0; seek < entry->fse_size; ) {
 		off64_t wsize;
 		int ret = 0;
@@ -431,9 +455,14 @@ fileset_openfile(fileset_t *fileset,
 			return (-1);
 	}
 
-	if (flag & O_CREAT)
+	if (flag & O_CREAT) {
 		entry->fse_flags |= FSE_EXISTS;
 
+		(void) ipc_mutex_lock(&fileset->fs_num_files_lock);
+		fileset->fs_num_act_files++;
+		(void) ipc_mutex_unlock(&fileset->fs_num_files_lock);
+	}
+
 	if (attrs & FLOW_ATTR_DSYNC) {
 #ifdef sun
 		open_attrs |= O_DSYNC;
@@ -486,6 +515,25 @@ fileset_pick(fileset_t *fileset, int flags, int tid)
 
 	(void) ipc_mutex_lock(&filebench_shm->shm_fileset_lock);
 
+	/* see if asking for impossible */
+	(void) ipc_mutex_lock(&fileset->fs_num_files_lock);
+	if (flags & FILESET_PICKEXISTS) {
+		if (fileset->fs_num_act_files == 0) {
+			(void) ipc_mutex_unlock(&fileset->fs_num_files_lock);
+			(void) ipc_mutex_unlock(
+			    &filebench_shm->shm_fileset_lock);
+			return (NULL);
+		}
+	} else if (flags & FILESET_PICKNOEXIST) {
+		if (fileset->fs_num_act_files == fileset->fs_realfiles) {
+			(void) ipc_mutex_unlock(&fileset->fs_num_files_lock);
+			(void) ipc_mutex_unlock(
+			    &filebench_shm->shm_fileset_lock);
+			return (NULL);
+		}
+	}
+	(void) ipc_mutex_unlock(&fileset->fs_num_files_lock);
+
 	while (entry == NULL) {
 
 		if ((flags & FILESET_PICKDIR) && (flags & FILESET_PICKRESET)) {
@@ -609,6 +657,9 @@ fileset_create(fileset_t *fileset)
 		return (-1);
 	}
 
+	/* declare all files currently non existant (single threaded code) */
+	fileset->fs_num_act_files = 0;
+
 #ifdef HAVE_RAW_SUPPORT
 	/* treat raw device as special case */
 	if (fileset->fs_attrs & FILESET_IS_RAW_DEV)
@@ -1002,6 +1053,7 @@ exists:
 		    fileset->fs_meandepth,
 		    (u_longlong_t)(fileset->fs_bytes / 1024UL / 1024UL));
 	}
+
 	return (0);
 }
 
@@ -1010,6 +1062,10 @@ exists:
  * fileset_sizegamma default values, and sets the fileset name to the
  * supplied name string. Puts the allocated fileset on the
  * master fileset list and returns a pointer to it.
+ *
+ * This routine implements the 'define fileset' calls found in a .f
+ * workload, such as in the following example:
+ * define fileset name=drew4ever, entries=$nfiles
  */
 fileset_t *
 fileset_define(avd_t name)
@@ -1028,6 +1084,10 @@ fileset_define(avd_t name)
 	filebench_log(LOG_DEBUG_IMPL,
 	    "Defining file %s", avd_get_str(name));
 
+	/* initialize fs_num_act_files lock */
+	(void) pthread_mutex_init(&fileset->fs_num_files_lock,
+	    ipc_mutexattr());
+
 	(void) ipc_mutex_lock(&filebench_shm->shm_fileset_lock);
 
 	fileset->fs_dirgamma = avd_int_alloc(1500);
diff --git a/usr/src/cmd/filebench/common/fileset.h b/usr/src/cmd/filebench/common/fileset.h
index 3ad69a61c9..c20bf28bf5 100644
--- a/usr/src/cmd/filebench/common/fileset.h
+++ b/usr/src/cmd/filebench/common/fileset.h
@@ -118,6 +118,10 @@ typedef struct fileset {
 	double		fs_meansize;	/* Specified mean file size */
 	int		fs_realfiles;	/* Actual files */
 	off64_t		fs_bytes;	/* Total space consumed by files */
+	fbint_t		fs_num_act_files;   /* total number of files */
+					    /* actually existing in the */
+					    /* host or server's file system */
+	pthread_mutex_t	fs_num_files_lock; /* lock for fs_num_act_files */
 	filesetentry_t	*fs_filelist;	/* List of files */
 	filesetentry_t	*fs_dirlist;	/* List of directories */
 	filesetentry_t	*fs_filefree;	/* Ptr to next free file */
diff --git a/usr/src/cmd/filebench/common/flowop.c b/usr/src/cmd/filebench/common/flowop.c
index 4aa46166c3..b75222cb23 100644
--- a/usr/src/cmd/filebench/common/flowop.c
+++ b/usr/src/cmd/filebench/common/flowop.c
@@ -324,6 +324,9 @@ flowop_destruct_all_flows(threadflow_t *threadflow)
 {
 	flowop_t *flowop;
 
+	/* wait a moment to give other threads a chance to stop too */
+	(void) sleep(1);
+
 	(void) ipc_mutex_lock(&threadflow->tf_lock);
 
 	/* prepare to call destruct flow routines, if necessary */
@@ -459,7 +462,7 @@ flowop_start(threadflow_t *threadflow)
 		}
 
 		/* Take it easy until everyone is ready to go */
-		if (!filebench_shm->shm_running) {
+		if (!filebench_shm->shm_procs_running) {
 			(void) sleep(1);
 			continue;
 		}
@@ -896,7 +899,7 @@ flowop_find(char *name)
 
 /*
  * Returns a pointer to the specified instance of flowop
- * "name" from the supplied list.
+ * "name" from the global list.
  */
 flowop_t *
 flowop_find_one(char *name, int instance)
@@ -923,6 +926,86 @@ flowop_find_one(char *name, int instance)
 }
 
 /*
+ * recursively searches through lists of flowops on a given thread
+ * and those on any included composite flowops for the named flowop.
+ * either returns with a pointer to the named flowop or NULL if it
+ * cannot be found.
+ */
+static flowop_t *
+flowop_recurse_search(char *path, char *name, flowop_t *list)
+{
+	flowop_t *test_flowop;
+	char fullname[MAXPATHLEN];
+
+	test_flowop = list;
+
+	/*
+	 * when searching a list of inner flowops, "path" is the fullname
+	 * of the containing composite flowop. Use it to form the
+	 * full name of the inner flowop to search for.
+	 */
+	if (path) {
+		if ((strlen(path) + strlen(name) + 1) > MAXPATHLEN) {
+			filebench_log(LOG_ERROR,
+			    "composite flowop path name %s.%s too long",
+			    path, name);
+			return (NULL);
+		}
+
+		/* create composite_name.name for recursive search */
+		(void) strcpy(fullname, path);
+		(void) strcat(fullname, ".");
+		(void) strcat(fullname, name);
+	} else {
+		(void) strcpy(fullname, name);
+	}
+
+	/*
+	 * loop through all flowops on the supplied tf_thrd_fops (flowop)
+	 * list or fo_comp_fops (inner flowop) list.
+	 */
+	while (test_flowop) {
+		if (strcmp(fullname, test_flowop->fo_name) == 0)
+			return (test_flowop);
+
+		if (test_flowop->fo_type == FLOW_TYPE_COMPOSITE) {
+			flowop_t *found_flowop;
+
+			found_flowop = flowop_recurse_search(
+			    test_flowop->fo_name, name,
+			    test_flowop->fo_comp_fops);
+
+			if (found_flowop)
+				return (found_flowop);
+		}
+		test_flowop = test_flowop->fo_exec_next;
+	}
+
+	/* not found here or on any child lists */
+	return (NULL);
+}
+
+/*
+ * Returns a pointer to flowop named "name" from the supplied tf_thrd_fops
+ * list of flowops. Returns the named flowop if found, or NULL.
+ */
+flowop_t *
+flowop_find_from_list(char *name, flowop_t *list)
+{
+	flowop_t *found_flowop;
+
+	flowop_find_barrier();
+
+	(void) ipc_mutex_lock(&filebench_shm->shm_flowop_lock);
+
+	found_flowop = flowop_recurse_search(NULL, name, list);
+
+	(void) ipc_mutex_unlock(&filebench_shm->shm_flowop_lock);
+
+	return (found_flowop);
+}
+
+/*
  * Composite flowop method. Does one pass through its list of
  * inner flowops per iteration.
  */
diff --git a/usr/src/cmd/filebench/common/flowop.h b/usr/src/cmd/filebench/common/flowop.h
index 5d6c83810e..f04e9c55e6 100644
--- a/usr/src/cmd/filebench/common/flowop.h
+++ b/usr/src/cmd/filebench/common/flowop.h
@@ -145,6 +145,7 @@ flowop_t *flowop_define(threadflow_t *, char *name, flowop_t *inherit,
     flowop_t **flowoplist_hdp, int instance, int type);
 flowop_t *flowop_find(char *name);
 flowop_t *flowop_find_one(char *name, int instance);
+flowop_t *flowop_find_from_list(char *name, flowop_t *list);
 void flowoplib_usage(void);
 void flowoplib_init(void);
 void flowop_delete_all(flowop_t **threadlist);
diff --git a/usr/src/cmd/filebench/common/flowop_library.c b/usr/src/cmd/filebench/common/flowop_library.c
index d884e48b24..5baf108f19 100644
--- a/usr/src/cmd/filebench/common/flowop_library.c
+++ b/usr/src/cmd/filebench/common/flowop_library.c
@@ -1021,6 +1021,29 @@ flowoplib_eventlimit(threadflow_t *threadflow, flowop_t *flowop)
 	return (FILEBENCH_OK);
 }
 
+static int
+flowoplib_event_find_target(threadflow_t *threadflow, flowop_t *flowop)
+{
+	if (flowop->fo_targetname[0] != '\0') {
+
+		/* Try to use statistics from specific flowop */
+		flowop->fo_targets =
+		    flowop_find_from_list(flowop->fo_targetname,
+		    threadflow->tf_thrd_fops);
+		if (flowop->fo_targets == NULL) {
+			filebench_log(LOG_ERROR,
+			    "limit target: could not find flowop %s",
+			    flowop->fo_targetname);
+			filebench_shutdown(1);
+			return (FILEBENCH_ERROR);
+		}
+	} else {
+		/* use total workload statistics */
+		flowop->fo_targets = NULL;
+	}
+	return (FILEBENCH_OK);
+}
+
 /*
  * Blocks the calling thread if the number of issued I/O
  * operations exceeds the number of posted events, thus
@@ -1042,12 +1065,33 @@ flowoplib_iopslimit(threadflow_t *threadflow, flowop_t *flowop)
 		filebench_log(LOG_DEBUG_IMPL, "rate %zx %s-%d locking",
 		    flowop, threadflow->tf_name, threadflow->tf_instance);
 		flowop->fo_initted = 1;
+
+		if (flowoplib_event_find_target(threadflow, flowop)
+		    == FILEBENCH_ERROR)
+			return (FILEBENCH_ERROR);
+
+		if (flowop->fo_targets && ((flowop->fo_targets->fo_attrs &
+		    (FLOW_ATTR_READ | FLOW_ATTR_WRITE)) == 0)) {
+			filebench_log(LOG_ERROR,
+			    "WARNING: Flowop %s does no IO",
+			    flowop->fo_targets->fo_name);
+			filebench_shutdown(1);
+			return (FILEBENCH_ERROR);
+		}
 	}
 
-	(void) ipc_mutex_lock(&controlstats_lock);
-	iops = (controlstats.fs_rcount +
-	    controlstats.fs_wcount);
-	(void) ipc_mutex_unlock(&controlstats_lock);
+	if (flowop->fo_targets) {
+		/*
+		 * Note that fs_count is already the sum of fs_rcount
+		 * and fs_wcount if looking at a single flowop.
+		 */
+		iops = flowop->fo_targets->fo_stats.fs_count;
+	} else {
+		(void) ipc_mutex_lock(&controlstats_lock);
+		iops = (controlstats.fs_rcount +
+		    controlstats.fs_wcount);
+		(void) ipc_mutex_unlock(&controlstats_lock);
+	}
 
 	/* Is this the first time around */
 	if (flowop->fo_tputlast == 0) {
@@ -1109,11 +1153,19 @@ flowoplib_opslimit(threadflow_t *threadflow, flowop_t *flowop)
 		filebench_log(LOG_DEBUG_IMPL, "rate %zx %s-%d locking",
 		    flowop, threadflow->tf_name, threadflow->tf_instance);
 		flowop->fo_initted = 1;
+
+		if (flowoplib_event_find_target(threadflow, flowop)
+		    == FILEBENCH_ERROR)
+			return (FILEBENCH_ERROR);
 	}
 
-	(void) ipc_mutex_lock(&controlstats_lock);
-	ops = controlstats.fs_count;
-	(void) ipc_mutex_unlock(&controlstats_lock);
+	if (flowop->fo_targets) {
+		ops = flowop->fo_targets->fo_stats.fs_count;
+	} else {
+		(void) ipc_mutex_lock(&controlstats_lock);
+		ops = controlstats.fs_count;
+		(void) ipc_mutex_unlock(&controlstats_lock);
+	}
 
 	/* Is this the first time around */
 	if (flowop->fo_tputlast == 0) {
@@ -1176,14 +1228,36 @@ flowoplib_bwlimit(threadflow_t *threadflow, flowop_t *flowop)
 		filebench_log(LOG_DEBUG_IMPL, "rate %zx %s-%d locking",
 		    flowop, threadflow->tf_name, threadflow->tf_instance);
 		flowop->fo_initted = 1;
+
+		if (flowoplib_event_find_target(threadflow, flowop)
+		    == FILEBENCH_ERROR)
+			return (FILEBENCH_ERROR);
+
+		if ((flowop->fo_targets) &&
+		    ((flowop->fo_targets->fo_attrs &
+		    (FLOW_ATTR_READ | FLOW_ATTR_WRITE)) == 0)) {
+			filebench_log(LOG_ERROR,
+			    "WARNING: Flowop %s does no Reads or Writes",
+			    flowop->fo_targets->fo_name);
+			filebench_shutdown(1);
+			return (FILEBENCH_ERROR);
+		}
 	}
 
-	(void) ipc_mutex_lock(&controlstats_lock);
-	bytes = (controlstats.fs_rbytes +
-	    controlstats.fs_wbytes);
-	(void) ipc_mutex_unlock(&controlstats_lock);
+	if (flowop->fo_targets) {
+		/*
+		 * Note that fs_bytes is already the sum of fs_rbytes
+		 * and fs_wbytes if looking at a single flowop.
+		 */
+		bytes = flowop->fo_targets->fo_stats.fs_bytes;
+	} else {
+		(void) ipc_mutex_lock(&controlstats_lock);
+		bytes = (controlstats.fs_rbytes +
+		    controlstats.fs_wbytes);
+		(void) ipc_mutex_unlock(&controlstats_lock);
+	}
 
-	/* Is this the first time around */
+	/* Is this the first time around? */
 	if (flowop->fo_tputlast == 0) {
 		flowop->fo_tputlast = bytes;
 		return (FILEBENCH_OK);
@@ -1240,15 +1314,40 @@ flowoplib_bwlimit(threadflow_t *threadflow, flowop_t *flowop)
 static int
 flowoplib_finishonbytes(threadflow_t *threadflow, flowop_t *flowop)
 {
-	uint64_t b;
-	uint64_t bytes = flowop->fo_constvalue; /* use constant value */
+	uint64_t bytes_io;		/* Bytes of I/O delivered so far */
+	uint64_t byte_lim = flowop->fo_constvalue;  /* Total Bytes desired */
+						    /* Uses constant value */
+
+	if (flowop->fo_initted == 0) {
+		filebench_log(LOG_DEBUG_IMPL, "rate %zx %s-%d locking",
+		    flowop, threadflow->tf_name, threadflow->tf_instance);
+		flowop->fo_initted = 1;
+
+		if (flowoplib_event_find_target(threadflow, flowop)
+		    == FILEBENCH_ERROR)
+			return (FILEBENCH_ERROR);
+
+		if ((flowop->fo_targets) &&
+		    ((flowop->fo_targets->fo_attrs &
+		    (FLOW_ATTR_READ | FLOW_ATTR_WRITE)) == 0)) {
+			filebench_log(LOG_ERROR,
+			    "WARNING: Flowop %s does no Reads or Writes",
+			    flowop->fo_targets->fo_name);
+			filebench_shutdown(1);
+			return (FILEBENCH_ERROR);
+		}
+	}
 
-	(void) ipc_mutex_lock(&controlstats_lock);
-	b = controlstats.fs_bytes;
-	(void) ipc_mutex_unlock(&controlstats_lock);
+	if (flowop->fo_targets) {
+		bytes_io = flowop->fo_targets->fo_stats.fs_bytes;
+	} else {
+		(void) ipc_mutex_lock(&controlstats_lock);
+		bytes_io = controlstats.fs_bytes;
+		(void) ipc_mutex_unlock(&controlstats_lock);
+	}
 
 	flowop_beginop(threadflow, flowop);
-	if (b > bytes) {
+	if (bytes_io > byte_lim) {
 		flowop_endop(threadflow, flowop, 0);
 		return (FILEBENCH_DONE);
 	}
@@ -1269,9 +1368,23 @@ flowoplib_finishoncount(threadflow_t *threadflow, flowop_t *flowop)
 	uint64_t ops;
 	uint64_t count = flowop->fo_constvalue; /* use constant value */
 
-	(void) ipc_mutex_lock(&controlstats_lock);
-	ops = controlstats.fs_count;
-	(void) ipc_mutex_unlock(&controlstats_lock);
+	if (flowop->fo_initted == 0) {
+		filebench_log(LOG_DEBUG_IMPL, "rate %zx %s-%d locking",
+		    flowop, threadflow->tf_name, threadflow->tf_instance);
+		flowop->fo_initted = 1;
+
+		if (flowoplib_event_find_target(threadflow, flowop)
+		    == FILEBENCH_ERROR)
+			return (FILEBENCH_ERROR);
+	}
+
+	if (flowop->fo_targets) {
+		ops = flowop->fo_targets->fo_stats.fs_count;
+	} else {
+		(void) ipc_mutex_lock(&controlstats_lock);
+		ops = controlstats.fs_count;
+		(void) ipc_mutex_unlock(&controlstats_lock);
+	}
 
 	flowop_beginop(threadflow, flowop);
 	if (ops >= count) {
@@ -1875,6 +1988,9 @@ flowoplib_deletefile(threadflow_t *threadflow, flowop_t *flowop)
 	(void) unlink(path);
 	flowop_endop(threadflow, flowop, 0);
 	file->fse_flags &= ~FSE_EXISTS;
+	(void) ipc_mutex_lock(&fileset->fs_num_files_lock);
+	fileset->fs_num_act_files--;
+	(void) ipc_mutex_unlock(&fileset->fs_num_files_lock);
 	(void) ipc_mutex_unlock(&file->fse_lock);
 
 	filebench_log(LOG_DEBUG_SCRIPT, "deleted file %s", file->fse_path);
diff --git a/usr/src/cmd/filebench/common/ipc.c b/usr/src/cmd/filebench/common/ipc.c
index e7593787b8..577bda7eec 100644
--- a/usr/src/cmd/filebench/common/ipc.c
+++ b/usr/src/cmd/filebench/common/ipc.c
@@ -336,6 +336,8 @@ ipc_init(void)
 	    ipc_mutexattr());
 	(void) pthread_mutex_init(&filebench_shm->shm_procflow_lock,
 	    ipc_mutexattr());
+	(void) pthread_mutex_init(&filebench_shm->shm_procs_running_lock,
+	    ipc_mutexattr());
 	(void) pthread_mutex_init(&filebench_shm->shm_threadflow_lock,
 	    ipc_mutexattr());
 	(void) pthread_mutex_init(&filebench_shm->shm_flowop_lock,
diff --git a/usr/src/cmd/filebench/common/ipc.h b/usr/src/cmd/filebench/common/ipc.h
index 38d3a01202..9ce44394d5 100644
--- a/usr/src/cmd/filebench/common/ipc.h
+++ b/usr/src/cmd/filebench/common/ipc.h
@@ -87,6 +87,7 @@ typedef struct filebench_shm {
 	pthread_mutex_t shm_msg_lock;
 	pthread_mutex_t shm_malloc_lock;
 	pthread_mutex_t shm_ism_lock;
+	pthread_mutex_t shm_procs_running_lock;	/* protects shm_procs_running */
 	pthread_rwlock_t shm_run_lock;
 	pthread_rwlock_t shm_flowop_find_lock;
 
@@ -119,7 +120,7 @@ typedef struct filebench_shm {
 	size_t		shm_allocated;
 	caddr_t		shm_addr;
 	char		*shm_ptr;
-	int		shm_running;
+	int		shm_procs_running;
 	int		shm_f_abort;
 	int		shm_rmode;
 	int		shm_1st_err;
diff --git a/usr/src/cmd/filebench/common/misc.c b/usr/src/cmd/filebench/common/misc.c
index 8d834f4cc4..553d28bcee 100644
--- a/usr/src/cmd/filebench/common/misc.c
+++ b/usr/src/cmd/filebench/common/misc.c
@@ -341,8 +341,7 @@ filebench_shutdown(int error) {
 		filebench_log(LOG_DEBUG_IMPL, "Shutdown");
 	}
 
-	if (filebench_shm->shm_running)
-		procflow_shutdown();
+	procflow_shutdown();
 
 	(void) unlink("/tmp/filebench_shm");
 	ipc_ismdelete();
diff --git a/usr/src/cmd/filebench/common/procflow.c b/usr/src/cmd/filebench/common/procflow.c
index 0209871359..5cfe559802 100644
--- a/usr/src/cmd/filebench/common/procflow.c
+++ b/usr/src/cmd/filebench/common/procflow.c
@@ -332,10 +332,10 @@ procflow_exec(char *name, int instance)
 		    "procflow_createproc exiting...");
 	}
 
+	(void) ipc_mutex_lock(&filebench_shm->shm_procs_running_lock);
+	filebench_shm->shm_procs_running --;
+	(void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
 	procflow->pf_running = 0;
-	(void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
-	filebench_shm->shm_running --;
-	(void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
 
 	return (ret);
 }
@@ -371,9 +371,21 @@ procflow_createnwait(void *nothing)
 		if (waitid(P_ALL, 0, &status, WEXITED) != 0)
 			pthread_exit(0);
 
+		(void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
 		/* if normal shutdown in progress, just quit */
 		if (filebench_shm->shm_f_abort)
+			(void) ipc_mutex_unlock(
+			    &filebench_shm->shm_procflow_lock);
+			pthread_exit(0);
+
+		/* if nothing running, exit */
+		if (filebench_shm->shm_procs_running == 0) {
+			filebench_shm->shm_f_abort = FILEBENCH_ABORT_RSRC;
+			(void) ipc_mutex_unlock(
+			    &filebench_shm->shm_procflow_lock);
 			pthread_exit(0);
+		}
+		(void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
 
 		if (status.si_code == CLD_EXITED) {
 			/* A process called exit(); check returned status */
@@ -391,11 +403,6 @@ procflow_createnwait(void *nothing)
 			filebench_shutdown(1);
 		}
 
-		/* nothing running, exit */
-		if (filebench_shm->shm_running == 0) {
-			filebench_shm->shm_f_abort = FILEBENCH_ABORT_RSRC;
-			pthread_exit(0);
-		}
 	}
 	/* NOTREACHED */
 	return (NULL);
@@ -507,7 +514,7 @@ procflow_delete(procflow_t *procflow, int wait_cnt)
 #ifdef USE_PROCESS_MODEL
 		(void) kill(procflow->pf_pid, SIGKILL);
 		filebench_log(LOG_DEBUG_SCRIPT,
-		    "Had to kill process %s-%d %d!",
+		    "procflow_delete: Had to kill process %s-%d %d!",
 		    procflow->pf_name,
 		    procflow->pf_instance,
 		    procflow->pf_pid);
@@ -605,7 +612,9 @@ procflow_allstarted()
 
 		procflow = procflow->pf_next;
 	}
-	filebench_shm->shm_running = running_procs;
+	(void) ipc_mutex_lock(&filebench_shm->shm_procs_running_lock);
+	filebench_shm->shm_procs_running = running_procs;
+	(void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
 
 	(void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
 
@@ -620,6 +629,7 @@ procflow_allstarted()
  * through the procflow list and deletes all procflows except
  * for the FLOW_MASTER procflow. Resets the f_abort flag when
  * finished.
+ *
  */
 void
 procflow_shutdown(void)
@@ -627,8 +637,16 @@ procflow_shutdown(void)
 	procflow_t *procflow = filebench_shm->shm_proclist;
 	int wait_cnt;
 
+	(void) ipc_mutex_lock(&filebench_shm->shm_procs_running_lock);
+	if (filebench_shm->shm_procs_running == 0) {
+		/* No processes running, so no need to do anything */
+		(void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
+		return;
+	}
+	filebench_shm->shm_procs_running = 0;
+	(void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
+
 	(void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
-	filebench_shm->shm_running = 0;
 	filebench_shm->shm_f_abort = 1;
 	wait_cnt = SHUTDOWN_WAIT_SECONDS;
 
diff --git a/usr/src/cmd/filebench/workloads/Makefile b/usr/src/cmd/filebench/workloads/Makefile
index d4a920c0f9..a056d540aa 100644
--- a/usr/src/cmd/filebench/workloads/Makefile
+++ b/usr/src/cmd/filebench/workloads/Makefile
@@ -58,6 +58,7 @@ WORKLOADS = \
 	randomread.f \
 	randomrw.f \
 	randomwrite.f \
+	ratelimcopyfiles.f \
 	singlestreamread.f \
 	singlestreamreaddirect.f \
 	singlestreamwrite.f \
diff --git a/usr/src/cmd/filebench/workloads/bringover.f b/usr/src/cmd/filebench/workloads/bringover.f
index a889cb49ad..6e1aaff7c2 100644
--- a/usr/src/cmd/filebench/workloads/bringover.f
+++ b/usr/src/cmd/filebench/workloads/bringover.f
@@ -32,9 +32,9 @@ set $filesize=16k
 set $iosize=1m
 set $nthreads=1
 
-set mode quit alldone
+set mode quit firstdone
 
-define fileset name=srcfiles,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc
+define fileset name=srcfiles,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc,paralloc
 define fileset name=destfiles,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth
 
 define process name=filereader,instances=1
@@ -50,8 +50,8 @@ define process name=filereader,instances=1
   }
 }
 
-echo  "Bringover Version 2.3 personality successfully loaded"
-usage "Usage: set \$dir=<dir>"
+echo  "Bringover Version 2.4 personality successfully loaded"
+usage "Usage: set \$dir=<dir>         defaults to $dir"
 usage "       set \$filesize=<size>   defaults to $filesize"
 usage "       set \$nfiles=<value>    defaults to $nfiles"
 usage "       set \$iosize=<size>     defaults to $iosize"
diff --git a/usr/src/cmd/filebench/workloads/copyfiles.f b/usr/src/cmd/filebench/workloads/copyfiles.f
index a47cb8f46f..fa77ddd3f6 100644
--- a/usr/src/cmd/filebench/workloads/copyfiles.f
+++ b/usr/src/cmd/filebench/workloads/copyfiles.f
@@ -31,9 +31,9 @@ set $filesize=16k
 set $iosize=1m
 set $nthreads=1
 
-set mode quit alldone
+set mode quit firstdone
 
-define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc=100
+define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc=100,paralloc
 define fileset name=destfiles,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth
 
 define process name=filereader,instances=1
@@ -49,8 +49,8 @@ define process name=filereader,instances=1
   }
 }
 
-echo  "CopyFiles Version 2.3 personality successfully loaded"
-usage "Usage: set \$dir=<dir>"
+echo  "CopyFiles Version 2.4 personality successfully loaded"
+usage "Usage: set \$dir=<dir>         defaults to $dir"
 usage "       set \$filesize=<size>   defaults to $filesize"
 usage "       set \$nfiles=<value>    defaults to $nfiles"
 usage "       set \$iosize=<size>     defaults to $iosize"
diff --git a/usr/src/cmd/filebench/workloads/createfiles.f b/usr/src/cmd/filebench/workloads/createfiles.f
index 198ecf3bfe..5905f6308e 100644
--- a/usr/src/cmd/filebench/workloads/createfiles.f
+++ b/usr/src/cmd/filebench/workloads/createfiles.f
@@ -31,7 +31,7 @@ set $filesize=16k
 set $iosize=1m
 set $nthreads=16
 
-set mode quit alldone
+set mode quit firstdone
 
 define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$meandirwidth
 
@@ -46,8 +46,8 @@ define process name=filecreate,instances=1
   }
 }
 
-echo  "Createfiles Version 2.3 personality successfully loaded"
-usage "Usage: set \$dir=<dir>"
+echo  "Createfiles Version 2.4 personality successfully loaded"
+usage "Usage: set \$dir=<dir>          defaults to $dir"
 usage "       set \$filesize=<size>    defaults to $filesize"
 usage "       set \$iosize=<size>      defaults to $iosize"
 usage "       set \$nfiles=<value>     defaults to $nfiles"
diff --git a/usr/src/cmd/filebench/workloads/deletefiles.f b/usr/src/cmd/filebench/workloads/deletefiles.f
index 5a75d1d2f8..4870c4732c 100644
--- a/usr/src/cmd/filebench/workloads/deletefiles.f
+++ b/usr/src/cmd/filebench/workloads/deletefiles.f
@@ -30,9 +30,9 @@ set $meandirwidth=100
 set $filesize=16k
 set $nthreads=16
 
-set mode quit alldone
+set mode quit firstdone
 
-define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$meandirwidth,prealloc=100
+define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$meandirwidth,prealloc=100,paralloc
 
 define process name=filedelete,instances=1
 {
@@ -43,8 +43,8 @@ define process name=filedelete,instances=1
   }
 }
 
-echo  "Deletefiles Version 2.2 personality successfully loaded"
-usage "Usage: set \$dir=<dir>"
+echo  "Deletefiles Version 2.3 personality successfully loaded"
+usage "Usage: set \$dir=<dir>          defaults to $dir"
 usage "       set \$filesize=<size>    defaults to $filesize"
 usage "       set \$nfiles=<value>     defaults to $nfiles"
 usage "       set \$nthreads=<value>   defaults to $nthreads"
diff --git a/usr/src/cmd/filebench/workloads/filemicro_delete.f b/usr/src/cmd/filebench/workloads/filemicro_delete.f
index c9cceabf11..206710fd11 100644
--- a/usr/src/cmd/filebench/workloads/filemicro_delete.f
+++ b/usr/src/cmd/filebench/workloads/filemicro_delete.f
@@ -36,7 +36,7 @@ set $nfiles=5000
 set $meandirwidth=100
 set $nthreads=16
 
-set mode quit alldone
+set mode quit firstdone
 
 define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$meandirwidth,prealloc=100,paralloc
 
@@ -50,8 +50,8 @@ define process name=filedelete,instances=1
   }
 }
 
-echo  "FileMicro-Delete Version 2.3 personality successfully loaded"
-usage "Usage: set \$dir=<dir>"
+echo  "FileMicro-Delete Version 2.4 personality successfully loaded"
+usage "Usage: set \$dir=<dir>           defaults to $dir"
 usage "       set \$count=<value>       defaults to $count"
 usage "       set \$filesize=<size>     defaults to $filesize"
 usage "       set \$nfiles=<value>      defaults to $nfiles"
diff --git a/usr/src/cmd/filebench/workloads/mongo.f b/usr/src/cmd/filebench/workloads/mongo.f
index fd9907de1f..f6d7ec6f2d 100644
--- a/usr/src/cmd/filebench/workloads/mongo.f
+++ b/usr/src/cmd/filebench/workloads/mongo.f
@@ -32,10 +32,10 @@ set $nthreads=1
 set $meaniosize=16k
 set $readiosize=1m
 
-set mode quit alldone
+set mode quit firstdone
 
-define fileset name=postset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc
-define fileset name=postsetdel,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc
+define fileset name=postset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc,paralloc
+define fileset name=postsetdel,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc,paralloc
 
 define process name=filereader,instances=1
 {
@@ -51,8 +51,8 @@ define process name=filereader,instances=1
   }
 }
 
-echo  "Mongo-like Version 2.2 personality successfully loaded"
-usage "Usage: set \$dir=<dir>"
+echo  "Mongo-like Version 2.3 personality successfully loaded"
+usage "Usage: set \$dir=<dir>          defaults to $dir"
 usage "       set \$filesize=<size>    defaults to $filesize"
 usage "       set \$nfiles=<value>     defaults to $nfiles"
 usage "       set \$dirwidth=<value>   defaults to $dirwidth"
diff --git a/usr/src/cmd/filebench/workloads/ratelimcopyfiles.f b/usr/src/cmd/filebench/workloads/ratelimcopyfiles.f
new file mode 100644
index 0000000000..1b23cbd583
--- /dev/null
+++ b/usr/src/cmd/filebench/workloads/ratelimcopyfiles.f
@@ -0,0 +1,72 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+#
+# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+# ident	"%Z%%M%	%I%	%E% SMI"
+
+# RateLimCopyFiles.f uses the iopslimit flowop with the target attribute
+# set to the writewholefile flowop to limit the rate to one writewholefile
+# operation per event. Without the target attribute set, the limit will
+# be one writewholefile OR readwholefile operation per event, so in effect
+# it will run at half the rate. Without the target attribute, this workload
+# is identical to copyfiles.f. Note that you do have to enable the event
+# generator for any of the rate limiting flowops to take effect, for example
+# by typing:
+#     eventget rate=10
+# at the go_filebench prompt to get ten events per second.
+#
+set $dir=/tmp
+set $dirwidth=20
+set $filesize=16k
+set $iosize=1m
+set $nfiles=1000
+set $nthreads=1
+
+set mode quit firstdone
+
+define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc=100
+define fileset name=destfiles,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth
+
+define process name=filereader,instances=1
+{
+  thread name=filereaderthread,memsize=10m,instances=$nthreads
+  {
+    flowop openfile name=openfile1,filesetname=bigfileset,fd=1
+    flowop readwholefile name=readfile1,fd=1,iosize=$iosize
+    flowop createfile name=createfile2,filesetname=destfiles,fd=2
+    flowop writewholefile name=writefile2,filesetname=destfiles,fd=2,srcfd=1,iosize=$iosize
+    flowop closefile name=closefile1,fd=1
+    flowop closefile name=closefile2,fd=2
+    flowop iopslimit name=iopslim1, target=writefile2
+  }
+}
+
+echo  "RateLimCopyFiles Version 1.0 personality successfully loaded"
+usage "Usage: set \$dir=<dir>"
+usage "       set \$filesize=<size>   defaults to $filesize"
+usage "       set \$nfiles=<value>    defaults to $nfiles"
+usage "       set \$iosize=<size>     defaults to $iosize"
+usage "       set \$dirwidth=<value>  defaults to $dirwidth"
+usage "       set \$nthreads=<value>  defaults to $nthreads"
+usage " "
+usage "       run"
diff --git a/usr/src/pkgdefs/SUNWfilebench/prototype_com b/usr/src/pkgdefs/SUNWfilebench/prototype_com
index 8415be41e5..d6d4d67ee6 100644
--- a/usr/src/pkgdefs/SUNWfilebench/prototype_com
+++ b/usr/src/pkgdefs/SUNWfilebench/prototype_com
@@ -80,6 +80,7 @@ f none usr/benchmarks/filebench/workloads/oltp.f 444 root bin
 f none usr/benchmarks/filebench/workloads/randomread.f 444 root bin
 f none usr/benchmarks/filebench/workloads/randomrw.f 444 root bin
 f none usr/benchmarks/filebench/workloads/randomwrite.f 444 root bin
+f none usr/benchmarks/filebench/workloads/ratelimcopyfiles.f 444 root bin
 f none usr/benchmarks/filebench/workloads/singlestreamread.f 444 root bin
 f none usr/benchmarks/filebench/workloads/singlestreamreaddirect.f 444 root bin
 f none usr/benchmarks/filebench/workloads/singlestreamwrite.f 444 root bin