6652464 FileBench needs an f-language definable composite flowop feature

10 files changed, 1081 insertions, 157 deletions

diff --git a/usr/src/cmd/filebench/common/filebench.h b/usr/src/cmd/filebench/common/filebench.h
index 5386425303..80e0d4156e 100644
--- a/usr/src/cmd/filebench/common/filebench.h
+++ b/usr/src/cmd/filebench/common/filebench.h
@@ -109,7 +109,7 @@ void filebench_shutdown(int error);
 #define	MIN(x, y) ((x) < (y) ? (x) : (y))
 #endif
 
-#define	FILEBENCH_VERSION	"1.2.4"
+#define	FILEBENCH_VERSION	"1.3.0"
 #define	FILEBENCHDIR	"/usr/benchmarks/filebench"
 #define	FILEBENCH_PROMPT	"filebench> "
 #define	MAX_LINE_LEN	1024
diff --git a/usr/src/cmd/filebench/common/flowop.c b/usr/src/cmd/filebench/common/flowop.c
index 1394152dd2..5ba66529cb 100644
--- a/usr/src/cmd/filebench/common/flowop.c
+++ b/usr/src/cmd/filebench/common/flowop.c
@@ -41,7 +41,10 @@
 #endif
 
 static flowop_t *flowop_define_common(threadflow_t *threadflow, char *name,
-    flowop_t *inherit, int instance, int type);
+    flowop_t *inherit, flowop_t **flowoplist_hdp, int instance, int type);
+static int flowop_composite(threadflow_t *threadflow, flowop_t *flowop);
+static int flowop_composite_init(flowop_t *flowop);
+static void flowop_composite_destruct(flowop_t *flowop);
 
 /*
  * A collection of flowop support functions. The actual code that
@@ -51,9 +54,54 @@ static flowop_t *flowop_define_common(threadflow_t *threadflow, char *name,
  * flowops, cyclically invoking the flowops on each threadflow's flowop
  * list, collecting statistics about flowop execution, and other
  * housekeeping duties are included in this file.
+ *
+ * User Defined Composite Flowops
+ *    The ability to define new flowops as lists of built-in or previously
+ * defined flowops has been added to Filebench. In a sense they are like
+ * in-line subroutines, which can have default attributes set at definition
+ * time and passed arguments at invocation time. Like other flowops (and
+ * unlike conventional subroutines) you can invoke them with an iteration
+ * count (the "iter" attribute), and they will loop through their associated
+ * list of flowops for "iter" number of times each time they are encountered
+ * in the thread or outer composite flowop which invokes them.
+ *
+ * Composite flowops are created with a "define" command, are given a name,
+ * optional default attributes, and local variable definitions on the
+ * "define" command line, followed by a brace enclosed list of flowops
+ * to execute. The enclosed flowops may include attributes that reference
+ * the local variables, as well as constants and global variables.
+ *
+ * Composite flowops are used pretty much like regular flowops, but you can
+ * also set local variables to constants or global variables ($local_var =
+ * [$var | $random_var | string | boolean | integer | double]) as part of
+ * the invocation. Thus each invocation can pass customized values to its
+ * inner flowops, greatly increasing their generality.
+ *
+ * All flowops are placed on a global, singly linked list, with fo_next
+ * being the link pointer for this list. The are also placed on a private
+ * list for the thread or composite flowop they are part of. The tf_thrd_fops
+ * pointer in the thread will point to the list of top level flowops in the
+ * thread, which are linked together by fo_exec_next. If any of these flowops
+ * are composite flowops, they will have a list of second level flowops rooted
+ * at the composite flowop's fo_comp_fops pointer. So, there is one big list
+ * of all flowops, and an n-arry tree of threads, composite flowops, and
+ * flowops, with composite flowops being the branch nodes in the tree.
+ *
+ * To illustrate, if we have three first level flowops, the first of which is
+ * a composite flowop consisting of two other flowops, we get:
+ *
+ * Thread->tf_thrd_fops -> flowop->fo_exec_next -> flowop->fo_exec_next
+ *			   flowop->fo_comp_fops		    |
+ *				    |			    V
+ *				    |			flowop->fo_exec_next
+ *				    |
+ *				    V
+ *				flowop->fo_exec_next -> flowop->fo_exec_next
+ *
+ * And all five flowops (plus others from any other threads) are on a global
+ * list linked with fo_next.
  */
 
-
 /*
  * Prints the name and instance number of each flowop in
  * the supplied list to the filebench log.
@@ -66,7 +114,7 @@ flowop_printlist(flowop_t *list)
 	while (flowop) {
 		filebench_log(LOG_DEBUG_IMPL, "flowop-list %s-%d",
 		    flowop->fo_name, flowop->fo_instance);
-		flowop = flowop->fo_threadnext;
+		flowop = flowop->fo_exec_next;
 	}
 	return (0);
 }
@@ -216,6 +264,47 @@ flowop_initflow(flowop_t *flowop)
 	return (0);
 }
 
+static int
+flowop_create_runtime_flowops(threadflow_t *threadflow, flowop_t **ops_list_ptr)
+{
+	flowop_t *flowop = *ops_list_ptr;
+
+	while (flowop) {
+		flowop_t *newflowop;
+
+		if (flowop == *ops_list_ptr)
+			*ops_list_ptr = NULL;
+
+		newflowop = flowop_define_common(threadflow, flowop->fo_name,
+		    flowop, ops_list_ptr, 1, 0);
+		if (newflowop == NULL)
+			return (FILEBENCH_ERROR);
+
+		/* check for fo_filename attribute, and resolve if present */
+		if (flowop->fo_filename) {
+			char *name;
+
+			name = avd_get_str(flowop->fo_filename);
+			newflowop->fo_fileset = fileset_find(name);
+
+			if (newflowop->fo_fileset == NULL) {
+				filebench_log(LOG_ERROR,
+				    "flowop %s: file %s not found",
+				    newflowop->fo_name, name);
+				filebench_shutdown(1);
+			}
+		}
+
+		if (flowop_initflow(newflowop) < 0) {
+			filebench_log(LOG_ERROR, "Flowop init of %s failed",
+			    newflowop->fo_name);
+		}
+
+		flowop = flowop->fo_exec_next;
+	}
+	return (FILEBENCH_OK);
+}
+
 /*
  * Calls the flowop's destruct function, pointed to by
  * flowop->fo_destruct.
@@ -245,13 +334,13 @@ flowop_destruct_all_flows(threadflow_t *threadflow)
 		return;
 	}
 
-	flowop = threadflow->tf_ops;
+	flowop = threadflow->tf_thrd_fops;
 	threadflow->tf_running = 0;
 	(void) ipc_mutex_unlock(&threadflow->tf_lock);
 
 	while (flowop) {
 		flowop_destructflow(flowop);
-		flowop = flowop->fo_threadnext;
+		flowop = flowop->fo_exec_next;
 	}
 }
 
@@ -295,7 +384,7 @@ flowop_start(threadflow_t *threadflow)
 	}
 	(void) ipc_mutex_unlock(&controlstats_lock);
 
-	flowop = threadflow->tf_ops;
+	flowop = threadflow->tf_thrd_fops;
 	threadflow->tf_stats.fs_stime = gethrtime();
 	flowop->fo_stats.fs_stime = gethrtime();
 
@@ -317,36 +406,11 @@ flowop_start(threadflow_t *threadflow)
 
 	/* Create the runtime flowops from those defined by the script */
 	(void) ipc_mutex_lock(&filebench_shm->shm_flowop_lock);
-	while (flowop) {
-		flowop_t *newflowop;
-
-		if (flowop == threadflow->tf_ops)
-			threadflow->tf_ops = NULL;
-		newflowop = flowop_define_common(threadflow, flowop->fo_name,
-		    flowop, 1, 0);
-		if (newflowop == NULL)
-			return;
-
-		/* check for fo_filename attribute, and resolve if present */
-		if (flowop->fo_filename) {
-			char *name;
-
-			name = avd_get_str(flowop->fo_filename);
-			newflowop->fo_fileset = fileset_find(name);
-
-			if (newflowop->fo_fileset == NULL) {
-				filebench_log(LOG_ERROR,
-				    "flowop %s: file %s not found",
-				    newflowop->fo_name, name);
-				filebench_shutdown(1);
-			}
-		}
-
-		if (flowop_initflow(newflowop) < 0) {
-			filebench_log(LOG_ERROR, "Flowop init of %s failed",
-			    newflowop->fo_name);
-		}
-		flowop = flowop->fo_threadnext;
+	if (flowop_create_runtime_flowops(threadflow, &threadflow->tf_thrd_fops)
+	    != FILEBENCH_OK) {
+		(void) ipc_mutex_unlock(&filebench_shm->shm_flowop_lock);
+		filebench_shutdown(1);
+		return;
 	}
 	(void) ipc_mutex_unlock(&filebench_shm->shm_flowop_lock);
 
@@ -354,7 +418,7 @@ flowop_start(threadflow_t *threadflow)
 	(void) pthread_rwlock_unlock(&filebench_shm->shm_flowop_find_lock);
 
 	/* Set to the start of the new flowop list */
-	flowop = threadflow->tf_ops;
+	flowop = threadflow->tf_thrd_fops;
 
 	threadflow->tf_abort = 0;
 	threadflow->tf_running = 1;
@@ -409,7 +473,7 @@ flowop_start(threadflow_t *threadflow)
 			flowop->fo_stats.fs_stime = gethrtime();
 
 		/* Execute the flowop for fo_iters times */
-		count = avd_get_int(flowop->fo_iters);
+		count = (int)avd_get_int(flowop->fo_iters);
 		for (i = 0; i < count; i++) {
 
 			filebench_log(LOG_DEBUG_SCRIPT, "%s: executing flowop "
@@ -497,11 +561,11 @@ flowop_start(threadflow_t *threadflow)
 		}
 
 		/* advance to next flowop */
-		flowop = flowop->fo_threadnext;
+		flowop = flowop->fo_exec_next;
 
 		/* but if at end of list, start over from the beginning */
 		if (flowop == NULL) {
-			flowop = threadflow->tf_ops;
+			flowop = threadflow->tf_thrd_fops;
 			threadflow->tf_stats.fs_count++;
 		}
 	}
@@ -539,31 +603,31 @@ flowop_delete(flowop_t **flowoplist, flowop_t *flowop)
 	/* Delete from thread's flowop list */
 	if (flowop == *flowoplist) {
 		/* First on list */
-		*flowoplist = flowop->fo_threadnext;
+		*flowoplist = flowop->fo_exec_next;
 		filebench_log(LOG_DEBUG_IMPL,
 		    "Delete0 flowop: (%s-%d)",
 		    flowop->fo_name,
 		    flowop->fo_instance);
 	} else {
-		while (entry->fo_threadnext) {
+		while (entry->fo_exec_next) {
 			filebench_log(LOG_DEBUG_IMPL,
 			    "Delete0 flowop: (%s-%d) == (%s-%d)",
-			    entry->fo_threadnext->fo_name,
-			    entry->fo_threadnext->fo_instance,
+			    entry->fo_exec_next->fo_name,
+			    entry->fo_exec_next->fo_instance,
 			    flowop->fo_name,
 			    flowop->fo_instance);
 
-			if (flowop == entry->fo_threadnext) {
+			if (flowop == entry->fo_exec_next) {
 				/* Delete */
 				filebench_log(LOG_DEBUG_IMPL,
 				    "Deleted0 flowop: (%s-%d)",
-				    entry->fo_threadnext->fo_name,
-				    entry->fo_threadnext->fo_instance);
-				entry->fo_threadnext =
-				    entry->fo_threadnext->fo_threadnext;
+				    entry->fo_exec_next->fo_name,
+				    entry->fo_exec_next->fo_instance);
+				entry->fo_exec_next =
+				    entry->fo_exec_next->fo_exec_next;
 				break;
 			}
-			entry = entry->fo_threadnext;
+			entry = entry->fo_exec_next;
 		}
 	}
 
@@ -620,25 +684,19 @@ flowop_delete_all(flowop_t **flowoplist)
 {
 	flowop_t *flowop = *flowoplist;
 
-	filebench_log(LOG_DEBUG_IMPL, "Deleting all flowops...");
+	(void) ipc_mutex_lock(&filebench_shm->shm_flowop_lock);
+
 	while (flowop) {
 		filebench_log(LOG_DEBUG_IMPL, "Deleting flowop (%s-%d)",
 		    flowop->fo_name, flowop->fo_instance);
-		flowop = flowop->fo_threadnext;
-	}
 
-	flowop = *flowoplist;
-
-	(void) ipc_mutex_lock(&filebench_shm->shm_flowop_lock);
-
-	while (flowop) {
 		if (flowop->fo_instance &&
 		    (flowop->fo_instance == FLOW_MASTER)) {
-			flowop = flowop->fo_threadnext;
+			flowop = flowop->fo_exec_next;
 			continue;
 		}
 		flowop_delete(flowoplist, flowop);
-		flowop = flowop->fo_threadnext;
+		flowop = flowop->fo_exec_next;
 	}
 
 	(void) ipc_mutex_unlock(&filebench_shm->shm_flowop_lock);
@@ -647,21 +705,21 @@ flowop_delete_all(flowop_t **flowoplist)
 /*
  * Allocates a flowop entity and initializes it with inherited
  * contents from the "inherit" flowop, if it is supplied, or
- * with zeros otherwise. In either case the file descriptor
- * (fo_fd) is set to -1, and the fo_next and fo_threadnext
+ * with zeros otherwise. In either case the fo_next and fo_exec_next
  * pointers are set to NULL, and fo_thread is set to point to
  * the owning threadflow. The initialized flowop is placed at
  * the head of the global flowop list, and also placed on the
- * tail of thethreadflow's tf_ops list. The routine locks the
- * flowop's fo_lock and leaves it held on return. If successful,
- * it returns a pointer to the allocated and initialized flowop,
- * otherwise NULL.
+ * tail of the supplied local flowop list, which will either
+ * be a threadflow's tf_thrd_fops list or a composite flowop's
+ * fo_comp_fops list. The routine locks the flowop's fo_lock and
+ * leaves it held on return. If successful, it returns a pointer
+ * to the allocated and initialized flowop, otherwise it returns NULL.
  *
  * filebench_shm->shm_flowop_lock must be held by caller.
  */
 static flowop_t *
 flowop_define_common(threadflow_t *threadflow, char *name, flowop_t *inherit,
-    int instance, int type)
+    flowop_t **flowoplist_hdp, int instance, int type)
 {
 	flowop_t *flowop;
 
@@ -685,7 +743,7 @@ flowop_define_common(threadflow_t *threadflow, char *name, flowop_t *inherit,
 		(void) pthread_mutex_init(&flowop->fo_lock, ipc_mutexattr());
 		(void) ipc_mutex_lock(&flowop->fo_lock);
 		flowop->fo_next = NULL;
-		flowop->fo_threadnext = NULL;
+		flowop->fo_exec_next = NULL;
 		filebench_log(LOG_DEBUG_IMPL,
 		    "flowop %s-%d calling init", name, instance);
 	} else {
@@ -711,22 +769,22 @@ flowop_define_common(threadflow_t *threadflow, char *name, flowop_t *inherit,
 	(void) strcpy(flowop->fo_name, name);
 	flowop->fo_instance = instance;
 
-	if (threadflow == NULL)
+	if (flowoplist_hdp == NULL)
 		return (flowop);
 
 	/* Add flowop to thread op list */
-	if (threadflow->tf_ops == NULL) {
-		threadflow->tf_ops = flowop;
-		flowop->fo_threadnext = NULL;
+	if (*flowoplist_hdp == NULL) {
+		*flowoplist_hdp = flowop;
+		flowop->fo_exec_next = NULL;
 	} else {
 		flowop_t *flowend;
 
 		/* Find the end of the thread list */
-		flowend = threadflow->tf_ops;
-		while (flowend->fo_threadnext != NULL)
-			flowend = flowend->fo_threadnext;
-		flowend->fo_threadnext = flowop;
-		flowop->fo_threadnext = NULL;
+		flowend = *flowoplist_hdp;
+		while (flowend->fo_exec_next != NULL)
+			flowend = flowend->fo_exec_next;
+		flowend->fo_exec_next = flowop;
+		flowop->fo_exec_next = NULL;
 	}
 
 	return (flowop);
@@ -739,18 +797,43 @@ flowop_define_common(threadflow_t *threadflow, char *name, flowop_t *inherit,
  */
 flowop_t *
 flowop_define(threadflow_t *threadflow, char *name, flowop_t *inherit,
-    int instance, int type)
+    flowop_t **flowoplist_hdp, int instance, int type)
 {
 	flowop_t	*flowop;
 
 	(void) ipc_mutex_lock(&filebench_shm->shm_flowop_lock);
 	flowop = flowop_define_common(threadflow, name,
-	    inherit, instance, type);
+	    inherit, flowoplist_hdp, instance, type);
+	(void) ipc_mutex_unlock(&filebench_shm->shm_flowop_lock);
+
+	if (flowop == NULL)
+		return (NULL);
+
+	(void) ipc_mutex_unlock(&flowop->fo_lock);
+	return (flowop);
+}
+
+/*
+ * Calls flowop_define_common() to allocate and initialize a
+ * composite flowop, and holds the shared flowop_lock during the call.
+ * It releases the created flowop's fo_lock when done.
+ */
+flowop_t *
+flowop_new_composite_define(char *name)
+{
+	flowop_t *flowop;
+
+	(void) ipc_mutex_lock(&filebench_shm->shm_flowop_lock);
+	flowop = flowop_define_common(NULL, name,
+	    NULL, NULL, 0, FLOW_TYPE_COMPOSITE);
 	(void) ipc_mutex_unlock(&filebench_shm->shm_flowop_lock);
 
 	if (flowop == NULL)
 		return (NULL);
 
+	flowop->fo_func = flowop_composite;
+	flowop->fo_init = flowop_composite_init;
+	flowop->fo_destruct = flowop_composite_destruct;
 	(void) ipc_mutex_unlock(&flowop->fo_lock);
 
 	return (flowop);
@@ -837,3 +920,108 @@ flowop_find_one(char *name, int instance)
 
 	return (test_flowop);
 }
+
+/*
+ * Composite flowop method. Does one pass through its list of
+ * inner flowops per iteration.
+ */
+static int
+flowop_composite(threadflow_t *threadflow, flowop_t *flowop)
+{
+	flowop_t	*inner_flowop;
+
+	/* get the first flowop in the list */
+	inner_flowop = flowop->fo_comp_fops;
+
+	/* make a pass through the list of sub flowops */
+	while (inner_flowop) {
+		int	i, count;
+
+		/* Abort if asked */
+		if (threadflow->tf_abort || filebench_shm->shm_f_abort)
+			return (FILEBENCH_DONE);
+
+		if (inner_flowop->fo_stats.fs_stime == 0)
+			inner_flowop->fo_stats.fs_stime = gethrtime();
+
+		/* Execute the flowop for fo_iters times */
+		count = (int)avd_get_int(inner_flowop->fo_iters);
+		for (i = 0; i < count; i++) {
+
+			filebench_log(LOG_DEBUG_SCRIPT, "%s: executing flowop "
+			    "%s-%d", threadflow->tf_name,
+			    inner_flowop->fo_name,
+			    inner_flowop->fo_instance);
+
+			switch ((*inner_flowop->fo_func)(threadflow,
+			    inner_flowop)) {
+
+			/* all done */
+			case FILEBENCH_DONE:
+				return (FILEBENCH_DONE);
+
+			/* quit if inner flowop limit reached */
+			case FILEBENCH_NORSC:
+				return (FILEBENCH_NORSC);
+
+			/* quit on inner flowop error */
+			case FILEBENCH_ERROR:
+				filebench_log(LOG_ERROR,
+				    "inner flowop %s failed",
+				    inner_flowop->fo_name);
+				return (FILEBENCH_ERROR);
+
+			/* otherwise keep going */
+			default:
+				break;
+			}
+
+		}
+
+		/* advance to next flowop */
+		inner_flowop = inner_flowop->fo_exec_next;
+	}
+
+	/* finished with this pass */
+	return (FILEBENCH_OK);
+}
+
+/*
+ * Composite flowop initialization. Creates runtime inner flowops
+ * from prototype inner flowops.
+ */
+static int
+flowop_composite_init(flowop_t *flowop)
+{
+	int err;
+
+	err = flowop_create_runtime_flowops(flowop->fo_thread,
+	    &flowop->fo_comp_fops);
+	if (err != FILEBENCH_OK)
+		return (err);
+
+	(void) ipc_mutex_unlock(&flowop->fo_lock);
+	return (0);
+}
+
+/*
+ * clean up inner flowops
+ */
+static void
+flowop_composite_destruct(flowop_t *flowop)
+{
+	flowop_t *inner_flowop = flowop->fo_comp_fops;
+
+	while (inner_flowop) {
+		filebench_log(LOG_DEBUG_IMPL, "Deleting inner flowop (%s-%d)",
+		    inner_flowop->fo_name, inner_flowop->fo_instance);
+
+		if (inner_flowop->fo_instance &&
+		    (inner_flowop->fo_instance == FLOW_MASTER)) {
+			inner_flowop = inner_flowop->fo_exec_next;
+			continue;
+		}
+		flowop_delete(&flowop->fo_comp_fops, inner_flowop);
+		inner_flowop = inner_flowop->fo_exec_next;
+	}
+}
diff --git a/usr/src/cmd/filebench/common/flowop.h b/usr/src/cmd/filebench/common/flowop.h
index d98b1d0ff8..5d6c83810e 100644
--- a/usr/src/cmd/filebench/common/flowop.h
+++ b/usr/src/cmd/filebench/common/flowop.h
@@ -56,8 +56,10 @@ typedef struct flowop {
 	char		fo_name[128];	/* Name */
 	int		fo_instance;	/* Instance number */
 	struct flowop	*fo_next;	/* Next in global list */
-	struct flowop	*fo_threadnext;	/* Next in thread's list */
+	struct flowop	*fo_exec_next;	/* Next in thread's or compfo's list */
 	struct flowop	*fo_resultnext;	/* List of flowops in result */
+	struct flowop	*fo_comp_fops;	/* List of flowops in composite fo */
+	var_t		*fo_lvar_list;	/* List of composite local vars */
 	struct threadflow *fo_thread;	/* Backpointer to thread */
 	int		(*fo_func)();	/* Method */
 	int		(*fo_init)();	/* Init Method */
@@ -118,22 +120,29 @@ typedef struct flowop {
 #define	FLOW_ATTR_READ		0x80
 #define	FLOW_ATTR_WRITE		0x100
 
-#define	FLOW_MASTER -1	    /* Declaration of thread from script */
+/* Flowop Instance Numbers */
+			    /* Worker flowops have instance numbers > 0 */
 #define	FLOW_DEFINITION 0   /* Prototype definition of flowop from library */
+#define	FLOW_INNER_DEF -1   /* Constructed proto flowops within composite */
+#define	FLOW_MASTER -2	    /* Master flowop based on flowop declaration */
+			    /* supplied within a thread definition */
 
-#define	FLOW_TYPES	5
+/* Flowop type definitions */
+
+#define	FLOW_TYPES	6
 #define	FLOW_TYPE_GLOBAL	0  /* Rolled up statistics */
 #define	FLOW_TYPE_IO		1  /* Op is an I/O, reflected in iops and lat */
 #define	FLOW_TYPE_AIO		2  /* Op is an async I/O, reflected in iops */
 #define	FLOW_TYPE_SYNC		3  /* Op is a sync event */
-#define	FLOW_TYPE_OTHER		4  /* Op is a something else */
+#define	FLOW_TYPE_COMPOSITE	4  /* Op is a composite flowop */
+#define	FLOW_TYPE_OTHER		5  /* Op is a something else */
 
 extern flowstat_t controlstats;
 extern pthread_mutex_t controlstats_lock;
 
 void flowop_init(void);
 flowop_t *flowop_define(threadflow_t *, char *name, flowop_t *inherit,
-    int instance, int type);
+    flowop_t **flowoplist_hdp, int instance, int type);
 flowop_t *flowop_find(char *name);
 flowop_t *flowop_find_one(char *name, int instance);
 void flowoplib_usage(void);
@@ -142,6 +151,7 @@ void flowop_delete_all(flowop_t **threadlist);
 void flowop_endop(threadflow_t *threadflow, flowop_t *flowop, int64_t bytes);
 void flowop_beginop(threadflow_t *threadflow, flowop_t *flowop);
 void flowop_destruct_all_flows(threadflow_t *threadflow);
+flowop_t *flowop_new_composite_define(char *name);
 void flowop_printall(void);
 
 #ifdef	__cplusplus
diff --git a/usr/src/cmd/filebench/common/flowop_library.c b/usr/src/cmd/filebench/common/flowop_library.c
index e13692b569..9a2bb8c192 100644
--- a/usr/src/cmd/filebench/common/flowop_library.c
+++ b/usr/src/cmd/filebench/common/flowop_library.c
@@ -233,7 +233,7 @@ flowoplib_init()
 		fl = &flowoplib_funcs[i];
 
 		if ((flowop = flowop_define(NULL,
-		    fl->fl_name, NULL, 0, fl->fl_type)) == 0) {
+		    fl->fl_name, NULL, NULL, 0, fl->fl_type)) == 0) {
 			filebench_log(LOG_ERROR,
 			    "failed to create flowop %s\n",
 			    fl->fl_name);
@@ -1503,7 +1503,7 @@ flowoplib_sempost(threadflow_t *threadflow, flowop_t *flowop)
 		int i;
 #endif /* HAVE_SYSV_SEM */
 		struct timespec timeout;
-		int value = avd_get_int(flowop->fo_value);
+		int value = (int)avd_get_int(flowop->fo_value);
 
 		if (target->fo_instance == FLOW_MASTER) {
 			target = target->fo_targetnext;
diff --git a/usr/src/cmd/filebench/common/ipc.h b/usr/src/cmd/filebench/common/ipc.h
index e60416f3ee..38d3a01202 100644
--- a/usr/src/cmd/filebench/common/ipc.h
+++ b/usr/src/cmd/filebench/common/ipc.h
@@ -98,6 +98,7 @@ typedef struct filebench_shm {
 	var_t		*shm_var_list;
 	var_t		*shm_var_dyn_list;
 	randdist_t	*shm_rand_list;
+	var_t		*shm_var_loc_list;
 	int		shm_debug_level;
 	hrtime_t	shm_epoch;
 	hrtime_t	shm_starttime;
diff --git a/usr/src/cmd/filebench/common/parser_gram.y b/usr/src/cmd/filebench/common/parser_gram.y
index cbfe0bae52..c8edfd5812 100644
--- a/usr/src/cmd/filebench/common/parser_gram.y
+++ b/usr/src/cmd/filebench/common/parser_gram.y
@@ -85,13 +85,16 @@ extern void yyerror(char *s);
 static void terminate(void);
 static cmd_t *alloc_cmd(void);
 static attr_t *alloc_attr(void);
+static attr_t *alloc_lvar_attr(var_t *var);
 static attr_t *get_attr(cmd_t *cmd, int64_t name);
 static attr_t *get_attr_integer(cmd_t *cmd, int64_t name);
 static attr_t *get_attr_bool(cmd_t *cmd, int64_t name);
+static void get_attr_lvars(cmd_t *cmd, flowop_t *flowop);
 static var_t *alloc_var(void);
 static var_t *get_var(cmd_t *cmd, int64_t name);
 static list_t *alloc_list();
 static probtabent_t *alloc_probtabent(void);
+static void add_lvar_to_list(var_t *newlvar, var_t **lvar_list);
 
 /* Info Commands */
 static void parser_list(cmd_t *);
@@ -105,6 +108,7 @@ static void parser_file_define(cmd_t *);
 static void parser_fileset_define(cmd_t *);
 static void parser_randvar_define(cmd_t *);
 static void parser_randvar_set(cmd_t *);
+static void parser_composite_flowop_define(cmd_t *);
 
 /* Create Commands */
 static void parser_proc_create(cmd_t *);
@@ -179,6 +183,7 @@ static void parser_abort(int arg);
 %token FSA_DIRGAMMA FSA_USEISM FSA_TYPE FSA_RANDTABLE FSA_RANDSRC FSA_RANDROUND
 %token FSA_RANDSEED FSA_RANDGAMMA FSA_RANDMEAN FSA_RANDMIN
 %token FSS_TYPE FSS_SEED FSS_GAMMA FSS_MEAN FSS_MIN FSS_SRC FSS_ROUND
+%token FSV_SET_LOCAL_VAR FSA_LVAR_ASSIGN
 %token FSA_ALLDONE FSA_FIRSTDONE FSA_TIMEOUT
 
 %type <ival> FSV_VAL_INT
@@ -188,6 +193,7 @@ static void parser_abort(int arg);
 %type <sval> FSV_VARIABLE
 %type <sval> FSV_RANDVAR
 %type <sval> FSK_ASSIGN
+%type <sval> FSV_SET_LOCAL_VAR
 
 %type <ival> FSC_LIST FSC_DEFINE FSC_SET FSC_LOAD FSC_RUN
 %type <ival> FSE_FILE FSE_PROC FSE_THREAD FSE_CLEAR FSC_HELP
@@ -198,7 +204,7 @@ static void parser_abort(int arg);
 
 %type <cmd> command inner_commands load_command run_command list_command
 %type <cmd> proc_define_command files_define_command randvar_define_command
-%type <cmd> debug_command create_command
+%type <cmd> fo_define_command debug_command create_command
 %type <cmd> sleep_command stats_command set_command shutdown_command
 %type <cmd> foreach_command log_command system_command flowop_command
 %type <cmd> eventgen_command quit_command flowop_list thread_list
@@ -208,10 +214,11 @@ static void parser_abort(int arg);
 %type <attr> fo_attr_op fo_attr_ops ev_attr_op ev_attr_ops
 %type <attr> randvar_attr_op randvar_attr_ops randvar_attr_typop
 %type <attr> randvar_attr_srcop attr_value attr_list_value
+%type <attr> comp_lvar_def comp_attr_op comp_attr_ops
 %type <list> integer_seplist string_seplist string_list var_string_list
 %type <list> var_string whitevar_string whitevar_string_list
 %type <ival> attrs_define_file attrs_define_thread attrs_flowop
-%type <ival> attrs_define_fileset attrs_define_proc attrs_eventgen
+%type <ival> attrs_define_fileset attrs_define_proc attrs_eventgen attrs_define_comp
 %type <ival> files_attr_name pt_attr_name fo_attr_name ev_attr_name
 %type <ival> randvar_attr_name FSA_TYPE randtype_name randvar_attr_param
 %type <ival> randsrc_name FSA_RANDSRC randvar_attr_tsp
@@ -265,6 +272,7 @@ command:
   proc_define_command
 | files_define_command
 | randvar_define_command
+| fo_define_command
 | debug_command
 | eventgen_command
 | create_command
@@ -951,6 +959,19 @@ randvar_define_command: FSC_DEFINE FSE_RAND randvar_attr_ops
 	$$->cmd_attr_list = $3;
 };
 
+fo_define_command: FSC_DEFINE FSC_FLOWOP comp_attr_ops FSK_OPENLST flowop_list FSK_CLOSELST
+{
+	if (($$ = alloc_cmd()) == NULL)
+		YYERROR;
+	$$->cmd = &parser_composite_flowop_define;
+	$$->cmd_list = $5;
+	$$->cmd_attr_list = $3;
+}
+| fo_define_command comp_attr_ops
+{
+	$1->cmd_attr_list = $2;
+};
+
 create_command: FSC_CREATE entity
 {
 	if (($$ = alloc_cmd()) == NULL)
@@ -1251,6 +1272,19 @@ fo_attr_ops: fo_attr_op
 	list_end->attr_next = $3;
 
 	$$ = $1;
+}
+| fo_attr_ops FSK_SEPLST comp_lvar_def
+{
+	attr_t *attr = NULL;
+	attr_t *list_end = NULL;
+
+	for (attr = $1; attr != NULL;
+	    attr = attr->attr_next)
+		list_end = attr; /* Find end of list */
+
+	list_end->attr_next = $3;
+
+	$$ = $1;
 };
 
 fo_attr_op: fo_attr_name FSK_ASSIGN attr_value
@@ -1296,7 +1330,6 @@ ev_attr_op: ev_attr_name FSK_ASSIGN attr_value
 	$$->attr_name = $1;
 };
 
-
 files_attr_name: attrs_define_file
 |attrs_define_fileset;
 
@@ -1407,6 +1440,79 @@ attrs_flowop:
 attrs_eventgen:
   FSA_RATE { $$ = FSA_RATE;};
 
+comp_attr_ops: comp_attr_op
+{
+	$$ = $1;
+}
+| comp_attr_ops FSK_SEPLST comp_attr_op
+{
+	attr_t *attr = NULL;
+	attr_t *list_end = NULL;
+
+	for (attr = $1; attr != NULL;
+	    attr = attr->attr_next)
+		list_end = attr; /* Find end of list */
+
+	list_end->attr_next = $3;
+
+	$$ = $1;
+}
+| comp_attr_ops FSK_SEPLST comp_lvar_def
+{
+	attr_t *attr = NULL;
+	attr_t *list_end = NULL;
+
+	for (attr = $1; attr != NULL;
+	    attr = attr->attr_next)
+		list_end = attr; /* Find end of list */
+
+	list_end->attr_next = $3;
+
+	$$ = $1;
+};
+
+comp_attr_op: attrs_define_comp FSK_ASSIGN attr_value
+{
+	$$ = $3;
+	$$->attr_name = $1;
+};
+
+comp_lvar_def: FSV_VARIABLE FSK_ASSIGN FSV_VAL_BOOLEAN
+{
+	if (($$ = alloc_lvar_attr(var_lvar_assign_boolean($1, $3))) == NULL)
+		YYERROR;
+}
+| FSV_VARIABLE FSK_ASSIGN FSV_VAL_INT
+{
+	if (($$ = alloc_lvar_attr(var_lvar_assign_integer($1, $3))) == NULL)
+		YYERROR;
+}
+| FSV_VARIABLE FSK_ASSIGN FSK_QUOTE FSV_WHITESTRING FSK_QUOTE
+{
+	if (($$ = alloc_lvar_attr(var_lvar_assign_string($1, $4))) == NULL)
+		YYERROR;
+}
+| FSV_VARIABLE FSK_ASSIGN FSV_STRING
+{
+	if (($$ = alloc_lvar_attr(var_lvar_assign_string($1, $3))) == NULL)
+		YYERROR;
+}
+| FSV_VARIABLE FSK_ASSIGN FSV_VARIABLE
+{
+	if (($$ = alloc_lvar_attr(var_lvar_assign_var($1, $3))) == NULL)
+		YYERROR;
+}
+| FSV_VARIABLE
+{
+	if (($$ = alloc_lvar_attr(var_lvar_alloc_local($1))) == NULL)
+		YYERROR;
+};
+
+
+attrs_define_comp:
+  FSA_NAME { $$ = FSA_NAME;}
+| FSA_ITERS { $$ = FSA_ITERS;};
+
 attr_value: FSV_STRING
 {
 	if (($$ = alloc_attr()) == NULL)
@@ -2053,7 +2159,7 @@ parser_thread_define(cmd_t *cmd, procflow_t *procflow, int procinstances)
 	for (inner_cmd = cmd->cmd_list; inner_cmd != NULL;
 	    inner_cmd = inner_cmd->cmd_next) {
 		parser_flowop_define(inner_cmd, threadflow,
-		    &threadflow->tf_ops, FLOW_MASTER);
+		    &threadflow->tf_thrd_fops, FLOW_MASTER);
 	}
 }
 
@@ -2149,6 +2255,123 @@ parser_flowop_get_attrs(cmd_t *cmd, flowop_t *flowop)
 	}
 }
 
+/*
+ * defines the FLOW_MASTER flowops within a FLOW_MASTER instance of
+ * a composit flowop. Default attributes from the FLOW_INNER_DEF instances
+ * of the composit flowop's inner flowops are used if set. Otherwise
+ * default attributes from the FLOW_MASTER instance of the composit flowop
+ * are used, which may include defaults from the original FLOW_DEFINITION
+ * of the composit flowop.
+ */
+static void
+parser_inner_flowop_define(threadflow_t *thread, flowop_t *comp0_flow,
+			   flowop_t *comp_mstr_flow)
+{
+	flowop_t *inner_flowtype, *inner_flowop;
+
+	/* follow flowop list, creating composit names */
+	inner_flowtype = comp0_flow->fo_comp_fops;
+	comp_mstr_flow->fo_comp_fops = NULL;
+
+	while (inner_flowtype) {
+		char fullname[MAXPATHLEN];
+
+		/* create composite_name.name for new flowop */
+		(void) strlcpy(fullname, comp_mstr_flow->fo_name, MAXPATHLEN);
+		(void) strlcat(fullname, ".", MAXPATHLEN);
+		(void) strlcat(fullname, inner_flowtype->fo_name,
+		    MAXPATHLEN);
+
+		if ((inner_flowop = flowop_define(thread, fullname,
+		    inner_flowtype, &comp_mstr_flow->fo_comp_fops,
+		    FLOW_MASTER, 0)) == NULL) {
+			filebench_log(LOG_ERROR,
+			    "define flowop: Failed to instantiate flowop %s\n",
+			    fullname);
+			filebench_shutdown(1);
+		}
+
+		/* if applicable, update filename attribute */
+		if (inner_flowop->fo_filename) {
+			char *name;
+
+			/* fix up avd_t */
+			avd_update(&inner_flowop->fo_filename,
+			    comp_mstr_flow->fo_lvar_list);
+
+			/* see if ready to get the file or fileset */
+			name = avd_get_str(inner_flowop->fo_filename);
+			if (name) {
+
+				inner_flowop->fo_fileset = fileset_find(name);
+
+				if (inner_flowop->fo_fileset == NULL) {
+					filebench_log(LOG_ERROR,
+					    "inr flowop %s: file %s not found",
+					    inner_flowop->fo_name, name);
+					filebench_shutdown(1);
+				}
+			}
+		}
+
+		/* update attributes from local variables */
+		avd_update(&inner_flowop->fo_iters,
+		    comp_mstr_flow->fo_lvar_list);
+
+		/* if the inner flowop is a composit flowop, recurse */
+		if (inner_flowtype->fo_type == FLOW_TYPE_COMPOSITE) {
+			var_t *newlvar, *proto_lvars, *lvar_ptr;
+
+			proto_lvars = inner_flowop->fo_lvar_list;
+			inner_flowop->fo_lvar_list = 0;
+
+			for (lvar_ptr = inner_flowtype->fo_lvar_list; lvar_ptr;
+			    lvar_ptr = lvar_ptr->var_next) {
+
+				if ((newlvar = var_lvar_alloc_local(
+				    lvar_ptr->var_name)) != NULL) {
+
+					add_lvar_to_list(newlvar,
+					    &inner_flowop->fo_lvar_list);
+
+					var_update_comp_lvars(newlvar,
+					    proto_lvars,
+					    comp_mstr_flow->fo_lvar_list);
+				}
+			}
+		  
+			parser_inner_flowop_define(thread,
+			    inner_flowtype,
+			    inner_flowop);
+
+			inner_flowtype = inner_flowtype->fo_exec_next;
+			continue;
+		}
+
+		avd_update(&inner_flowop->fo_iosize,
+		    comp_mstr_flow->fo_lvar_list);
+		avd_update(&inner_flowop->fo_wss,
+		    comp_mstr_flow->fo_lvar_list);
+		avd_update(&inner_flowop->fo_iters,
+		    comp_mstr_flow->fo_lvar_list);
+		avd_update(&inner_flowop->fo_value,
+		    comp_mstr_flow->fo_lvar_list);
+		avd_update(&inner_flowop->fo_random,
+		    comp_mstr_flow->fo_lvar_list);
+		avd_update(&inner_flowop->fo_dsync,
+		    comp_mstr_flow->fo_lvar_list);
+		avd_update(&inner_flowop->fo_rotatefd,
+		    comp_mstr_flow->fo_lvar_list);
+		avd_update(&inner_flowop->fo_blocking,
+		    comp_mstr_flow->fo_lvar_list);
+		avd_update(&inner_flowop->fo_directio,
+		    comp_mstr_flow->fo_lvar_list);
+		avd_update(&inner_flowop->fo_highwater,
+		    comp_mstr_flow->fo_lvar_list);
+
+		inner_flowtype = inner_flowtype->fo_exec_next;
+	}
+}
 
 /*
  * Calls flowop_define() to allocate a flowop with the supplied name.
@@ -2206,22 +2429,75 @@ parser_flowop_define(cmd_t *cmd, threadflow_t *thread,
 	}
 
 	if ((flowop = flowop_define(thread, name,
-	    flowop_type, category, 0)) == NULL) {
+	    flowop_type, flowoplist_hdp, category, 0)) == NULL) {
+		filebench_log(LOG_ERROR,
+		    "define flowop: Failed to instantiate flowop %s\n",
+		    cmd->cmd_name);
+		filebench_shutdown(1);
+	}
+
+	/* Iterations */
+	if (attr = get_attr_integer(cmd, FSA_ITERS))
+		flowop->fo_iters = attr->attr_avd;
+	else
+		flowop->fo_iters = avd_int_alloc(1);
+
+
+	/* if this is a use of a composit flowop, create inner FLOW MASTERS */
+	if (flowop_type->fo_type == FLOW_TYPE_COMPOSITE) {
+		get_attr_lvars(cmd, flowop);
+		if (category == FLOW_MASTER)
+			parser_inner_flowop_define(thread,
+			    flowop_type, flowop);
+	}
+	else {
+		parser_flowop_get_attrs(cmd, flowop);
+	}
+}
+
+static void
+parser_composite_flowop_define(cmd_t *cmd)
+{
+	flowop_t *flowop;
+	cmd_t *inner_cmd;
+	char *name;
+	attr_t *attr;
+
+	/* Get the name of the flowop */
+	if (attr = get_attr(cmd, FSA_NAME)) {
+		name = avd_get_str(attr->attr_avd);
+	} else {
+		filebench_log(LOG_ERROR,
+		    "define flowop: Composit flowop specifies no name");
+
+		filebench_shutdown(1);
+	}
+
+	if ((flowop = flowop_new_composite_define(name)) == NULL) {
 		filebench_log(LOG_ERROR,
 		    "define flowop: Failed to instantiate flowop %s\n",
 		    cmd->cmd_name);
 		filebench_shutdown(1);
 	}
 
+	/* place any local var_t variables on the flowop's local list */
+	get_attr_lvars(cmd, flowop);
+
 	/* Iterations */
 	if (attr = get_attr_integer(cmd, FSA_ITERS))
 		flowop->fo_iters = attr->attr_avd;
 	else
 		flowop->fo_iters = avd_int_alloc(1);
 
-	parser_flowop_get_attrs(cmd, flowop);
+	/* define inner flowops */
+	for (inner_cmd = cmd->cmd_list; inner_cmd != NULL;
+	    inner_cmd = inner_cmd->cmd_next) {
+		parser_flowop_define(inner_cmd, NULL,
+		    &flowop->fo_comp_fops, FLOW_INNER_DEF);
+	}
 }
 
+
 /*
  * Calls fileset_define() to allocate a fileset with the supplied name and
  * initializes the fileset's pathname attribute, and optionally the
@@ -3252,8 +3528,6 @@ parser_randvar_set(cmd_t *cmd)
 		{
 			int disttype = (int)avd_get_int(value);
 
-			printf("parser_randvar_set: changing type to %d\n",
-			    disttype);
 			rndp->rnd_type &= (~RAND_TYPE_MASK);
 
 			switch (disttype) {
@@ -3274,9 +3548,6 @@ parser_randvar_set(cmd_t *cmd)
 		{
 			int randsrc = (int)avd_get_int(value);
 
-			printf("parser_randvar_set: changing source to %d\n",
-			    randsrc);
-
 			rndp->rnd_type &=
 			    (~(RAND_SRC_URANDOM | RAND_SRC_GENERATOR));
 
@@ -3351,7 +3622,7 @@ free_cmd(cmd_t *cmd)
  * a pointer to the attr_t.
  */
 static attr_t *
-alloc_attr()
+alloc_attr(void)
 {
 	attr_t *attr;
 
@@ -3381,6 +3652,24 @@ alloc_probtabent(void)
 }
 
 /*
+ * Allocates an attr_t structure and puts the supplied var_t into
+ * its attr_avd location, and sets its name to FSA_LVAR_ASSIGN
+ */
+static attr_t *
+alloc_lvar_attr(var_t *var)
+{
+	attr_t *attr;
+
+	if ((attr = alloc_attr()) == NULL)
+		return (NULL);
+
+	attr->attr_name = FSA_LVAR_ASSIGN;
+	attr->attr_avd = (avd_t)var;
+
+	return (attr);
+}
+
+/*
  * Searches the attribute list for the command for the named attribute type.
  * The attribute list is created by the parser from the list of attributes
  * supplied with certain commands, such as the define and flowop commands.
@@ -3490,6 +3779,66 @@ get_attr_bool(cmd_t *cmd, int64_t name)
 }
 
 /*
+ * removes the newly allocated local var from the shared local var
+ * list, then puts it at the head of the private local var list
+ * supplied as the second argument.
+ */
+static void
+add_lvar_to_list(var_t *newlvar, var_t **lvar_list)
+{
+	var_t *prev;
+
+	/* remove from shared local list, if there */
+	if (newlvar == filebench_shm->shm_var_loc_list) {
+		/* on top of list, just grap */
+		filebench_shm->shm_var_loc_list = newlvar->var_next;
+	} else {
+		/* find newvar on list and remove */
+		for (prev = filebench_shm->shm_var_loc_list; prev;
+		    prev = prev->var_next) {
+			if (prev->var_next == newlvar)
+				prev->var_next = newlvar->var_next;
+		}
+	}
+	newlvar->var_next = NULL;
+
+	/* add to flowop private local list at head */
+	newlvar->var_next = *lvar_list;
+	*lvar_list = newlvar;
+}
+
+/*
+ * Searches the attribute list for the command for any allocated local
+ * variables. The attribute list is created by the parser from the list of
+ * attributes supplied with certain commands, such as the define and flowop
+ * commands. Places all found local vars onto the flowop's local variable
+ * list. 
+ */
+static void
+get_attr_lvars(cmd_t *cmd, flowop_t *flowop)
+{
+	attr_t *attr;
+	var_t *list_tail, *orig_lvar_list;
+
+	/* save the local var list */
+	orig_lvar_list = flowop->fo_lvar_list;
+
+	for (attr = cmd->cmd_attr_list; attr != NULL;
+	    attr = attr->attr_next) {
+
+		if (attr->attr_name == FSA_LVAR_ASSIGN) {
+			var_t *newvar, *prev;
+
+			if ((newvar = (var_t *)attr->attr_avd) == NULL)
+				continue;
+
+			add_lvar_to_list(newvar, &flowop->fo_lvar_list);
+			var_update_comp_lvars(newvar, orig_lvar_list, NULL);
+		}
+	}
+}
+
+/*
  * Allocates memory for a list_t structure, initializes it to zero, and
  * returns a pointer to it. On failure, returns NULL.
  */
diff --git a/usr/src/cmd/filebench/common/threadflow.c b/usr/src/cmd/filebench/common/threadflow.c
index 6c99e339d9..d5c1965298 100644
--- a/usr/src/cmd/filebench/common/threadflow.c
+++ b/usr/src/cmd/filebench/common/threadflow.c
@@ -271,7 +271,7 @@ threadflow_delete(threadflow_t **threadlist, threadflow_t *threadflow,
 		    threadflow->tf_instance);
 
 		threadflow_kill(threadflow, wait_cnt);
-		flowop_delete_all(&threadflow->tf_ops);
+		flowop_delete_all(&threadflow->tf_thrd_fops);
 		*threadlist = threadflow->tf_next;
 		ipc_free(FILEBENCH_THREADFLOW, (char *)threadflow);
 		return (0);
@@ -292,7 +292,7 @@ threadflow_delete(threadflow_t **threadlist, threadflow_t *threadflow,
 			    entry->tf_next->tf_name,
 			    entry->tf_next->tf_instance);
 			threadflow_kill(entry->tf_next, wait_cnt);
-			flowop_delete_all(&entry->tf_next->tf_ops);
+			flowop_delete_all(&entry->tf_next->tf_thrd_fops);
 			ipc_free(FILEBENCH_THREADFLOW, (char *)threadflow);
 			entry->tf_next = entry->tf_next->tf_next;
 			return (0);
diff --git a/usr/src/cmd/filebench/common/threadflow.h b/usr/src/cmd/filebench/common/threadflow.h
index 119c368a1b..125280e816 100644
--- a/usr/src/cmd/filebench/common/threadflow.h
+++ b/usr/src/cmd/filebench/common/threadflow.h
@@ -82,7 +82,7 @@ typedef struct threadflow {
 	pthread_mutex_t	tf_lock;	/* Mutex around threadflow */
 	avd_t		tf_instances;	/* Number of instances for this flow */
 	struct threadflow *tf_next;	/* Next on proc list */
-	struct flowop	*tf_ops;	/* Flowop list */
+	struct flowop	*tf_thrd_fops;	/* Flowop list */
 	caddr_t		tf_mem;		/* Private Memory */
 	avd_t		tf_memsize;	/* Private Memory size attribute */
 	fbint_t		tf_constmemsize; /* constant copy of memory size */
diff --git a/usr/src/cmd/filebench/common/vars.c b/usr/src/cmd/filebench/common/vars.c
index 38fad4d540..cc3f3e6d32 100644
--- a/usr/src/cmd/filebench/common/vars.c
+++ b/usr/src/cmd/filebench/common/vars.c
@@ -69,6 +69,7 @@ static var_t *var_find_dynamic(char *name);
  * and integers of vars, and said components of avd_t.
  */
 
+
 /*
  * returns a pointer to a string indicating the type of data contained
  * in the supplied attribute variable descriptor.
@@ -460,6 +461,11 @@ avd_alloc_var_ptr(var_t *var)
 		avd->avd_val.randptr = var->var_val.randptr;
 		break;
 
+	case VAR_TYPE_INDVAR_SET:
+		avd->avd_type = AVD_IND_VAR;
+		avd->avd_val.varptr = var->var_val.varptr;
+		break;
+
 	default:
 		avd->avd_type = AVD_IND_VAR;
 		avd->avd_val.varptr = var;
@@ -500,7 +506,7 @@ avd_str_alloc(char *string)
 /*
  * Allocates a var (var_t) from interprocess shared memory.
  * Places the allocated var on the end of the globally shared
- * var_list. Finally, the routine allocates a string containing
+ * shm_var_list. Finally, the routine allocates a string containing
  * a copy of the supplied "name" string. If any allocations
  * fails, returns NULL, otherwise it returns a pointer to the
  * newly allocated var.
@@ -535,6 +541,12 @@ var_alloc_cmn(char *name, int var_type)
 		var_listp = &filebench_shm->shm_var_dyn_list;
 		break;
 
+	case VAR_TYPE_LOCAL:
+		/* place on head of shared local list */
+		newvar->var_next = filebench_shm->shm_var_loc_list;
+		filebench_shm->shm_var_loc_list = newvar;
+		return (newvar);
+
 	default:
 		var_listp = &filebench_shm->shm_var_list;
 		break;
@@ -552,9 +564,25 @@ var_alloc_cmn(char *name, int var_type)
 }
 
 /*
+ * Allocates a var (var_t) from interprocess shared memory after
+ * first adjusting the name to elminate the leading $. Places the
+ * allocated var temporarily on the end of the globally
+ * shared var_loc_list. If the allocation fails, returns NULL,
+ * otherwise it returns a pointer to the newly allocated var.
+ */
+var_t *
+var_lvar_alloc_local(char *name)
+{
+	if (name[0] == '$')
+		name += 1;
+
+	return (var_alloc_cmn(name, VAR_TYPE_LOCAL));
+}
+
+/*
  * Allocates a var (var_t) from interprocess shared memory and
  * places the allocated var on the end of the globally shared
- * var_list. If the allocation fails, returns NULL, otherwise
+ * shm_var_list. If the allocation fails, returns NULL, otherwise
  * it returns a pointer to the newly allocated var.
  */
 static var_t *
@@ -566,7 +594,7 @@ var_alloc(char *name)
 /*
  * Allocates a var (var_t) from interprocess shared memory.
  * Places the allocated var on the end of the globally shared
- * var_dyn_list. If the allocation fails, returns NULL, otherwise
+ * shm_var_dyn_list. If the allocation fails, returns NULL, otherwise
  * it returns a pointer to the newly allocated var.
  */
 static var_t *
@@ -576,15 +604,22 @@ var_alloc_dynamic(char *name)
 }
 
 /*
- * Searches for var_t with name "name" in the master var_list.
- * If successful, returns a pointer to the var_t, otherwise
- * returns NULL.
+ * Searches for var_t with name "name" in the shm_var_loc_list,
+ * then, if not found, in the global shm_var_list. If a matching
+ * local or global var is found, returns a pointer to the var_t,
+ * otherwise returns NULL.
  */
 static var_t *
 var_find(char *name)
 {
 	var_t *var;
 
+	for (var = filebench_shm->shm_var_loc_list; var != NULL;
+	    var = var->var_next) {
+		if (strcmp(var->var_name, name) == 0)
+			return (var);
+	}
+
 	for (var = filebench_shm->shm_var_list; var != NULL;
 	    var = var->var_next) {
 		if (strcmp(var->var_name, name) == 0)
@@ -595,6 +630,40 @@ var_find(char *name)
 }
 
 /*
+ * Searches for var_t with name "name" in the supplied shm_var_list.
+ * If not found there, checks the global list. If still
+ * unsuccessful, returns NULL. Otherwise returns a pointer to the var_t.
+ */
+static var_t *
+var_find_list_only(char *name, var_t *var_list)
+{
+	var_t *var;
+
+	for (var = var_list; var != NULL; var = var->var_next) {
+		if (strcmp(var->var_name, name) == 0)
+			return (var);
+	}
+
+	return (NULL);
+}
+
+/*
+ * Searches for var_t with name "name" in the supplied shm_var_list.
+ * If not found there, checks the global list. If still
+ * unsuccessful, returns NULL. Otherwise returns a pointer to the var_t.
+ */
+static var_t *
+var_find_list(char *name, var_t *var_list)
+{
+	var_t *var;
+
+	if ((var = var_find_list_only(name, var_list)) != NULL)
+		return (var);
+	else
+		return (var_find(name));
+}
+
+/*
  * Searches for the named var, and, if found, sets its
  * var_val.boolean's value to that of the supplied boolean.
  * If not found, the routine allocates a new var and sets
@@ -959,13 +1028,64 @@ var_randvar_to_string(char *name, int param_name)
 }
 
 /*
+ * Copies the value stored in the source string into the destination
+ * string. Returns -1 if any problems encountered, 0 otherwise.
+ */
+static int
+var_copy(var_t *dst_var, var_t *src_var) {
+
+	if (VAR_HAS_BOOLEAN(src_var)) {
+		VAR_SET_BOOL(dst_var, src_var->var_val.boolean);
+		filebench_log(LOG_DEBUG_SCRIPT,
+		    "Assign var %s=%s", dst_var->var_name,
+		    dst_var->var_val.boolean?"true":"false");
+	}
+
+	if (VAR_HAS_INTEGER(src_var)) {
+		VAR_SET_INT(dst_var, src_var->var_val.integer);
+		filebench_log(LOG_DEBUG_SCRIPT,
+		    "Assign var %s=%llu", dst_var->var_name,
+		    (u_longlong_t)dst_var->var_val.integer);
+	}
+
+	if (VAR_HAS_DOUBLE(src_var)) {
+		VAR_SET_DBL(dst_var, src_var->var_val.dbl_flt);
+		filebench_log(LOG_DEBUG_SCRIPT,
+		    "Assign var %s=%lf", dst_var->var_name,
+		    dst_var->var_val.dbl_flt);
+	}
+
+	if (VAR_HAS_STRING(src_var)) {
+		char *strptr;
+
+		if ((strptr =
+		    ipc_stralloc(src_var->var_val.string)) == NULL) {
+			filebench_log(LOG_ERROR,
+			    "Cannot assign string for variable %s",
+			    dst_var->var_name);
+			return (-1);
+		}
+		VAR_SET_STR(dst_var, strptr);
+		filebench_log(LOG_DEBUG_SCRIPT,
+		    "Assign var %s=%s", dst_var->var_name,
+		    dst_var->var_val.string);
+	}
+
+	if (VAR_HAS_INDVAR(src_var)) {
+		VAR_SET_INDVAR(dst_var, src_var->var_val.varptr);
+		filebench_log(LOG_DEBUG_SCRIPT,
+		    "Assign var %s to var %s", dst_var->var_name,
+		    src_var->var_name);
+	}
+	return (0);
+}
+
+/*
  * Searches for the var named "name", and if not found
- * allocates it. The then extracts the var_string from
- * the var named "string" and copies it into the var_string
- * of the var "name", after first allocating a piece of
- * interprocess shared string memory. If the var "name"
- * cannot be found or allocated, or the var "string" cannot
- * be found, the routine returns -1, otherwise it returns 0.
+ * allocates it. The then copies the value from
+ * the src_var into the destination var "name"
+ * If the var "name" cannot be found or allocated, or the var "src_name"
+ * cannot be found, the routine returns -1, otherwise it returns 0.
  */
 int
 var_assign_var(char *name, char *src_name)
@@ -996,40 +1116,7 @@ var_assign_var(char *name, char *src_name)
 		return (-1);
 	}
 
-	if (VAR_HAS_BOOLEAN(src_var)) {
-		VAR_SET_BOOL(dst_var, src_var->var_val.boolean);
-		filebench_log(LOG_VERBOSE,
-		    "Assign var %s=%d", name, src_var->var_val.boolean);
-	}
-
-	if (VAR_HAS_INTEGER(src_var)) {
-		VAR_SET_INT(dst_var, src_var->var_val.integer);
-		filebench_log(LOG_VERBOSE,
-		    "Assign var %s=%llu",
-		    name, (u_longlong_t)src_var->var_val.integer);
-	}
-
-	if (VAR_HAS_DOUBLE(src_var)) {
-		VAR_SET_DBL(dst_var, src_var->var_val.dbl_flt);
-		filebench_log(LOG_VERBOSE,
-		    "Assign var %s=%lf", name, src_var->var_val.dbl_flt);
-	}
-
-	if (VAR_HAS_STRING(src_var)) {
-		char *strptr;
-
-		if ((strptr =
-		    ipc_stralloc(src_var->var_val.string)) == NULL) {
-			filebench_log(LOG_ERROR,
-			    "Cannot assign variable %s",
-			    name);
-			return (-1);
-		}
-		VAR_SET_STR(dst_var, strptr);
-		filebench_log(LOG_VERBOSE,
-		    "Assign var %s=%s", name, src_var->var_val.string);
-	}
-	return (0);
+	return (var_copy(dst_var, src_var));
 }
 
 /*
@@ -1039,7 +1126,7 @@ var_assign_var(char *name, char *src_name)
  * before the copy. Space for the string in the var comes
  * from interprocess shared memory. If the var "name"
  * cannot be found or allocated, or the memory for the
- * var_string copy of "string" cannot be allocated, the
+ * var_val.string copy of "string" cannot be allocated, the
  * routine returns -1, otherwise it returns 0.
  */
 int
@@ -1079,6 +1166,197 @@ var_assign_string(char *name, char *string)
 }
 
 /*
+ * Allocates a local var. The then extracts the var_string from
+ * the var named "string" and copies it into the var_string
+ * of the var "name", after first allocating a piece of
+ * interprocess shared string memory. Returns a pointer to the
+ * newly allocated local var or NULL on error.
+ */
+var_t *
+var_lvar_assign_var(char *name, char *src_name)
+{
+	var_t *dst_var, *src_var;
+
+	src_name += 1;
+
+	if ((src_var = var_find(src_name)) == NULL) {
+		filebench_log(LOG_ERROR,
+		    "Cannot find source variable %s", src_name);
+		return (NULL);
+	}
+
+	dst_var = var_lvar_alloc_local(name);
+
+	if (dst_var == NULL) {
+		filebench_log(LOG_ERROR, "Cannot assign variable %s",
+		    name);
+		return (NULL);
+	}
+
+	/*
+	 * if referencing another local var which is currently
+	 * empty, indirect to it
+	 */
+	if ((src_var->var_type & VAR_TYPE_MASK) == VAR_TYPE_LOCAL) {
+		VAR_SET_INDVAR(dst_var, src_var);
+		filebench_log(LOG_DEBUG_SCRIPT,
+		    "Assign local var %s to %s", name, src_name);
+		return (dst_var);
+	}
+
+	if (VAR_HAS_BOOLEAN(src_var)) {
+		VAR_SET_BOOL(dst_var, src_var->var_val.boolean);
+		filebench_log(LOG_DEBUG_SCRIPT,
+		    "Assign var (%s, %p)=%s", name,
+		    dst_var, src_var->var_val.boolean?"true":"false");
+	} else if (VAR_HAS_INTEGER(src_var)) {
+		VAR_SET_INT(dst_var, src_var->var_val.integer);
+		filebench_log(LOG_DEBUG_SCRIPT,
+		    "Assign var (%s, %p)=%llu", name,
+		    dst_var, (u_longlong_t)src_var->var_val.integer);
+	} else if (VAR_HAS_STRING(src_var)) {
+		char *strptr;
+
+		if ((strptr = ipc_stralloc(src_var->var_val.string)) == NULL) {
+			filebench_log(LOG_ERROR,
+			    "Cannot assign variable %s",
+			    name);
+			return (NULL);
+		}
+		VAR_SET_STR(dst_var, strptr);
+		filebench_log(LOG_DEBUG_SCRIPT,
+		    "Assign var (%s, %p)=%s", name,
+		    dst_var, src_var->var_val.string);
+	} else if (VAR_HAS_DOUBLE(src_var)) {
+		/* LINTED E_ASSIGMENT_CAUSE_LOSS_PREC */
+		VAR_SET_INT(dst_var, src_var->var_val.dbl_flt);
+		filebench_log(LOG_DEBUG_SCRIPT,
+		    "Assign var (%s, %p)=%8.2f", name,
+		    dst_var, src_var->var_val.dbl_flt);
+	} else if (VAR_HAS_RANDDIST(src_var)) {
+		VAR_SET_RAND(dst_var, src_var->var_val.randptr);
+		filebench_log(LOG_DEBUG_SCRIPT,
+		    "Assign var (%s, %p)=%llu", name,
+		    dst_var, (u_longlong_t)src_var->var_val.integer);
+	}
+
+	return (dst_var);
+}
+
+/*
+ * the routine allocates a new local var and sets
+ * its var_boolean's value to that of the supplied
+ * boolean. It returns a pointer to the new local var
+ */
+var_t *
+var_lvar_assign_boolean(char *name, boolean_t bool)
+{
+	var_t *var;
+
+	var = var_lvar_alloc_local(name);
+
+	if (var == NULL) {
+		filebench_log(LOG_ERROR, "Cannot assign variable %s",
+		    name);
+		return (NULL);
+	}
+
+	VAR_SET_BOOL(var, bool);
+
+	filebench_log(LOG_DEBUG_SCRIPT, "Assign integer %s=%s",
+	    name, bool ? "true" : "false");
+
+	return (var);
+}
+
+/*
+ * the routine allocates a new local var and sets
+ * its var_integers's value to that of the supplied
+ * integer. It returns a pointer to the new local var
+ */
+var_t *
+var_lvar_assign_integer(char *name, fbint_t integer)
+{
+	var_t *var;
+
+	var = var_lvar_alloc_local(name);
+
+	if (var == NULL) {
+		filebench_log(LOG_ERROR, "Cannot assign variable %s",
+		    name);
+		return (NULL);
+	}
+
+	VAR_SET_INT(var, integer);
+
+	filebench_log(LOG_DEBUG_SCRIPT, "Assign integer %s=%llu",
+	    name, (u_longlong_t)integer);
+
+	return (var);
+}
+
+/*
+ * the routine allocates a new local var and sets
+ * its var_dbl_flt value to that of the supplied
+ * double precission floating point number. It returns
+ * a pointer to the new local var
+ */
+var_t *
+var_lvar_assign_double(char *name, double dbl)
+{
+	var_t *var;
+
+	var = var_lvar_alloc_local(name);
+
+	if (var == NULL) {
+		filebench_log(LOG_ERROR, "Cannot assign variable %s",
+		    name);
+		return (NULL);
+	}
+
+	VAR_SET_DBL(var, dbl);
+
+	filebench_log(LOG_DEBUG_SCRIPT, "Assign integer %s=%8.2f", name, dbl);
+
+	return (var);
+}
+
+/*
+ * Like var_lvar_assign_integer, only this routine copies the
+ * supplied "string" into the var named "name". If the var
+ * named "name" cannot be found then it is first allocated
+ * before the copy. Space for the string in the var comes
+ * from interprocess shared memory. The allocated local var
+ * is returned at as a char *, or NULL on error.
+ */
+var_t *
+var_lvar_assign_string(char *name, char *string)
+{
+	var_t *var;
+	char *strptr;
+
+	var = var_lvar_alloc_local(name);
+
+	if (var == NULL) {
+		filebench_log(LOG_ERROR, "Cannot assign variable %s",
+		    name);
+		return (NULL);
+	}
+
+	if ((strptr = ipc_stralloc(string)) == NULL) {
+		filebench_log(LOG_ERROR, "Cannot assign variable %s",
+		    name);
+		return (NULL);
+	}
+	VAR_SET_STR(var, strptr);
+
+	filebench_log(LOG_DEBUG_SCRIPT,
+	    "Lvar_assign_string (%s, %p)=%s", name, var, string);
+
+	return (var);
+}
+
+/*
  * Tests to see if the supplied variable name without the portion after
  * the last period is that of a random variable. If it is, it returns
  * the number of characters to backspace to skip the period and field
@@ -1198,7 +1476,7 @@ var_find_environment(var_t *var)
  * Look up special variables. The "name" argument is used to find
  * the desired special var and fill it with an appropriate string
  * value. Looks for an already allocated var of the same name on
- * the var_dyn_list. If not found a new dynamic var is allocated.
+ * the shm_var_dyn_list. If not found a new dynamic var is allocated.
  * if the name begins with '{', it is an internal variable, and
  * var_find_internal() is called. If the name begins with '(' it
  * is an environment varable, and var_find_environment() is
@@ -1248,3 +1526,74 @@ var_find_dynamic(char *name)
 
 	return (NULL);
 }
+
+/*
+ * replace the avd_t attribute value descriptor in the new FLOW_MASTER flowop
+ * that points to a local variable with a new avd_t containing
+ * the actual value from the local variable.
+ */
+void
+avd_update(avd_t *avdp, var_t *lvar_list)
+{
+	var_t *old_lvar, *new_lvar;
+
+	if ((*avdp)->avd_type == AVD_IND_VAR) {
+
+		/* Make sure there is a local var */
+		if ((old_lvar = (*avdp)->avd_val.varptr) == NULL) {
+			filebench_log(LOG_ERROR,
+			    "avd_update: local var not found");
+			return;
+		}
+	} else {
+		/* Empty or not indirect, so no update needed */
+		return;
+	}
+
+	/*  allocate a new avd using the new or old lvar contents */
+	if ((new_lvar =
+	    var_find_list(old_lvar->var_name, lvar_list)) != NULL)
+		(*avdp) = avd_alloc_var_ptr(new_lvar);
+	else
+		(*avdp) = avd_alloc_var_ptr(old_lvar);
+}
+
+void
+var_update_comp_lvars(var_t *newlvar, var_t *proto_comp_vars,
+    var_t *mstr_lvars)
+{
+	var_t *proto_lvar;
+
+	/* find the prototype lvar from the inherited list */
+	proto_lvar = var_find_list_only(newlvar->var_name, proto_comp_vars);
+
+	if (proto_lvar == NULL)
+		return;
+
+	/*
+	 * if the new local variable has not already been assigned
+	 * a value, try to copy a value from the prototype local variable
+	 */
+	if ((newlvar->var_type & VAR_TYPE_SET_MASK) == 0) {
+
+		/* copy value from prototype lvar to new lvar */
+		(void) var_copy(newlvar, proto_lvar);
+	}
+
+	/* If proto lvar is indirect, see if we can colapse indirection */
+	if (VAR_HAS_INDVAR(proto_lvar)) {
+		var_t *uplvp;
+
+		uplvp = (var_t *)proto_lvar->var_val.varptr;
+
+		/* search for more current uplvar on comp master list */
+		if (mstr_lvars) {
+			uplvp = var_find_list_only(
+			    uplvp->var_name, mstr_lvars);
+			VAR_SET_INDVAR(newlvar, uplvp);
+		}
+
+		if (VAR_HAS_INDVAR(uplvp))
+			VAR_SET_INDVAR(newlvar, uplvp->var_val.varptr);
+	}
+}
diff --git a/usr/src/cmd/filebench/common/vars.h b/usr/src/cmd/filebench/common/vars.h
index af12a801fc..ede8dca983 100644
--- a/usr/src/cmd/filebench/common/vars.h
+++ b/usr/src/cmd/filebench/common/vars.h
@@ -87,18 +87,21 @@ typedef struct var {
 		double		dbl_flt;
 		char		*string;
 		struct randdist *randptr;
+		struct var	*varptr;
 	} var_val;
 } var_t;
 
 #define	VAR_TYPE_GLOBAL		0x00	/* global variable */
 #define	VAR_TYPE_DYNAMIC	0x01	/* Dynamic variable */
 #define	VAR_TYPE_RANDOM		0x02	/* random variable */
+#define	VAR_TYPE_LOCAL		0x03	/* Local variable */
 #define	VAR_TYPE_MASK		0x0f
 #define	VAR_TYPE_BOOL_SET	0x10	/* var contains a boolean */
 #define	VAR_TYPE_INT_SET	0x20	/* var contains an integer */
 #define	VAR_TYPE_STR_SET	0x30	/* var contains a string */
 #define	VAR_TYPE_DBL_SET	0x40	/* var contains a double */
 #define	VAR_TYPE_RAND_SET	0x50	/* var contains a randdist pointer */
+#define	VAR_TYPE_INDVAR_SET	0x60    /* var points to another local var */
 #define	VAR_TYPE_SET_MASK	0xf0
 
 #define	VAR_HAS_BOOLEAN(vp) \
@@ -116,6 +119,9 @@ typedef struct var {
 #define	VAR_HAS_RANDDIST(vp) \
 	(((vp)->var_type & VAR_TYPE_SET_MASK) == VAR_TYPE_RAND_SET)
 
+#define	VAR_HAS_INDVAR(vp) \
+	(((vp)->var_type & VAR_TYPE_SET_MASK) == VAR_TYPE_INDVAR_SET)
+
 #define	VAR_SET_BOOL(vp, val)	\
 	{			\
 		(vp)->var_val.boolean = (val); \
@@ -134,41 +140,62 @@ typedef struct var {
 	{			\
 		(vp)->var_val.dbl_flt = (val); \
 		(vp)->var_type = \
-		(((vp)->var_type & (~VAR_TYPE_SET_MASK)) | VAR_TYPE_DBL_SET); \
+		    (((vp)->var_type & (~VAR_TYPE_SET_MASK)) | \
+		    VAR_TYPE_DBL_SET); \
 	}
 
 #define	VAR_SET_STR(vp, val)	\
 	{			\
 		(vp)->var_val.string = (val); \
 		(vp)->var_type = \
-		(((vp)->var_type & (~VAR_TYPE_SET_MASK)) | VAR_TYPE_STR_SET); \
+		    (((vp)->var_type & (~VAR_TYPE_SET_MASK)) | \
+		    VAR_TYPE_STR_SET); \
 	}
 
 #define	VAR_SET_RAND(vp, val)	\
 	{			\
 		(vp)->var_val.randptr = (val); \
 		(vp)->var_type = \
-		(((vp)->var_type & (~VAR_TYPE_SET_MASK)) | VAR_TYPE_RAND_SET);\
+		    (((vp)->var_type & (~VAR_TYPE_SET_MASK)) | \
+		    VAR_TYPE_RAND_SET); \
+	}
+
+#define	VAR_SET_INDVAR(vp, val)	\
+	{			\
+		(vp)->var_val.varptr = (val); \
+		(vp)->var_type = \
+		    (((vp)->var_type & (~VAR_TYPE_SET_MASK)) | \
+		    VAR_TYPE_INDVAR_SET); \
 	}
 
 avd_t avd_bool_alloc(boolean_t bool);
 avd_t avd_int_alloc(fbint_t integer);
 avd_t avd_str_alloc(char *string);
+boolean_t avd_get_bool(avd_t);
+fbint_t avd_get_int(avd_t);
+double avd_get_dbl(avd_t);
+char *avd_get_str(avd_t);
+void avd_update(avd_t *avdp, var_t *lvar_list);
 avd_t var_ref_attr(char *name);
 int var_assign_boolean(char *name, boolean_t bool);
 int var_assign_integer(char *name, fbint_t integer);
+int var_assign_double(char *name, double dbl);
 int var_assign_string(char *name, char *string);
 int var_assign_var(char *name, char *string);
+void var_update_comp_lvars(var_t *newlvar, var_t *proto_comp_vars,
+    var_t *mstr_lvars);
 var_t *var_define_randvar(char *name);
 var_t *var_find_randvar(char *name);
 boolean_t var_to_boolean(char *name);
 fbint_t var_to_integer(char *name);
+var_t *var_lvar_alloc_local(char *name);
+var_t *var_lvar_assign_boolean(char *name, boolean_t);
+var_t *var_lvar_assign_integer(char *name, fbint_t);
+var_t *var_lvar_assign_double(char *name, double);
+var_t *var_lvar_assign_string(char *name, char *string);
+var_t *var_lvar_assign_var(char *name, char *src_name);
 char *var_to_string(char *name);
 char *var_randvar_to_string(char *name, int param);
-boolean_t avd_get_bool(avd_t);
-fbint_t avd_get_int(avd_t);
-double avd_get_dbl(avd_t);
-char *avd_get_str(avd_t);
 int var_is_set4_randvar(char *name);
 
 #ifdef	__cplusplus