6708713 FileBench needs to run in multi-client mode

13 files changed, 1161 insertions, 284 deletions

diff --git a/usr/src/cmd/filebench/Makefile.com b/usr/src/cmd/filebench/Makefile.com
index dbd3114a43..80b75065c9 100644
--- a/usr/src/cmd/filebench/Makefile.com
+++ b/usr/src/cmd/filebench/Makefile.com
@@ -22,7 +22,6 @@
 # Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
-#ident	"%Z%%M%	%I%	%E% SMI"
 
 .KEEP_STATE:
 
@@ -39,6 +38,7 @@ SRCS = \
 	gamma_dist.c \
         ipc.c \
         misc.c \
+	multi_client_sync.c \
         procflow.c \
         stats.c \
         threadflow.c \
@@ -58,7 +58,7 @@ CLEANFILES += parser_gram.c parser_gram.h parser_lex.c y.tab.h y.tab.c
 CPPFLAGS += -I. -I../common
 CFLAGS += $(CCVERBOSE) $(CTF_FLAGS)
 CFLAGS64 += $(CCVERBOSE) $(CTF_FLAGS)
-LDLIBS += -lkstat -lm -ltecla
+LDLIBS += -lkstat -lm -ltecla -lsocket -lnsl
 
 LFLAGS = -t -v
 YFLAGS = -d
diff --git a/usr/src/cmd/filebench/common/filebench.h b/usr/src/cmd/filebench/common/filebench.h
index e0610a0478..d31df501c2 100644
--- a/usr/src/cmd/filebench/common/filebench.h
+++ b/usr/src/cmd/filebench/common/filebench.h
@@ -117,7 +117,7 @@ void filebench_shutdown(int error);
 #define	MIN(x, y) ((x) < (y) ? (x) : (y))
 #endif
 
-#define	FILEBENCH_VERSION	"1.3.4"
+#define	FILEBENCH_VERSION	"1.4.0"
 #define	FILEBENCHDIR	"/usr/benchmarks/filebench"
 #define	FILEBENCH_PROMPT	"filebench> "
 #define	MAX_LINE_LEN	1024
diff --git a/usr/src/cmd/filebench/common/fileset.c b/usr/src/cmd/filebench/common/fileset.c
index be81898860..99ee2887c9 100644
--- a/usr/src/cmd/filebench/common/fileset.c
+++ b/usr/src/cmd/filebench/common/fileset.c
@@ -429,7 +429,7 @@ fileset_alloc_thread(filesetentry_t *entry)
  */
 int
 fileset_openfile(fileset_t *fileset,
-    filesetentry_t *entry, int flag, int mode, int attrs)
+    filesetentry_t *entry, int flag, int filemode, int attrs)
 {
 	char path[MAXPATHLEN];
 	char dir[MAXPATHLEN];
@@ -462,7 +462,7 @@ fileset_openfile(fileset_t *fileset,
 #endif
 	}
 
-	if ((fd = open64(path, flag | open_attrs, mode)) < 0) {
+	if ((fd = open64(path, flag | open_attrs, filemode)) < 0) {
 		filebench_log(LOG_ERROR,
 		    "Failed to open file %s: %s",
 		    path, strerror(errno));
@@ -747,7 +747,7 @@ fileset_create(fileset_t *fileset)
 	char *fileset_name;
 	int randno;
 	int preallocated = 0;
-	int reusing = 0;
+	int reusing;
 
 	if ((fileset_path = avd_get_str(fileset->fs_path)) == NULL) {
 		filebench_log(LOG_ERROR, "%s path not set",
@@ -769,34 +769,47 @@ fileset_create(fileset_t *fileset)
 
 	/* XXX Add check to see if there is enough space */
 
-	/* Remove existing */
+	/* set up path to fileset */
 	(void) strcpy(path, fileset_path);
 	(void) strcat(path, "/");
 	(void) strcat(path, fileset_name);
-	if ((stat64(path, &sb) == 0) && (strlen(path) > 3) &&
-	    (strlen(avd_get_str(fileset->fs_path)) > 2)) {
-		if (!avd_get_bool(fileset->fs_reuse)) {
-			char cmd[MAXPATHLEN];
-
-			(void) snprintf(cmd, sizeof (cmd), "rm -rf %s", path);
-			(void) system(cmd);
-			filebench_log(LOG_VERBOSE,
-			    "Removed any existing %s %s in %llu seconds",
-			    fileset_entity_name(fileset), fileset_name,
-			    (u_longlong_t)(((gethrtime() - start) /
-			    1000000000) + 1));
-		} else {
-			/* we are re-using */
-			reusing = 1;
-			filebench_log(LOG_VERBOSE, "Re-using %s %s.",
-			    fileset_entity_name(fileset), fileset_name);
-		}
+
+	/* if exists and resusing, then don't create new */
+	if (((stat64(path, &sb) == 0)&& (strlen(path) > 3) &&
+	    (strlen(avd_get_str(fileset->fs_path)) > 2)) &&
+	    avd_get_bool(fileset->fs_reuse)) {
+		reusing = 1;
+	} else {
+		reusing = 0;
 	}
-	(void) mkdir(path, 0755);
 
-	/* make the filesets directory tree */
-	if (fileset_create_subdirs(fileset, path) == FILEBENCH_ERROR)
-		return (FILEBENCH_ERROR);
+	if (!reusing) {
+		char cmd[MAXPATHLEN];
+
+		/* Remove existing */
+		(void) snprintf(cmd, sizeof (cmd), "rm -rf %s", path);
+		(void) system(cmd);
+		filebench_log(LOG_VERBOSE,
+		    "Removed any existing %s %s in %llu seconds",
+		    fileset_entity_name(fileset), fileset_name,
+		    (u_longlong_t)(((gethrtime() - start) /
+		    1000000000) + 1));
+	} else {
+		/* we are re-using */
+		filebench_log(LOG_VERBOSE, "Re-using %s %s.",
+		    fileset_entity_name(fileset), fileset_name);
+	}
+
+	/* make the filesets directory tree unless in reuse mode */
+	if (!reusing && (avd_get_bool(fileset->fs_prealloc))) {
+		filebench_log(LOG_INFO,
+		    "making tree for filset %s", path);
+
+		(void) mkdir(path, 0755);
+
+		if (fileset_create_subdirs(fileset, path) == FILEBENCH_ERROR)
+			return (FILEBENCH_ERROR);
+	}
 
 	start = gethrtime();
 
@@ -811,13 +824,16 @@ fileset_create(fileset_t *fileset)
 
 	while (entry = fileset_pick(fileset, pickflags, 0)) {
 		pthread_t tid;
+		int newrand;
 
 		pickflags = FILESET_PICKUNIQUE;
 
 		/* entry doesn't need to be locked during initialization */
 		fileset_unbusy(entry, FALSE, FALSE);
 
-		if (rand() < randno)
+		newrand = rand();
+
+		if (newrand < randno)
 			continue;
 
 		preallocated++;
diff --git a/usr/src/cmd/filebench/common/multi_client_sync.c b/usr/src/cmd/filebench/common/multi_client_sync.c
new file mode 100644
index 0000000000..459f18229f
--- /dev/null
+++ b/usr/src/cmd/filebench/common/multi_client_sync.c
@@ -0,0 +1,111 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+
+#include "filebench.h"
+#include "multi_client_sync.h"
+#include <netdb.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <errno.h>
+
+#define	MCS_NAMELENGTH	128
+#define	MCS_MSGLENGTH	(MCS_NAMELENGTH * 8)
+
+static int mc_sync_sock_id;
+static char this_client_name[MCS_NAMELENGTH];
+
+/*
+ * Open a socket to the master synchronization host
+ */
+int
+mc_sync_open_sock(char *master_name, int master_port, char *my_name)
+{
+	struct sockaddr_in client_in;
+	struct sockaddr_in master_in;
+	struct hostent master_info;
+	int error_num;
+	char buffer[MCS_MSGLENGTH];
+
+	(void) strncpy(this_client_name, my_name, MCS_NAMELENGTH);
+	if ((mc_sync_sock_id = socket(PF_INET, SOCK_STREAM, 0)) == -1) {
+		filebench_log(LOG_ERROR, "could not create a client socket");
+		return (FILEBENCH_ERROR);
+	}
+
+	client_in.sin_family = AF_INET;
+	client_in.sin_port = INADDR_ANY;
+	client_in.sin_addr.s_addr = INADDR_ANY;
+
+	if (bind(mc_sync_sock_id, (struct sockaddr *)&client_in,
+	    sizeof (client_in)) == -1) {
+		filebench_log(LOG_ERROR, "could not bind to client socket");
+		return (FILEBENCH_ERROR);
+	}
+
+	if (gethostbyname_r(master_name, &master_info, buffer, MCS_MSGLENGTH,
+	    &error_num) == NULL) {
+		filebench_log(LOG_ERROR, "could not locate sync master");
+		return (FILEBENCH_ERROR);
+	}
+
+	master_in.sin_family = AF_INET;
+	master_in.sin_port = htons((uint16_t)master_port);
+	(void) memcpy(&master_in.sin_addr.s_addr, *master_info.h_addr_list,
+	    sizeof (master_in.sin_addr.s_addr));
+
+	if (connect(mc_sync_sock_id, (struct sockaddr *)&master_in,
+	    sizeof (master_in)) == -1) {
+		filebench_log(LOG_ERROR,
+		    "connection refused to sync master, error %d", errno);
+		return (FILEBENCH_ERROR);
+	}
+
+	return (FILEBENCH_OK);
+}
+
+/*
+ * Send a synchronization message and wait for a reply
+ */
+int
+mc_sync_synchronize(int sync_point)
+{
+	char msg[MCS_MSGLENGTH];
+	int amnt;
+
+	(void) snprintf(msg, MCS_MSGLENGTH,
+	    "cmd=SYNC,id=xyzzy,name=%s,sample=%d\n",
+	    this_client_name, sync_point);
+	(void) send(mc_sync_sock_id, msg, strlen(msg), 0);
+
+	amnt = 0;
+	msg[0] = 0;
+
+	while (strchr(msg, '\n') == NULL)
+		amnt += recv(mc_sync_sock_id, msg, sizeof (msg), 0);
+
+	filebench_log(LOG_INFO, "sync point %d succeeded!\n", sync_point);
+	return (FILEBENCH_OK);
+}
diff --git a/usr/src/cmd/filebench/common/multi_client_sync.h b/usr/src/cmd/filebench/common/multi_client_sync.h
new file mode 100644
index 0000000000..5c3961f474
--- /dev/null
+++ b/usr/src/cmd/filebench/common/multi_client_sync.h
@@ -0,0 +1,36 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_MULTI_CLIENT_SYNC_H
+#define	_MULTI_CLIENT_SYNC_H
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <inet/ip.h>
+
+int mc_sync_open_sock(char *master_name, int master_port, char *client_name);
+int mc_sync_synchronize(int synch_point);
+
+#endif /* _MULTI_CLIENT_SYNC_H */
diff --git a/usr/src/cmd/filebench/common/parser_gram.y b/usr/src/cmd/filebench/common/parser_gram.y
index a58ee5557e..7e4510cf9d 100644
--- a/usr/src/cmd/filebench/common/parser_gram.y
+++ b/usr/src/cmd/filebench/common/parser_gram.y
@@ -26,10 +26,6 @@
  */
 
 %{
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-%}
-
-%{
 
 #include <stdlib.h>
 #include <stdio.h>
@@ -57,6 +53,7 @@
 #ifdef HAVE_LIBTECLA
 #include "auto_comp.h"
 #endif
+#include "multi_client_sync.h"
 
 int dofile = FS_FALSE;
 static const char cmdname[] = "filebench";
@@ -135,6 +132,7 @@ static void parser_log(cmd_t *cmd);
 static void parser_statscmd(cmd_t *cmd);
 static void parser_statsdump(cmd_t *cmd);
 static void parser_statsxmldump(cmd_t *cmd);
+static void parser_statsmultidump(cmd_t *cmd);
 static void parser_echo(cmd_t *cmd);
 static void parser_usage(cmd_t *cmd);
 static void parser_vars(cmd_t *cmd);
@@ -143,6 +141,8 @@ static void parser_system(cmd_t *cmd);
 static void parser_statssnap(cmd_t *cmd);
 static void parser_directory(cmd_t *cmd);
 static void parser_eventgen(cmd_t *cmd);
+static void parser_enable_mc(cmd_t *cmd);
+static void parser_domultisync(cmd_t *cmd);
 static void parser_run(cmd_t *cmd);
 static void parser_run_variable(cmd_t *cmd);
 static void parser_help(cmd_t *cmd);
@@ -169,12 +169,13 @@ static void parser_version(cmd_t *cmd);
 %token FSC_LIST FSC_DEFINE FSC_EXEC FSC_QUIT FSC_DEBUG FSC_CREATE
 %token FSC_SLEEP FSC_STATS FSC_FOREACH FSC_SET FSC_SHUTDOWN FSC_LOG
 %token FSC_SYSTEM FSC_FLOWOP FSC_EVENTGEN FSC_ECHO FSC_LOAD FSC_RUN
-%token FSC_USAGE FSC_HELP FSC_VARS FSC_VERSION
+%token FSC_USAGE FSC_HELP FSC_VARS FSC_VERSION FSC_ENABLE FSC_DOMULTISYNC
 %token FSV_STRING FSV_VAL_INT FSV_VAL_BOOLEAN FSV_VARIABLE FSV_WHITESTRING
 %token FSV_RANDUNI FSV_RANDTAB FSV_RANDVAR FSV_URAND FSV_RAND48
 %token FST_INT FST_BOOLEAN
 %token FSE_FILE FSE_PROC FSE_THREAD FSE_CLEAR FSE_ALL FSE_SNAP FSE_DUMP
 %token FSE_DIRECTORY FSE_COMMAND FSE_FILESET FSE_XMLDUMP FSE_RAND FSE_MODE
+%token FSE_MULTI FSE_MULTIDUMP
 %token FSK_SEPLST FSK_OPENLST FSK_CLOSELST FSK_ASSIGN FSK_IN FSK_QUOTE
 %token FSK_DIRSEPLST
 %token FSA_SIZE FSA_PREALLOC FSA_PARALLOC FSA_PATH FSA_REUSE
@@ -184,7 +185,8 @@ static void parser_version(cmd_t *cmd);
 %token FSA_HIGHWATER FSA_DIRECTIO FSA_DIRWIDTH FSA_FD FSA_SRCFD FSA_ROTATEFD
 %token FSA_NAMELENGTH FSA_FILESIZE FSA_ENTRIES FSA_FILESIZEGAMMA FSA_DIRDEPTHRV
 %token FSA_DIRGAMMA FSA_USEISM FSA_TYPE FSA_RANDTABLE FSA_RANDSRC FSA_RANDROUND
-%token FSA_RANDSEED FSA_RANDGAMMA FSA_RANDMEAN FSA_RANDMIN
+%token FSA_RANDSEED FSA_RANDGAMMA FSA_RANDMEAN FSA_RANDMIN FSA_MASTER
+%token FSA_CLIENT
 %token FSS_TYPE FSS_SEED FSS_GAMMA FSS_MEAN FSS_MIN FSS_SRC FSS_ROUND
 %token FSV_SET_LOCAL_VAR FSA_LVAR_ASSIGN
 %token FSA_ALLDONE FSA_FIRSTDONE FSA_TIMEOUT
@@ -198,7 +200,8 @@ static void parser_version(cmd_t *cmd);
 %type <sval> FSK_ASSIGN
 %type <sval> FSV_SET_LOCAL_VAR
 
-%type <ival> FSC_LIST FSC_DEFINE FSC_SET FSC_LOAD FSC_RUN
+%type <ival> FSC_LIST FSC_DEFINE FSC_SET FSC_LOAD FSC_RUN FSC_ENABLE
+%type <ival> FSC_DOMULTISYNC
 %type <ival> FSE_FILE FSE_PROC FSE_THREAD FSE_CLEAR FSC_HELP FSC_VERSION
 
 %type <sval> name
@@ -212,20 +215,21 @@ static void parser_version(cmd_t *cmd);
 %type <cmd> foreach_command log_command system_command flowop_command
 %type <cmd> eventgen_command quit_command flowop_list thread_list
 %type <cmd> thread echo_command usage_command help_command vars_command
-%type <cmd> version_command
+%type <cmd> version_command enable_command multisync_command
 
 %type <attr> files_attr_op files_attr_ops pt_attr_op pt_attr_ops
 %type <attr> fo_attr_op fo_attr_ops ev_attr_op ev_attr_ops
 %type <attr> randvar_attr_op randvar_attr_ops randvar_attr_typop
 %type <attr> randvar_attr_srcop attr_value attr_list_value
 %type <attr> comp_lvar_def comp_attr_op comp_attr_ops
+%type <attr> enable_multi_ops enable_multi_op multisync_op
 %type <list> integer_seplist string_seplist string_list var_string_list
 %type <list> var_string whitevar_string whitevar_string_list
 %type <ival> attrs_define_file attrs_define_thread attrs_flowop
 %type <ival> attrs_define_fileset attrs_define_proc attrs_eventgen attrs_define_comp
 %type <ival> files_attr_name pt_attr_name fo_attr_name ev_attr_name
 %type <ival> randvar_attr_name FSA_TYPE randtype_name randvar_attr_param
-%type <ival> randsrc_name FSA_RANDSRC randvar_attr_tsp
+%type <ival> randsrc_name FSA_RANDSRC randvar_attr_tsp em_attr_name
 %type <ival> FSS_TYPE FSS_SEED FSS_GAMMA FSS_MEAN FSS_MIN FSS_SRC
 
 %type <rndtb>  probtabentry_list probtabentry
@@ -295,6 +299,8 @@ command:
 | stats_command
 | system_command
 | version_command
+| enable_command
+| multisync_command
 | quit_command;
 
 foreach_command: FSC_FOREACH
@@ -454,6 +460,27 @@ vars_command: FSC_VARS
 	$$->cmd = parser_printvars;
 };
 
+enable_command: FSC_ENABLE FSE_MULTI
+{
+	if (($$ = alloc_cmd()) == NULL)
+		YYERROR;
+
+	$$->cmd = parser_enable_mc;
+}
+| enable_command  enable_multi_ops
+{
+	$1->cmd_attr_list = $2;
+};
+
+multisync_command: FSC_DOMULTISYNC multisync_op
+{
+	if (($$ = alloc_cmd()) == NULL)
+		YYERROR;
+
+	$$->cmd = parser_domultisync;
+	$$->cmd_attr_list = $2;
+}
+
 string_list: FSV_VARIABLE
 {
 	if (($$ = alloc_list()) == NULL)
@@ -870,6 +897,13 @@ stats_command: FSC_STATS FSE_SNAP
 
 	$$->cmd_param_list = $3;
 	$$->cmd = parser_statsxmldump;
+}| FSC_STATS FSE_MULTIDUMP whitevar_string_list
+{
+	if (($$ = alloc_cmd()) == NULL)
+		YYERROR;
+
+	$$->cmd_param_list = $3;
+	$$->cmd = parser_statsmultidump;
 };
 
 quit_command: FSC_QUIT
@@ -1342,6 +1376,37 @@ ev_attr_op: ev_attr_name FSK_ASSIGN attr_value
 	$$->attr_name = $1;
 };
 
+/* attribute parsing for enable multiple client command */
+enable_multi_ops: enable_multi_op
+{
+	$$ = $1;
+}
+| enable_multi_ops FSK_SEPLST enable_multi_op
+{
+	attr_t *attr = NULL;
+	attr_t *list_end = NULL;
+
+	for (attr = $1; attr != NULL;
+	    attr = attr->attr_next)
+		list_end = attr; /* Find end of list */
+
+	list_end->attr_next = $3;
+
+	$$ = $1;
+};
+
+enable_multi_op: em_attr_name FSK_ASSIGN attr_value
+{
+	$$ = $3;
+	$$->attr_name = $1;
+}
+
+multisync_op: FSA_VALUE FSK_ASSIGN attr_value
+{
+	$$ = $3;
+	$$->attr_name = FSA_VALUE;
+}
+
 files_attr_name: attrs_define_file
 |attrs_define_fileset;
 
@@ -1452,6 +1517,10 @@ attrs_flowop:
 attrs_eventgen:
   FSA_RATE { $$ = FSA_RATE;};
 
+em_attr_name:
+  FSA_MASTER { $$ = FSA_MASTER;};
+| FSA_CLIENT { $$ = FSA_CLIENT;};
+
 comp_attr_ops: comp_attr_op
 {
 	$$ = $1;
@@ -2555,6 +2624,27 @@ parser_fileset_define_common(cmd_t *cmd)
 
 	fileset->fs_path = pathname;
 
+	/* How much should we preallocate? */
+	if ((attr = get_attr_integer(cmd, FSA_PREALLOC)) &&
+	    attr->attr_avd) {
+		if (AVD_IS_RANDOM(attr->attr_avd)) {
+			filebench_log(LOG_ERROR,
+			    "define fileset: Prealloc attr cannot be random");
+			filebench_shutdown(1);
+		}
+		fileset->fs_preallocpercent = attr->attr_avd;
+	} else if (attr && !attr->attr_avd) {
+		fileset->fs_preallocpercent = avd_int_alloc(100);
+	} else {
+		fileset->fs_preallocpercent = avd_int_alloc(0);
+	}
+
+	/* Should we preallocate? */
+	if (attr = get_attr_bool(cmd, FSA_PREALLOC))
+		fileset->fs_prealloc = attr->attr_avd;
+	else
+		fileset->fs_prealloc = avd_bool_alloc(FALSE);
+
 	/* Should we prealloc in parallel? */
 	if (attr = get_attr_bool(cmd, FSA_PARALLOC))
 		fileset->fs_paralloc = attr->attr_avd;
@@ -2562,21 +2652,21 @@ parser_fileset_define_common(cmd_t *cmd)
 		fileset->fs_paralloc = avd_bool_alloc(FALSE);
 
 	/* Should we reuse the existing file? */
-	if (attr = get_attr_bool(cmd, FSA_REUSE)) {
+	if (attr = get_attr_bool(cmd, FSA_REUSE))
 		fileset->fs_reuse = attr->attr_avd;
-	} else
+	else
 		fileset->fs_reuse = avd_bool_alloc(FALSE);
 
 	/* Should we leave in cache? */
-	if (attr = get_attr_bool(cmd, FSA_CACHED)) {
+	if (attr = get_attr_bool(cmd, FSA_CACHED))
 		fileset->fs_cached = attr->attr_avd;
-	} else
+	else
 		fileset->fs_cached = avd_bool_alloc(FALSE);
 
 	/* Get the mean or absolute size of the file */
-	if (attr = get_attr_integer(cmd, FSA_SIZE)) {
+	if (attr = get_attr_integer(cmd, FSA_SIZE))
 		fileset->fs_size = attr->attr_avd;
-	} else
+	else
 		fileset->fs_size = avd_int_alloc(0);
 
 	return (fileset);
@@ -2584,9 +2674,8 @@ parser_fileset_define_common(cmd_t *cmd)
 
 /*
  * Calls parser_fileset_define_common() to allocate a fileset with
- * one entry and optionally the fileset_prealloc. Sets the
- * fileset_preallocpercent, fileset_entries, fileset_dirwidth,
- * fileset_dirgamma, and fileset_sizegamma attributes
+ * one entry and optionally the fileset_prealloc. sets the fileset_entries,
+ * fileset_dirwidth, fileset_dirgamma, and fileset_sizegamma attributes
  * to appropriate values for emulating the old "fileobj" entity
  */
 static void
@@ -2614,17 +2703,6 @@ parser_file_define(cmd_t *cmd)
 	/* Set the dir and size gammas to 0 */
 	fileset->fs_dirgamma = avd_int_alloc(0);
 	fileset->fs_sizegamma = avd_int_alloc(0);
-
-	/* Does file need to be preallocated? */
-	if (attr = get_attr_bool(cmd, FSA_PREALLOC)) {
-		/* yes */
-		fileset->fs_prealloc = attr->attr_avd;
-		fileset->fs_preallocpercent = avd_int_alloc(100);
-	} else {
-		/* no */
-		fileset->fs_prealloc = avd_bool_alloc(FALSE);
-		fileset->fs_preallocpercent = avd_int_alloc(0);
-	}
 }
 
 /*
@@ -2645,28 +2723,6 @@ parser_fileset_define(cmd_t *cmd)
 		filebench_shutdown(1);
 		return;
 	}
-
-	/* How much should we preallocate? */
-	if ((attr = get_attr_integer(cmd, FSA_PREALLOC)) &&
-	    attr->attr_avd) {
-		if (AVD_IS_RANDOM(attr->attr_avd)) {
-			filebench_log(LOG_ERROR,
-			    "define fileset: Prealloc attr cannot be random");
-			filebench_shutdown(1);
-		}
-		fileset->fs_preallocpercent = attr->attr_avd;
-	} else if (attr && !attr->attr_avd) {
-		fileset->fs_preallocpercent = avd_int_alloc(100);
-	} else {
-		fileset->fs_preallocpercent = avd_int_alloc(0);
-	}
-
-	/* Should we preallocate? */
-	if (attr = get_attr_bool(cmd, FSA_PREALLOC)) {
-		fileset->fs_prealloc = attr->attr_avd;
-	} else
-		fileset->fs_prealloc = avd_bool_alloc(FALSE);
-
 	/* Get the number of files in the fileset */
 	if (attr = get_attr_integer(cmd, FSA_ENTRIES)) {
 		fileset->fs_entries = attr->attr_avd;
@@ -2954,6 +3010,52 @@ parser_printvars(cmd_t *cmd)
 }
 
 /*
+ * Establishes multi-client synchronization socket with synch server.
+ */
+static void
+parser_enable_mc(cmd_t *cmd)
+{
+	attr_t *attr;
+	char *master;
+	char *client;
+
+	if (attr= get_attr(cmd, FSA_MASTER)) {
+		master = avd_get_str(attr->attr_avd);
+	} else {
+		filebench_log(LOG_ERROR,
+		    "enable multi: no master specified");
+		return;
+	}
+
+	if (attr= get_attr(cmd, FSA_CLIENT)) {
+		client = avd_get_str(attr->attr_avd);
+	} else {
+		filebench_log(LOG_ERROR,
+		    "enable multi: no client specified");
+		return;
+	}
+
+	mc_sync_open_sock(master, 8001, client);
+}
+
+/*
+ * Exchanges multi-client synchronization message with synch server.
+ */
+static void
+parser_domultisync(cmd_t *cmd)
+{
+	attr_t *attr;
+	fbint_t value;
+
+	if (attr = get_attr(cmd, FSA_VALUE))
+		value = avd_get_int(attr->attr_avd);
+	else
+		value = 1;
+
+	mc_sync_synchronize((int)value);
+}
+
+/*
  * Used by the SET command to add a var and default value string to the
  * varstr string. It allocates a new, larger varstr string, copies the
  * old contents of varstr into it, then adds the new var string on the end.
@@ -3332,6 +3434,30 @@ parser_statsdump(cmd_t *cmd)
 }
 
 /*
+ * Same as statsdump, but outputs in a computer friendly format.
+ */
+static void
+parser_statsmultidump(cmd_t *cmd)
+{
+	char *string;
+
+	if (cmd->cmd_param_list == NULL)
+		return;
+
+	string = parser_list2string(cmd->cmd_param_list);
+
+	if (string == NULL)
+		return;
+
+	filebench_log(LOG_VERBOSE,
+	    "Stats dump to file '%s'", string);
+
+	stats_multidump(string);
+
+	free(string);
+}
+
+/*
  * Same as parser_statsdump, but in xml format.
  */
 static void
diff --git a/usr/src/cmd/filebench/common/parser_lex.l b/usr/src/cmd/filebench/common/parser_lex.l
index 67e7dd82a0..5485899877 100644
--- a/usr/src/cmd/filebench/common/parser_lex.l
+++ b/usr/src/cmd/filebench/common/parser_lex.l
@@ -27,10 +27,6 @@
  */
 
 %{
-#pragma	ident	"%Z%%M%	%I%	%E% SMI"
-%}
-
-%{
 
 #include <stdlib.h>
 #include <sys/types.h>
@@ -70,7 +66,9 @@ eventgen                { return FSC_EVENTGEN; }
 create                  { return FSC_CREATE; }
 define			{ return FSC_DEFINE; }
 debug                   { return FSC_DEBUG; }
+domultisync		{ return FSC_DOMULTISYNC; }
 echo                    { return FSC_ECHO; }
+enable			{ return FSC_ENABLE; }
 exit		        { return FSC_QUIT; }
 foreach                 { return FSC_FOREACH; }
 flowop		        { return FSC_FLOWOP; }
@@ -100,14 +98,15 @@ clear                   { return FSE_CLEAR; }
 snap                    { return FSE_SNAP; }
 dump                    { return FSE_DUMP; }
 xmldump                 { return FSE_XMLDUMP; }
+multidump               { return FSE_MULTIDUMP; }
 all                     { return FSE_ALL; }
 mode                    { return FSE_MODE; }
+multi			{ return FSE_MULTI; }
 
 cached                  { return FSA_CACHED; }
 dirwidth                { return FSA_DIRWIDTH; }
 dirdepthrv              { return FSA_DIRDEPTHRV; }
 dirgamma                { return FSA_DIRGAMMA; }
-namelength              { return FSA_NAMELENGTH; }
 filesize                { return FSA_SIZE; }
 filesizegamma           { return FSA_FILESIZEGAMMA; }
 directio                { return FSA_DIRECTIO; }
@@ -120,8 +119,11 @@ filesetname             { return FSA_FILE; }
 instances               { return FSA_INSTANCES;}                  
 iosize                  { return FSA_IOSIZE; }
 iters                   { return FSA_ITERS;}
+master			{ return FSA_MASTER; }
+client			{ return FSA_CLIENT; }
 memsize                 { return FSA_MEMSIZE; }
 name                    { return FSA_NAME;}
+namelength              { return FSA_NAMELENGTH; }
 nice                    { return FSA_NICE;}
 entries                 { return FSA_ENTRIES;}
 prealloc                { return FSA_PREALLOC; }
diff --git a/usr/src/cmd/filebench/common/stats.c b/usr/src/cmd/filebench/common/stats.c
index 36edc7d3da..42236d3bcc 100644
--- a/usr/src/cmd/filebench/common/stats.c
+++ b/usr/src/cmd/filebench/common/stats.c
@@ -23,8 +23,6 @@
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include "config.h"
 
 #include <stdio.h>
@@ -547,7 +545,7 @@ stats_snap(void)
 	free(str);
 
 	filebench_log(LOG_INFO,
-	    "\nIO Summary:      %5d ops %5.1lf ops/s, (%0.0lf/%0.0lf r/w) "
+	    "\nIO Summary:      %5d ops, %5.1lf ops/s, (%0.0lf/%0.0lf r/w) "
 	    "%5.1lfmb/s, %6.0fus cpu/op, %5.1fms latency",
 	    iostat->fs_count + aiostat->fs_count,
 	    (iostat->fs_count + aiostat->fs_count) /
@@ -762,6 +760,85 @@ stats_xmldump(char *filename)
 }
 
 /*
+ * same as stats_dump, but in computer friendly format
+ */
+void
+stats_multidump(char *filename)
+{
+	flowstat_t *iostat = &globalstats[FLOW_TYPE_IO];
+	flowstat_t *aiostat = &globalstats[FLOW_TYPE_AIO];
+	flowop_t *flowop;
+
+	/* don't dump stats if run ended in error */
+	if (filebench_shm->shm_f_abort == FILEBENCH_ABORT_ERROR)
+		return;
+
+	(void) strcpy(filebench_shm->shm_dump_filename, filename);
+
+	filebench_log(LOG_INFO, "in statsmultidump %s", filename);
+
+	if (filebench_shm->shm_dump_fd > 0) {
+		(void) close(filebench_shm->shm_dump_fd);
+		filebench_shm->shm_dump_fd = -1;
+	}
+
+	filebench_log(LOG_DUMP, "Flowop totals:");
+
+	flowop = filebench_shm->shm_flowoplist;
+	while (flowop) {
+
+		if (flowop->fo_instance != FLOW_MASTER) {
+			flowop = flowop->fo_next;
+			continue;
+		}
+
+		filebench_log(LOG_DUMP,
+		    "%s\t%1.0lf\t%1.1lf\t%1.1f\t%1.0f",
+		    flowop->fo_name,
+		    flowop->fo_stats.fs_count /
+		    ((globalstats->fs_etime - globalstats->fs_stime) / FSECS),
+		    (flowop->fo_stats.fs_bytes / (1024 * 1024)) /
+		    ((globalstats->fs_etime - globalstats->fs_stime) / FSECS),
+		    flowop->fo_stats.fs_count ?
+		    flowop->fo_stats.fs_mstate[FLOW_MSTATE_LAT] /
+		    (flowop->fo_stats.fs_count * 1000000.0) : 0,
+		    flowop->fo_stats.fs_count ?
+		    flowop->fo_stats.fs_mstate[FLOW_MSTATE_CPU] /
+		    (flowop->fo_stats.fs_count * 1000.0) : 0);
+
+		flowop = flowop->fo_next;
+	}
+
+	filebench_log(LOG_DUMP, "");
+	filebench_log(LOG_DUMP,
+	    "IO Summary:\n%d\t%1.1lf\t%1.0lf\t%1.0lf\t%1.1lf\t%1.0f\t%1.1f\n",
+
+	    iostat->fs_count + aiostat->fs_count,
+
+	    (iostat->fs_count + aiostat->fs_count) /
+	    ((globalstats->fs_etime - globalstats->fs_stime) / FSECS),
+
+	    (iostat->fs_rcount + aiostat->fs_rcount) /
+	    ((globalstats->fs_etime - globalstats->fs_stime) / FSECS),
+
+	    (iostat->fs_wcount + aiostat->fs_wcount) /
+	    ((globalstats->fs_etime - globalstats->fs_stime) / FSECS),
+
+	    ((iostat->fs_bytes + aiostat->fs_bytes) / (1024 * 1024)) /
+	    ((globalstats->fs_etime - globalstats->fs_stime) / FSECS),
+
+	    (iostat->fs_rcount + iostat->fs_wcount +
+	    aiostat->fs_rcount + aiostat->fs_wcount) ?
+	    (iostat->fs_syscpu / 1000.0) /
+	    (iostat->fs_rcount + iostat->fs_wcount +
+	    aiostat->fs_rcount + aiostat->fs_wcount) : 0,
+
+	    (iostat->fs_rcount + iostat->fs_wcount) ?
+	    iostat->fs_mstate[FLOW_MSTATE_LAT] /
+	    ((iostat->fs_rcount + iostat->fs_wcount) * 1000000.0) : 0);
+}
+
+/*
  * Clears all the statistics variables (fo_stats) for every defined flowop.
  * It also creates a global flowstat table if one doesn't already exist and
  * clears it.
diff --git a/usr/src/cmd/filebench/common/stats.h b/usr/src/cmd/filebench/common/stats.h
index 917b3a584f..346a4c05a3 100644
--- a/usr/src/cmd/filebench/common/stats.h
+++ b/usr/src/cmd/filebench/common/stats.h
@@ -19,15 +19,13 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef	_FB_STATS_H
 #define	_FB_STATS_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include "config.h"
 #ifdef HAVE_STDINT_H
 #include <stdint.h>
@@ -43,6 +41,7 @@ void stats_clear(void);
 void stats_snap(void);
 void stats_dump(char *filename);
 void stats_xmldump(char *filename);
+void stats_multidump(char *filename);
 
 #ifndef HAVE_HRTIME
 /* typedef uint64_t hrtime_t; */
diff --git a/usr/src/cmd/filebench/config/Makefile b/usr/src/cmd/filebench/config/Makefile
index 462bee5394..5697c531bf 100644
--- a/usr/src/cmd/filebench/config/Makefile
+++ b/usr/src/cmd/filebench/config/Makefile
@@ -19,17 +19,16 @@
 # CDDL HEADER END
 #
 #
-# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
-# ident	"%Z%%M%	%I%	%E% SMI"
 
 .KEEP_STATE:
 
 include ../../Makefile.cmd
 
 CONFIGS = fileio.prof filemacro.prof filemicro.prof generic.func \
-seqread.prof randomread.prof
+seqread.prof randomread.prof multi_fileserver.prof
 ROOTUSRBENCHDIR = $(ROOT)/usr/benchmarks
 ROOTUSRBENCHFBCONFIGDIR = $(ROOTUSRBENCHDIR)/filebench/config
 FBCONFIGS = $(CONFIGS:%=$(ROOTUSRBENCHFBCONFIGDIR)/%)
diff --git a/usr/src/cmd/filebench/config/generic.func b/usr/src/cmd/filebench/config/generic.func
index b0ae54d402..4bc82c9a0e 100644
--- a/usr/src/cmd/filebench/config/generic.func
+++ b/usr/src/cmd/filebench/config/generic.func
@@ -19,10 +19,9 @@
 # CDDL HEADER END
 #
 #
-# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
-# ident	"%Z%%M%	%I%	%E% SMI"
 
 sub pre_run {
         # Initialize filebench to appropriate personality, create files
@@ -43,8 +42,7 @@ sub pre_run {
 
 sub post_run {
         my $statsbase = get_STATSBASE();
-        # Shutdown processes and quit filebench
-        op_quit();
+
         # Create a html summary of the run
 	system ("cd $statsbase; " . get_FILEBENCH() . "/scripts/filebench_compare $statsbase")
 }
diff --git a/usr/src/cmd/filebench/config/multi_fileserver.prof b/usr/src/cmd/filebench/config/multi_fileserver.prof
new file mode 100644
index 0000000000..f5fee327c0
--- /dev/null
+++ b/usr/src/cmd/filebench/config/multi_fileserver.prof
@@ -0,0 +1,52 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+#
+# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+# Example multi-client fileserver workload. Three clients named "client1",
+# "client2" and "client3" access one file server whose shared directory is
+# mounted on each client under the pathname "/theserver". This will run the
+# fileserver workload on each of the clients, using seperate filesets for
+# each server.
+
+MULTICLIENT {
+	targetpath = /theserver;
+	clients = client1, client2, client3;
+}
+
+DEFAULTS {
+	runtime = 60;
+        dir = /tmp;
+        stats = /tmp;
+	filesystem = nfs;
+	description = "fileserver nfs";
+}
+
+CONFIG fileserver {
+	function = generic;
+	personality = fileserver;
+	nfiles = 1000;
+	meandirwidth = 20;
+	filesize = 16k;
+	nthreads = 1;
+	meaniosize = 2k;
+}
diff --git a/usr/src/cmd/filebench/fbscript/filebench.pl b/usr/src/cmd/filebench/fbscript/filebench.pl
index ee20cd55d4..20e71774c3 100755
--- a/usr/src/cmd/filebench/fbscript/filebench.pl
+++ b/usr/src/cmd/filebench/fbscript/filebench.pl
@@ -20,22 +20,31 @@
 # CDDL HEADER END
 #
 #
-# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
-# ident	"%Z%%M%	%I%	%E% SMI"
-#
 
 use POSIX;
+use Socket;
 
-my $QUIT = 0;
+my $MULTI_CLIENT = 0;
 my $USE_XANADU = 0;
 my $TIMEOUT = 60;
+my $EOL = "\n";
 my $FILEBENCH = "/usr/benchmarks/filebench";
 my $PROG = "/usr/benchmarks/filebench/bin/go_filebench";
+my $SHAREDFILEALLOCATOR;
+my $TARGETPATH;
+my $TARGETDIR;
+my $FB_MASTERPATH;
+my $STATSBASE;
+my $CONFNAME;
 my $FSCRIPT;
 my $SCRIPT_NO;
+my @CLIENTLIST = ();
+my %CLIENTHASH = ();
 my @CONFLIST;
+my %MULTIDATA = ();
 my %DEFDATA = ();
 my %CONFDATA = ();
 my %STATSHASH = ();
@@ -66,8 +75,24 @@ sub get_CONFNAME {
     return ($CONFNAME);
 }
 
-sub get_CONFNAME {
-    return ($CONFNAME);
+sub multi_putval {
+    my ($key) = shift;
+    my ($val) = shift;
+    @{MULTIDATA{$key}} = ();
+    push(@{ $MULTIDATA{$key} }, $val);
+}
+
+sub multi_getval {
+    my ($key) = shift;
+    return ("@{$MULTIDATA{$key}}");
+}
+
+sub multi_exists {
+    my ($key) = shift;
+    if (exists($MULTIDATA{$key})) {
+	return (1);
+    }
+    return (0);
 }
 
 sub conf_getval {
@@ -106,18 +131,24 @@ sub op_init {
 
 sub op_load {
     my ($workload) = shift;
-    my $scriptname = conf_reqval("statsdir") . "/thisrun.f";
-    if($workload ne '') {
-      	   open (FSCRIPT, ">$scriptname");
-      	   chmod (0755, $scriptname);
-      	   print FSCRIPT "#!$PROG -f\n\n";
-           # Load the df
-           print FSCRIPT "load $workload\n";
-           # Load the user defined defaults
-           op_load_defaults();
+    $scriptname = conf_reqval("statsdir") . "/thisrun.f";
 
-           # Create the associated files and filesets
-           print FSCRIPT "create filesets\n";	
+    if($workload ne '') {
+	print ("Creating Client Script " . $scriptname . "\n");
+	open (FSCRIPT, ">$scriptname");
+	chmod (0755, $scriptname);
+	print FSCRIPT "#!$PROG -f\n\n";
+	# Load the df
+	print FSCRIPT "load $workload\n";
+	# Load the user defined defaults
+	op_load_defaults();
+
+	# enable multiclient, if needed
+	if ($MULTI_CLIENT == 1) {
+	    print FSCRIPT "enable multi master=".multi_getval("masterhost").", client=".conf_getval("myname")."\n";
+	}
+	# Create the associated files and filesets
+	print FSCRIPT "create filesets\n";	
 
     }
     $SCRIPT_NO = 1;
@@ -161,18 +192,12 @@ sub op_msg {
 }
 
 sub op_quit {
-    if($QUIT) {
-	# Shutdown the appropriate processes
-	print FSCRIPT "shutdown processes\n";
+    # Shutdown the appropriate processes
+    print FSCRIPT "shutdown processes\n";
 
-	# Quit filebench
-        print FSCRIPT "quit\n";
-	close(FSCRIPT);
-	print "Running " . conf_reqval("statsdir") . "/thisrun.f\n";
-	system (conf_reqval("statsdir") . "/thisrun.f");
-    } else {
-        print STDOUT "ERROR: pre-mature call to op_quit\n";
-    }
+    # Quit filebench
+    print FSCRIPT "quit\n";
+    close(FSCRIPT);
 }
 
 sub op_statsdir {
@@ -209,145 +234,156 @@ sub op_indiv_stats {
 sub op_stats {
     my ($time) = shift;
     my ($statsfile) = shift;
+    my $mstrstatsdir = $STATSBASE."/".$CONFNAME;
+
+    if ($MULTI_CLIENT == 1) {
+	print FSCRIPT "domultisync value=1\n";
+    }
 
-        # Create the associated processes and start them running
-        print FSCRIPT "create processes\n";
-
-	if (($time ne '') && ($statsfile ne '')) {
-	    # Clear the current statistics buffers
-	    print FSCRIPT "stats clear\n";
-
-	    # Start external statistics collection (if any)
-	    # Note all statistics arrays MUST be the same length !
-	    if (@ext_stats != ()) {
-	    	if (($#ext_stats == $#file_stats) && ($#ext_stats == $#arg_stats)) {
-		        $script = conf_reqval("statsdir") . "/stats$SCRIPT_NO.sh";
-		        open (RUNSCRIPT, ">$script");
-		        chmod (0755, $script);
-		        print FSCRIPT "system \"$script\"\n";
-  		        $SCRIPT_NO++;
-			$index=0;
-			foreach my $ext (@ext_stats) {
-	    			print RUNSCRIPT "$FILEBENCH/scripts/collect_$ext $ext $file_stats[$index] ";
-				print RUNSCRIPT  conf_reqval("statsdir");
-				print RUNSCRIPT " $time $FILEBENCH $arg_stats[$index] &\n";
-				$index++;
-			}
+    # Create the associated processes and start them running
+    print FSCRIPT "create processes\n";
+
+    if (($time ne '') && ($statsfile ne '')) {
+	# Clear the current statistics buffers
+	print FSCRIPT "stats clear\n";
+
+	# Start external statistics collection (if any)
+	# Note all statistics arrays MUST be the same length !
+	if (@ext_stats != ()) {
+	    if (($#ext_stats == $#file_stats) && ($#ext_stats == $#arg_stats)) {
+		$script = $mstrstatsdir . "/stats$SCRIPT_NO.sh";
+		open (RUNSCRIPT, ">$script");
+		chmod (0755, $script);
+		print FSCRIPT "system \"$script\"\n";
+		$SCRIPT_NO++;
+		$index=0;
+		foreach my $ext (@ext_stats) {
+		    print RUNSCRIPT "$FILEBENCH/scripts/collect_$ext $ext $file_stats[$index] ";
+		    print RUNSCRIPT  $mstrstatsdir;
+		    print RUNSCRIPT " $time $FILEBENCH $arg_stats[$index] &\n";
+		    $index++;
 		}
 	    }
-            close(RUNSCRIPT);
-
-	    # Sleep for the run time
-	    print FSCRIPT "sleep $time\n";
-
-	    # Snap the statistics
-	    print FSCRIPT "stats snap\n";
-
-            # Dump the statistics to a raw file - out required due to filename constraint
-	    print FSCRIPT "stats dump \"$statsfile.out\"\n";	
-
-	    # Statistics reaping occurs here
-	    if (@ext_stats != ()) {
-	    	if (($#ext_stats == $#file_stats) && ($#ext_stats == $#arg_stats)) {
-		    $script = conf_reqval("statsdir") . "/stats$SCRIPT_NO.sh";
-		    open (RUNSCRIPT, ">$script");
-		    chmod (0755, $script);
-		    print FSCRIPT "system \"$script\"\n";
-		    $SCRIPT_NO++;
-		    foreach my $ext (@ext_stats) {
-	                print RUNSCRIPT "$FILEBENCH/scripts/kill_stats $ext &\n";
-	            }
-	            close(RUNSCRIPT);
+	}
+	close(RUNSCRIPT);
+
+	# Sleep for the run time
+	print FSCRIPT "sleep $time\n";
+
+	# Snap the statistics
+	print FSCRIPT "stats snap\n";
+
+	# Dump the statistics to a raw file - out required due to filename constraint
+	if ($MULTI_CLIENT == 1) {
+	    print FSCRIPT "domultisync value=2\n";
+	    print FSCRIPT "stats multidump \"$statsfile.out\"\n";
+	} else {
+	    print FSCRIPT "stats dump \"$statsfile.out\"\n";
+	}
+
+	# Statistics reaping occurs here
+	if (@ext_stats != ()) {
+	    if (($#ext_stats == $#file_stats) && ($#ext_stats == $#arg_stats)) {
+		$script = $mstrstatsdir . "/stats$SCRIPT_NO.sh";
+		open (RUNSCRIPT, ">$script");
+		chmod (0755, $script);
+		print FSCRIPT "system \"$script\"\n";
+		$SCRIPT_NO++;
+		foreach my $ext (@ext_stats) {
+		    print RUNSCRIPT "$FILEBENCH/scripts/kill_stats $ext &\n";
 		}
-	    }
-	
-            # Dump the statistics to a Xanadu compatible XML file
-	    if ($USE_XANADU) {
-	        op_xmlstats($statsfile);
-
-	        $script = conf_reqval("statsdir") . "/stats$SCRIPT_NO.pl";
-	        open (RUNSCRIPT, ">$script");
-	        chmod (0755, $script);
-	        print FSCRIPT "system \"$script\"\n";
-	        $SCRIPT_NO++;
-
-                # The following loop adds the benchpoint run parameters and statistics into the filebench XML file
-                # We capture the meta data from the start of the filebench xml file
-                print RUNSCRIPT "#!/usr/bin/perl\n";
-		print RUNSCRIPT "\$phase=1;\n";
-                print RUNSCRIPT "open(STATSFILE,\"<".conf_reqval("statsdir")."/$statsfile.xml\");\n";
-                print RUNSCRIPT "open(OSTATSFILE,\">".conf_reqval("statsdir")."/$statsfile.new.xml\");\n";
-                print RUNSCRIPT "while (<STATSFILE>) {\n";
-		print RUNSCRIPT "\t\$temp=\$_;\n";
-                print RUNSCRIPT "\tif ((!((/.*meta.*/) || (/.*stat_doc.*/))) && (\$phase == 1)) {\n";
-                print RUNSCRIPT "\t\topen(XMLFILE,\"<".conf_reqval("statsdir")."/$statsfile.config.xml\");\n";
-                print RUNSCRIPT "\t\twhile (<XMLFILE>) {\n";
-                print RUNSCRIPT "\t\t\tprint OSTATSFILE \$_;\n";
-                print RUNSCRIPT "\t\t}\n";
-                print RUNSCRIPT "\t\tclose(XMLFILE);\n";
-		print RUNSCRIPT "\t\t\$phase++;\n";
-                print RUNSCRIPT "\t}\n";
-                print RUNSCRIPT "\tprint OSTATSFILE \$temp;\n";
-                print RUNSCRIPT "}\n";
-                print RUNSCRIPT "close(STATSFILE);\n";
-                print RUNSCRIPT "close(OSTATSFILE);\n";
-                print RUNSCRIPT "unlink(\"".conf_reqval("statsdir")."/$statsfile.xml\");\n";
-                print RUNSCRIPT "unlink(\"".conf_reqval("statsdir")."/$statsfile.config.xml\");\n";
-                print RUNSCRIPT "system(\"mv ".conf_reqval("statsdir")."/$statsfile.new.xml ".conf_reqval("statsdir")."/$statsfile.xml\");\n";
-
-	        $script = conf_reqval("statsdir") . "/stats$SCRIPT_NO.sh";
-	        open (RUNSCRIPT, ">$script");
-	        chmod (0755, $script);
-	        print FSCRIPT "system \"$script\"\n";
-	        $SCRIPT_NO++;
-
-    	        print RUNSCRIPT "mkdir ".conf_reqval("statsdir")."/xml\n";
-    	        print RUNSCRIPT "mkdir ".conf_reqval("statsdir")."/html\n";
-
-    	        print RUNSCRIPT "mv ".conf_reqval("statsdir")."/$statsfile.xml ".conf_reqval("statsdir")."/xml/$statsfile.xml\n";
-
-	        # Process XML file using Xanadu 2
-	        print RUNSCRIPT "$FILEBENCH/xanadu/scripts/xanadu import ".conf_reqval("statsdir")." ".conf_reqval("statsdir")."/xml ".conf_reqval("function")."-".conf_reqval("statsdir")."\n";
-	        print RUNSCRIPT "$FILEBENCH/xanadu/scripts/xanadu export ".conf_reqval("statsdir")."/xml ".conf_reqval("statsdir")."/html\n";
-                close(RUNSCRIPT);
+		close(RUNSCRIPT);
 	    }
 	}
-	return(0);	
+
+	# Dump the statistics to a Xanadu compatible XML file
+	if ($USE_XANADU) {
+	    op_xmlstats($statsfile);
+
+	    $script = $mstrstatsdir . "/stats$SCRIPT_NO.pl";
+	    open (RUNSCRIPT, ">$script");
+	    chmod (0755, $script);
+	    print FSCRIPT "system \"$script\"\n";
+	    $SCRIPT_NO++;
+
+	    # The following loop adds the benchpoint run parameters and statistics into the filebench XML file
+	    # We capture the meta data from the start of the filebench xml file
+	    print RUNSCRIPT "#!/usr/bin/perl\n";
+	    print RUNSCRIPT "\$phase=1;\n";
+	    print RUNSCRIPT "open(STATSFILE,\"<".$mstrstatsdir."/$statsfile.xml\");\n";
+	    print RUNSCRIPT "open(OSTATSFILE,\">".$mstrstatsdir."/$statsfile.new.xml\");\n";
+	    print RUNSCRIPT "while (<STATSFILE>) {\n";
+	    print RUNSCRIPT "\t\$temp=\$_;\n";
+	    print RUNSCRIPT "\tif ((!((/.*meta.*/) || (/.*stat_doc.*/))) && (\$phase == 1)) {\n";
+	    print RUNSCRIPT "\t\topen(XMLFILE,\"<".$mstrstatsdir."/$statsfile.config.xml\");\n";
+	    print RUNSCRIPT "\t\twhile (<XMLFILE>) {\n";
+	    print RUNSCRIPT "\t\t\tprint OSTATSFILE \$_;\n";
+	    print RUNSCRIPT "\t\t}\n";
+	    print RUNSCRIPT "\t\tclose(XMLFILE);\n";
+	    print RUNSCRIPT "\t\t\$phase++;\n";
+	    print RUNSCRIPT "\t}\n";
+	    print RUNSCRIPT "\tprint OSTATSFILE \$temp;\n";
+	    print RUNSCRIPT "}\n";
+	    print RUNSCRIPT "close(STATSFILE);\n";
+	    print RUNSCRIPT "close(OSTATSFILE);\n";
+	    print RUNSCRIPT "unlink(\"".$mstrstatsdir."/$statsfile.xml\");\n";
+	    print RUNSCRIPT "unlink(\"".$mstrstatsdir."/$statsfile.config.xml\");\n";
+	    print RUNSCRIPT "system(\"mv ".$mstrstatsdir."/$statsfile.new.xml ".$mstrstatsdir."/$statsfile.xml\");\n";
+
+	    $script = $mstrstatsdir . "/stats$SCRIPT_NO.sh";
+	    open (RUNSCRIPT, ">$script");
+	    chmod (0755, $script);
+	    print FSCRIPT "system \"$script\"\n";
+	    $SCRIPT_NO++;
+
+	    print RUNSCRIPT "mkdir ".$mstrstatsdir."/xml\n";
+	    print RUNSCRIPT "mkdir ".$mstrstatsdir."/html\n";
+
+	    print RUNSCRIPT "mv ".$mstrstatsdir."/$statsfile.xml ".$mstrstatsdir."/xml/$statsfile.xml\n";
+
+	    # Process XML file using Xanadu 2
+	    print RUNSCRIPT "$FILEBENCH/xanadu/scripts/xanadu import ".$mstrstatsdir." ".$mstrstatsdir."/xml ".conf_reqval("function")."-".$mstrstatsdir."\n";
+	    print RUNSCRIPT "$FILEBENCH/xanadu/scripts/xanadu export ".$mstrstatsdir."/xml ".$mstrstatsdir."/html\n";
+	    close(RUNSCRIPT);
+	}
+    }
+    return(0);	
 }
 
 sub op_xmlstats {
     my ($statsfile) = shift;
-	if($statsfile ne '') {
-	    	print FSCRIPT "stats xmldump \"$statsfile.xml\"\n";	
-
-		# The following loop adds the benchpoint run parameters and statistics into a temporary XML file
-		open(OSTATSFILE,">".conf_reqval("statsdir")."/$statsfile.config.xml");
-		%CONFHASH = conf_hash();
-		# There is no test for whether CONFHASH contains no keys 
-		# The following two lines is to obtain the stats run directory name for xanadu meta data
-		print OSTATSFILE "<meta name=\"RunId\" value=\"".conf_reqval("function")."-".conf_reqval("statsdir")."\"/>\n";
-		print OSTATSFILE "<stat_group name=\"Benchpoint Configuration\">\n";
-		print OSTATSFILE "<cell_list>\n";
-		foreach $k (keys(%CONFHASH)) {
-			print OSTATSFILE "<cell>@{ $CONFHASH{$k} }</cell>\n";
-		}
-		print OSTATSFILE "</cell_list>\n";
-		print OSTATSFILE "<dim_list>\n";
-		print OSTATSFILE "<dim>\n";
-		print OSTATSFILE "<dimval>Value</dimval>\n";
-		print OSTATSFILE "</dim>\n";
-		print OSTATSFILE "<dim>\n";
-		foreach $k (keys(%CONFHASH)) {
-			print OSTATSFILE "<dimval>$k</dimval>\n";
-		}
-		print OSTATSFILE "</dim>\n";
-		print OSTATSFILE "</dim_list>\n";
-		print OSTATSFILE "</stat_group>\n";
-		close(OSTATSFILE);
-
-		return(0);	
+    my $mstrstatsdir = $STATSBASE."/".$CONFNAME;
+    if($statsfile ne '') {
+	print FSCRIPT "stats xmldump \"$statsfile.xml\"\n";	
+
+	# The following loop adds the benchpoint run parameters and statistics into a temporary XML file
+	open(OSTATSFILE,">".$mstrstatsdir."/$statsfile.config.xml");
+	%CONFHASH = conf_hash();
+	# There is no test for whether CONFHASH contains no keys 
+	# The following two lines is to obtain the stats run directory name for xanadu meta data
+	print OSTATSFILE "<meta name=\"RunId\" value=\"".conf_reqval("function")."-".$mstrstatsdir."\"/>\n";
+	print OSTATSFILE "<stat_group name=\"Benchpoint Configuration\">\n";
+	print OSTATSFILE "<cell_list>\n";
+	foreach $k (keys(%CONFHASH)) {
+	    print OSTATSFILE "<cell>@{ $CONFHASH{$k} }</cell>\n";
+	}
+	print OSTATSFILE "</cell_list>\n";
+	print OSTATSFILE "<dim_list>\n";
+	print OSTATSFILE "<dim>\n";
+	print OSTATSFILE "<dimval>Value</dimval>\n";
+	print OSTATSFILE "</dim>\n";
+	print OSTATSFILE "<dim>\n";
+	foreach $k (keys(%CONFHASH)) {
+	    print OSTATSFILE "<dimval>$k</dimval>\n";
 	}
-	return(1);	
+	print OSTATSFILE "</dim>\n";
+	print OSTATSFILE "</dim_list>\n";
+	print OSTATSFILE "</stat_group>\n";
+	close(OSTATSFILE);
+
+	return(0);	
+    }
+    return(1);	
 }
 
 sub op_command {
@@ -397,14 +433,17 @@ sub op_load_defaults {
 #    }
 
     # Cater for the user defined defaults
-#    foreach my $var (@vars) {
-
-
-    # Cater for the user defined defaults
     foreach $var (keys(%CONFDATA)) {
         if (conf_exists($var)) {
             $var =~ s/ //g;
             my $val = conf_getval($var);
+
+	    if (($SHAREDFILEALLOCATOR) and ($var eq "sharedprealloc")) {
+		if (conf_reqval("myname") ne $SHAREDFILEALLOCATOR) {
+		    $val = "0";
+		}
+	    }
+
             op_set($var, $val);
 	}
     }
@@ -416,7 +455,7 @@ sub op_load_defaults {
 
 sub parse_profile {
     my ($profile) = shift;
-    my ($config_section, $default_section);
+    my ($config_section, $default_section, $multi_section);
     
     open(CFILE, "$profile") or 
 	die "ERROR: couldn't open profile";
@@ -429,12 +468,32 @@ sub parse_profile {
 	if($line =~ /^#/ or $line eq "") {
 	} else {
 	    if($line =~ /}/) {
+		if($multi_section == 1) {
+		    $multi_section = 0;
+		}
 		if($default_section == 1) {
 		    $default_section = 0;
 		}
 		if($config_section == 1) {
 		    $config_section = 0;
 		}
+	    } elsif($multi_section) {
+		$line =~ /(.+) = (.+);/;
+		my $opt = $1;
+		my $val = $2;
+		chomp($opt);
+		chomp($val);
+		my @vals = ();
+		# Check to see if this needs to be a list
+		if($val =~ /,/) {
+		    push(@vals, $+) while $val =~
+			m{"([^\"\\]*(?:\\.[^\"\\]*)*)",? | ([^,]+),? | , }gx;
+		    push(@vals, undef) if substr($val, -1,1) eq ',';
+		    @{ $MULTIDATA{$opt} } = @vals;
+		} else {
+		    @{MULTIDATA{$opt}} = ();
+		    push(@{ $MULTIDATA{$opt} }, $val);		   
+		}	       
 	    } elsif($default_section) {
 		$line =~ /(.+) = (.+);/;
 		my $opt = $1;
@@ -447,7 +506,7 @@ sub parse_profile {
 		    push(@vals, $+) while $val =~
 			m{"([^\"\\]*(?:\\.[^\"\\]*)*)",? | ([^,]+),? | , }gx;
 		    push(@vals, undef) if substr($val, -1,1) eq ',';
-		    @{ $DEFDATA{$opt} }  = @vals;
+		    @{ $DEFDATA{$opt} } = @vals;
 		} else {
 		    @{CONFDATA{$opt}} = ();
 		    push(@{ $DEFDATA{$opt} }, $val);		   
@@ -458,6 +517,9 @@ sub parse_profile {
 	 	    $config =~ s/CONFIG[ 	]+(.+) {/$1/;
 		    push(@CONFLIST, $config);
 		    $config_section = 1;
+		} elsif($line =~ /MULTICLIENT {/) {
+		    $multi_section = 1;
+		    $MULTI_CLIENT = 1;
 		} elsif($line =~ /DEFAULTS {/) {
 		    $default_section = 1;
 		}
@@ -520,6 +582,107 @@ sub parse_config {
     
     # Bye, bye
     #close(CFILE) or die "ERROR: config file closing difficulties";
+    return \%confdata;
+}
+
+sub build_run
+{
+    # The following function is taken from the user's function file
+    pre_run();
+
+    # Set the global statistics directory for this run
+    op_statsdir();
+
+    # The following function is taken from the user's function file
+    bm_run();
+
+    # Finish and close the .f script
+    op_quit();
+}
+
+# statistics aggregation section
+my %FLOWOPVALS;
+my @SUMMARYVALS;
+
+sub init_combined_stats
+{
+    %FLOWOPVALS = ();
+    @SUMMARYVALS = (0,0,0,0,0,0);
+}
+
+sub add_2combstats
+{
+    my ($confname) = shift;
+    my ($thisclient) = shift;
+    my $clstatdir;
+    my $flowopmode = 0;
+    my $summarymode = 0;
+
+    print "adding in stats for client: $thisclient, configuration: $confname\n";
+
+    $clstatdir = multi_getval("masterpath")."/".$thisclient;
+
+    print "from: ".$clstatdir."/stats.".$confname.".out\n";
+    open (CLSTATS, $clstatdir."/stats.".$confname.".out");
+    while(<CLSTATS>) {
+	my ($line) = $_;
+	chomp($line);
+	if (($flowopmode == 0) and ($summarymode == 0)) {
+	    if ($line =~ /^Flowop totals:/) {
+		$flowopmode = 1;
+		next;
+	    }
+	    if ($line =~ /^IO Summary:/) {
+		$summarymode = 1;
+		next;
+	    }
+	}
+	if ($line eq "") {
+	    $flowopmode = 0;
+	    $summarymode = 0;
+	    next;
+	}
+
+	# get the good stuff
+	if ($flowopmode == 1) {
+	    my @elementlist;
+	    my @valuelist;
+	    my $flkey;
+	    my $vallistref = [];
+
+	    @elementlist = split('	', $line);
+	    $flkey = $elementlist[0];
+	    @valuelist = @elementlist[1..$#elementlist];
+
+	    if (exists($FLOWOPVALS{$flkey})) {
+		my $numvals;
+
+		$vallistref = $FLOWOPVALS{$flkey};
+		$numvals = @{$vallistref};
+		for (my $idx = 0; $idx < $numvals; $idx++) {
+		    $vallistref->[$idx] += $valuelist[$idx];
+		}
+	    } else {
+		# newly found flowop name
+		$vallistref = [@valuelist];
+		$FLOWOPVALS{$flkey} = $vallistref;
+	    }
+	    next;
+	}
+
+	# get final totals
+	if ($summarymode == 1) {
+	    my @valuelist;
+
+	    @valuelist = split('	', $line);
+
+	    for (my $idx = 0; $idx <= $#valuelist; $idx++) {
+		$SUMMARYVALS[$idx] += $valuelist[$idx];
+	    }
+	    next;
+	}
+    }
+    close (CLSTATS);
 }
 
 sub print_usage
@@ -527,6 +690,166 @@ sub print_usage
     print "Usage:\n\tfilebench <profile name>\n\tfilebench -c <stat_dir> ...\n";
 }
 
+sub dump_combined_stats
+{
+    my ($confname) = shift;
+    my $totvalsref = [];
+    my $flkey;
+    use FileHandle;
+
+## set up output formating info
+format flowoplinefrm =
+@<<<<<<<<<<<<<<<<<<< @#######ops/s @###.#mb/s @#####.#ms/op @#######us/op-cpu
+$flkey, $totvalsref->[0], $totvalsref->[1], $totvalsref->[2]/$#CLIENTLIST, $totvalsref->[3]/$#CLIENTLIST
+.
+
+format summarylinefrm =
+
+IO Summary: @#######ops, @#####.#ops/s, (@####/@#### r/w) @#####.#mb/s, @######us cpu/op, @####.#ms latency
+$SUMMARYVALS[0], $SUMMARYVALS[1], $SUMMARYVALS[2], $SUMMARYVALS[3], $SUMMARYVALS[4], $SUMMARYVALS[5], $SUMMARYVALS[6]
+.
+
+    open (SUMSTATS, ">$STATSBASE/$confname/stats.$confname.out");
+    print "Per-Operation Breakdown:\n";
+    print SUMSTATS "Per-Operation Breakdown:\n";
+
+    format_name  STDOUT "flowoplinefrm";
+    format_name  SUMSTATS "flowoplinefrm";
+
+    foreach $flkey (keys %FLOWOPVALS) {
+
+	$totvalsref = $FLOWOPVALS{$flkey};
+
+	write STDOUT;
+	write SUMSTATS;
+    }
+
+    format_name  STDOUT "summarylinefrm";
+    format_name  SUMSTATS "summarylinefrm";
+
+    write STDOUT;
+    write SUMSTATS;
+    close (SUMSTATS);
+}
+
+#
+# polls the synchronization socket for each client in turn every 5 seconds,
+# then sends synch responses once all clients have "checked in". The
+# sample number in the received sync requests must match the sequence
+# number supplied with the call.
+#
+sub sync_receive
+{
+    my $seqnum = shift;
+#    my @cl_list;
+    my %cl_hash = ();
+    %cl_hash = %CLIENTHASH;
+
+    my $count = @CLIENTLIST;
+    print "waiting for sync message: $seqnum from $count clients\n";
+    while ($count > 0) {
+	my $rcv_str = "";
+
+	sleep 5;
+
+	foreach my $client_name (keys %cl_hash)
+	{
+	    my $clientdata = $CLIENTHASH{$client_name};
+	    my $client_hndl = $$clientdata[0];
+	    print "recv sync: $client_name undefined handle\n" unless defined($client_hndl);
+	    my $client_iaddr = $$clientdata[1];
+	    my $sn = $$clientdata[2];
+	    my $rtn = 0;
+
+	    do {
+		my $tmp_str;
+		$rtn = recv($client_hndl, $tmp_str, 80, MSG_DONTWAIT);
+		if (defined($rtn)) {
+		    $rcv_str = $rcv_str.$tmp_str;
+		}   
+	    } until (!defined($rtn) || ($rcv_str =~ /$EOL/s ));
+
+	    if (defined($rtn)) {
+		my %ophash = ();
+		my $ok;
+
+		my @oplist = split /,/,$rcv_str;
+		foreach my $opent (@oplist)
+		{
+		    my ($op, $val) = split /=/,$opent;
+		    $ophash{$op} = $val;
+		}
+		$ok = ($sn == $seqnum);
+		$ok &&= defined((my $cmd_val = $ophash{"cmd"}));
+		$ok &&= defined((my $samp_val = $ophash{"sample"}));
+		if ($ok && ($cmd_val eq "SYNC") && ($samp_val == $seqnum))
+		{
+		    delete $cl_hash{$client_name};
+		    $count--;
+		    print "received a sync request from $client_name\n";
+		    ${$CLIENTHASH{$client_name}}[2] = ($sn + 1);
+		} else {
+		    print "received invalid sync request string [".rcv_str."] from client $client_name\n";
+		}
+	    }
+	}
+    }
+    print "received all sync requests for seq $seqnum, sending responses\n";
+    foreach my $client_name (@CLIENTLIST)
+    {
+	my $clientdata = $CLIENTHASH{$client_name};
+	my $client_hndl = $$clientdata[0];
+	print "send resp: $client_name undefined handle\n" unless defined($client_hndl);
+
+	send ($client_hndl, "rsp=PASS,sample=$seqnum\n", 0);
+    }
+}
+
+#
+# waits for all known clients to connect, then calls sync_recieve(1) to
+# sync_receive(N) to wait for N sync requests for designated sync points
+# 1..N.
+#
+sub sync_server
+{
+    my $port = shift || 8001;
+    my $proto = getprotobyname('tcp');
+    my $paddr;
+    my $count;
+
+    socket(Server, PF_INET, SOCK_STREAM, $proto)     || die "socket: $!";
+    setsockopt(Server, SOL_SOCKET, SO_REUSEADDR,
+	       pack("l", 1))			     || die "setsockopt: $1";
+    bind(Server, sockaddr_in($port, INADDR_ANY))     || die "bind: $1";
+    listen(Server, SOMAXCONN)			     || die "listen: $1";
+
+# wait for connection requests from clients
+    print "sync: Waiting for ".@CLIENTLIST." Clients\n";
+    for ($count = @CLIENTLIST; $count > 0; $count--) {
+	$paddr = accept(my $client_hndl, Server);
+	die "bad socket address" unless $paddr;
+
+	my ($port, $iaddr) = sockaddr_in($paddr);
+	my $cl_name = gethostbyaddr($iaddr, AF_INET);
+
+	if (!exists($CLIENTHASH{$cl_name})) {
+	    die "sync from unknown client $cl_name";
+	}
+
+	print "received sync connection from client: $cl_name\n";
+	${$CLIENTHASH{$cl_name}}[0] = $client_hndl;
+	${$CLIENTHASH{$cl_name}}[1] = $iaddr;
+    }
+
+# indicate that all clients have checked in
+    sync_receive(1);
+    if (conf_exists("runtime") == 1) {
+	my $runtime =  conf_getval("runtime");
+	sleep $runtime;
+    }
+    sync_receive(2);
+}
+
 ##############################################################################
 ## Main program
 ##############################################################################
@@ -556,51 +879,189 @@ parse_profile("$PROFILENAME.prof");
 %CONFDATA = ();
 %CONFDATA = %DEFDATA;
 
-# Setup the statistics base directory
-$STATSBASE = conf_reqval("stats");
-my $filesystem = conf_reqval("filesystem");
+# get the name of the host this script is running on
 my $hostname = `hostname`;
 chomp($hostname);
+
+# Check for Multi-Client operation
+if ($MULTI_CLIENT == 1) {
+
+    if (multi_exists("targetpath")) {
+	$TARGETPATH = multi_getval("targetpath");
+    } else {
+	print "ERROR: Target pathname required for multi-client operation\n";
+	exit(1);
+    }
+
+    if (multi_exists("clients")) {
+	@CLIENTLIST = split(' ',multi_getval("clients"));
+    } else {
+	print "ERROR: client list required for multi-client operation\n";
+	exit(1);
+    }
+
+    if (multi_exists("sharefiles")) {
+	$SHAREDFILEALLOCATOR = multi_getval("sharefiles");
+    } else {
+	$SHAREDFILEALLOCATOR = "";
+    }
+
+    $TARGETDIR = $TARGETPATH.conf_getval("dir");
+
+    # Setup the multi client statistics base directory
+    $STATSBASE = $TARGETPATH.conf_reqval("stats");
+
+    multi_putval("masterhost", $hostname) unless multi_exists("masterhost");
+    multi_putval("masterpath", $STATSBASE) unless multi_exists("masterpath");
+
+    # create a path for filebench.pl to use to access the master directory
+    $FB_MASTERPATH = multi_getval("masterpath");
+
+    print "Target PathName = $TARGETPATH, path = ".multi_getval("masterpath")."\n";
+
+} else {
+    # Setup the single client statistics base directory
+    $STATSBASE = conf_reqval("stats");
+}
+
+my $filesystem = conf_reqval("filesystem");
 $STATSBASE = $STATSBASE . "/$hostname-$filesystem-$PROFILENAME-";
 my $timestamp = strftime "%b_%e_%Y-%Hh_%Mm_%Ss", localtime;
 $timestamp =~ s/ //;
 $STATSBASE = $STATSBASE . $timestamp;
 
-foreach $CONFNAME (@CONFLIST) {
+foreach $config_name (@CONFLIST)
+{
     %CONFDATA = ();
     %CONFDATA = %DEFDATA;
-    parse_config("$CONFNAME");
+    $CONFNAME = $config_name;
+    parse_config("$config_name");
     my $function = conf_reqval("function");
+    my $statsdir;
+
     if (-f "$function.func") {
 	require "$function.func";
     } else {
 	require "$FILEBENCH/config/$function.func";
     }
-    $QUIT = 0;
 
-    # Setup the statistics directory
-    $statsdir = $STATSBASE . "/" . $CONFNAME;
-    push(@{ $CONFDATA{"statsdir"} }, $statsdir);		   
-    system("mkdir -p $statsdir");
-
-    # The following function is taken from the user's function file
-    pre_run();
+    # Setup the final statistics directory
+    system("mkdir -p $STATSBASE");
 
     # Leave a log of the run info	
     open (RUNLOG, ">$STATSBASE/thisrun.prof");
     print RUNLOG "# " . conf_reqval("description") . "\n";
     close (RUNLOG);
-    system ("cat $PROFILENAME.prof >>$STATSBASE/thisrun.prof");
 
-    # Set the global statistics directory for this run
-    op_statsdir();
+    system ("cat $PROFILENAME.prof >>".$STATSBASE."/thisrun.prof");
 
-    # The following function is taken from the user's function file
-    bm_run();
-   
-    $QUIT = 1;
+    $statsdir = $STATSBASE . "/" . $config_name;
+    system("mkdir -p $statsdir");
+    system("chmod a+w $statsdir");
+
+    if ($MULTI_CLIENT == 1) {
+	my @pidlist;
+	my %multi_confdata;
+	my $procpid;
+	my $syncclients = "";
+
+	%multi_confdata = %CONFDATA;
+
+	foreach my $thisclient (@CLIENTLIST) {
+	    my $tmpdir;
+	    my $tmpstatdir;
+	    my @clientdata;
+
+	    %CONFDATA = ();
+	    %CONFDATA = %multi_confdata;
+	    printf "building client: " . $thisclient . "\n";
+
+	    # Setup the statistics directory for each client
+	    $tmpstatdir = multi_getval("masterpath")."/".$thisclient;
+
+	    if ($SHAREDFILEALLOCATOR) {
+		$tmpdir = $TARGETDIR;
+	    } else {
+		$tmpdir = $TARGETDIR."/".$thisclient;
+	    }
+
+# add info to client hash
+	    @clientdata = ();
+	    $clientdata[2] = 1;
+	    $CLIENTHASH{$thisclient} = \@clientdata;
+	    $syncclients = $syncclients." --client ".$thisclient;
+
+	    push(@{ $CONFDATA{"myname"} }, $thisclient);
+	    push(@{ $CONFDATA{"statsdir"} }, $tmpstatdir);
+	    system("mkdir -p ".$FB_MASTERPATH."/".$thisclient);
+	    system("chmod 0777 ".$FB_MASTERPATH."/".$thisclient);
+
+	    # modify dir config variable for multiclient
+	    if (conf_exists("dir")) {
+		@{$CONFDATA{"dir"}} = ($tmpdir);
+	    }
+	    build_run();
+	}
+
+	# Begin the RUN!!!
+	print "Running " . $STATSBASE . "\n";
+
+	#spawn the synchronization server
+	print "Starting sync server on host ".$hostname."\n";
+	if ($procpid = fork) {
+	    push(@pidlist, $procpid);
+	} else {
+	    sync_server();
+	    exit(0);
+	}
+
+	sleep(3);
+
+	# remotely execute the run on each client
+	foreach $thisclient (@CLIENTLIST) {
+	    if($procpid = fork) {
+		push(@pidlist, $procpid);
+	    } else {
+		if ($thisclient eq $hostname) {
+		    print "Starting local client: $thisclient\n";
+		    system(multi_getval("masterpath")."/".$thisclient."/thisrun.f");
+		} else {
+		    print "Starting remote client: $thisclient\n";
+		    system("ssh ".$thisclient." ".multi_getval("masterpath")."/".$thisclient."/thisrun.f >> ".multi_getval("masterpath")."/".$thisclient."/runs.out");
+		}
+		exit(0);
+	    }
+	}
+
+	# wait for all of them to finish
+	foreach $procpid (@pidlist) {
+	    waitpid($procpid, 0);
+	}
+
+	init_combined_stats();
+
+	foreach $thisclient (@CLIENTLIST) {
+	    add_2combstats($config_name, $thisclient);
+	}
+
+	# dump the combined client stats
+	dump_combined_stats($config_name);
+
+    } else {
+	push(@{ $CONFDATA{"statsdir"} }, $statsdir);
+
+	build_run();
+
+	# Execute the run
+	print "Running " . conf_reqval("statsdir") . "/thisrun.f\n";
+	system ($statsdir."/thisrun.f");
+
+
+    }
 
-    # The following function is taken from the user's function file
-    post_run();
-    print "\n";
 }
+
+# The following function is taken from the user's function file
+post_run();
+
+print "\n";