diff options
Diffstat (limited to 'usr/src')
94 files changed, 18819 insertions, 0 deletions
diff --git a/usr/src/Makefile.lint b/usr/src/Makefile.lint index 1d326418c7..4da8cc8309 100644 --- a/usr/src/Makefile.lint +++ b/usr/src/Makefile.lint @@ -122,6 +122,7 @@ COMMON_SUBDIRS = \ cmd/fdformat \ cmd/fgrep \ cmd/file \ + cmd/filebench \ cmd/find \ cmd/fmthard \ cmd/fmtmsg \ diff --git a/usr/src/Makefile.master b/usr/src/Makefile.master index d7c358f9c2..8b4832a09f 100644 --- a/usr/src/Makefile.master +++ b/usr/src/Makefile.master @@ -158,6 +158,7 @@ ELFDUMP= /usr/ccs/bin/elfdump M4= /usr/ccs/bin/m4 STRIP= /usr/ccs/bin/strip LEX= /usr/ccs/bin/lex +FLEX= $(SFW_ROOT)/bin/flex YACC= /usr/ccs/bin/yacc CPP= /usr/lib/cpp JAVAC= $(JAVA_ROOT)/bin/javac diff --git a/usr/src/cmd/Makefile b/usr/src/cmd/Makefile index 3272df2aad..b3173b0190 100644 --- a/usr/src/cmd/Makefile +++ b/usr/src/cmd/Makefile @@ -158,6 +158,7 @@ COMMON_SUBDIRS= \ filesync \ fgrep \ file \ + filebench \ find \ fm \ fmli \ diff --git a/usr/src/cmd/filebench/Makefile b/usr/src/cmd/filebench/Makefile new file mode 100644 index 0000000000..fd2294c39f --- /dev/null +++ b/usr/src/cmd/filebench/Makefile @@ -0,0 +1,58 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +#ident "%Z%%M% %I% %E% SMI" + +include ../Makefile.cmd + +SUBDIRS = \ + config \ + fbscript \ + scripts \ + workloads + +$(64ONLY)SUBDIRS += $(MACH) +$(BUILD64)SUBDIRS += $(MACH64) + +PROG = go_filebench +ROOTCMDDIR = $(ROOT)/usr/benchmarks/filebench/bin + +.KEEP_STATE: + +all := TARGET = all +clean := TARGET = clean +clobber := TARGET = clobber +install := TARGET = install +lint := TARGET = lint + +all clean clobber lint: $(SUBDIRS) + +install: $(SUBDIRS) + -$(RM) $(ROOTCMD) + -$(LN) $(ISAEXEC) $(ROOTCMD) + +$(SUBDIRS): FRC + @cd $@; pwd; $(MAKE) $(TARGET) + +FRC: diff --git a/usr/src/cmd/filebench/Makefile.com b/usr/src/cmd/filebench/Makefile.com new file mode 100644 index 0000000000..35e44e8d57 --- /dev/null +++ b/usr/src/cmd/filebench/Makefile.com @@ -0,0 +1,98 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +#ident "%Z%%M% %I% %E% SMI" + +.KEEP_STATE: + +include ../../Makefile.cmd +include ../../Makefile.targ + +SRCS = \ + auto_comp.c \ + eventgen.c \ + fileobj.c \ + fileset.c \ + flowop.c \ + flowop_library.c \ + gamma_dist.c \ + ipc.c \ + misc.c \ + procflow.c \ + stats.c \ + threadflow.c \ + utils.c \ + vars.c + +PROG = go_filebench +ROOTFBBINDIR = $(ROOT)/usr/benchmarks/filebench/bin +OBJS = $(SRCS:%.c=%.o) parser_gram.o parser_lex.o +LINTFLAGS += -erroff=E_FUNC_ARG_UNUSED -erroff=E_NAME_DEF_NOT_USED2 \ + -erroff=E_NAME_USED_NOT_DEF2 +LINTFLAGS64 += -erroff=E_FUNC_ARG_UNUSED -erroff=E_NAME_DEF_NOT_USED2 \ + -erroff=E_NAME_USED_NOT_DEF2 +LINTFILES = $(SRCS:%.c=%.ln) +CLEANFILES += parser_gram.c parser_gram.h parser_lex.c y.tab.h y.tab.c + +CPPFLAGS += -I. -I../common +CFLAGS += $(CCVERBOSE) $(CTF_FLAGS) +CFLAGS64 += $(CCVERBOSE) $(CTF_FLAGS) +LDLIBS += -lkstat -lm -ltecla + +LFLAGS = -t -v +YFLAGS = -d + +.PARALLEL: $(OBJS) $(LINTFILES) + +all: $(PROG) + +$(PROG): $(OBJS) + $(LINK.c) $(OBJS) -o $@ $(LDLIBS) + $(CTFMERGE) -L VERSION -o $@ $(OBJS) + $(POST_PROCESS) + +parser_lex.c: ../common/parser_lex.l + $(FLEX) $(LFLAGS) ../common/parser_lex.l > $@ + +parser_gram.c: ../common/parser_gram.y + $(YACC) $(YFLAGS) ../common/parser_gram.y + $(MV) y.tab.c parser_gram.c + $(MV) y.tab.h parser_gram.h + +%.o: %.c + $(COMPILE.c) $< + $(CTFCONVERT_O) + +%.o: ../common/%.c + $(COMPILE.c) $< + $(CTFCONVERT_O) + +clean: + $(RM) $(OBJS) $(LINTFILES) $(CLEANFILES) + +%.ln: ../common/%.c + $(LINT.c) -c $< + +lint: $(LINTFILES) + $(LINT.c) $(LINTFILES) $(LDLIBS) diff --git a/usr/src/cmd/filebench/amd64/Makefile b/usr/src/cmd/filebench/amd64/Makefile new file mode 100644 index 0000000000..2d96a03113 --- /dev/null +++ b/usr/src/cmd/filebench/amd64/Makefile @@ -0,0 +1,38 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +#ident "%Z%%M% %I% %E% SMI" + +include ../Makefile.com + +FBBINDIRamd64 = $(ROOTFBBINDIR)/amd64 +PROGamd64 = $(PROG:%=$(FBBINDIRamd64)/%) + +$(FBBINDIRamd64): + $(INS.dir) + +$(FBBINDIRamd64)/%: % + $(INS.file) + +install: all .WAIT $(FBBINDIRamd64) .WAIT $(PROGamd64) diff --git a/usr/src/cmd/filebench/common/auto_comp.c b/usr/src/cmd/filebench/common/auto_comp.c new file mode 100644 index 0000000000..1c0a0b9074 --- /dev/null +++ b/usr/src/cmd/filebench/common/auto_comp.c @@ -0,0 +1,1044 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include <sys/types.h> +#include <dirent.h> +#include <strings.h> +#include "filebench.h" +#include "auto_comp.h" + +#define VARNAME_MAXLEN 128 +#define FILENAME_MAXLEN 128 +#define MALLOC_STEP 64 + +#define CSUF_CMD " " +#define CSUF_ARG " " +#define CSUF_LVARNAME "=" +#define CSUF_RVARNAME "," +#define CSUF_ATTRNAME "=" + +#define ATTR_LIST_SEP ',' +#define ATTR_ASSIGN_OP '=' +#define VAR_ASSIGN_OP '=' +#define VAR_PREFIX '$' + +#ifndef HAVE_BOOLEAN_T +typedef enum { B_FALSE, B_TRUE } boolean_t; +#endif + +typedef char ac_fname_t[FILENAME_MAXLEN]; + +typedef struct ac_fname_cache { + ac_fname_t *fnc_buf; + int fnc_bufsize; + time_t fnc_mtime; +} ac_fname_cache_t; + +typedef enum ac_match_result { + MATCH_DONE, + MATCH_CONT +} ac_match_result_t; + +/* + * We parse an user input line into multiple blank separated strings. + * The last string is always the one user wants to complete, the other + * preceding strings set up the context on how to complete the last one. + * + * ac_str_t repsents one such a string, which can be of the following + * types: + * + * STRTYPE_COMPLETE - the string is one of the preceding strings. + * STRTYPE_INCOMPLETE - the string is the one being completed, user + * has inputted at least one character for it. + * STRTYPE_NULL - the string is the one being completed, user + * has inputted nothing for it. + * + * ac_str_t structure has the following members: + * + * startp - the start position of the string in the user input buffer + * endp - the end position of the string in the user input buffer + * strtype - the type of the string. It can be of the following values: + * STRTYPE_COMPLETE, STRTYPE_INCOMPLETE, STRTYPE_NULL, + * and STRTYPE_INVALID. + */ + +typedef enum ac_strtype { + STRTYPE_COMPLETE, + STRTYPE_INCOMPLETE, + STRTYPE_NULL, + STRTYPE_INVALID +} ac_strtype_t; + +typedef struct ac_str { + const char *startp; + const char *endp; + ac_strtype_t strtype; +} ac_str_t; + +#define STR_NUM 3 + +typedef struct ac_inputline { + ac_str_t strs[STR_NUM]; +} ac_inputline_t; + +/* + * ac_iter represents a general interface to access a list of values for + * matching user input string. The structure has the following methods: + * + * bind - bind the iterator to a list, and save a pointer to user + * passed space in nlistpp. + * reset - reset internal index pointer to point to list head. + * get_nextstr - this is the method that does the real work. It + * walks through the list and returns string associated with + * the current item. It can also return some other thing the + * caller is interested via the user passed space pointed by + * nlistpp. In our case, that is a pointer to a list which + * contains all possible values for the next string in user + * input. + * + * It has the following data members: + * + * listp - a pointer to the list to be iterated through + * curp - index pointer to maintain position when iterating + * nlistpp - a pointer to user passed space for returning a list of + * values for the next string in user input + */ + +typedef struct ac_iter { + void *listp; + void *curp; + void *nlistpp; + void (*bind)(struct ac_iter *, void *, void *); + void (*reset)(struct ac_iter *); + const char *(*get_nextstr)(struct ac_iter *); +} ac_iter_t; + +/* + * We consider a filebench command is composed of a sequence of tokens + * (ie., command name, argument name, attribute name, etc.). Many of + * these tokens have limited string values. These values, as well as + * their dependencies, are used to complete user input string. + * + * There are the following tokens: + * + * TOKTYPE_CMD - command name + * TOKTYPE_ARG - argument name + * TOKTYPE_ATTRNAME - attribute name + * TOKTYPE_ATTRVAL - attribute value + * TOKTYPE_LVARNAME - variable name, used on left side of assign + * operator + * TOKTYPE_RVARNAME - variable name, used on right side of assign + * operator + * TOKTYPE_VARVAL - variable value + * TOKTYPE_LOADFILE - load file name + * TOKTYPE_ATTRLIST - pseudo token type for attribute list + * TOKTYPE_VARLIST - pseudo token type for variable list + * TOKTYPE_NULL - pseudo token type for aborting auto-completion + * + * The reason why there are two different token types for variable name + * is because, depending on its position, there are different requirements + * on how to do completion and display matching results. See more details + * in lvarname_iter and rvarname_iter definition. + * + * Attribute list and variable list are not really a single token. Instead + * they contain multiple tokens, and thus have different requirements on + * how to complete them. TOKTYPE_ATTRLIST and TOKTYPE_VARLIST are + * introduced to to solve this issue. See more details below on + * get_curtok() function in ac_tokinfo_t structure. + * + * ac_tokval_t represents a string value a token can have. The structure + * also contains a pointer to a ac_tvlist_t structure, which represents + * all possible values for the next token in the same command. + * + * str - the token's string value + * nlistp - a list which contains string values for the token + * that follows + * + * ac_tvlist_t represents all possible values for a token. These values + * are stored in an ac_tokval_t array. The structure also has a member + * toktype, which is used to index an ac_tokinfo_t array to get the + * information on how to access and use the associated value list. + * + * vals - a list of string values for this token + * toktype - the token's type + * + * ac_tokinfo_t contains information on how to access and use the + * string values of a specific token. Among them, the most important + * thing is an iterator to access the value list. The reason to use + * iterator is to encapsulate list implementation details. That is + * necessary because some tokens have dynamic values(for example, + * argument of load command), which cannot be predefined using + * ac_tokval_t array. + * + * ac_tokinfo_t structure has the following members: + * + * toktype - token type + * iter - iterator to access the token's value list + * cont_suffix - continuation suffix for this token. See note 1 + * below on what is continuation suffix. + * get_curtok - a function to parse a multi-token user string. + * It parse that string and returns the word being + * completed and its token type. See note 2 below. + * + * Notes: + * + * 1) Continuation suffix is a convenient feature provided by libtecla. + * A continuation suffix is a string which is automatically appended + * to a fully completed string. For example, if a command name is + * fully completed, a blank space will be appended to it. This is + * very convenient because it not only saves typing, but also gives + * user an indication to continue. + * + * 2) get_curtok() function is a trick to support user input strings + * which have multiple tokens. Take attribute list as an example, + * although we defined a token type TOKTYPE_ATTRLIST for it, it is + * not a token actually, instead it contains multiple tokens like + * attribute name, attribute value, etc., and attribute value can + * be either a literal string or a variable name prefixed with a + * '$' sign. For this reason, get_curtok() function is needed to + * parse that string to get the word being completed and its token + * type so that we can match the word against with the proper value + * list. + */ + +typedef enum ac_toktype { + TOKTYPE_CMD, + TOKTYPE_ARG, + TOKTYPE_ATTRNAME, + TOKTYPE_ATTRVAL, + TOKTYPE_LVARNAME, + TOKTYPE_RVARNAME, + TOKTYPE_VARVAL, + TOKTYPE_LOADFILE, + TOKTYPE_ATTRLIST, + TOKTYPE_VARLIST, + TOKTYPE_NULL +} ac_toktype_t; + +typedef ac_toktype_t (*ac_get_curtok_func_t)(ac_str_t *); + +typedef struct ac_tokinfo { + ac_toktype_t toktype; + ac_iter_t *iter; + char *cont_suffix; + ac_get_curtok_func_t get_curtok; +} ac_tokinfo_t; + +typedef struct ac_tokval { + char *str; + struct ac_tvlist *nlistp; +} ac_tokval_t; + +typedef struct ac_tvlist { + ac_tokval_t *vals; + ac_toktype_t toktype; +} ac_tvlist_t; + +/* + * Variables and prototypes + */ + +static void common_bind(ac_iter_t *, void *, void *); +static void common_reset(ac_iter_t *); +static void varname_bind(ac_iter_t *, void *, void *); +static void loadfile_bind(ac_iter_t *, void *, void *); +static const char *get_next_tokval(ac_iter_t *); +static const char *get_next_lvarname(ac_iter_t *); +static const char *get_next_rvarname(ac_iter_t *); +static const char *get_next_loadfile(ac_iter_t *); +static ac_toktype_t parse_attr_list(ac_str_t *); +static ac_toktype_t parse_var_list(ac_str_t *); + +static ac_iter_t tokval_iter = { + NULL, + NULL, + NULL, + common_bind, + common_reset, + get_next_tokval +}; + +static ac_iter_t lvarname_iter = { + NULL, + NULL, + NULL, + varname_bind, + common_reset, + get_next_lvarname +}; + +static ac_iter_t rvarname_iter = { + NULL, + NULL, + NULL, + varname_bind, + common_reset, + get_next_rvarname +}; + +static ac_iter_t loadfile_iter = { + NULL, + NULL, + NULL, + loadfile_bind, + common_reset, + get_next_loadfile +}; + +/* + * Note: We use toktype to index into this array, so for each toktype, + * there must be one element in the array, and in the same order + * as that toktype is defined in ac_toktype. + */ +static ac_tokinfo_t token_info[] = { + { TOKTYPE_CMD, &tokval_iter, CSUF_CMD, NULL }, + { TOKTYPE_ARG, &tokval_iter, CSUF_ARG, NULL }, + { TOKTYPE_ATTRNAME, &tokval_iter, CSUF_ATTRNAME, NULL }, + { TOKTYPE_ATTRVAL, NULL, NULL, NULL }, + { TOKTYPE_LVARNAME, &lvarname_iter, CSUF_LVARNAME, NULL }, + { TOKTYPE_RVARNAME, &rvarname_iter, CSUF_RVARNAME, NULL }, + { TOKTYPE_VARVAL, NULL, NULL, NULL }, + { TOKTYPE_LOADFILE, &loadfile_iter, CSUF_ARG, NULL }, + { TOKTYPE_ATTRLIST, NULL, NULL, parse_attr_list }, + { TOKTYPE_VARLIST, NULL, NULL, parse_var_list }, + { TOKTYPE_NULL, NULL, NULL, NULL } +}; + +static ac_tokval_t event_attrnames[] = { + { "rate", NULL}, + { NULL, NULL} +}; + +static ac_tvlist_t event_attrs = { + event_attrnames, + TOKTYPE_ATTRLIST +}; + +static ac_tokval_t file_attrnames[] = { + { "path", NULL }, + { "reuse", NULL }, + { "prealloc", NULL }, + { "paralloc", NULL }, + { NULL, NULL } +}; + +static ac_tvlist_t file_attrs = { + file_attrnames, + TOKTYPE_ATTRLIST +}; + +static ac_tokval_t fileset_attrnames[] = { + { "size", NULL }, + { "path", NULL }, + { "dirwidth", NULL }, + { "prealloc", NULL }, + { "filesizegamma", NULL }, + { "dirgamma", NULL }, + { "cached", NULL }, + { "entries", NULL }, + { NULL, NULL } +}; + +static ac_tvlist_t fileset_attrs = { + fileset_attrnames, + TOKTYPE_ATTRLIST +}; + +static ac_tokval_t process_attrnames[] = { + { "nice", NULL }, + { "instances", NULL }, + { NULL, NULL } +}; + +static ac_tvlist_t process_attrs = { + process_attrnames, + TOKTYPE_ATTRLIST +}; + +static ac_tokval_t create_argnames[] = { + { "file", NULL }, + { "fileset", NULL }, + { "process", NULL }, + { NULL, NULL } +}; + +static ac_tvlist_t create_args = { + create_argnames, + TOKTYPE_ARG +}; + +static ac_tokval_t define_argnames[] = { + { "file", &file_attrs }, + { "fileset", &fileset_attrs }, + { "process", &process_attrs }, + { NULL, NULL } +}; + +static ac_tvlist_t define_args = { + define_argnames, + TOKTYPE_ARG +}; + +static ac_tvlist_t load_args = { + NULL, + TOKTYPE_LOADFILE +}; + +static ac_tvlist_t set_args = { + NULL, + TOKTYPE_VARLIST +}; + +static ac_tokval_t shutdown_argnames[] = { + { "process", NULL }, + { NULL, NULL } +}; + +static ac_tvlist_t shutdown_args = { + shutdown_argnames, + TOKTYPE_ARG +}; + +static ac_tokval_t stats_argnames[] = { + { "clear", NULL }, + { "directory", NULL }, + { "command", NULL }, + { "dump", NULL }, + { "xmldump", NULL }, + { NULL, NULL } +}; + +static ac_tvlist_t stats_args = { + stats_argnames, + TOKTYPE_ARG +}; + +static ac_tokval_t fb_cmdnames[] = { + { "create", &create_args }, + { "define", &define_args }, + { "debug", NULL }, + { "echo", NULL }, + { "eventgen", &event_attrs }, + { "foreach", NULL }, + { "help", NULL }, + { "list", NULL }, + { "load", &load_args }, + { "log", NULL }, + { "quit", NULL }, + { "run", NULL }, + { "set", &set_args }, + { "shutdown", &shutdown_args }, + { "sleep", NULL }, + { "stats", &stats_args }, + { "system", NULL }, + { "usage", NULL }, + { "vars", NULL }, + { NULL, NULL }, +}; + +static ac_tvlist_t fb_cmds = { + fb_cmdnames, + TOKTYPE_CMD +}; + +static ac_fname_cache_t loadnames = { NULL, 0, 0 }; + +static int search_loadfiles(ac_fname_cache_t *); +static void parse_user_input(const char *, int, ac_inputline_t *); +static int compare_string(ac_str_t *, const char *, boolean_t, const char **); +static ac_match_result_t match_string(WordCompletion *, const char *, int, + ac_str_t *, ac_iter_t *, const char *); + +/* + * Bind the iterator to the passed list + */ +static void +common_bind(ac_iter_t *iterp, void *listp, void *nlistpp) +{ + iterp->listp = listp; + iterp->nlistpp = nlistpp; +} + +/* + * Reset index pointer to point to list head + */ +static void +common_reset(ac_iter_t *iterp) +{ + iterp->curp = iterp->listp; +} + +/* + * Walk through an array of ac_tokval_t structures and return string + * of each item. + */ +static const char * +get_next_tokval(ac_iter_t *iterp) +{ + ac_tokval_t *listp = iterp->listp; /* list head */ + ac_tokval_t *curp = iterp->curp; /* index pointer */ + /* user passed variable for returning value list for next token */ + ac_tvlist_t **nlistpp = iterp->nlistpp; + const char *p; + + if (listp == NULL || curp == NULL) + return (NULL); + + /* get the current item's string */ + p = curp->str; + + /* + * save the current item's address into a user passed variable + */ + if (nlistpp != NULL) + *nlistpp = curp->nlistp; + + /* advance the index pointer */ + iterp->curp = ++curp; + + return (p); +} + +/* + * Bind the iterator to filebench_shm->var_list + */ +static void +varname_bind(ac_iter_t *iterp, void *listp, void * nlistpp) +{ + iterp->listp = filebench_shm->var_list; + iterp->nlistpp = nlistpp; +} + +/* + * Walk through a linked list of var_t type structures and return name + * of each variable with a preceding '$' sign + */ +static const char * +get_next_lvarname(ac_iter_t *iterp) +{ + static char buf[VARNAME_MAXLEN]; + + var_t *listp = iterp->listp; /* list head */ + var_t *curp = iterp->curp; /* index pointer */ + /* User passed variable for returning value list for next token */ + ac_tvlist_t **nlistpp = iterp->nlistpp; + const char *p; + + if (listp == NULL || curp == NULL) + return (NULL); + + /* Get current variable's name, copy it to buf, with a '$' prefix */ + p = curp->var_name; + (void) snprintf(buf, sizeof (buf), "$%s", p); + + /* No information for the next input string */ + if (nlistpp != NULL) + *nlistpp = NULL; + + /* Advance the index pointer */ + iterp->curp = curp->var_next; + + return (buf); +} + +/* + * Walk through a linked list of var_t type structures and return name + * of each variable + */ +static const char * +get_next_rvarname(ac_iter_t *iterp) +{ + var_t *listp = iterp->listp; /* list head */ + var_t *curp = iterp->curp; /* index pointer */ + /* User passed variable for returning value list for next item */ + ac_tvlist_t **nlistpp = iterp->nlistpp; + const char *p; + + if (listp == NULL || curp == NULL) + return (NULL); + + /* Get current variable's name */ + p = curp->var_name; + + /* No information for the next input string */ + if (nlistpp != NULL) + *nlistpp = NULL; + + /* Advance the index pointer */ + iterp->curp = curp->var_next; + + return (p); +} + +/* + * Bind the iterator to loadnames.fnc_buf, which is an ac_fname_t array + * and contains up-to-date workload file names. The function calls + * search_loadfiles() to update the cache before the binding. + */ +static void +loadfile_bind(ac_iter_t *iterp, void *listp, void * nlistpp) +{ + /* Check loadfile name cache, update it if needed */ + (void) search_loadfiles(&loadnames); + + iterp->listp = loadnames.fnc_buf; + iterp->nlistpp = nlistpp; +} + +/* + * Walk through a string(ac_fname_t, more exactly) array and return each + * string, until a NULL iterm is encountered. + */ +static const char * +get_next_loadfile(ac_iter_t *iterp) +{ + ac_fname_t *listp = iterp->listp; /* list head */ + ac_fname_t *curp = iterp->curp; /* index pointer */ + /* User passed variable for returning value list for next item */ + ac_tvlist_t **nlistpp = iterp->nlistpp; + const char *p; + + if (listp == NULL || curp == NULL) + return (NULL); + + /* + * Get current file name. If an NULL item is encountered, it means + * this is the end of the list. In that case, we need to set p to + * NULL to indicate to the caller that the end of the list is reached. + */ + p = (char *)curp; + if (*p == NULL) + p = NULL; + + /* No information for the next input string */ + if (nlistpp != NULL) + *nlistpp = NULL; + + /* Advance the index pointer */ + iterp->curp = ++curp; + + return (p); +} + +/* + * Search for available workload files in workload direcotry and + * update workload name cache. + */ +static int +search_loadfiles(ac_fname_cache_t *fnamecache) +{ + DIR *dirp; + struct dirent *fp; + struct stat dstat; + time_t mtime; + ac_fname_t *buf; + int bufsize = MALLOC_STEP; + int len, i; + + if (stat(FILEBENCHDIR"/workloads", &dstat) != 0) + return (-1); + mtime = dstat.st_mtime; + + /* Return if there is no change since last time */ + if (mtime == fnamecache->fnc_mtime) + return (0); + + /* Get loadfile names and cache it */ + if ((buf = malloc(sizeof (ac_fname_t) * bufsize)) == NULL) + return (-1); + if ((dirp = opendir(FILEBENCHDIR"/workloads")) == NULL) + return (-1); + i = 0; + while ((fp = readdir(dirp)) != NULL) { + len = strlen(fp->d_name); + if (len <= 2 || (fp->d_name)[len - 2] != '.' || + (fp->d_name)[len - 1] != 'f') + continue; + + if (i == bufsize) { + bufsize += MALLOC_STEP; + if ((buf = realloc(buf, sizeof (ac_fname_t) * + bufsize)) == NULL) + return (-1); + } + + (void) snprintf(buf[i], FILENAME_MAXLEN, "%s", fp->d_name); + if (len -2 <= FILENAME_MAXLEN - 1) { + /* Remove .f suffix in file name */ + buf[i][len -2] = NULL; + } + i++; + } + /* Added a NULL iterm as the array's terminator */ + buf[i][0] = NULL; + + if (fnamecache->fnc_bufsize != 0) + free(fnamecache->fnc_buf); + fnamecache->fnc_buf = buf; + fnamecache->fnc_bufsize = bufsize; + fnamecache->fnc_mtime = mtime; + + return (0); +} + +/* + * Parse user input line into a list of blank separated strings, and + * save the result in the passed ac_inputline_t structure. line and word_end + * parameters are passed from libtecla library. line points to user input + * buffer, and word_end is the index of the last character of user input. + */ +static void +parse_user_input(const char *line, int word_end, ac_inputline_t *input) +{ + const char *p = line; + int i; + + /* Reset all fileds */ + for (i = 0; i < STR_NUM; i++) { + input->strs[i].startp = NULL; + input->strs[i].endp = NULL; + input->strs[i].strtype = STRTYPE_INVALID; + } + + /* + * Parse user input. We don't use word_end to do boundary checking, + * instead we take advantage of the fact that the passed line + * parameter is always terminated by '\0'. + */ + for (i = 0; i < STR_NUM; i++) { + /* Skip leading blank spaces */ + while (*p == ' ') + p++; + + if (*p == NULL) { + /* + * User input nothing for the string being input + * before he pressed TAB. We use STR_NULL flag + * to indicate this so that match_str() will list + * all available candidates. + */ + input->strs[i].startp = p; + input->strs[i].strtype = STRTYPE_NULL; + return; + } + + /* Recoard the start and end of the string */ + input->strs[i].startp = p; + while ((*p != ' ') && (*p != NULL)) + p++; + input->strs[i].endp = p - 1; + + if (*p == NULL) { + input->strs[i].strtype = STRTYPE_INCOMPLETE; + return; + } else { + /* The string is followed by a blank space */ + input->strs[i].strtype = STRTYPE_COMPLETE; + } + } +} + +/* + * Parse an input string which is an attribue list, get the current word + * user wants to complete, and return its token type. + * + * An atribute list has the following format: + * + * name1=val,name2=$var,... + * + * The function modifies the passed acstr string on success to point to + * the word being completed. + */ +static ac_toktype_t +parse_attr_list(ac_str_t *acstr) +{ + const char *p; + + if (acstr->strtype == STRTYPE_COMPLETE) { + /* + * User has input a complete string for attribute list + * return TOKTYPE_NULL to abort the matching. + */ + return (TOKTYPE_ATTRLIST); + } else if (acstr->strtype == STRTYPE_NULL) { + /* + * User haven't input anything for the attribute list, + * he must be trying to list all attribute names. + */ + return (TOKTYPE_ATTRNAME); + } + + /* + * The string may contain multiple comma separated "name=value" + * items. Try to find the last one and move startp to point to it. + */ + for (p = acstr->endp; p >= acstr->startp && *p != ATTR_LIST_SEP; p--) {} + + if (p == acstr->endp) { + /* + * The last character of the string is ',', which means + * user is trying to list all attribute names. + */ + acstr->startp = p + 1; + acstr->strtype = STRTYPE_NULL; + return (TOKTYPE_ATTRNAME); + } else if (p > acstr->startp) { + /* + * Found ',' between starp and endp, move startp pointer + * to point to the last item. + */ + acstr->startp = p + 1; + } + + /* + * Now startp points to the last "name=value" item. Search in + * the characters user has input for this item: + * + * a) if there isn't '=' character, user is inputting attribute name + * b) if there is a '=' character and it is followed by a '$', + * user is inputting variable name + * c) if there is a '=' character and it isn't followed by a '$', + * user is inputting a literal string as attribute value. + */ + for (p = acstr->startp; p <= acstr->endp; p++) { + if (*p == ATTR_ASSIGN_OP) { + /* Found "=" operator in the string */ + if (*(p + 1) == VAR_PREFIX) { + acstr->startp = p + 2; + if (*acstr->startp != NULL) + acstr->strtype = STRTYPE_INCOMPLETE; + else + acstr->strtype = STRTYPE_NULL; + return (TOKTYPE_RVARNAME); + } else { + return (TOKTYPE_ATTRVAL); + } + } + } + + /* Didn't find '=' operator, the string must be an attribute name */ + return (TOKTYPE_ATTRNAME); +} + +/* + * Parse an input string which is a variable list, get the current word + * user wants to complete, and return its token type. + * + * A varaible list has the following format: + * + * $varname=value + * + * The function modifies the passed acstr string on success to point to + * the word being completed. + */ +static ac_toktype_t +parse_var_list(ac_str_t *acstr) +{ + const char *p; + + if (acstr->strtype == STRTYPE_COMPLETE) { + /* + * User has input a complete string for var list + * return TOKTYPE_NULL to abort the matching. + */ + return (TOKTYPE_NULL); + } else if (acstr->strtype == STRTYPE_NULL) { + /* + * User haven't input anything for the attribute list, + * he must be trying to list all available var names. + */ + return (TOKTYPE_LVARNAME); + } + + /* + * Search in what user has input: + * + * a) if there isn't a '=' character, user is inputting var name + * b) if there is a '=' character, user is inputting var value + */ + for (p = acstr->startp; p <= acstr->endp; p++) { + if (*p == VAR_ASSIGN_OP) + return (TOKTYPE_VARVAL); + } + + /* Didn't find '=' operator, user must be inputting an var name */ + return (TOKTYPE_LVARNAME); +} + +/* + * Compare two strings acstr and str. acstr is a string of ac_str_t type, + * str is a normal string. If issub is B_TRUE, the function checks if + * acstr is a sub-string of str, starting from index 0; otherwise it checks + * if acstr and str are exactly the same. + * + * The function returns 0 on success and -1 on failure. When it succeeds, + * it also set restp to point to the rest part of the normal string. + */ +static int +compare_string(ac_str_t *acstr, const char *str, boolean_t issub, + const char **restp) +{ + const char *p, *q; + + for (p = acstr->startp, q = str; (p <= acstr->endp) && (*q != '\0'); + p++, q++) { + if (*p != *q) + return (-1); + } + + if (p == acstr->endp + 1) { + if (*q == '\0' || issub == B_TRUE) { + if (restp != NULL) + *restp = q; + return (0); + } + } + + return (-1); +} + +/* + * Use the passed iterp iterator to access a list of string values to + * look for those matches with acstr, an user input string to be completed. + * + * cpl, line, work_end, and cont_suffix are parameters needed by + * cpl_add_completion(), which adds matched entries to libtecla. + * + * Since user input line may have multiple strings, the function is + * expected to be called multiple times to match those strings one + * by one until the last one is reached. + * + * The multi-step matching process also means the function should provide + * a way to indicate to the caller whether to continue or abort the + * whole matching process. The function does that with the following + * return values: + * + * MATCH_DONE - the matching for the whole user input is done. This + * can mean either some items are found or none is found. + * In either case, the caller shouldn't continue to + * match the rest strings, either because there is + * no strings left, or because the matching for the + * current string failed so there is no need to check + * further. + * MATCH_CONT - the matching for the current string succeeds, but + * user needs to continue to match the rest strings. + */ +static ac_match_result_t +match_string(WordCompletion *cpl, const char *line, int word_end, + ac_str_t *acstr, ac_iter_t *iterp, const char *cont_suffix) +{ + const char *str, *restp; + + iterp->reset(iterp); + + if (acstr->strtype == STRTYPE_COMPLETE) { + while ((str = iterp->get_nextstr(iterp)) != NULL) { + if (!compare_string(acstr, str, B_FALSE, NULL)) { + /* Continue to check rest strings */ + return (MATCH_CONT); + } + } + } else if (acstr->strtype == STRTYPE_NULL) { + /* User input nothing. List all available strings */ + while ((str = iterp->get_nextstr(iterp)) != NULL) { + (void) cpl_add_completion(cpl, line, + acstr->startp - line, word_end, str, + NULL, cont_suffix); + } + } else if (acstr->strtype == STRTYPE_INCOMPLETE) { + while ((str = iterp->get_nextstr(iterp)) != NULL) { + if (!compare_string(acstr, str, B_TRUE, &restp)) { + /* It matches! Add it. */ + (void) cpl_add_completion(cpl, line, + acstr->startp - line, word_end, restp, + NULL, cont_suffix); + } + } + } + + return (MATCH_DONE); +} + +/* + * This is the interface between filebench and libtecla for auto- + * completion. It is called by libtecla whenever user initiates a + * auto-completion request(ie., pressing TAB key). + * + * The function calls parse_user_input() to parse user input into + * multiple strings, then it calls match_string() to match each + * string in user input in sequence until either the last string + * is reached and completed or the the matching fails. + */ +CPL_MATCH_FN(command_complete) +{ + ac_inputline_t inputline; + ac_tvlist_t *clistp = &fb_cmds, *nlistp; + ac_toktype_t toktype; + ac_iter_t *iterp; + char *cont_suffix; + ac_get_curtok_func_t get_curtok; + int i, ret; + + /* Parse user input and save the result in inputline variable. */ + parse_user_input(line, word_end, &inputline); + + /* + * Match each string in user input against the proper token's + * value list, and continue the loop until either the last string + * is reached and completed or the matching aborts. + */ + for (i = 0; i < STR_NUM && + inputline.strs[i].strtype != STRTYPE_INVALID && clistp != NULL; + i++) { + toktype = clistp->toktype; + + /* + * If the current stirng can contain multiple tokens, modify + * the stirng to point to the word being input and return + * its token type. + */ + get_curtok = token_info[toktype].get_curtok; + if (get_curtok != NULL) + toktype = (*get_curtok)(&inputline.strs[i]); + + iterp = token_info[toktype].iter; + cont_suffix = token_info[toktype].cont_suffix; + /* Return if there is no completion info for the token */ + if (iterp == NULL) + break; + + iterp->bind(iterp, clistp->vals, &nlistp); + /* Match user string against the token's list */ + ret = match_string(cpl, line, word_end, &inputline.strs[i], + iterp, cont_suffix); + if (ret == MATCH_DONE) + return (0); + clistp = nlistp; + } + + return (0); +} diff --git a/usr/src/cmd/filebench/common/auto_comp.h b/usr/src/cmd/filebench/common/auto_comp.h new file mode 100644 index 0000000000..e526f7650b --- /dev/null +++ b/usr/src/cmd/filebench/common/auto_comp.h @@ -0,0 +1,43 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +#ifndef _FB_AUTOCOMP_H +#define _FB_AUTOCOMP_H + +#include <libtecla.h> + +#ifdef __cplusplus +extern "C" { +#endif + +extern CPL_MATCH_FN(command_complete); + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_AUTOCOMP_H */ diff --git a/usr/src/cmd/filebench/common/config.h b/usr/src/cmd/filebench/common/config.h new file mode 100644 index 0000000000..034ee88a89 --- /dev/null +++ b/usr/src/cmd/filebench/common/config.h @@ -0,0 +1,83 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_CONFIG_H +#define _FB_CONFIG_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#ifdef __cplusplus +extern "C" { +#endif + +#define HAVE_AIO 1 +#define HAVE_AIOCB64_T 1 +#define HAVE_AIOWAITN 1 +#define HAVE_AIO_H 1 +#define HAVE_CADDR_T 1 +#define HAVE_FORK 1 +#define HAVE_FORK1 1 +#define HAVE_HRTIME 1 +#define HAVE_LIBKSTAT 1 +#define HAVE_LWPS 1 +#define HAVE_MKSTEMP 1 +#define HAVE_OFF64_T 1 +#define HAVE_PROCFS 1 +#define HAVE_PROCSCOPE_PTHREADS 1 +#define HAVE_PTHREAD 1 +#define HAVE_PTHREAD_MUTEXATTR_SETPROTOCOL 1 +#define HAVE_ROBUST_MUTEX 1 +#define HAVE_SEMTIMEDOP 1 +#define HAVE_SETRLIMIT 1 +#define HAVE_SHM_SHARE_MMU 1 +#define HAVE_SIGSEND 1 +#define HAVE_STDINT_H 1 +#define HAVE_SYSV_SEM 1 +#define HAVE_SEM_RMID 1 +#define HAVE_SYS_INT_LIMITS_H 1 +#define HAVE_UINT64_MAX 1 +#define HAVE_UINT_T 1 +#define HAVE_BOOLEAN_T 1 +#define HAVE_LIBTECLA 1 +#define USE_PROCESS_MODEL 1 + +/* Define to 1 if you have the <libaio.h> header file. */ +/* #undefHAVE_LIBAIO_H */ + +/* Checking if you have /proc/stat */ +/* #undef HAVE_PROC_STAT */ + +/* Define to 1 if you have the <sys/async.h> header file. */ +/* #undef HAVE_SYS_ASYNC_H */ + +/* Define if you want support for RDTSC. */ +/* #undef USE_RDTSC */ + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_CONFIG_H */ diff --git a/usr/src/cmd/filebench/common/eventgen.c b/usr/src/cmd/filebench/common/eventgen.c new file mode 100644 index 0000000000..bff586e17d --- /dev/null +++ b/usr/src/cmd/filebench/common/eventgen.c @@ -0,0 +1,173 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +/* + * The event generator in this module is the producer half of a + * metering system which blocks flows using consumer routines in the + * flowop_library.c module. Four routines in that module can limit rates + * by event rate (flowoplib_eventlimit), by I/O operations rate + * (flowoplib_iopslimit()), by operations rate (flowoplib_opslimit), + * or by I/O bandwidth limit (flowoplib_bwlimit). By setting appropriate + * event generation rates, required calls per second, I/O ops per second, + * file system ops per second, or I/O bandwidth per second limits can + * be set. Note, the generated events are shared with all consumer + * flowops, of which their will be one for each process / thread + * instance which has a consumer flowop defined in it. + */ + +#include <sys/time.h> +#include "vars.h" +#include "eventgen.h" +#include "filebench.h" +#include "flowop.h" +#include "ipc.h" + +/* + * Prints "how to use" information for the eventgen module + */ +void +eventgen_usage(void) +{ + (void) fprintf(stderr, "eventgen rate=<rate>\n"); + (void) fprintf(stderr, "\n"); +} + +/* + * The producer side of the event system. + * Once eventgen_hz has been set by eventgen_setrate(), + * the routine sends eventgen_hz events per second until + * the program terminates. Events are posted by incrementing + * filebench_shm->eventgen_q by the number of generated + * events then signalling the condition variable + * filebench_shm->eventgen_cv to indicate to event consumers + * that more events are available. + * + * Eventgen_thread attempts to sleep for 10 event periods, + * then, once awakened, determines how many periods actually + * passed since sleeping, and issues a set of events equal + * to the number of periods that it slept, thus keeping the + * average rate at the requested rate. + */ +static void +eventgen_thread(void) +{ + hrtime_t last; + + last = gethrtime(); + + /* CONSTCOND */ + while (1) { + struct timespec sleeptime; + hrtime_t delta; + int count; + + if (filebench_shm->eventgen_hz == 0) { + (void) sleep(1); + continue; + } + /* Sleep for 10xperiod */ + sleeptime.tv_sec = 0; + sleeptime.tv_nsec = 1000000000UL / filebench_shm->eventgen_hz; + sleeptime.tv_nsec *= 10; + if (sleeptime.tv_nsec < 1000UL) + sleeptime.tv_nsec = 1000UL; + sleeptime.tv_sec = sleeptime.tv_nsec / 1000000000UL; + if (sleeptime.tv_sec > 0) + sleeptime.tv_nsec -= (sleeptime.tv_sec * 1000000000UL); + (void) nanosleep(&sleeptime, NULL); + delta = gethrtime() - last; + last = gethrtime(); + count = (filebench_shm->eventgen_hz * delta) / 1000000000; + + filebench_log(LOG_DEBUG_SCRIPT, + "delta %lldms count %d", delta / 1000000, count); + + /* Send 'count' events */ + (void) ipc_mutex_lock(&filebench_shm->eventgen_lock); + /* Keep the producer with a max of 5 second depth */ + if (filebench_shm->eventgen_q < + (5 * filebench_shm->eventgen_hz)) + filebench_shm->eventgen_q += count; + + (void) pthread_cond_signal(&filebench_shm->eventgen_cv); + + (void) ipc_mutex_unlock(&filebench_shm->eventgen_lock); + } +} + +/* + * Creates a thread to run the event generator eventgen_thread + * routine. Shuts down filebench if the eventgen thread cannot + * be created. + */ +void +eventgen_init(void) +{ + /* + * Linux does not like it if the first + * argument to pthread_create is null. It actually + * segv's. -neel + */ + pthread_t tid; + + if (pthread_create(&tid, NULL, + (void *(*)(void*))eventgen_thread, 0) != 0) { + filebench_log(LOG_ERROR, "create timer thread failed: %s", + strerror(errno)); + filebench_shutdown(1); + } +} + +/* + * Puts the current event rate in the integer portion of the + * supplied var_t. Returns a pointer to the var_t. + */ +var_t * +eventgen_ratevar(var_t *var) +{ + var->var_integer = filebench_shm->eventgen_hz; + return (var); +} + +/* + * Sets the event generator rate to that supplied by + * vinteger_t rate. + */ +void +eventgen_setrate(vinteger_t rate) +{ + filebench_shm->eventgen_hz = rate; +} + +/* + * Turns off the event generator by setting the rate to zero + */ +void +eventgen_reset(void) +{ + filebench_shm->eventgen_q = 0; +} diff --git a/usr/src/cmd/filebench/common/eventgen.h b/usr/src/cmd/filebench/common/eventgen.h new file mode 100644 index 0000000000..21296614e6 --- /dev/null +++ b/usr/src/cmd/filebench/common/eventgen.h @@ -0,0 +1,50 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_EVENTGEN_H +#define _FB_EVENTGEN_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" +#include "vars.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void eventgen_init(void); +void eventgen_setrate(vinteger_t rate); +var_t *eventgen_ratevar(var_t *var); +void eventgen_usage(void); +void eventgen_reset(void); + +#define EVENTGEN_VAR "rate" + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_EVENTGEN_H */ diff --git a/usr/src/cmd/filebench/common/filebench.h b/usr/src/cmd/filebench/common/filebench.h new file mode 100644 index 0000000000..0e6ce15e8a --- /dev/null +++ b/usr/src/cmd/filebench/common/filebench.h @@ -0,0 +1,127 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_FILEBENCH_H +#define _FB_FILEBENCH_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" + +#include <stdio.h> +#include <string.h> + +#include "vars.h" +#include "misc.h" +#include "flowop.h" +#include "ipc.h" + +#ifdef HAVE_STDINT_H +#include <stdint.h> +#endif + + +#ifdef __STDC__ +#include <stdarg.h> +#define __V(x) x +#ifndef __P +#define __P(x) x +#endif +#else +#include <varargs.h> +#define __V(x) (va_alist) va_dcl +#define __P(x) () +#define const +#endif + +#include <sys/times.h> + +#ifdef HAVE_SYS_INT_LIMITS_H +#include <sys/int_limits.h> +#endif /* HAVE_SYS_INT_LIMITS_H */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern pid_t pid; +extern int errno; +extern char *execname; +extern int noproc; + +void filebench_init(); +void filebench_log __V((int level, const char *fmt, ...)); +void filebench_shutdown(int error); + +#ifndef HAVE_UINT64_MAX +#define UINT64_MAX (((off64_t)1UL<<63UL) - 1UL) +#endif + +#define FILEBENCH_RANDMAX64 UINT64_MAX +#define FILEBENCH_RANDMAX32 UINT32_MAX + +#if defined(_LP64) || (__WORDSIZE == 64) +#define filebench_randomno filebench_randomno64 +#define FILEBENCH_RANDMAX FILEBENCH_RANDMAX64 +#else +#define filebench_randomno filebench_randomno32 +#define FILEBENCH_RANDMAX FILEBENCH_RANDMAX32 +#endif +int filebench_randomno32(uint32_t *, uint32_t, uint32_t); +int filebench_randomno64(uint64_t *, uint64_t, uint64_t); + +#define KB (1024LL) +#define MB (KB * KB) +#define GB (KB * MB) + +#define MMAP_SIZE (1024UL * 1024UL * 1024UL) + +#ifndef MIN +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#endif + +#define FILEBENCH_VERSION "1.0.0" +#define FILEBENCHDIR "/usr/benchmarks/filebench" +#define FILEBENCH_PROMPT "filebench> " +#define MAX_LINE_LEN 1024 +#define MAX_CMD_HIST 128 + +/* For MacOSX */ +#ifndef HAVE_OFF64_T +#define mmap64 mmap +#define off64_t off_t +#define open64 open +#define stat64 stat +#define pread64 pread +#define pwrite64 pwrite +#define lseek64 lseek +#define fstat64 fstat +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_FILEBENCH_H */ diff --git a/usr/src/cmd/filebench/common/fileobj.c b/usr/src/cmd/filebench/common/fileobj.c new file mode 100644 index 0000000000..88241f5d2b --- /dev/null +++ b/usr/src/cmd/filebench/common/fileobj.c @@ -0,0 +1,476 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + + +#include <fcntl.h> +#include <pthread.h> +#include <errno.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> + +#ifdef HAVE_UTILITY_H +#include <utility.h> +#endif + +#include "vars.h" +#include "filebench.h" +#include "fileobj.h" + +/* + * File objects, of type fileobj_t, are entities which contain + * information about filebench files, one file per fileobj. The + * information includes the file's name, its pathname, its size, and + * a set of attributes indicating whether to create, preallocate, + * allocate in parallel, reuse, and cache the file or not. All fileobj + * are kept on a global linked list in shared memory. + */ + +/* + * Prints syntax for specifying file object parameters. + */ +void +fileobj_usage() +{ + (void) fprintf(stderr, + "define file name=<name>,path=<pathname>,size=<size>\n"); + (void) fprintf(stderr, " [,paralloc]\n"); + (void) fprintf(stderr, " [,prealloc]\n"); + (void) fprintf(stderr, " [,reuse]\n"); + (void) fprintf(stderr, "\n"); +} + +/* + * Frees up memory mapped file region of supplied size. + * The file descriptor "fd" indicates which memory mapped + * file. If successful, returns 0. Otherwise returns -1 if + * "size" is zero, or -1 times the number of times msync() + * failed. + */ +static int +fileobj_freemem(int fd, off64_t size) +{ + off64_t left; + int ret = -1; + + for (left = size; left > 0; left -= MMAP_SIZE) { + off64_t thismapsize; + caddr_t addr; + + thismapsize = MIN(MMAP_SIZE, left); + addr = mmap64(0, thismapsize, PROT_READ|PROT_WRITE, + MAP_SHARED, fd, size - left); + ret += msync(addr, thismapsize, MS_INVALIDATE); + (void) munmap(addr, thismapsize); + } + + return (ret); +} + +/* + * Creates the file associated with a fileobj (file object) + * and writes fo_size bytes to it. The bytes are written + * FILE_ALLOC_BLOCK bytes at a time. However, if the file + * already exists, is full size, and the fo_reuse flag is + * set, then the file is just left alone. + */ +static int +fileobj_prealloc(fileobj_t *fileobj) +{ + off64_t seek; + int fd; + char *buf; + int exists; + struct stat64 sb; + char name[MAXPATHLEN]; + + if (*fileobj->fo_path == NULL) { + filebench_log(LOG_ERROR, "File path not set"); + return (-1); + } + + (void) strcpy(name, *fileobj->fo_path); + (void) mkdir(name, 0777); + (void) strcat(name, "/"); + (void) strcat(name, fileobj->fo_name); + + if ((fd = open64(name, O_RDWR)) < 0) { + filebench_log(LOG_ERROR, + "Failed to find file %s for pre-allocation: %s", + name, strerror(errno)); + return (-1); + } + + /* If it's a raw device */ + exists = (fstat64(fd, &sb) == 0); +#ifdef RAW + if (exists && sb.st_rdev) + return (0); +#endif + + if (integer_isset(fileobj->fo_reuse) && exists) { + if (sb.st_size == (off64_t)*fileobj->fo_size) { + filebench_log(LOG_INFO, "Re-using file %s", name); + if (!integer_isset(fileobj->fo_cached)) + (void) fileobj_freemem(fd, *fileobj->fo_size); + (void) close(fd); + return (0); + } else if (sb.st_size > (off64_t)*fileobj->fo_size) { + /* reuse, but too large */ + filebench_log(LOG_INFO, + "Truncating & Re-using file %s", name); + (void) ftruncate64(fd, (off64_t)*fileobj->fo_size); + if (!integer_isset(fileobj->fo_cached)) + (void) fileobj_freemem(fd, *fileobj->fo_size); + (void) close(fd); + return (0); + } + } + + if ((buf = (char *)malloc(FILE_ALLOC_BLOCK)) == NULL) { + (void) close(fd); + return (-1); + } + + for (seek = 0; seek < (off64_t)*fileobj->fo_size; ) { + off64_t wsize; + int ret = 0; + + /* Write FILE_ALLOC_BLOCK's worth except on last write */ + wsize = MIN((off64_t)*fileobj->fo_size - seek, + FILE_ALLOC_BLOCK); + + ret = write(fd, buf, wsize); + if (ret != wsize) { + filebench_log(LOG_ERROR, + "Failed to pre-allocate file %s: %s", + name, strerror(errno)); + (void) close(fd); + return (-1); + } + seek += wsize; + } + + filebench_log(LOG_INFO, + "Pre-allocated file %s", name); + + (void) fsync(fd); + if (!integer_isset(fileobj->fo_cached)) + (void) fileobj_freemem(fd, *fileobj->fo_size); + (void) close(fd); + return (0); +} + +/* + * Creates the file portion of a fileobj (file object) and + * leaves it empty. If the file already exists and the fo_reuse + * flag is set, the file will be retained, otherwise any + * existing file will first be deleted, and then a new + * (empty) file will be created. + */ +static int +fileobj_createfile(fileobj_t *fileobj) +{ + int fd; + struct stat64 sb; + int exists; + char name[MAXPATHLEN]; + + if (*fileobj->fo_path == NULL) { + filebench_log(LOG_ERROR, "File path not set"); + return (-1); + } + + (void) strcpy(name, *fileobj->fo_path); + (void) mkdir(name, 0777); + (void) strcat(name, "/"); + (void) strcat(name, fileobj->fo_name); + + /* If it's a raw device */ + exists = (stat64(name, &sb) == 0); +#ifdef RAW + if (exists && sb.st_rdev) + return (0); +#endif + + /* + * If we are re-using the file, then just free up the cache and + * return + */ + if (integer_isset(fileobj->fo_reuse) && exists) { + fd = open64(name, O_RDWR); + (void) fsync(fd); + (void) fileobj_freemem(fd, *fileobj->fo_size); + (void) close(fd); + return (fd < 0); + } + + filebench_log(LOG_DEBUG_IMPL, "Creating file %s", name); + (void) unlink(name); + if ((fd = open64(name, O_RDWR | O_CREAT, 0666)) < 0) { + filebench_log(LOG_ERROR, + "Failed to create file %s: %s", + name, strerror(errno)); + return (-1); + } + (void) fsync(fd); + (void) close(fd); + + return (fd < 0); +} + +/* + * Creates and optionally preallocates the actual file + * portions of all fileobjs found on the filelist maintained + * in shared memory. If fo_paralloc is set, a thread will + * be spawned to preallocate the fileobj in parallel with + * that of other fileobjs. The routine waits for all such + * threads to finish before exiting. + */ +int +fileobj_init() +{ + fileobj_t *fileobj = filebench_shm->filelist; + int nthreads = 0; + int ret = 0; + + (void) ipc_mutex_lock(&filebench_shm->fileobj_lock); + + filebench_log(LOG_INFO, + "Creating/pre-allocating files"); + + while (fileobj) { + + /* Create files */ + if (*fileobj->fo_create) + ret += fileobj_createfile(fileobj); + + /* Preallocate files */ + if (integer_isset(fileobj->fo_prealloc)) { + + if (!integer_isset(fileobj->fo_paralloc)) { + ret += fileobj_prealloc(fileobj); + fileobj = fileobj->fo_next; + continue; + } + + if (pthread_create(&fileobj->fo_tid, NULL, + (void *(*)(void*))fileobj_prealloc, + fileobj) != 0) { + filebench_log(LOG_ERROR, + "File prealloc thread create failed"); + filebench_shutdown(1); + } else { + nthreads++; + } + } + + fileobj = fileobj->fo_next; + } + + /* Wait for allocations to finish */ + if (nthreads) { + filebench_log(LOG_INFO, + "Waiting for preallocation threads to complete..."); + fileobj = filebench_shm->filelist; + while (fileobj) { + ret += pthread_join(fileobj->fo_tid, NULL); + fileobj = fileobj->fo_next; + } + } + + (void) ipc_mutex_unlock(&filebench_shm->fileobj_lock); + + return (ret); +} + +/* + * Allocates a fileobj (file object) in shared memory, sets + * it's name to "name" and places it on the shared filelist. + * It also returns a pointer to the fileobj. + */ +fileobj_t * +fileobj_define(char *name) +{ + fileobj_t *fileobj; + + if (name == NULL) + return (NULL); + + /* allocate a fileobj from shared memory */ + if ((fileobj = (fileobj_t *)ipc_malloc(FILEBENCH_FILEOBJ)) == NULL) { + filebench_log(LOG_ERROR, + "fileobj_define: Can't malloc fileobj"); + return (NULL); + } + + filebench_log(LOG_DEBUG_IMPL, "Defining file %s", name); + + (void) ipc_mutex_lock(&filebench_shm->fileobj_lock); + + /* Add fileobj to global list */ + if (filebench_shm->filelist == NULL) { + filebench_shm->filelist = fileobj; + fileobj->fo_next = NULL; + } else { + fileobj->fo_next = filebench_shm->filelist; + filebench_shm->filelist = fileobj; + } + + (void) ipc_mutex_unlock(&filebench_shm->fileobj_lock); + + /* name the new fileobj */ + (void) strcpy(fileobj->fo_name, name); + + return (fileobj); +} + +/* + * Opens the file associated with a fileobj. The "attrs" + * integer supplies optional attributes to the open64 call + * used to actually open the file. The file is opened in + * read/write mode, and may be opened in synchronous mode + * if FLOW_ATTR_DSYNC is set in "attrs", and be set to + * DIRECTIO_ON if FLOW_ATTR_DIRECTIO is set in "attrs". + * The file descriptor integer returned by open64 is + * returned to the caller. + */ +int +fileobj_open(fileobj_t *fileobj, int attrs) +{ + int open_attrs = 0; + int fd; + char name[MAXPATHLEN]; + + if (*fileobj->fo_path == NULL) { + filebench_log(LOG_ERROR, "File path not set"); + return (-1); + } + + (void) strcpy(name, *fileobj->fo_path); + (void) mkdir(name, 0777); + (void) strcat(name, "/"); + (void) strcat(name, fileobj->fo_name); + + if (attrs & FLOW_ATTR_DSYNC) { +#ifdef sun + open_attrs |= O_DSYNC; +#else + open_attrs |= O_FSYNC; +#endif + } + + fd = open64(name, O_RDWR | open_attrs, 0666); + filebench_log(LOG_DEBUG_SCRIPT, "open file %s flags %d = %d", + *fileobj->fo_path, open_attrs, fd); + + if (fd < 0) { + filebench_log(LOG_ERROR, + "Failed to open %s: %s", + name, + strerror(errno)); + } + + /* if running on Solaris, decide whether to use buffered io or not */ +#ifdef sun + if (attrs & FLOW_ATTR_DIRECTIO) + (void) directio(fd, DIRECTIO_ON); + else + (void) directio(fd, DIRECTIO_OFF); +#endif + + return (fd); +} + +/* + * Searches the shared "filelist" for the named filobj. + * Returns a pointer to the fileobj if found, otherwise NULL. + */ +fileobj_t * +fileobj_find(char *name) +{ + fileobj_t *fileobj = filebench_shm->filelist; + + (void) ipc_mutex_lock(&filebench_shm->fileobj_lock); + + while (fileobj) { + + if (strcmp(name, fileobj->fo_name) == 0) { + (void) ipc_mutex_unlock(&filebench_shm->fileobj_lock); + return (fileobj); + } + fileobj = fileobj->fo_next; + } + (void) ipc_mutex_unlock(&filebench_shm->fileobj_lock); + + return (NULL); +} + +/* + * Iterates over all the file objects in the filelist, + * executing the supplied command "*cmd()" on them. Also + * indicates to the executed command if it is the first + * time the command has been executed since the current + * call to fileobj_iter. + */ +void +fileobj_iter(int (*cmd)(fileobj_t *fileobj, int first)) +{ + fileobj_t *fileobj = filebench_shm->filelist; + int count = 0; + + (void) ipc_mutex_lock(&filebench_shm->fileobj_lock); + + while (fileobj) { + cmd(fileobj, count == 0); + fileobj = fileobj->fo_next; + count++; + } + + (void) ipc_mutex_unlock(&filebench_shm->fileobj_lock); +} + +/* + * Prints information to the filebench log about the file + * object. Also prints a header on the first call. + */ +int +fileobj_print(fileobj_t *fileobj, int first) +{ + if (first) { + filebench_log(LOG_INFO, "%10s %32s %8s", + "File Name", + "Path Name", + "Size"); + } + + filebench_log(LOG_INFO, "%10s %32s %8ld", + fileobj->fo_name, + *fileobj->fo_path, + *fileobj->fo_size); + return (0); +} diff --git a/usr/src/cmd/filebench/common/fileobj.h b/usr/src/cmd/filebench/common/fileobj.h new file mode 100644 index 0000000000..c23bfaa39f --- /dev/null +++ b/usr/src/cmd/filebench/common/fileobj.h @@ -0,0 +1,76 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_FILEOBJ_H +#define _FB_FILEOBJ_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/param.h> +#include <sys/resource.h> +#include <pthread.h> + +#include "vars.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct fileobj { + char fo_name[128]; /* Name */ + struct fileobj *fo_next; /* Next in list */ + pthread_t fo_tid; /* Thread id, for par alloc */ + var_string_t fo_path; /* Pathname in fs */ + var_integer_t fo_size; /* Initial size */ + var_integer_t fo_create; /* Attr */ + var_integer_t fo_prealloc; /* Attr */ + var_integer_t fo_paralloc; /* Attr */ + var_integer_t fo_reuse; /* Attr */ + var_integer_t fo_cached; /* Attr */ + int fo_attrs; /* Attributes */ +} fileobj_t; + +#define FILE_ALLOC_BLOCK (off64_t)1024 * 1024 + +fileobj_t *fileobj_define(char *); +fileobj_t *fileobj_find(char *); +int fileobj_init(void); +int fileobj_open(fileobj_t *fileobj, int attrs); +void fileobj_iter(int (*cmd)(fileobj_t *, int)); +int fileobj_print(fileobj_t *fileobj, int first); +void fileobj_usage(void); + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_FILEOBJ_H */ diff --git a/usr/src/cmd/filebench/common/fileset.c b/usr/src/cmd/filebench/common/fileset.c new file mode 100644 index 0000000000..83f08fd043 --- /dev/null +++ b/usr/src/cmd/filebench/common/fileset.c @@ -0,0 +1,913 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + + +#include <fcntl.h> +#include <pthread.h> +#include <errno.h> +#include <math.h> +#include <libgen.h> +#include <sys/mman.h> +#include "fileset.h" +#include "filebench.h" +#include "gamma_dist.h" + +/* + * File sets, of type fileset_t, are entities which contain + * information about collections of files and subdirectories in Filebench. + * The fileset, once populated, consists of a tree of fileset entries of + * type filesetentry_t which specify files and directories. The fileset + * is rooted in a directory specified by fs_path, and once the populated + * fileset has been created, has a tree of directories and files + * corresponding to the fileset's filesetentry tree. + */ + +/* + * Removes the last file or directory name from a pathname. + * Basically removes characters from the end of the path by + * setting them to \0 until a forward slash '/' is + * encountered. It also removes the forward slash. + */ +static char * +trunc_dirname(char *dir) +{ + char *s = dir + strlen(dir); + + while (s != dir) { + int c = *s; + + *s = 0; + if (c == '/') + break; + s--; + } + return (dir); +} + +/* + * Prints a list of allowed options and how to specify them. + */ +void +fileset_usage(void) +{ + (void) fprintf(stderr, "define fileset name=<name>,path=<pathname>," + "entries=<number>\n"); + (void) fprintf(stderr, " [,dirwidth=[width]\n"); + (void) fprintf(stderr, " [,dirgamma=[100-10000] " + "(Gamma * 1000)\n"); + (void) fprintf(stderr, + " [,sizegamma=[100-10000] (Gamma * 1000)\n"); + (void) fprintf(stderr, + " [,prealloc=[percent]]\n"); + (void) fprintf(stderr, " [,reuse]\n"); + (void) fprintf(stderr, "\n"); +} + +/* + * Frees up memory mapped file region of supplied size. The + * file descriptor "fd" indicates which memory mapped file. + * If successful, returns 0. Otherwise returns -1 if "size" + * is zero, or -1 times the number of times msync() failed. + */ +static int +fileset_freemem(int fd, off64_t size) +{ + off64_t left; + int ret = 0; + + for (left = size; left > 0; left -= MMAP_SIZE) { + off64_t thismapsize; + caddr_t addr; + + thismapsize = MIN(MMAP_SIZE, left); + addr = mmap64(0, thismapsize, PROT_READ|PROT_WRITE, + MAP_SHARED, fd, size - left); + ret += msync(addr, thismapsize, MS_INVALIDATE); + (void) munmap(addr, thismapsize); + } + return (ret); +} + +/* + * Creates a path string from the filesetentry_t "*entry" + * and all of its parent's path names. The resulting path + * is a concatination of all the individual parent paths. + * Allocates memory for the path string and returns a + * pointer to it. + */ +char * +fileset_resolvepath(filesetentry_t *entry) +{ + filesetentry_t *fsep = entry; + char path[MAXPATHLEN]; + char pathtmp[MAXPATHLEN]; + char *s; + + *path = 0; + while (fsep->fse_parent) { + (void) strcpy(pathtmp, "/"); + (void) strcat(pathtmp, fsep->fse_path); + (void) strcat(pathtmp, path); + (void) strcpy(path, pathtmp); + fsep = fsep->fse_parent; + } + + s = malloc(strlen(path) + 1); + (void) strcpy(s, path); + return (s); +} + +/* + * Creates multiple nested directories as required by the + * supplied path. Starts at the end of the path, creating + * a list of directories to mkdir, up to the root of the + * path, then mkdirs them one at a time from the root on down. + */ +static int +fileset_mkdir(char *path, int mode) +{ + char *p; + char *dirs[65536]; + int i = 0; + + if ((p = strdup(path)) == NULL) + goto null_str; + + /* + * Fill an array of subdirectory path names until either we + * reach the root or encounter an already existing subdirectory + */ + /* CONSTCOND */ + while (1) { + struct stat64 sb; + + if (stat64(p, &sb) == 0) + break; + if (strlen(p) < 3) + break; + if ((dirs[i] = strdup(p)) == NULL) { + free(p); + goto null_str; + } + + (void) trunc_dirname(p); + i++; + } + + /* Make the directories, from closest to root downwards. */ + for (--i; i >= 0; i--) { + (void) mkdir(dirs[i], mode); + free(dirs[i]); + } + + free(p); + return (0); + +null_str: + /* clean up */ + for (--i; i >= 0; i--) + free(dirs[i]); + + filebench_log(LOG_ERROR, + "Failed to create directory path %s: Out of memory", path); + + return (-1); +} + + +/* + * First creates the parent directories of the file using + * fileset_mkdir(). Then Optionally sets the O_DSYNC flag + * and opens the file with open64(). It unlocks the fileset + * entry lock, sets the DIRECTIO_ON or DIRECTIO_OFF flags + * as requested, and returns the file descriptor integer + * for the opened file. + */ +int +fileset_openfile(fileset_t *fileset, + filesetentry_t *entry, int flag, int mode, int attrs) +{ + char path[MAXPATHLEN]; + char dir[MAXPATHLEN]; + char *pathtmp; + struct stat64 sb; + int fd; + int open_attrs = 0; + + *path = 0; + (void) strcpy(path, *fileset->fs_path); + (void) strcat(path, "/"); + (void) strcat(path, fileset->fs_name); + pathtmp = fileset_resolvepath(entry); + (void) strcat(path, pathtmp); + (void) strcpy(dir, path); + free(pathtmp); + (void) trunc_dirname(dir); + + /* If we are going to create a file, create the parent dirs */ + if ((flag & O_CREAT) && (stat64(dir, &sb) != 0)) { + if (fileset_mkdir(dir, 0755) == -1) + return (-1); + } + + if (flag & O_CREAT) + entry->fse_flags |= FSE_EXISTS; + + if (attrs & FLOW_ATTR_DSYNC) { +#ifdef sun + open_attrs |= O_DSYNC; +#else + open_attrs |= O_FSYNC; +#endif + } + + if ((fd = open64(path, flag | open_attrs, mode)) < 0) { + filebench_log(LOG_ERROR, + "Failed to open file %s: %s", + path, strerror(errno)); + (void) ipc_mutex_unlock(&entry->fse_lock); + return (-1); + } + (void) ipc_mutex_unlock(&entry->fse_lock); + +#ifdef sun + if (attrs & FLOW_ATTR_DIRECTIO) + (void) directio(fd, DIRECTIO_ON); + else + (void) directio(fd, DIRECTIO_OFF); +#endif + + return (fd); +} + + +/* + * Selects a fileset entry from a fileset. If the + * FILESET_PICKDIR flag is set it will pick a directory + * entry, otherwise a file entry. The FILESET_PICKRESET + * flag will cause it to reset the free list to the + * overall list (file or directory). The FILESET_PICKUNIQUE + * flag will take an entry off of one of the free (unused) + * lists (file or directory), otherwise the entry will be + * picked off of one of the rotor lists (file or directory). + * The FILESET_PICKEXISTS will insure that only extant + * (FSE_EXISTS) state files are selected, while + * FILESET_PICKNOEXIST insures that only non extant + * (not FSE_EXISTS) state files are selected. + */ +filesetentry_t * +fileset_pick(fileset_t *fileset, int flags, int tid) +{ + filesetentry_t *entry = NULL; + filesetentry_t *first = NULL; + + (void) ipc_mutex_lock(&filebench_shm->fileset_lock); + + while (entry == NULL) { + + if ((flags & FILESET_PICKDIR) && (flags & FILESET_PICKRESET)) { + entry = fileset->fs_dirlist; + while (entry) { + entry->fse_flags |= FSE_FREE; + entry = entry->fse_dirnext; + } + fileset->fs_dirfree = fileset->fs_dirlist; + } + + if (!(flags & FILESET_PICKDIR) && (flags & FILESET_PICKRESET)) { + entry = fileset->fs_filelist; + while (entry) { + entry->fse_flags |= FSE_FREE; + entry = entry->fse_filenext; + } + fileset->fs_filefree = fileset->fs_filelist; + } + + if (flags & FILESET_PICKUNIQUE) { + if (flags & FILESET_PICKDIR) { + entry = fileset->fs_dirfree; + if (entry == NULL) + goto empty; + fileset->fs_dirfree = entry->fse_dirnext; + } else { + entry = fileset->fs_filefree; + if (entry == NULL) + goto empty; + fileset->fs_filefree = entry->fse_filenext; + } + entry->fse_flags &= ~FSE_FREE; + } else { + if (flags & FILESET_PICKDIR) { + entry = fileset->fs_dirrotor; + if (entry == NULL) + fileset->fs_dirrotor = + entry = fileset->fs_dirlist; + fileset->fs_dirrotor = entry->fse_dirnext; + } else { + entry = fileset->fs_filerotor[tid]; + if (entry == NULL) + fileset->fs_filerotor[tid] = + entry = fileset->fs_filelist; + fileset->fs_filerotor[tid] = + entry->fse_filenext; + } + } + + if (first == entry) + goto empty; + + if (first == NULL) + first = entry; + + /* Return locked entry */ + (void) ipc_mutex_lock(&entry->fse_lock); + + /* If we ask for an existing file, go round again */ + if ((flags & FILESET_PICKEXISTS) && + !(entry->fse_flags & FSE_EXISTS)) { + (void) ipc_mutex_unlock(&entry->fse_lock); + entry = NULL; + } + + /* If we ask for not an existing file, go round again */ + if ((flags & FILESET_PICKNOEXIST) && + (entry->fse_flags & FSE_EXISTS)) { + (void) ipc_mutex_unlock(&entry->fse_lock); + entry = NULL; + } + } + + (void) ipc_mutex_unlock(&filebench_shm->fileset_lock); + filebench_log(LOG_DEBUG_SCRIPT, "Picked file %s", entry->fse_path); + return (entry); + +empty: + (void) ipc_mutex_unlock(&filebench_shm->fileset_lock); + return (NULL); +} + +/* + * Given a fileset "fileset", create the associated files as + * specified in the attributes of the fileset. The fileset is + * rooted in a directory whose pathname is in fs_path. If the + * directory exists, meaning that there is already a fileset, + * and the fs_reuse attribute is false, then remove it and all + * its contained files and subdirectories. Next, the routine + * creates a root directory for the fileset. All the file type + * filesetentries are cycled through creating as needed + * their containing subdirectory trees in the filesystem and + * creating actual files for fs_preallocpercent of them. The + * created files are filled with fse_size bytes of unitialized + * data. The routine returns -1 on errors, 0 on success. + */ +static int +fileset_create(fileset_t *fileset) +{ + filesetentry_t *entry; + char path[MAXPATHLEN]; + char *buf; + struct stat64 sb; + int pickflags = FILESET_PICKUNIQUE | FILESET_PICKRESET; + hrtime_t start = gethrtime(); + int preallocated = 0; + int reusing = 0; + + if ((buf = (char *)malloc(FILE_ALLOC_BLOCK)) == NULL) + return (-1); + + if (*fileset->fs_path == NULL) { + filebench_log(LOG_ERROR, "Fileset path not set"); + return (-1); + } + + /* XXX Add check to see if there is enough space */ + + /* Remove existing */ + (void) strcpy(path, *fileset->fs_path); + (void) strcat(path, "/"); + (void) strcat(path, fileset->fs_name); + if ((stat64(path, &sb) == 0) && (strlen(path) > 3) && + (strlen(*fileset->fs_path) > 2)) { + if (!integer_isset(fileset->fs_reuse)) { + char cmd[MAXPATHLEN]; + + (void) snprintf(cmd, sizeof (cmd), "rm -rf %s", path); + (void) system(cmd); + filebench_log(LOG_VERBOSE, + "Removed any existing fileset %s in %d seconds", + fileset->fs_name, + ((gethrtime() - start) / 1000000000) + 1); + } else { + /* we are re-using */ + reusing = 1; + filebench_log(LOG_VERBOSE, + "Re-using fileset %s on %s file system.", + fileset->fs_name, sb.st_fstype); + } + } + (void) mkdir(path, 0755); + + start = gethrtime(); + + filebench_log(LOG_VERBOSE, "Creating fileset %s...", + fileset->fs_name); + + while (entry = fileset_pick(fileset, pickflags, 0)) { + char dir[MAXPATHLEN]; + char *pathtmp; + off64_t seek; + int fd; + int randno; + + pickflags = FILESET_PICKUNIQUE; + + entry->fse_flags &= ~FSE_EXISTS; + + if (!integer_isset(fileset->fs_prealloc)) { + (void) ipc_mutex_unlock(&entry->fse_lock); + continue; + } + + *path = 0; + (void) strcpy(path, *fileset->fs_path); + (void) strcat(path, "/"); + (void) strcat(path, fileset->fs_name); + pathtmp = fileset_resolvepath(entry); + (void) strcat(path, pathtmp); + (void) strcpy(dir, path); + free(pathtmp); + + (void) trunc_dirname(dir); + + if (stat64(dir, &sb) != 0) { + if (fileset_mkdir(dir, 0775) == -1) { + (void) ipc_mutex_unlock(&entry->fse_lock); + return (-1); + } + } + + randno = ((RAND_MAX * (100 - *(fileset->fs_preallocpercent))) + / 100); + + if (rand() < randno) { + (void) ipc_mutex_unlock(&entry->fse_lock); + continue; + } + + preallocated++; + + filebench_log(LOG_DEBUG_IMPL, "Populated %s", entry->fse_path); + + /* see if reusing and this file exists */ + if (reusing && (stat64(path, &sb) == 0)) { + if ((fd = open64(path, O_RDWR)) < 0) { + filebench_log(LOG_INFO, + "Attempted but failed to Re-use file %s", + path); + (void) ipc_mutex_unlock(&entry->fse_lock); + return (-1); + } + + if (sb.st_size == (off64_t)entry->fse_size) { + filebench_log(LOG_INFO, + "Re-using file %s", path); + + if (!integer_isset(fileset->fs_cached)) + (void) fileset_freemem(fd, + entry->fse_size); + + entry->fse_flags |= FSE_EXISTS; + (void) close(fd); + (void) ipc_mutex_unlock(&entry->fse_lock); + continue; + + } else if (sb.st_size > (off64_t)entry->fse_size) { + /* reuse, but too large */ + filebench_log(LOG_INFO, + "Truncating & re-using file %s", path); + + (void) ftruncate64(fd, + (off64_t)entry->fse_size); + + if (!integer_isset(fileset->fs_cached)) + (void) fileset_freemem(fd, + entry->fse_size); + + entry->fse_flags |= FSE_EXISTS; + (void) close(fd); + (void) ipc_mutex_unlock(&entry->fse_lock); + continue; + } + } else { + + /* No file or not reusing, so create */ + if ((fd = open64(path, O_RDWR | O_CREAT, 0644)) < 0) { + filebench_log(LOG_ERROR, + "Failed to pre-allocate file %s: %s", + path, strerror(errno)); + + return (-1); + } + } + + entry->fse_flags |= FSE_EXISTS; + + for (seek = 0; seek < entry->fse_size; ) { + off64_t wsize; + int ret = 0; + + /* + * Write FILE_ALLOC_BLOCK's worth, + * except on last write + */ + wsize = MIN(entry->fse_size - seek, FILE_ALLOC_BLOCK); + + ret = write(fd, buf, wsize); + if (ret != wsize) { + filebench_log(LOG_ERROR, + "Failed to pre-allocate file %s: %s", + path, strerror(errno)); + (void) close(fd); + return (-1); + } + seek += wsize; + } + + if (!integer_isset(fileset->fs_cached)) + (void) fileset_freemem(fd, entry->fse_size); + + (void) close(fd); + (void) ipc_mutex_unlock(&entry->fse_lock); + + filebench_log(LOG_DEBUG_IMPL, + "Pre-allocated file %s size %lld", path, entry->fse_size); + } + filebench_log(LOG_VERBOSE, + "Preallocated %d of %lld of fileset %s in %d seconds", + preallocated, + *(fileset->fs_entries), + fileset->fs_name, + ((gethrtime() - start) / 1000000000) + 1); + + free(buf); + return (0); +} + +/* + * Adds an entry to the fileset's file list. Single threaded so + * no locking needed. + */ +static void +fileset_insfilelist(fileset_t *fileset, filesetentry_t *entry) +{ + if (fileset->fs_filelist == NULL) { + fileset->fs_filelist = entry; + entry->fse_filenext = NULL; + } else { + entry->fse_filenext = fileset->fs_filelist; + fileset->fs_filelist = entry; + } +} + +/* + * Adds an entry to the fileset's directory list. Single + * threaded so no locking needed. + */ +static void +fileset_insdirlist(fileset_t *fileset, filesetentry_t *entry) +{ + if (fileset->fs_dirlist == NULL) { + fileset->fs_dirlist = entry; + entry->fse_dirnext = NULL; + } else { + entry->fse_dirnext = fileset->fs_dirlist; + fileset->fs_dirlist = entry; + } +} + +/* + * Obtaines a filesetentry entity for a file to be placed in a + * (sub)directory of a fileset. The size of the file may be + * specified by fs_meansize, or calculated from a gamma + * distribution of parameter fs_sizegamma and of mean size + * fs_meansize. The filesetentry entity is placed on the file + * list in the specified parent filesetentry entity, which may + * be a directory filesetentry, or the root filesetentry in the + * fileset. It is also placed on the fileset's list of all + * contained files. Returns 0 if successful or -1 if ipc memory + * for the path string cannot be allocated. + */ +static int +fileset_populate_file(fileset_t *fileset, filesetentry_t *parent, int serial) +{ + char tmpname[16]; + filesetentry_t *entry; + double drand; + double gamma; + + if ((entry = (filesetentry_t *)ipc_malloc(FILEBENCH_FILESETENTRY)) + == NULL) { + filebench_log(LOG_ERROR, + "fileset_populate_file: Can't malloc filesetentry"); + return (-1); + } + + (void) pthread_mutex_init(&entry->fse_lock, ipc_mutexattr()); + entry->fse_parent = parent; + entry->fse_fileset = fileset; + entry->fse_flags |= FSE_FREE; + fileset_insfilelist(fileset, entry); + + (void) snprintf(tmpname, sizeof (tmpname), "%08d", serial); + if ((entry->fse_path = (char *)ipc_pathalloc(tmpname)) == NULL) { + filebench_log(LOG_ERROR, + "fileset_populate_file: Can't alloc path string"); + return (-1); + } + + gamma = *(fileset->fs_sizegamma) / 1000.0; + + if (gamma > 0) { + drand = gamma_dist_knuth(gamma, fileset->fs_meansize / gamma); + entry->fse_size = (off64_t)drand; + } else { + entry->fse_size = (off64_t)fileset->fs_meansize; + } + + fileset->fs_bytes += entry->fse_size; + + fileset->fs_realfiles++; + return (0); +} + +/* + * Creates a directory node in a fileset, by obtaining a + * filesetentry entity for the node and initializing it + * according to parameters of the fileset. It determines a + * directory tree depth and directory width, optionally using + * a gamma distribution. If its calculated depth is less then + * its actual depth in the directory tree, it becomes a leaf + * node and files itself with "width" number of file type + * filesetentries, otherwise it files itself with "width" + * number of directory type filesetentries, using recursive + * calls to fileset_populate_subdir. The end result of the + * initial call to this routine is a tree of directories of + * random width and varying depth with sufficient leaf + * directories to contain all required files. + * Returns 0 on success. Returns -1 if ipc path string memory + * cannot be allocated and returns an error code (currently + * also -1) from calls to fileset_populate_file or recursive + * calls to fileset_populate_subdir. + */ +static int +fileset_populate_subdir(fileset_t *fileset, filesetentry_t *parent, + int serial, double depth) +{ + double randepth, drand, ranwidth, gamma; + int isleaf = 0; + char tmpname[16]; + filesetentry_t *entry; + int i; + + depth += 1; + + /* Create dir node */ + if ((entry = (filesetentry_t *)ipc_malloc(FILEBENCH_FILESETENTRY)) + == NULL) { + filebench_log(LOG_ERROR, + "fileset_populate_subdir: Can't malloc filesetentry"); + return (-1); + } + + (void) pthread_mutex_init(&entry->fse_lock, ipc_mutexattr()); + + (void) snprintf(tmpname, sizeof (tmpname), "%08d", serial); + if ((entry->fse_path = (char *)ipc_pathalloc(tmpname)) == NULL) { + filebench_log(LOG_ERROR, + "fileset_populate_subdir: Can't alloc path string"); + return (-1); + } + + entry->fse_parent = parent; + entry->fse_flags |= FSE_DIR | FSE_FREE; + fileset_insdirlist(fileset, entry); + + gamma = *(fileset->fs_dirgamma) / 1000.0; + if (gamma > 0) { + drand = gamma_dist_knuth(gamma, fileset->fs_meandepth / gamma); + randepth = (int)drand; + } else { + randepth = (int)fileset->fs_meandepth; + } + + gamma = *(fileset->fs_sizegamma) / 1000.0; + + if (gamma > 0) { + drand = gamma_dist_knuth(gamma, fileset->fs_meanwidth / gamma); + ranwidth = drand; + } else { + ranwidth = fileset->fs_meanwidth; + } + + if (randepth == 0) + randepth = 1; + if (ranwidth == 0) + ranwidth = 1; + if (depth >= randepth) + isleaf = 1; + + /* + * Create directory of random width according to distribution, or + * if root directory, continue until #files required + */ + for (i = 1; + ((parent == NULL) || (i < ranwidth + 1)) && + (fileset->fs_realfiles < *(fileset->fs_entries)); i++) { + int ret = 0; + + if (parent && isleaf) + ret = fileset_populate_file(fileset, entry, i); + else + ret = fileset_populate_subdir(fileset, entry, i, depth); + + if (ret != 0) + return (ret); + } + return (0); +} + +/* + * Populates a fileset with files and subdirectory entries. Uses + * the supplied fs_dirwidth and fs_entries (number of files) to + * calculate the required fs_meandepth (of subdirectories) and + * initialize the fs_meanwidth and fs_meansize variables. Then + * calls fileset_populate_subdir() to do the recursive + * subdirectory entry creation and leaf file entry creation. All + * of the above is skipped if the fileset has already been + * populated. Returns 0 on success, or an error code from the + * call to fileset_populate_subdir if that call fails. + */ +static int +fileset_populate(fileset_t *fileset) +{ + int nfiles; + int meandirwidth = *(fileset->fs_dirwidth); + int ret; + + /* Skip if already populated */ + if (fileset->fs_bytes > 0) + goto exists; + + /* + * Input params are: + * # of files + * ave # of files per dir + * max size of dir + * # ave size of file + * max size of file + */ + nfiles = *(fileset->fs_entries); + fileset->fs_meandepth = log(nfiles) / log(meandirwidth); + fileset->fs_meanwidth = meandirwidth; + fileset->fs_meansize = *(fileset->fs_size); + + if ((ret = fileset_populate_subdir(fileset, NULL, 1, 0)) != 0) + return (ret); + + +exists: + filebench_log(LOG_VERBOSE, "Fileset %s: %lld files, " + "avg dir = %.1lf, avg depth = %.1lf, mbytes=%lld", + fileset->fs_name, + *(fileset->fs_entries), + fileset->fs_meanwidth, + fileset->fs_meandepth, + fileset->fs_bytes / 1024UL / 1024UL); + + return (0); +} + +/* + * Allocates a fileset instance, initializes fs_dirgamma and + * fs_sizegamma default values, and sets the fileset name to the + * supplied name string. Puts the allocated fileset on the + * master fileset list and returns a pointer to it. + */ +fileset_t * +fileset_define(char *name) +{ + fileset_t *fileset; + + if (name == NULL) + return (NULL); + + if ((fileset = (fileset_t *)ipc_malloc(FILEBENCH_FILESET)) == NULL) { + filebench_log(LOG_ERROR, + "fileset_define: Can't malloc fileset"); + return (NULL); + } + + filebench_log(LOG_DEBUG_IMPL, "Defining file %s", name); + + (void) ipc_mutex_lock(&filebench_shm->fileset_lock); + + fileset->fs_dirgamma = integer_alloc(1500); + fileset->fs_sizegamma = integer_alloc(1500); + + /* Add fileset to global list */ + if (filebench_shm->filesetlist == NULL) { + filebench_shm->filesetlist = fileset; + fileset->fs_next = NULL; + } else { + fileset->fs_next = filebench_shm->filesetlist; + filebench_shm->filesetlist = fileset; + } + + (void) ipc_mutex_unlock(&filebench_shm->fileset_lock); + + (void) strcpy(fileset->fs_name, name); + + return (fileset); +} + +/* + * If supplied with a pointer to a fileset and the fileset's + * fs_prealloc flag is set, calls fileset_populate() to populate + * the fileset with filesetentries, then calls fileset_create() + * to make actual directories and files for the filesetentries. + * Otherwise, it applies fileset_populate() and fileset_create() + * to all the filesets on the master fileset list. It always + * returns zero (0) if one fileset is populated / created, + * otherwise it returns the sum of returned values from + * fileset_create() and fileset_populate(), which + * will be a negative one (-1) times the number of + * fileset_create() calls which failed. + */ +int +fileset_createset(fileset_t *fileset) +{ + fileset_t *list; + int ret = 0; + + if (fileset && integer_isset(fileset->fs_prealloc)) { + if ((ret = fileset_populate(fileset)) != 0) + return (ret); + return (fileset_create(fileset)); + } + + list = filebench_shm->filesetlist; + while (list) { + ret += fileset_populate(list); + ret += fileset_create(list); + list = list->fs_next; + } + + return (ret); +} + +/* + * Searches through the master fileset list for the named fileset. + * If found, returns pointer to same, otherwise returns NULL. + */ +fileset_t * +fileset_find(char *name) +{ + fileset_t *fileset = filebench_shm->filesetlist; + + (void) ipc_mutex_lock(&filebench_shm->fileset_lock); + + while (fileset) { + if (strcmp(name, fileset->fs_name) == 0) { + (void) ipc_mutex_unlock(&filebench_shm->fileset_lock); + return (fileset); + } + fileset = fileset->fs_next; + } + (void) ipc_mutex_unlock(&filebench_shm->fileset_lock); + + return (NULL); +} diff --git a/usr/src/cmd/filebench/common/fileset.h b/usr/src/cmd/filebench/common/fileset.h new file mode 100644 index 0000000000..336fa1722f --- /dev/null +++ b/usr/src/cmd/filebench/common/fileset.h @@ -0,0 +1,129 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_FILESET_H +#define _FB_FILESET_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" + +#ifndef HAVE_OFF64_T +/* + * We are probably on linux. + * According to http://www.suse.de/~aj/linux_lfs.html, defining the + * above, automatically changes type of off_t to off64_t. so let + * us use only off_t as off64_t is not defined + */ +#define off64_t off_t +#endif /* HAVE_OFF64_T */ + + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/param.h> +#include <sys/resource.h> +#include <pthread.h> + +#include "fileobj.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define FSE_MAXTID 16384 + +#define FSE_MAXPATHLEN 16 +#define FSE_DIR 0x1 +#define FSE_FREE 0x2 +#define FSE_EXISTS 0x4 +#define FSE_BUSY 0x8 + +typedef struct filesetentry { + struct filesetentry *fse_next; + struct filesetentry *fse_parent; + struct filesetentry *fse_filenext; /* List of files */ + struct filesetentry *fse_dirnext; /* List of directories */ + struct fileset *fse_fileset; /* Parent fileset */ + pthread_mutex_t fse_lock; + char *fse_path; + int fse_depth; + off64_t fse_size; + int fse_flags; +} filesetentry_t; + +#define FILESET_PICKANY 0x1 /* Pick any file from the set */ +#define FILESET_PICKUNIQUE 0x2 /* Pick a unique file from set until empty */ +#define FILESET_PICKRESET 0x4 /* Reset FILESET_PICKUNIQUE selection list */ +#define FILESET_PICKDIR 0x8 /* Pick a directory */ +#define FILESET_PICKEXISTS 0x10 /* Pick an existing file */ +#define FILESET_PICKNOEXIST 0x20 /* Pick a file that doesn't exist */ + +typedef struct fileset { + struct fileset *fs_next; /* Next in list */ + char fs_name[128]; /* Name */ + pthread_t fs_tid; /* Thread id, for par alloc */ + var_string_t fs_path; /* Pathname prefix in fs */ + var_integer_t fs_entries; /* Set size */ + var_integer_t fs_preallocpercent; /* Prealloc size */ + int fs_attrs; /* Attributes */ + var_integer_t fs_dirwidth; /* Explicit or 0 for distribution */ + var_integer_t fs_size; /* Explicit or 0 for distribution */ + var_integer_t fs_dirgamma; /* Dirwidth Gamma distribution (* 1000) */ + var_integer_t fs_sizegamma; /* Filesize Gamma distribution (* 1000) */ + var_integer_t fs_create; /* Attr */ + var_integer_t fs_prealloc; /* Attr */ + var_integer_t fs_cached; /* Attr */ + var_integer_t fs_reuse; /* Attr */ + double fs_meandepth; /* Computed mean depth */ + double fs_meanwidth; /* Specified mean dir width */ + double fs_meansize; /* Specified mean file size */ + int fs_realfiles; /* Actual files */ + off64_t fs_bytes; /* Space potentially consumed by all files */ + filesetentry_t *fs_filelist; /* List of files */ + filesetentry_t *fs_dirlist; /* List of directories */ + filesetentry_t *fs_filefree; /* Ptr to next free file */ + filesetentry_t *fs_dirfree; /* Ptr to next free directory */ + filesetentry_t *fs_filerotor[FSE_MAXTID]; /* next file to select */ + filesetentry_t *fs_dirrotor; /* Ptr to next directory to select */ +} fileset_t; + +int fileset_createset(fileset_t *); +int fileset_openfile(fileset_t *fileset, filesetentry_t *entry, + int flag, int mode, int attrs); +fileset_t *fileset_define(char *); +fileset_t *fileset_find(char *name); +filesetentry_t *fileset_pick(fileset_t *fileset, int flags, int tid); +char *fileset_resolvepath(filesetentry_t *entry); +void fileset_usage(void); + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_FILESET_H */ diff --git a/usr/src/cmd/filebench/common/flowop.c b/usr/src/cmd/filebench/common/flowop.c new file mode 100644 index 0000000000..99602c446c --- /dev/null +++ b/usr/src/cmd/filebench/common/flowop.c @@ -0,0 +1,725 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" + +#ifdef HAVE_LWPS +#include <sys/lwp.h> +#endif +#include <fcntl.h> +#include "filebench.h" +#include "flowop.h" +#include "stats.h" + +#ifdef LINUX_PORT +#include <sys/types.h> +#include <linux/unistd.h> +#endif + +static flowop_t *flowop_define_common(threadflow_t *threadflow, char *name, + flowop_t *inherit, int instance, int type); + + +/* + * A collection of flowop support functions. The actual code that + * implements the various flowops is in flowop_library.c. + * + * Routines for defining, creating, initializing and destroying + * flowops, cyclically invoking the flowops on each threadflow's flowop + * list, collecting statistics about flowop execution, and other + * housekeeping duties are included in this file. + */ + + +/* + * Prints the name and instance number of each flowop in + * the supplied list to the filebench log. + */ +int +flowop_printlist(flowop_t *list) +{ + flowop_t *flowop = list; + + while (flowop) { + filebench_log(LOG_DEBUG_IMPL, "flowop-list %s-%d", + flowop->fo_name, flowop->fo_instance); + flowop = flowop->fo_threadnext; + } + return (0); +} + +#define TIMESPEC_TO_HRTIME(s, e) (((e.tv_sec - s.tv_sec) * 1000000000LL) + \ + (e.tv_nsec - s.tv_nsec)) +/* + * Puts current high resolution time in start time entry + * for threadflow and may also calculate running filebench + * overhead statistics. + */ +void +flowop_beginop(threadflow_t *threadflow, flowop_t *flowop) +{ +#ifdef HAVE_PROCFS + if ((noproc == 0) && (threadflow->tf_lwpusagefd == 0)) { + char procname[128]; + + (void) snprintf(procname, sizeof (procname), + "/proc/%d/lwp/%d/lwpusage", pid, _lwp_self()); + threadflow->tf_lwpusagefd = open(procname, O_RDONLY); + } + + (void) pread(threadflow->tf_lwpusagefd, + &threadflow->tf_susage, + sizeof (struct prusage), 0); + + /* Compute overhead time in this thread around op */ + if (threadflow->tf_eusage.pr_stime.tv_nsec) { + flowop->fo_stats.fs_mstate[FLOW_MSTATE_OHEAD] += + TIMESPEC_TO_HRTIME(threadflow->tf_eusage.pr_utime, + threadflow->tf_susage.pr_utime) + + TIMESPEC_TO_HRTIME(threadflow->tf_eusage.pr_ttime, + threadflow->tf_susage.pr_ttime) + + TIMESPEC_TO_HRTIME(threadflow->tf_eusage.pr_stime, + threadflow->tf_susage.pr_stime); + } +#endif + /* Start of op for this thread */ + threadflow->tf_stime = gethrtime(); +} + +flowstat_t controlstats; +static int controlstats_zeroed = 0; + +/* + * Updates flowop's latency statistics, using saved start + * time and current high resolution time. Updates flowop's + * io count and transferred bytes statistics. Also updates + * threadflow's and flowop's cumulative read or write byte + * and io count statistics. + */ +void +flowop_endop(threadflow_t *threadflow, flowop_t *flowop) +{ + hrtime_t t; + + flowop->fo_stats.fs_mstate[FLOW_MSTATE_LAT] += + (gethrtime() - threadflow->tf_stime); +#ifdef HAVE_PROCFS + if ((pread(threadflow->tf_lwpusagefd, &threadflow->tf_eusage, + sizeof (struct prusage), 0)) != sizeof (struct prusage)) + filebench_log(LOG_ERROR, "cannot read /proc"); + + t = + TIMESPEC_TO_HRTIME(threadflow->tf_susage.pr_utime, + threadflow->tf_eusage.pr_utime) + + TIMESPEC_TO_HRTIME(threadflow->tf_susage.pr_ttime, + threadflow->tf_eusage.pr_ttime) + + TIMESPEC_TO_HRTIME(threadflow->tf_susage.pr_stime, + threadflow->tf_eusage.pr_stime); + flowop->fo_stats.fs_mstate[FLOW_MSTATE_CPU] += t; + + flowop->fo_stats.fs_mstate[FLOW_MSTATE_WAIT] += + TIMESPEC_TO_HRTIME(threadflow->tf_susage.pr_tftime, + threadflow->tf_eusage.pr_tftime) + + TIMESPEC_TO_HRTIME(threadflow->tf_susage.pr_dftime, + threadflow->tf_eusage.pr_dftime) + + TIMESPEC_TO_HRTIME(threadflow->tf_susage.pr_kftime, + threadflow->tf_eusage.pr_kftime) + + TIMESPEC_TO_HRTIME(threadflow->tf_susage.pr_kftime, + threadflow->tf_eusage.pr_kftime) + + TIMESPEC_TO_HRTIME(threadflow->tf_susage.pr_slptime, + threadflow->tf_eusage.pr_slptime); +#endif + + flowop->fo_stats.fs_count++; + flowop->fo_stats.fs_bytes += *flowop->fo_iosize; + if ((flowop->fo_type & FLOW_TYPE_IO) || + (flowop->fo_type & FLOW_TYPE_AIO)) { + controlstats.fs_count++; + controlstats.fs_bytes += *flowop->fo_iosize; + } + if (flowop->fo_attrs & FLOW_ATTR_READ) { + threadflow->tf_stats.fs_rbytes += *flowop->fo_iosize; + threadflow->tf_stats.fs_rcount++; + flowop->fo_stats.fs_rcount++; + controlstats.fs_rbytes += *flowop->fo_iosize; + controlstats.fs_rcount++; + } else if (flowop->fo_attrs & FLOW_ATTR_WRITE) { + threadflow->tf_stats.fs_wbytes += *flowop->fo_iosize; + threadflow->tf_stats.fs_wcount++; + flowop->fo_stats.fs_wcount++; + controlstats.fs_wbytes += *flowop->fo_iosize; + controlstats.fs_wcount++; + } +} + +/* + * Calls the flowop's initialization function, pointed to by + * flowop->fo_init. + */ +static int +flowop_initflow(flowop_t *flowop) +{ + if ((*flowop->fo_init)(flowop) < 0) { + filebench_log(LOG_ERROR, "flowop %s-%d init failed", + flowop->fo_name, flowop->fo_instance); + return (-1); + } + return (0); +} + +/* + * The final initialization and main execution loop for the + * worker threads. Sets threadflow and flowop start times, + * waits for all process to start, then creates the runtime + * flowops from those defined by the F language workload + * script. It does some more initialization, then enters a + * loop to repeatedly execute the flowops on the flowop list + * until an abort condition is detected, at which time it exits. + * This is the starting routine for the new worker thread + * created by threadflow_createthread(), and is not currently + * called from anywhere else. + */ +void +flowop_start(threadflow_t *threadflow) +{ + flowop_t *flowop; + size_t memsize; + int ret = 0; + + pid = getpid(); + +#ifdef HAVE_PROCFS + if (noproc == 0) { + char procname[128]; + long ctl[2] = {PCSET, PR_MSACCT}; + int pfd; + + (void) snprintf(procname, sizeof (procname), + "/proc/%d/lwp/%d/lwpctl", pid, _lwp_self()); + pfd = open(procname, O_WRONLY); + (void) pwrite(pfd, &ctl, sizeof (ctl), 0); + (void) close(pfd); + } +#endif + + if (!controlstats_zeroed) { + (void) memset(&controlstats, 0, sizeof (controlstats)); + controlstats_zeroed = 1; + } + + flowop = threadflow->tf_ops; + threadflow->tf_stats.fs_stime = gethrtime(); + flowop->fo_stats.fs_stime = gethrtime(); + + /* Hold the flowop find lock as reader to prevent lookups */ + (void) pthread_rwlock_rdlock(&filebench_shm->flowop_find_lock); + + /* + * Block until all processes have started, acting like + * a barrier. The original filebench process initially + * holds the run_lock as a reader, preventing any of the + * threads from obtaining the writer lock, and hence + * passing this point. Once all processes and threads + * have been created, the original process unlocks + * run_lock, allowing each waiting thread to lock + * and then immediately unlock it, then begin running. + */ + (void) pthread_rwlock_wrlock(&filebench_shm->run_lock); + (void) pthread_rwlock_unlock(&filebench_shm->run_lock); + + /* Create the runtime flowops from those defined by the script */ + (void) ipc_mutex_lock(&filebench_shm->flowop_lock); + while (flowop) { + flowop_t *newflowop; + + if (flowop == threadflow->tf_ops) + threadflow->tf_ops = NULL; + newflowop = flowop_define_common(threadflow, flowop->fo_name, + flowop, 1, 0); + if (newflowop == NULL) + return; + if (flowop_initflow(newflowop) < 0) { + filebench_log(LOG_ERROR, "Flowop init of %s failed", + newflowop->fo_name); + } + flowop = flowop->fo_threadnext; + } + (void) ipc_mutex_unlock(&filebench_shm->flowop_lock); + + /* Release the find lock as reader to allow lookups */ + (void) pthread_rwlock_unlock(&filebench_shm->flowop_find_lock); + + /* Set to the start of the new flowop list */ + flowop = threadflow->tf_ops; + + threadflow->tf_abort = 0; + threadflow->tf_running = 1; + + /* If we are going to use ISM, allocate later */ + if (threadflow->tf_attrs & THREADFLOW_USEISM) { + threadflow->tf_mem = + ipc_ismmalloc((size_t)*threadflow->tf_memsize); + } else { + threadflow->tf_mem = malloc((size_t)*threadflow->tf_memsize); + } + + memsize = *threadflow->tf_memsize; + (void) memset(threadflow->tf_mem, 0, memsize); + filebench_log(LOG_DEBUG_SCRIPT, "Thread allocated %d bytes", memsize); + +#ifdef HAVE_LWPS + filebench_log(LOG_DEBUG_SCRIPT, "Thread %zx (%d) started", + threadflow, + _lwp_self()); +#endif + + /* Main filebench worker loop */ + /* CONSTCOND */ + while (1) { + int i; + + /* Abort if asked */ + if (threadflow->tf_abort || filebench_shm->f_abort) { + (void) ipc_mutex_lock(&threadflow->tf_lock); + threadflow->tf_running = 0; + (void) ipc_mutex_unlock(&threadflow->tf_lock); + break; + } + + /* Be quiet while stats are gathered */ + if (filebench_shm->bequiet) { + (void) sleep(1); + continue; + } + + /* Take it easy until everyone is ready to go */ + if (!filebench_shm->allrunning) + (void) sleep(1); + + if (flowop->fo_stats.fs_stime == 0) + flowop->fo_stats.fs_stime = gethrtime(); + + if (flowop == NULL) { + filebench_log(LOG_ERROR, "flowop_read null flowop"); + return; + } + + if (threadflow->tf_memsize == 0) { + filebench_log(LOG_ERROR, + "Zero memory size for thread %s", + threadflow->tf_name); + return; + } + + filebench_log(LOG_DEBUG_SCRIPT, "%s: executing flowop %s-%d", + threadflow->tf_name, flowop->fo_name, flowop->fo_instance); + + /* Execute the flowop for fo_iters times */ + for (i = 0; i < *flowop->fo_iters; i++) { + filebench_log(LOG_DEBUG_SCRIPT, "%s: executing flowop " + "%s-%d", threadflow->tf_name, flowop->fo_name, + flowop->fo_instance); + ret = (*flowop->fo_func)(threadflow, flowop); + filebench_log(LOG_DEBUG_SCRIPT, "%s: executing flowop " + "%s-%d", threadflow->tf_name, flowop->fo_name, + flowop->fo_instance); + + /* Return value > 0 means "stop the filebench run" */ + if (ret > 0) { + filebench_log(LOG_VERBOSE, + "%s: exiting flowop %s-%d", + threadflow->tf_name, flowop->fo_name, + flowop->fo_instance); + (void) ipc_mutex_lock(&threadflow->tf_lock); + threadflow->tf_abort = 1; + filebench_shm->f_abort = 1; + threadflow->tf_running = 0; + (void) ipc_mutex_unlock(&threadflow->tf_lock); + break; + } + /* + * Return value < 0 means "flowop failed, stop the + * filebench run" + */ + if (ret < 0) { + filebench_log(LOG_ERROR, "flowop %s failed", + flowop->fo_name); + (void) ipc_mutex_lock(&threadflow->tf_lock); + threadflow->tf_abort = 1; + filebench_shm->f_abort = 1; + threadflow->tf_running = 0; + (void) ipc_mutex_unlock(&threadflow->tf_lock); + break; + } + } + + /* advance to next flowop */ + flowop = flowop->fo_threadnext; + + /* but if at end of list, start over from the beginning */ + if (flowop == NULL) { + flowop = threadflow->tf_ops; + threadflow->tf_stats.fs_count++; + } + } + +#ifdef HAVE_LWPS + filebench_log(LOG_DEBUG_SCRIPT, "Thread %d exiting", + _lwp_self()); +#endif + + pthread_exit(&ret); +} + +void +flowop_init(void) +{ + flowoplib_init(); +} + +/* + * Calls the flowop's destruct function, pointed to by + * flowop->fo_destruct. + */ +static void +flowop_destructflow(flowop_t *flowop) +{ + (*flowop->fo_destruct)(flowop); +} + +/* + * Delete the designated flowop from the thread's flowop list. + * After removal from the list, the flowop is destroyed with + * flowop_destructflow(). + */ +static void +flowop_delete(flowop_t **flowoplist, flowop_t *flowop) +{ + flowop_t *entry = *flowoplist; + int found = 0; + + filebench_log(LOG_DEBUG_IMPL, "Deleting flowop (%s-%d)", + flowop->fo_name, + flowop->fo_instance); + + /* Delete from thread's flowop list */ + if (flowop == *flowoplist) { + /* First on list */ + *flowoplist = flowop->fo_threadnext; + filebench_log(LOG_DEBUG_IMPL, + "Delete0 flowop: (%s-%d)", + flowop->fo_name, + flowop->fo_instance); + } else { + while (entry->fo_threadnext) { + filebench_log(LOG_DEBUG_IMPL, + "Delete0 flowop: (%s-%d) == (%s-%d)", + entry->fo_threadnext->fo_name, + entry->fo_threadnext->fo_instance, + flowop->fo_name, + flowop->fo_instance); + + if (flowop == entry->fo_threadnext) { + /* Delete */ + filebench_log(LOG_DEBUG_IMPL, + "Deleted0 flowop: (%s-%d)", + entry->fo_threadnext->fo_name, + entry->fo_threadnext->fo_instance); + entry->fo_threadnext = + entry->fo_threadnext->fo_threadnext; + break; + } + entry = entry->fo_threadnext; + } + } + + /* Call destructor */ + flowop_destructflow(flowop); + +#ifdef HAVE_PROCFS + /* Close /proc stats */ + if (flowop->fo_thread) + (void) close(flowop->fo_thread->tf_lwpusagefd); +#endif + + /* Delete from global list */ + entry = filebench_shm->flowoplist; + + if (flowop == filebench_shm->flowoplist) { + /* First on list */ + filebench_shm->flowoplist = flowop->fo_next; + found = 1; + } else { + while (entry->fo_next) { + filebench_log(LOG_DEBUG_IMPL, + "Delete flowop: (%s-%d) == (%s-%d)", + entry->fo_next->fo_name, + entry->fo_next->fo_instance, + flowop->fo_name, + flowop->fo_instance); + + if (flowop == entry->fo_next) { + /* Delete */ + entry->fo_next = entry->fo_next->fo_next; + found = 1; + break; + } + + entry = entry->fo_next; + } + } + if (found) { + filebench_log(LOG_DEBUG_IMPL, + "Deleted flowop: (%s-%d)", + flowop->fo_name, + flowop->fo_instance); + ipc_free(FILEBENCH_FLOWOP, (char *)flowop); + } else { + filebench_log(LOG_DEBUG_IMPL, "Flowop %s-%d not found!", + flowop->fo_name, + flowop->fo_instance); + } +} + +/* + * Deletes all the flowops from a flowop list. + */ +void +flowop_delete_all(flowop_t **flowoplist) +{ + flowop_t *flowop = *flowoplist; + + filebench_log(LOG_DEBUG_IMPL, "Deleting all flowops..."); + while (flowop) { + filebench_log(LOG_DEBUG_IMPL, "Deleting all flowops (%s-%d)", + flowop->fo_name, flowop->fo_instance); + flowop = flowop->fo_threadnext; + } + + flowop = *flowoplist; + + (void) ipc_mutex_lock(&filebench_shm->flowop_lock); + + while (flowop) { + if (flowop->fo_instance && + (flowop->fo_instance == FLOW_MASTER)) { + flowop = flowop->fo_threadnext; + continue; + } + flowop_delete(flowoplist, flowop); + flowop = flowop->fo_threadnext; + } + + (void) ipc_mutex_unlock(&filebench_shm->flowop_lock); +} + +/* + * Allocates a flowop entity and initializes it with inherited + * contents from the "inherit" flowop, if it is supplied, or + * with zeros otherwise. In either case the file descriptor + * (fo_fd) is set to -1, and the fo_next and fo_threadnext + * pointers are set to NULL, and fo_thread is set to point to + * the owning threadflow. The initialized flowop is placed at + * the head of the global flowop list, and also placed on the + * tail of thethreadflow's tf_ops list. The routine locks the + * flowop's fo_lock and leaves it held on return. If successful, + * it returns a pointer to the allocated and initialized flowop, + * otherwise NULL. + * + * filebench_shm->flowop_lock must be held by caller. + */ +static flowop_t * +flowop_define_common(threadflow_t *threadflow, char *name, flowop_t *inherit, + int instance, int type) +{ + flowop_t *flowop; + + if (name == NULL) + return (NULL); + + if ((flowop = (flowop_t *)ipc_malloc(FILEBENCH_FLOWOP)) == NULL) { + filebench_log(LOG_ERROR, + "flowop_define: Can't malloc flowop"); + return (NULL); + } + + filebench_log(LOG_DEBUG_IMPL, "defining flowops %s-%d, addr %zx", + name, instance, flowop); + + if (flowop == NULL) + return (NULL); + + if (inherit) { + (void) memcpy(flowop, inherit, sizeof (flowop_t)); + (void) pthread_mutex_init(&flowop->fo_lock, ipc_mutexattr()); + (void) ipc_mutex_lock(&flowop->fo_lock); + flowop->fo_next = NULL; + flowop->fo_threadnext = NULL; + flowop->fo_fd = -1; + filebench_log(LOG_DEBUG_IMPL, + "flowop %s-%d calling init", name, instance); + } else { + (void) memset(flowop, 0, sizeof (flowop_t)); + flowop->fo_fd = -1; + flowop->fo_iters = integer_alloc(1); + flowop->fo_type = type; + (void) pthread_mutex_init(&flowop->fo_lock, ipc_mutexattr()); + (void) ipc_mutex_lock(&flowop->fo_lock); + } + + /* Create backpointer to thread */ + flowop->fo_thread = threadflow; + + /* Add flowop to global list */ + if (filebench_shm->flowoplist == NULL) { + filebench_shm->flowoplist = flowop; + flowop->fo_next = NULL; + } else { + flowop->fo_next = filebench_shm->flowoplist; + filebench_shm->flowoplist = flowop; + } + + (void) strcpy(flowop->fo_name, name); + flowop->fo_instance = instance; + + if (threadflow == NULL) + return (flowop); + + /* Add flowop to thread op list */ + if (threadflow->tf_ops == NULL) { + threadflow->tf_ops = flowop; + flowop->fo_threadnext = NULL; + } else { + flowop_t *flowend; + + /* Find the end of the thread list */ + flowend = threadflow->tf_ops; + while (flowend->fo_threadnext != NULL) + flowend = flowend->fo_threadnext; + flowend->fo_threadnext = flowop; + flowop->fo_threadnext = NULL; + } + + return (flowop); +} + +/* + * Calls flowop_define_common() to allocate and initialize a + * flowop, and holds the shared flowop_lock during the call. + * It releases the created flowop's fo_lock when done. + */ +flowop_t * +flowop_define(threadflow_t *threadflow, char *name, flowop_t *inherit, + int instance, int type) +{ + flowop_t *flowop; + + (void) ipc_mutex_lock(&filebench_shm->flowop_lock); + flowop = flowop_define_common(threadflow, name, + inherit, instance, type); + (void) ipc_mutex_unlock(&filebench_shm->flowop_lock); + + if (flowop == NULL) + return (NULL); + + (void) ipc_mutex_unlock(&flowop->fo_lock); + + return (flowop); +} + +/* + * Attempts to take a write lock on the flowop_find_lock that is + * defined in interprocess shared memory. Since each call to + * flowop_start() holds a read lock on flowop_find_lock, this + * routine effectively blocks until all instances of + * flowop_start() have finished. The flowop_find() routine calls + * this routine so that flowops won't be searched for until all + * flowops have been created by flowop_start. + */ +static void +flowop_find_barrier(void) +{ + /* Block on wrlock to ensure find waits for all creates */ + (void) pthread_rwlock_wrlock(&filebench_shm->flowop_find_lock); + (void) pthread_rwlock_unlock(&filebench_shm->flowop_find_lock); +} + +/* + * Returns a list of flowops named "name" from the master + * flowop list. + */ +flowop_t * +flowop_find(char *name) +{ + flowop_t *flowop = filebench_shm->flowoplist; + flowop_t *result = NULL; + + flowop_find_barrier(); + + (void) ipc_mutex_lock(&filebench_shm->flowop_lock); + + while (flowop) { + if (strcmp(name, flowop->fo_name) == 0) { + + /* Add flowop to result list */ + if (result == NULL) { + result = flowop; + flowop->fo_resultnext = NULL; + } else { + flowop->fo_resultnext = result; + result = flowop; + } + } + flowop = flowop->fo_next; + } + + (void) ipc_mutex_unlock(&filebench_shm->flowop_lock); + + + return (result); +} + +/* + * Returns a pointer to the specified instance of flowop + * "name" from the list returned by flowop_find(). + */ +flowop_t * +flowop_find_one(char *name, int instance) +{ + flowop_t *result; + + result = flowop_find(name); + + while (result) { + if ((strcmp(name, result->fo_name) == 0) && + (instance == result->fo_instance)) + break; + result = result->fo_next; + } + + return (result); +} diff --git a/usr/src/cmd/filebench/common/flowop.h b/usr/src/cmd/filebench/common/flowop.h new file mode 100644 index 0000000000..57ac4bc703 --- /dev/null +++ b/usr/src/cmd/filebench/common/flowop.h @@ -0,0 +1,144 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_FLOWOP_H +#define _FB_FLOWOP_H + + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/param.h> +#include <sys/resource.h> +#include <pthread.h> +#ifndef HAVE_SYSV_SEM +#include <semaphore.h> +#endif +#include "stats.h" +#include "threadflow.h" +#include "vars.h" +#include "fileset.h" +#include "filebench.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct flowop { + char fo_name[128]; /* Name */ + int fo_instance; /* Instance number */ + struct flowop *fo_next; /* Next in global list */ + struct flowop *fo_threadnext; /* Next in thread's list */ + struct flowop *fo_resultnext; /* List of flowops in result */ + struct threadflow *fo_thread; /* Backpointer to thread */ + int (*fo_func)(); /* Method */ + int (*fo_init)(); /* Init Method */ + void (*fo_destruct)(); /* Destructor Method */ + int fo_type; /* Type */ + int fo_attrs; /* Flow op attribute */ + fileobj_t *fo_file; /* File for op */ + fileset_t *fo_fileset; /* Fileset for op */ + int fo_fd; /* File descriptor */ + int fo_fdnumber; /* User specified file descriptor */ + int fo_srcfdnumber; /* User specified src file descriptor */ + var_integer_t fo_iosize; /* Size of operation */ + var_integer_t fo_wss; /* Flow op working set size */ + char fo_targetname[128]; /* Target, for wakeup etc... */ + struct flowop *fo_targets; /* List of targets matching name */ + struct flowop *fo_targetnext; /* List of targets matching name */ + var_integer_t fo_iters; /* Number of iterations of op */ + var_integer_t fo_value; /* Attr */ + var_integer_t fo_sequential; /* Attr */ + var_integer_t fo_random; /* Attr */ + var_integer_t fo_stride; /* Attr */ + var_integer_t fo_backwards; /* Attr */ + var_integer_t fo_dsync; /* Attr */ + var_integer_t fo_blocking; /* Attr */ + var_integer_t fo_directio; /* Attr */ + var_integer_t fo_rotatefd; /* Attr */ + flowstat_t fo_stats; /* Flow statistics */ + pthread_cond_t fo_cv; /* Block/wakeup cv */ + pthread_mutex_t fo_lock; /* Mutex around flowop */ + char *fo_buf; /* Per-flowop buffer */ +#ifdef HAVE_SYSV_SEM + int fo_semid_lw; /* sem id */ + int fo_semid_hw; /* sem id for highwater block */ +#else + sem_t fo_sem; /* sem_t for posix semaphores */ +#endif /* HAVE_SYSV_SEM */ + var_integer_t fo_highwater; /* value of highwater paramter */ + void *fo_idp; /* id, for sems etc */ + hrtime_t fo_timestamp; /* for ratecontrol, etc... */ + int fo_initted; /* Set to one if initialized */ + int64_t fo_tputbucket; /* Throughput bucket, for limiter */ + uint64_t fo_tputlast; /* Throughput count, for delta's */ + +} flowop_t; + +/* Flow Op Attrs */ +#define FLOW_ATTR_SEQUENTIAL 0x1 +#define FLOW_ATTR_RANDOM 0x2 +#define FLOW_ATTR_STRIDE 0x4 +#define FLOW_ATTR_BACKWARDS 0x8 +#define FLOW_ATTR_DSYNC 0x10 +#define FLOW_ATTR_BLOCKING 0x20 +#define FLOW_ATTR_DIRECTIO 0x40 +#define FLOW_ATTR_READ 0x80 +#define FLOW_ATTR_WRITE 0x100 + +#define FLOW_MASTER -1 /* Declaration of thread from script */ +#define FLOW_DEFINITION 0 /* Prototype definition of flowop from library */ + +#define FLOW_TYPES 5 +#define FLOW_TYPE_GLOBAL 0 /* Rolled up statistics */ +#define FLOW_TYPE_IO 1 /* Op is an I/O, reflected in iops and lat */ +#define FLOW_TYPE_AIO 2 /* Op is an async I/O, reflected in iops */ +#define FLOW_TYPE_SYNC 3 /* Op is a sync event */ +#define FLOW_TYPE_OTHER 4 /* Op is a something else */ + +extern flowstat_t controlstats; + +void flowop_init(void); +flowop_t *flowop_define(threadflow_t *, char *name, flowop_t *inherit, + int instance, int type); +flowop_t *flowop_find(char *name); +flowop_t *flowop_find_one(char *name, int instance); +void flowoplib_usage(void); +void flowoplib_init(void); +void flowop_delete_all(flowop_t **threadlist); +void flowop_endop(threadflow_t *threadflow, flowop_t *flowop); +void flowop_beginop(threadflow_t *threadflow, flowop_t *flowop); + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_FLOWOP_H */ diff --git a/usr/src/cmd/filebench/common/flowop_library.c b/usr/src/cmd/filebench/common/flowop_library.c new file mode 100644 index 0000000000..78f0a64b63 --- /dev/null +++ b/usr/src/cmd/filebench/common/flowop_library.c @@ -0,0 +1,2437 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" + +#include <sys/types.h> +#ifdef HAVE_SYS_ASYNCH_H +#include <sys/asynch.h> +#endif +#include <sys/ipc.h> +#include <sys/sem.h> +#include <sys/errno.h> +#include <sys/time.h> +#include <inttypes.h> +#include <fcntl.h> + +#ifdef HAVE_UTILITY_H +#include <utility.h> +#endif /* HAVE_UTILITY_H */ + +#ifdef HAVE_AIO +#include <aio.h> +#endif /* HAVE_AIO */ + +#ifdef HAVE_LIBAIO_H +#include <libaio.h> +#endif /* HAVE_LIBAIO_H */ + +#ifdef HAVE_SYS_ASYNC_H +#include <sys/asynch.h> +#endif /* HAVE_SYS_ASYNC_H */ + +#ifdef HAVE_AIO_H +#include <aio.h> +#endif /* HAVE_AIO_H */ + +#ifndef HAVE_UINT_T +#define uint_t unsigned int +#endif /* HAVE_UINT_T */ + +#ifndef HAVE_AIOCB64_T +#define aiocb64 aiocb +#endif /* HAVE_AIOCB64_T */ + +#ifndef HAVE_SYSV_SEM +#include <semaphore.h> +#endif /* HAVE_SYSV_SEM */ + +#include "filebench.h" +#include "flowop.h" +#include "fileset.h" + +/* + * These routines implement the flowops from the f language. Each + * flowop has has a name such as "read", and a set of function pointers + * to call for initialization, execution and destruction of the flowop. + * The table flowoplib_funcs[] contains a flowoplib struct for each + * implemented flowop. Most flowops use a generic initialization function + * and all currently use a generic destruction function. All flowop + * functions referenced from the table are in this file, though, of + * course, they often call functions from other files. + * + * The flowop_init() routine uses the flowoplib_funcs[] table to + * create an initial set of "instance 0" flowops, one for each type of + * flowop, from which all other flowops are derived. These "instance 0" + * flowops are initialized with information from the table including + * pointers for their fo_init, fo_func and fo_destroy functions. When + * a flowop definition is encountered in an f language script, the + * "type" of flowop, such as "read" is used to search for the + * "instance 0" flowop named "read", then a new flowop is allocated + * which inherits its function pointers and other initial properties + * from the instance 0 flowop, and is given a new name as specified + * by the "name=" attribute. + */ + +static int flowoplib_init_generic(flowop_t *flowop); +static void flowoplib_destruct_generic(flowop_t *flowop); +static int flowoplib_fdnum(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_write(threadflow_t *threadflow, flowop_t *flowop); +#ifdef HAVE_AIO +static int flowoplib_aiowrite(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_aiowait(threadflow_t *threadflow, flowop_t *flowop); +#endif +static int flowoplib_read(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_block_init(flowop_t *flowop); +static int flowoplib_block(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_wakeup(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_hog(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_delay(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_sempost(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_sempost_init(flowop_t *flowop); +static int flowoplib_semblock(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_semblock_init(flowop_t *flowop); +static void flowoplib_semblock_destruct(flowop_t *flowop); +static int flowoplib_eventlimit(threadflow_t *, flowop_t *flowop); +static int flowoplib_bwlimit(threadflow_t *, flowop_t *flowop); +static int flowoplib_iopslimit(threadflow_t *, flowop_t *flowop); +static int flowoplib_opslimit(threadflow_t *, flowop_t *flowop); +static int flowoplib_openfile(threadflow_t *, flowop_t *flowop); +static int flowoplib_openfile_common(threadflow_t *, flowop_t *flowop, int fd); +static int flowoplib_createfile(threadflow_t *, flowop_t *flowop); +static int flowoplib_closefile(threadflow_t *, flowop_t *flowop); +static int flowoplib_fsync(threadflow_t *, flowop_t *flowop); +static int flowoplib_readwholefile(threadflow_t *, flowop_t *flowop); +static int flowoplib_writewholefile(threadflow_t *, flowop_t *flowop); +static int flowoplib_appendfile(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_appendfilerand(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_deletefile(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_statfile(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_finishoncount(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_finishonbytes(threadflow_t *threadflow, flowop_t *flowop); +static int flowoplib_fsyncset(threadflow_t *threadflow, flowop_t *flowop); + +typedef struct flowoplib { + int fl_type; + int fl_attrs; + char *fl_name; + int (*fl_init)(); + int (*fl_func)(); + void (*fl_destruct)(); +} flowoplib_t; + +static flowoplib_t flowoplib_funcs[] = { + FLOW_TYPE_IO, FLOW_ATTR_WRITE, "write", flowoplib_init_generic, + flowoplib_write, flowoplib_destruct_generic, + FLOW_TYPE_IO, FLOW_ATTR_READ, "read", flowoplib_init_generic, + flowoplib_read, flowoplib_destruct_generic, +#ifdef HAVE_AIO + FLOW_TYPE_AIO, FLOW_ATTR_WRITE, "aiowrite", flowoplib_init_generic, + flowoplib_aiowrite, flowoplib_destruct_generic, + FLOW_TYPE_AIO, 0, "aiowait", flowoplib_init_generic, + flowoplib_aiowait, flowoplib_destruct_generic, +#endif + FLOW_TYPE_SYNC, 0, "block", flowoplib_block_init, + flowoplib_block, flowoplib_destruct_generic, + FLOW_TYPE_SYNC, 0, "wakeup", flowoplib_init_generic, + flowoplib_wakeup, flowoplib_destruct_generic, + FLOW_TYPE_SYNC, 0, "semblock", flowoplib_semblock_init, + flowoplib_semblock, flowoplib_semblock_destruct, + FLOW_TYPE_SYNC, 0, "sempost", flowoplib_sempost_init, + flowoplib_sempost, flowoplib_destruct_generic, + FLOW_TYPE_OTHER, 0, "hog", flowoplib_init_generic, + flowoplib_hog, flowoplib_destruct_generic, + FLOW_TYPE_OTHER, 0, "delay", flowoplib_init_generic, + flowoplib_delay, flowoplib_destruct_generic, + FLOW_TYPE_OTHER, 0, "eventlimit", flowoplib_init_generic, + flowoplib_eventlimit, flowoplib_destruct_generic, + FLOW_TYPE_OTHER, 0, "bwlimit", flowoplib_init_generic, + flowoplib_bwlimit, flowoplib_destruct_generic, + FLOW_TYPE_OTHER, 0, "iopslimit", flowoplib_init_generic, + flowoplib_iopslimit, flowoplib_destruct_generic, + FLOW_TYPE_OTHER, 0, "opslimit", flowoplib_init_generic, + flowoplib_opslimit, flowoplib_destruct_generic, + FLOW_TYPE_OTHER, 0, "finishoncount", flowoplib_init_generic, + flowoplib_finishoncount, flowoplib_destruct_generic, + FLOW_TYPE_OTHER, 0, "finishonbytes", flowoplib_init_generic, + flowoplib_finishonbytes, flowoplib_destruct_generic, + FLOW_TYPE_IO, 0, "openfile", flowoplib_init_generic, + flowoplib_openfile, flowoplib_destruct_generic, + FLOW_TYPE_IO, 0, "createfile", flowoplib_init_generic, + flowoplib_createfile, flowoplib_destruct_generic, + FLOW_TYPE_IO, 0, "closefile", flowoplib_init_generic, + flowoplib_closefile, flowoplib_destruct_generic, + FLOW_TYPE_IO, 0, "fsync", flowoplib_init_generic, + flowoplib_fsync, flowoplib_destruct_generic, + FLOW_TYPE_IO, 0, "fsyncset", flowoplib_init_generic, + flowoplib_fsyncset, flowoplib_destruct_generic, + FLOW_TYPE_IO, 0, "statfile", flowoplib_init_generic, + flowoplib_statfile, flowoplib_destruct_generic, + FLOW_TYPE_IO, FLOW_ATTR_READ, "readwholefile", flowoplib_init_generic, + flowoplib_readwholefile, flowoplib_destruct_generic, + FLOW_TYPE_IO, FLOW_ATTR_WRITE, "appendfile", flowoplib_init_generic, + flowoplib_appendfile, flowoplib_destruct_generic, + FLOW_TYPE_IO, FLOW_ATTR_WRITE, "appendfilerand", flowoplib_init_generic, + flowoplib_appendfilerand, flowoplib_destruct_generic, + FLOW_TYPE_IO, 0, "deletefile", flowoplib_init_generic, + flowoplib_deletefile, flowoplib_destruct_generic, + FLOW_TYPE_IO, FLOW_ATTR_WRITE, "writewholefile", flowoplib_init_generic, + flowoplib_writewholefile, flowoplib_destruct_generic +}; + +/* + * Loops through the master list of flowops defined in this + * module, and creates and initializes a flowop for each one + * by calling flowop_define. As a side effect of calling + * flowop define, the created flowops are placed on the + * master flowop list. All created flowops are set to + * instance "0". + */ +void +flowoplib_init() +{ + int nops = sizeof (flowoplib_funcs) / sizeof (flowoplib_t); + int i; + + for (i = 0; i < nops; i++) { + flowop_t *flowop; + flowoplib_t *fl; + + fl = &flowoplib_funcs[i]; + + if ((flowop = flowop_define(NULL, + fl->fl_name, NULL, 0, fl->fl_type)) == 0) { + filebench_log(LOG_ERROR, + "failed to create flowop %s\n", + fl->fl_name); + filebench_shutdown(1); + } + + flowop->fo_func = fl->fl_func; + flowop->fo_init = fl->fl_init; + flowop->fo_destruct = fl->fl_destruct; + flowop->fo_attrs = fl->fl_attrs; + } +} + +static int +flowoplib_init_generic(flowop_t *flowop) +{ + (void) ipc_mutex_unlock(&flowop->fo_lock); + return (0); +} + +/* ARGSUSED */ +static void +flowoplib_destruct_generic(flowop_t *flowop) +{ +} + +/* + * Generates a file attribute from flags in the supplied flowop. + * Sets FLOW_ATTR_DIRECTIO and/or FLOW_ATTR_DSYNC as needed. + */ +static int +flowoplib_fileattrs(flowop_t *flowop) +{ + int attrs = 0; + + if (*flowop->fo_directio) + attrs |= FLOW_ATTR_DIRECTIO; + + if (*flowop->fo_dsync) + attrs |= FLOW_ATTR_DSYNC; + + return (attrs); +} + +/* + * Searches for a file descriptor. Tries the flowop's + * fo_fdnumber first and returns with it if it has been + * explicitly set (greater than 0). It next checks to + * see if a rotating file descriptor policy is in effect, + * and if not returns the fdnumber regardless of what + * it is. (note that if it is 0, it just selects to the + * default file descriptor in the threadflow's tf_fd + * array). If the rotating fd policy is in effect, it + * cycles from the end of the tf_fd array to one location + * beyond the maximum needed by the number of entries in + * the associated fileset on each invocation, then starts + * over from the end. + * + * The routine returns an index into the threadflow's + * tf_fd table where the actual file descriptor will be + * found. Note: the calling routine must not call this + * routine if the flowop does not have a fileset, and the + * flowop's fo_fdnumber is zero and fo_rotatefd is + * asserted, or an addressing fault may occur. + */ +int +flowoplib_fdnum(threadflow_t *threadflow, flowop_t *flowop) +{ + /* If the script sets the fd explicitly */ + if (flowop->fo_fdnumber > 0) + return (flowop->fo_fdnumber); + + /* If the flowop defaults to persistent fd */ + if (!integer_isset(flowop->fo_rotatefd)) + return (flowop->fo_fdnumber); + + /* Rotate the fd on each flowop invocation */ + if (*(flowop->fo_fileset->fs_entries) > (THREADFLOW_MAXFD / 2)) { + filebench_log(LOG_ERROR, "Out of file descriptors in flowop %s" + " (too many files : %d", flowop->fo_name, + *(flowop->fo_fileset->fs_entries)); + return (-1); + } + + /* First time around */ + if (threadflow->tf_fdrotor == 0) + threadflow->tf_fdrotor = THREADFLOW_MAXFD; + + /* One fd for every file in the set */ + if (*(flowop->fo_fileset->fs_entries) == + (THREADFLOW_MAXFD - threadflow->tf_fdrotor)) + threadflow->tf_fdrotor = THREADFLOW_MAXFD; + + + threadflow->tf_fdrotor--; + filebench_log(LOG_DEBUG_IMPL, "selected fd = %d", + threadflow->tf_fdrotor); + return (threadflow->tf_fdrotor); +} + +/* + * Emulate posix read / pread. If the flowop has a fileset, + * a file descriptor number index is fetched, otherwise a + * supplied fileobj file is used. In either case the specified + * file will be opened if not already open. If the flowop has + * neither a fileset or fileobj, an error is logged and -1 + * returned. + * + * The actual read is done to a random offset in the + * threadflow's thread memory (tf_mem), with a size set by + * fo_iosize and at either a random disk offset within the + * working set size, or at the next sequential location. If + * any errors are encountered, -1 is returned, if successful, + * 0 is returned. + */ +static int +flowoplib_read(threadflow_t *threadflow, flowop_t *flowop) +{ + size_t memoffset; + vinteger_t wss; + long memsize, round; + int filedesc; + int ret; + + if (flowop->fo_fileset || flowop->fo_fdnumber) { + int fd = flowoplib_fdnum(threadflow, flowop); + + if (fd == -1) + return (-1); + + if (threadflow->tf_fd[fd] == 0) { + (void) flowoplib_openfile_common(threadflow, + flowop, fd); + filebench_log(LOG_DEBUG_IMPL, "read opened file %s", + threadflow->tf_fse[fd]->fse_path); + } + filedesc = threadflow->tf_fd[fd]; + if (*flowop->fo_wss == 0) + wss = threadflow->tf_fse[fd]->fse_size; + else + wss = *flowop->fo_wss; + } else { + if (flowop->fo_file == NULL) { + filebench_log(LOG_ERROR, "flowop NULL file"); + return (-1); + } + if (flowop->fo_fd < 0) + flowop->fo_fd = fileobj_open(flowop->fo_file, + flowoplib_fileattrs(flowop)); + + if (flowop->fo_fd < 0) { + filebench_log(LOG_ERROR, "failed to open file %s", + flowop->fo_file->fo_name); + return (-1); + } + filedesc = flowop->fo_fd; + if (*flowop->fo_wss == 0) + wss = *flowop->fo_file->fo_size; + else + wss = *flowop->fo_wss; + } + + if (*flowop->fo_iosize == 0) { + filebench_log(LOG_ERROR, "zero iosize for thread %s", + flowop->fo_name); + return (-1); + } + + memsize = *threadflow->tf_memsize; + round = *flowop->fo_iosize; + + if (filebench_randomno(&memoffset, memsize, round) == -1) { + filebench_log(LOG_ERROR, + "tf_memsize smaller than IO size for thread %s", + flowop->fo_name); + return (-1); + } + + if (*flowop->fo_random) { + uint64_t fileoffset; + + if (filebench_randomno64(&fileoffset, wss, + *flowop->fo_iosize) == -1) { + filebench_log(LOG_ERROR, + "file size smaller than IO size for thread %s", + flowop->fo_name); + return (-1); + } + + (void) flowop_beginop(threadflow, flowop); + if ((ret = pread64(filedesc, threadflow->tf_mem + memoffset, + *flowop->fo_iosize, (off64_t)fileoffset)) == -1) { + (void) flowop_endop(threadflow, flowop); + filebench_log(LOG_ERROR, + "read file %s failed, offset %lld " + "memoffset %zd: %s", + flowop->fo_file->fo_name, + fileoffset, memoffset, strerror(errno)); + flowop_endop(threadflow, flowop); + return (-1); + } + (void) flowop_endop(threadflow, flowop); + + if ((ret == 0)) + (void) lseek64(filedesc, 0, SEEK_SET); + + } else { + (void) flowop_beginop(threadflow, flowop); + if ((ret = read(filedesc, threadflow->tf_mem + memoffset, + *flowop->fo_iosize)) == -1) { + filebench_log(LOG_ERROR, + "read file %s failed, memoffset %zd: %s", + flowop->fo_file->fo_name, + memoffset, strerror(errno)); + (void) flowop_endop(threadflow, flowop); + return (-1); + } + (void) flowop_endop(threadflow, flowop); + + if ((ret == 0)) + (void) lseek64(filedesc, 0, SEEK_SET); + } + + return (0); +} + +#ifdef HAVE_AIO + +/* + * Asynchronous write section. An Asynchronous IO element + * (aiolist_t) is used to associate the asynchronous write request with + * its subsequent completion. This element includes a aiocb64 struct + * that is used by posix aio_xxx calls to track the asynchronous writes. + * The flowops aiowrite and aiowait result in calls to these posix + * aio_xxx system routines to do the actual asynchronous write IO + * operations. + */ + + +/* + * Allocates an asynchronous I/O list (aio, of type + * aiolist_t) element. Adds it to the flowop thread's + * threadflow aio list. Returns a pointer to the element. + */ +static aiolist_t * +aio_allocate(flowop_t *flowop) +{ + aiolist_t *aiolist; + + if ((aiolist = malloc(sizeof (aiolist_t))) == NULL) { + filebench_log(LOG_ERROR, "malloc aiolist failed"); + filebench_shutdown(1); + } + + /* Add to list */ + if (flowop->fo_thread->tf_aiolist == NULL) { + flowop->fo_thread->tf_aiolist = aiolist; + aiolist->al_next = NULL; + } else { + aiolist->al_next = flowop->fo_thread->tf_aiolist; + flowop->fo_thread->tf_aiolist = aiolist; + } + return (aiolist); +} + +/* + * Searches for the aiolist element that has a matching + * completion block, aiocb. If none found returns -1. If + * found, removes the aiolist element from flowop thread's + * list and returns 0. + */ +static int +aio_deallocate(flowop_t *flowop, struct aiocb64 *aiocb) +{ + aiolist_t *aiolist = flowop->fo_thread->tf_aiolist; + aiolist_t *previous = NULL; + aiolist_t *match = NULL; + + if (aiocb == NULL) { + filebench_log(LOG_ERROR, "null aiocb deallocate"); + return (0); + } + + while (aiolist) { + if (aiocb == &(aiolist->al_aiocb)) { + match = aiolist; + break; + } + previous = aiolist; + aiolist = aiolist->al_next; + } + + if (match == NULL) + return (-1); + + /* Remove from the list */ + if (previous) + previous->al_next = match->al_next; + else + flowop->fo_thread->tf_aiolist = match->al_next; + + return (0); +} + +/* + * Emulate posix aiowrite(). Determines which file to use, + * either one file of a fileset, or the file associated + * with a fileobj, allocates and fills an aiolist_t element + * for the write, and issues the asynchronous write. This + * operation is only valid for random IO, and returns an + * error if the flowop is set for sequential IO. Returns 0 + * on success, -1 on any encountered error. + */ +static int +flowoplib_aiowrite(threadflow_t *threadflow, flowop_t *flowop) +{ + size_t memoffset; + vinteger_t wss; + long memsize, round; + int filedesc; + + if (flowop->fo_fileset || flowop->fo_fdnumber) { + int fd = flowoplib_fdnum(threadflow, flowop); + + if (fd == -1) + return (-1); + + if (threadflow->tf_fd[fd] == 0) { + (void) flowoplib_openfile_common(threadflow, + flowop, fd); + filebench_log(LOG_DEBUG_IMPL, + "writefile opened file %s", + threadflow->tf_fse[fd]->fse_path); + } + filedesc = threadflow->tf_fd[fd]; + if (*flowop->fo_wss == 0) + wss = threadflow->tf_fse[fd]->fse_size; + else + wss = *flowop->fo_wss; + } else { + if (flowop->fo_file == NULL) { + filebench_log(LOG_ERROR, "flowop NULL file"); + return (-1); + } + if (flowop->fo_fd < 0) + flowop->fo_fd = fileobj_open(flowop->fo_file, + flowoplib_fileattrs(flowop)); + + if (flowop->fo_fd < 0) { + filebench_log(LOG_ERROR, "failed to open file %s", + flowop->fo_file->fo_name); + return (-1); + } + filedesc = flowop->fo_fd; + if (*flowop->fo_wss == 0) + wss = *flowop->fo_file->fo_size; + else + wss = *flowop->fo_wss; + } + + if (*flowop->fo_iosize == 0) { + filebench_log(LOG_ERROR, "zero iosize for thread %s", + flowop->fo_name); + return (-1); + } + + memsize = *threadflow->tf_memsize; + round = *flowop->fo_iosize; + + /* Select memory offset for IO */ + if (filebench_randomno(&memoffset, memsize, round) == -1) { + filebench_log(LOG_ERROR, + "tf_memsize smaller than IO size for thread %s", + flowop->fo_name); + return (-1); + } + + if (*flowop->fo_random) { + uint64_t fileoffset; + struct aiocb64 *aiocb; + aiolist_t *aiolist; + + if (filebench_randomno64(&fileoffset, + wss, *flowop->fo_iosize) == -1) { + filebench_log(LOG_ERROR, + "file size smaller than IO size for thread %s", + flowop->fo_name); + return (-1); + } + + aiolist = aio_allocate(flowop); + aiolist->al_type = AL_WRITE; + aiocb = &aiolist->al_aiocb; + + aiocb->aio_fildes = filedesc; + aiocb->aio_buf = threadflow->tf_mem + memoffset; + aiocb->aio_nbytes = *flowop->fo_iosize; + aiocb->aio_offset = (off64_t)fileoffset; + aiocb->aio_reqprio = 0; + + filebench_log(LOG_DEBUG_IMPL, + "aio fd=%d, bytes=%lld, offset=%lld", + filedesc, *flowop->fo_iosize, fileoffset); + + flowop_beginop(threadflow, flowop); + if (aio_write64(aiocb) < 0) { + filebench_log(LOG_ERROR, "aiowrite failed: %s", + strerror(errno)); + filebench_shutdown(1); + } + flowop_endop(threadflow, flowop); + } else { + return (-1); + } + + return (0); +} + + + +#define MAXREAP 4096 + +/* + * Emulate posix aiowait(). Waits for the completion of half the + * outstanding asynchronous IOs, or a single IO, which ever is + * larger. The routine will return after a sufficient number of + * completed calls issued by any thread in the procflow have + * completed, or a 1 second timout elapses. All completed + * IO operations are deleted from the thread's aiolist. + */ +static int +flowoplib_aiowait(threadflow_t *threadflow, flowop_t *flowop) +{ + struct aiocb64 **worklist; + aiolist_t *aio = flowop->fo_thread->tf_aiolist; + int uncompleted = 0; + + worklist = calloc(MAXREAP, sizeof (struct aiocb64 *)); + + /* Count the list of pending aios */ + while (aio) { + uncompleted++; + aio = aio->al_next; + } + + do { + uint_t ncompleted = 0; + uint_t todo; + struct timespec timeout; + int inprogress; + int i; + + /* Wait for half of the outstanding requests */ + timeout.tv_sec = 1; + timeout.tv_nsec = 0; + + if (uncompleted > MAXREAP) + todo = MAXREAP; + else + todo = uncompleted / 2; + + if (todo == 0) + todo = 1; + + flowop_beginop(threadflow, flowop); + +#ifdef HAVE_AIOWAITN + if ((aio_waitn64((struct aiocb64 **)worklist, + MAXREAP, &todo, &timeout) == -1) && + errno && (errno != ETIME)) { + filebench_log(LOG_ERROR, + "aiowait failed: %s, outstanding = %d, " + "ncompleted = %d ", + strerror(errno), uncompleted, todo); + } + + ncompleted = todo; + /* Take the completed I/Os from the list */ + inprogress = 0; + for (i = 0; i < ncompleted; i++) { + if ((aio_return64(worklist[i]) == -1) && + (errno == EINPROGRESS)) { + inprogress++; + continue; + } + if (aio_deallocate(flowop, worklist[i]) < 0) { + filebench_log(LOG_ERROR, "Could not remove " + "aio from list "); + flowop_endop(threadflow, flowop); + return (-1); + } + } + + uncompleted -= ncompleted; + uncompleted += inprogress; + +#else + + for (ncompleted = 0, inprogress = 0, + aio = flowop->fo_thread->tf_aiolist; + ncompleted < todo, aio != NULL; aio = aio->al_next) { + + result = aio_error64(&aio->al_aiocb); + + if (result == EINPROGRESS) { + inprogress++; + continue; + } + + if ((aio_return64(&aio->al_aiocb) == -1) || result) { + filebench_log(LOG_ERROR, "aio failed: %s", + strerror(result)); + continue; + } + + ncompleted++; + + if (aio_deallocate(flowop, &aio->al_aiocb) < 0) { + filebench_log(LOG_ERROR, "Could not remove aio " + "from list "); + flowop_endop(threadflow, flowop); + return (-1); + } + } + + uncompleted -= ncompleted; + +#endif + filebench_log(LOG_DEBUG_SCRIPT, + "aio2 completed %d ios, uncompleted = %d, inprogress = %d", + ncompleted, uncompleted, inprogress); + + } while (uncompleted > MAXREAP); + + flowop_endop(threadflow, flowop); + + free(worklist); + + return (0); +} + +#endif /* HAVE_AIO */ + +/* + * Initializes a "flowop_block" flowop. Specifically, it + * initializes the flowop's fo_cv and unlocks the fo_lock. + */ +static int +flowoplib_block_init(flowop_t *flowop) +{ + filebench_log(LOG_DEBUG_IMPL, "flow %s-%d block init address %zx", + flowop->fo_name, flowop->fo_instance, &flowop->fo_cv); + (void) pthread_cond_init(&flowop->fo_cv, ipc_condattr()); + (void) ipc_mutex_unlock(&flowop->fo_lock); + + return (0); +} + +/* + * Blocks the threadflow until woken up by flowoplib_wakeup. + * The routine blocks on the flowop's fo_cv condition variable. + */ +static int +flowoplib_block(threadflow_t *threadflow, flowop_t *flowop) +{ + filebench_log(LOG_DEBUG_IMPL, "flow %s-%d blocking at address %zx", + flowop->fo_name, flowop->fo_instance, &flowop->fo_cv); + (void) ipc_mutex_lock(&flowop->fo_lock); + + flowop_beginop(threadflow, flowop); + (void) pthread_cond_wait(&flowop->fo_cv, &flowop->fo_lock); + flowop_endop(threadflow, flowop); + + filebench_log(LOG_DEBUG_IMPL, "flow %s-%d unblocking", + flowop->fo_name, flowop->fo_instance); + + (void) ipc_mutex_unlock(&flowop->fo_lock); + + return (0); +} + +/* + * Wakes up one or more target blocking flowops. + * Sends broadcasts on the fo_cv condition variables of all + * flowops on the target list, except those that are + * FLOW_MASTER flowops. The target list consists of all + * flowops whose name matches this flowop's "fo_targetname" + * attribute. The target list is generated on the first + * invocation, and the run will be shutdown if no targets + * are found. Otherwise the routine always returns 0. + */ +static int +flowoplib_wakeup(threadflow_t *threadflow, flowop_t *flowop) +{ + flowop_t *target; + + /* if this is the first wakeup, create the wakeup list */ + if (flowop->fo_targets == NULL) { + flowop_t *result = flowop_find(flowop->fo_targetname); + + flowop->fo_targets = result; + if (result == NULL) { + filebench_log(LOG_ERROR, + "wakeup: could not find op %s for thread %s", + flowop->fo_targetname, + threadflow->tf_name); + filebench_shutdown(1); + } + while (result) { + result->fo_targetnext = + result->fo_resultnext; + result = result->fo_resultnext; + } + } + + target = flowop->fo_targets; + + /* wakeup the targets */ + while (target) { + if (target->fo_instance == FLOW_MASTER) { + target = target->fo_targetnext; + continue; + } + filebench_log(LOG_DEBUG_IMPL, + "wakeup flow %s-%d at address %zx", + target->fo_name, + target->fo_instance, + &target->fo_cv); + + flowop_beginop(threadflow, flowop); + (void) ipc_mutex_lock(&target->fo_lock); + (void) pthread_cond_broadcast(&target->fo_cv); + (void) ipc_mutex_unlock(&target->fo_lock); + flowop_endop(threadflow, flowop); + + target = target->fo_targetnext; + } + + return (0); +} + +/* + * "think time" routines. the "hog" routine consumes cpu cycles as + * it "thinks", while the "delay" flowop simply calls sleep() to delay + * for a given number of seconds without consuming cpu cycles. + */ + + +/* + * Consumes CPU cycles and memory bandwidth by looping for + * flowop->fo_value times. With each loop sets memory location + * threadflow->tf_mem to 1. + */ +static int +flowoplib_hog(threadflow_t *threadflow, flowop_t *flowop) +{ + uint64_t value = *flowop->fo_value; + int i; + + flowop_beginop(threadflow, flowop); + filebench_log(LOG_DEBUG_IMPL, "hog enter"); + for (i = 0; i < value; i++) + *(threadflow->tf_mem) = 1; + flowop_endop(threadflow, flowop); + filebench_log(LOG_DEBUG_IMPL, "hog exit"); + return (0); +} + + +/* + * Delays for fo_value seconds. + */ +static int +flowoplib_delay(threadflow_t *threadflow, flowop_t *flowop) +{ + int value = *flowop->fo_value; + + flowop_beginop(threadflow, flowop); + (void) sleep(value); + flowop_endop(threadflow, flowop); + return (0); +} + +/* + * Rate limiting routines. This is the event consuming half of the + * event system. Each of the four following routines will limit the rate + * to one unit of either calls, issued I/O operations, issued filebench + * operations, or I/O bandwidth. Since there is only one event generator, + * the events will be divided amoung multiple instances of an event + * consumer, and further divided among different consumers if more than + * one has been defined. There is no mechanism to enforce equal sharing + * of events. + */ + +/* + * Completes one invocation per posted event. If eventgen_q + * has an event count greater than zero, one will be removed + * (count decremented), otherwise the calling thread will + * block until another event has been posted. Always returns 0 + */ +static int +flowoplib_eventlimit(threadflow_t *threadflow, flowop_t *flowop) +{ + /* Immediately bail if not set/enabled */ + if (filebench_shm->eventgen_hz == 0) + return (0); + + if (flowop->fo_initted == 0) { + filebench_log(LOG_DEBUG_IMPL, "rate %zx %s-%d locking", + flowop, threadflow->tf_name, threadflow->tf_instance); + flowop->fo_initted = 1; + } + + flowop_beginop(threadflow, flowop); + while (filebench_shm->eventgen_hz) { + (void) ipc_mutex_lock(&filebench_shm->eventgen_lock); + if (filebench_shm->eventgen_q > 0) { + filebench_shm->eventgen_q--; + (void) ipc_mutex_unlock(&filebench_shm->eventgen_lock); + break; + } + (void) pthread_cond_wait(&filebench_shm->eventgen_cv, + &filebench_shm->eventgen_lock); + (void) ipc_mutex_unlock(&filebench_shm->eventgen_lock); + } + flowop_endop(threadflow, flowop); + return (0); +} + +/* + * Blocks the calling thread if the number of issued I/O + * operations exceeds the number of posted events, thus + * limiting the average I/O operation rate to the rate + * specified by eventgen_hz. Always returns 0. + */ +static int +flowoplib_iopslimit(threadflow_t *threadflow, flowop_t *flowop) +{ + uint64_t iops; + uint64_t delta; + int events; + + /* Immediately bail if not set/enabled */ + if (filebench_shm->eventgen_hz == 0) + return (0); + + if (flowop->fo_initted == 0) { + filebench_log(LOG_DEBUG_IMPL, "rate %zx %s-%d locking", + flowop, threadflow->tf_name, threadflow->tf_instance); + flowop->fo_initted = 1; + } + + iops = (controlstats.fs_rcount + + controlstats.fs_wcount); + + /* Is this the first time around */ + if (flowop->fo_tputlast == 0) { + flowop->fo_tputlast = iops; + return (0); + } + + delta = iops - flowop->fo_tputlast; + flowop->fo_tputbucket -= delta; + flowop->fo_tputlast = iops; + + /* No need to block if the q isn't empty */ + if (flowop->fo_tputbucket >= 0LL) { + flowop_endop(threadflow, flowop); + return (0); + } + + iops = flowop->fo_tputbucket * -1; + events = iops; + + flowop_beginop(threadflow, flowop); + while (filebench_shm->eventgen_hz) { + + (void) ipc_mutex_lock(&filebench_shm->eventgen_lock); + if (filebench_shm->eventgen_q >= events) { + filebench_shm->eventgen_q -= events; + (void) ipc_mutex_unlock(&filebench_shm->eventgen_lock); + flowop->fo_tputbucket += events; + break; + } + (void) pthread_cond_wait(&filebench_shm->eventgen_cv, + &filebench_shm->eventgen_lock); + (void) ipc_mutex_unlock(&filebench_shm->eventgen_lock); + } + flowop_endop(threadflow, flowop); + + return (0); +} + +/* + * Blocks the calling thread if the number of issued filebench + * operations exceeds the number of posted events, thus limiting + * the average filebench operation rate to the rate specified by + * eventgen_hz. Always returns 0. + */ +static int +flowoplib_opslimit(threadflow_t *threadflow, flowop_t *flowop) +{ + uint64_t ops; + uint64_t delta; + int events; + + /* Immediately bail if not set/enabled */ + if (filebench_shm->eventgen_hz == 0) + return (0); + + if (flowop->fo_initted == 0) { + filebench_log(LOG_DEBUG_IMPL, "rate %zx %s-%d locking", + flowop, threadflow->tf_name, threadflow->tf_instance); + flowop->fo_initted = 1; + } + + ops = controlstats.fs_count; + + /* Is this the first time around */ + if (flowop->fo_tputlast == 0) { + flowop->fo_tputlast = ops; + return (0); + } + + delta = ops - flowop->fo_tputlast; + flowop->fo_tputbucket -= delta; + flowop->fo_tputlast = ops; + + /* No need to block if the q isn't empty */ + if (flowop->fo_tputbucket >= 0LL) { + flowop_endop(threadflow, flowop); + return (0); + } + + ops = flowop->fo_tputbucket * -1; + events = ops; + + flowop_beginop(threadflow, flowop); + while (filebench_shm->eventgen_hz) { + (void) ipc_mutex_lock(&filebench_shm->eventgen_lock); + if (filebench_shm->eventgen_q >= events) { + filebench_shm->eventgen_q -= events; + (void) ipc_mutex_unlock(&filebench_shm->eventgen_lock); + flowop->fo_tputbucket += events; + break; + } + (void) pthread_cond_wait(&filebench_shm->eventgen_cv, + &filebench_shm->eventgen_lock); + (void) ipc_mutex_unlock(&filebench_shm->eventgen_lock); + } + flowop_endop(threadflow, flowop); + + return (0); +} + + +/* + * Blocks the calling thread if the number of bytes of I/O + * issued exceeds one megabyte times the number of posted + * events, thus limiting the average I/O byte rate to one + * megabyte times the event rate as set by eventgen_hz. + * Always retuns 0. + */ +static int +flowoplib_bwlimit(threadflow_t *threadflow, flowop_t *flowop) +{ + uint64_t bytes; + uint64_t delta; + int events; + + /* Immediately bail if not set/enabled */ + if (filebench_shm->eventgen_hz == 0) + return (0); + + if (flowop->fo_initted == 0) { + filebench_log(LOG_DEBUG_IMPL, "rate %zx %s-%d locking", + flowop, threadflow->tf_name, threadflow->tf_instance); + flowop->fo_initted = 1; + } + + bytes = (controlstats.fs_rbytes + + controlstats.fs_wbytes); + + /* Is this the first time around */ + if (flowop->fo_tputlast == 0) { + flowop->fo_tputlast = bytes; + return (0); + } + + delta = bytes - flowop->fo_tputlast; + flowop->fo_tputbucket -= delta; + flowop->fo_tputlast = bytes; + + /* No need to block if the q isn't empty */ + if (flowop->fo_tputbucket >= 0LL) { + flowop_endop(threadflow, flowop); + return (0); + } + + bytes = flowop->fo_tputbucket * -1; + events = (bytes / MB) + 1; + + filebench_log(LOG_DEBUG_IMPL, "%lld bytes, %lld events", + bytes, events); + + flowop_beginop(threadflow, flowop); + while (filebench_shm->eventgen_hz) { + (void) ipc_mutex_lock(&filebench_shm->eventgen_lock); + if (filebench_shm->eventgen_q >= events) { + filebench_shm->eventgen_q -= events; + (void) ipc_mutex_unlock(&filebench_shm->eventgen_lock); + flowop->fo_tputbucket += (events * MB); + break; + } + (void) pthread_cond_wait(&filebench_shm->eventgen_cv, + &filebench_shm->eventgen_lock); + (void) ipc_mutex_unlock(&filebench_shm->eventgen_lock); + } + flowop_endop(threadflow, flowop); + + return (0); +} + +/* + * These flowops terminate a benchmark run when either the specified + * number of bytes of I/O (flowoplib_finishonbytes) or the specified + * number of I/O operations (flowoplib_finishoncount) have been generated. + */ + + +/* + * Stop filebench run when specified number of I/O bytes have been + * transferred. Compares controlstats.fs_bytes with *flowop->value, + * and if greater returns 1, stopping the run, if not, returns 0 + * to continue running. + */ +static int +flowoplib_finishonbytes(threadflow_t *threadflow, flowop_t *flowop) +{ + uint64_t b; + uint64_t bytes = *flowop->fo_value; + + b = controlstats.fs_bytes; + + flowop_beginop(threadflow, flowop); + if (b > bytes) { + flowop_endop(threadflow, flowop); + return (1); + } + flowop_endop(threadflow, flowop); + + return (0); +} + +/* + * Stop filebench run when specified number of I/O operations have + * been performed. Compares controlstats.fs_count with *flowop->value, + * and if greater returns 1, stopping the run, if not, returns 0 to + * continue running. + */ +static int +flowoplib_finishoncount(threadflow_t *threadflow, flowop_t *flowop) +{ + uint64_t ops; + uint64_t count = *flowop->fo_value; + + ops = controlstats.fs_count; + + flowop_beginop(threadflow, flowop); + if (ops > count) { + flowop_endop(threadflow, flowop); + return (1); + } + flowop_endop(threadflow, flowop); + + return (0); +} + +/* + * Semaphore synchronization using either System V semaphores or + * posix semaphores. If System V semaphores are available, they will be + * used, otherwise posix semaphores will be used. + */ + + +/* + * Initializes the filebench "block on semaphore" flowop. + * If System V semaphores are implemented, the routine + * initializes the System V semaphore subsystem if it hasn't + * already been initialized, also allocates a pair of semids + * and initializes the highwater System V semaphore. + * If no System V semaphores, then does nothing special. + * Returns -1 if it cannot acquire a set of System V semphores + * or if the initial post to the semaphore set fails. Returns 0 + * on success. + */ +static int +flowoplib_semblock_init(flowop_t *flowop) +{ + +#ifdef HAVE_SYSV_SEM + int semid; + struct sembuf sbuf[2]; + int highwater; + + ipc_seminit(); + + flowop->fo_semid_lw = ipc_semidalloc(); + flowop->fo_semid_hw = ipc_semidalloc(); + + filebench_log(LOG_DEBUG_IMPL, "flow %s-%d semblock init semid=%x", + flowop->fo_name, flowop->fo_instance, flowop->fo_semid_lw); + + /* + * Raise the number of the hw queue, causing the posting side to + * block if queue is > 2 x blocking value + */ + if ((semid = semget(filebench_shm->semkey, FILEBENCH_NSEMS, 0)) == -1) { + filebench_log(LOG_ERROR, "semblock init lookup %x failed: %s", + filebench_shm->semkey, + strerror(errno)); + return (-1); + } + + if ((highwater = flowop->fo_semid_hw) == 0) + highwater = *flowop->fo_value; + + filebench_log(LOG_DEBUG_IMPL, "setting highwater to : %d", highwater); + + sbuf[0].sem_num = highwater; + sbuf[0].sem_op = *flowop->fo_highwater; + sbuf[0].sem_flg = 0; + if ((semop(semid, &sbuf[0], 1) == -1) && errno) { + filebench_log(LOG_ERROR, "semblock init post failed: %s (%d," + "%d)", strerror(errno), sbuf[0].sem_num, sbuf[0].sem_op); + return (-1); + } +#else + filebench_log(LOG_DEBUG_IMPL, + "flow %s-%d semblock init with posix semaphore", + flowop->fo_name, flowop->fo_instance); + + sem_init(&flowop->fo_sem, 1, 0); +#endif /* HAVE_SYSV_SEM */ + + if (!(*flowop->fo_blocking)) + (void) ipc_mutex_unlock(&flowop->fo_lock); + + return (0); +} + +/* + * Releases the semids for the System V semaphore allocated + * to this flowop. If not using System V semaphores, then + * it is effectively just a no-op. Always returns 0. + */ +static void +flowoplib_semblock_destruct(flowop_t *flowop) +{ +#ifdef HAVE_SYSV_SEM + ipc_semidfree(flowop->fo_semid_lw); + ipc_semidfree(flowop->fo_semid_hw); +#else + sem_destroy(&flowop->fo_sem); +#endif /* HAVE_SYSV_SEM */ +} + +/* + * Attempts to pass a System V or posix semaphore as appropriate, + * and blocks if necessary. Returns -1 if a set of System V + * semphores is not available or cannot be acquired, or if the initial + * post to the semaphore set fails. Returns 0 on success. + */ +static int +flowoplib_semblock(threadflow_t *threadflow, flowop_t *flowop) +{ + +#ifdef HAVE_SYSV_SEM + struct sembuf sbuf[2]; + int value = *flowop->fo_value; + int semid; + struct timespec timeout; + + if ((semid = semget(filebench_shm->semkey, FILEBENCH_NSEMS, 0)) == -1) { + filebench_log(LOG_ERROR, "lookup semop %x failed: %s", + filebench_shm->semkey, + strerror(errno)); + return (-1); + } + + filebench_log(LOG_DEBUG_IMPL, + "flow %s-%d sem blocking on id %x num %x value %d", + flowop->fo_name, flowop->fo_instance, semid, + flowop->fo_semid_hw, value); + + /* Post, decrement the increment the hw queue */ + sbuf[0].sem_num = flowop->fo_semid_hw; + sbuf[0].sem_op = value; + sbuf[0].sem_flg = 0; + sbuf[1].sem_num = flowop->fo_semid_lw; + sbuf[1].sem_op = value * -1; + sbuf[1].sem_flg = 0; + timeout.tv_sec = 600; + timeout.tv_nsec = 0; + + if (*flowop->fo_blocking) + (void) ipc_mutex_unlock(&flowop->fo_lock); + + flowop_beginop(threadflow, flowop); + +#ifdef HAVE_SEMTIMEDOP + (void) semtimedop(semid, &sbuf[0], 1, &timeout); + (void) semtimedop(semid, &sbuf[1], 1, &timeout); +#else + (void) semop(semid, &sbuf[0], 1); + (void) semop(semid, &sbuf[1], 1); +#endif /* HAVE_SEMTIMEDOP */ + + if (*flowop->fo_blocking) + (void) ipc_mutex_lock(&flowop->fo_lock); + + flowop_endop(threadflow, flowop); + +#else + int value = *flowop->fo_value; + int i; + + filebench_log(LOG_DEBUG_IMPL, + "flow %s-%d sem blocking on posix semaphore", + flowop->fo_name, flowop->fo_instance); + + /* Decrement sem by value */ + for (i = 0; i < value; i++) { + if (sem_wait(&flowop->fo_sem) == -1) { + filebench_log(LOG_ERROR, "semop wait failed"); + return (-1); + } + } + + filebench_log(LOG_DEBUG_IMPL, "flow %s-%d sem unblocking", + flowop->fo_name, flowop->fo_instance); +#endif /* HAVE_SYSV_SEM */ + + return (0); +} + +/* + * Calls ipc_seminit(), and does so whether System V semaphores + * are available or not. Hence it will cause ipc_seminit to log errors + * if they are not. Always returns 0. + */ +/* ARGSUSED */ +static int +flowoplib_sempost_init(flowop_t *flowop) +{ +#ifdef HAVE_SYSV_SEM + ipc_seminit(); +#endif /* HAVE_SYSV_SEM */ + return (0); +} + +/* + * Post to a System V or posix semaphore as appropriate. + * On the first call for a given flowop instance, this routine + * will use the fo_targetname attribute to locate all semblock + * flowops that are expecting posts from this flowop. All + * target flowops on this list will have a post operation done + * to their semaphores on each call. + */ +static int +flowoplib_sempost(threadflow_t *threadflow, flowop_t *flowop) +{ + flowop_t *target; + + filebench_log(LOG_DEBUG_IMPL, + "sempost flow %s-%d", + flowop->fo_name, + flowop->fo_instance); + + /* if this is the first post, create the post list */ + if (flowop->fo_targets == NULL) { + flowop_t *result = flowop_find(flowop->fo_targetname); + + flowop->fo_targets = result; + + if (result == NULL) { + filebench_log(LOG_ERROR, + "sempost: could not find op %s for thread %s", + flowop->fo_targetname, + threadflow->tf_name); + filebench_shutdown(1); + } + + while (result) { + result->fo_targetnext = + result->fo_resultnext; + result = result->fo_resultnext; + } + } + + target = flowop->fo_targets; + + flowop_beginop(threadflow, flowop); + /* post to the targets */ + while (target) { +#ifdef HAVE_SYSV_SEM + struct sembuf sbuf[2]; + int semid; + int blocking; +#else + int i; +#endif /* HAVE_SYSV_SEM */ + int value = *flowop->fo_value; + struct timespec timeout; + + if (target->fo_instance == FLOW_MASTER) { + target = target->fo_targetnext; + continue; + } + +#ifdef HAVE_SYSV_SEM + + filebench_log(LOG_DEBUG_IMPL, + "sempost flow %s-%d num %x", + target->fo_name, + target->fo_instance, + target->fo_semid_lw); + /* ipc_mutex_lock(&target->fo_lock); */ + + if ((semid = semget(filebench_shm->semkey, + FILEBENCH_NSEMS, 0)) == -1) { + filebench_log(LOG_ERROR, + "lookup semop %x failed: %s", + filebench_shm->semkey, + strerror(errno)); + /* ipc_mutex_unlock(&target->fo_lock); */ + return (-1); + } + + sbuf[0].sem_num = target->fo_semid_lw; + sbuf[0].sem_op = value; + sbuf[0].sem_flg = 0; + sbuf[1].sem_num = target->fo_semid_hw; + sbuf[1].sem_op = value * -1; + sbuf[1].sem_flg = 0; + timeout.tv_sec = 600; + timeout.tv_nsec = 0; + + if (*flowop->fo_blocking) + blocking = 1; + else + blocking = 0; + +#ifdef HAVE_SEMTIMEDOP + if ((semtimedop(semid, &sbuf[0], blocking + 1, + &timeout) == -1) && (errno && (errno != EAGAIN))) { +#else + if ((semop(semid, &sbuf[0], blocking + 1) == -1) && + (errno && (errno != EAGAIN))) { +#endif /* HAVE_SEMTIMEDOP */ + filebench_log(LOG_ERROR, "semop post failed: %s", + strerror(errno)); + /* ipc_mutex_unlock(&target->fo_lock); */ + return (-1); + } + + filebench_log(LOG_DEBUG_IMPL, + "flow %s-%d finished posting", + target->fo_name, target->fo_instance); +#else + filebench_log(LOG_DEBUG_IMPL, + "sempost flow %s-%d to posix semaphore", + target->fo_name, + target->fo_instance); + + /* Increment sem by value */ + for (i = 0; i < value; i++) { + if (sem_post(&target->fo_sem) == -1) { + filebench_log(LOG_ERROR, "semop post failed"); + /* ipc_mutex_unlock(&target->fo_lock); */ + return (-1); + } + } + + filebench_log(LOG_DEBUG_IMPL, "flow %s-%d unblocking", + target->fo_name, target->fo_instance); +#endif /* HAVE_SYSV_SEM */ + + target = target->fo_targetnext; + } + flowop_endop(threadflow, flowop); + + return (0); +} + + +/* + * Section for exercising create / open / close / delete operations + * on files within a fileset. For proper operation, the flowop attribute + * "fd", which sets the fo_fdnumber field in the flowop, must be used + * so that the same file is opened and later closed. "fd" is an index + * into a pair of arrays maintained by threadflows, one of which + * contains the operating system assigned file descriptors and the other + * a pointer to the filesetentry whose file the file descriptor + * references. An openfile flowop defined without fd being set will use + * the default (0) fd or, if specified, rotate through fd indices, but + * createfile and closefile must use the default or a specified fd. + * Meanwhile deletefile picks and arbitrary file to delete, regardless + * of fd attribute. + */ + +/* + * XXX Making file selection more consistent among the flowops might good + */ + + +/* + * Emulates (and actually does) file open. Obtains a file descriptor + * index, then calls flowoplib_openfile_common() to open. Returns -1 + * if not file descriptor is found or flowoplib_openfile_common + * encounters an error, otherwise 0. + */ +static int +flowoplib_openfile(threadflow_t *threadflow, flowop_t *flowop) +{ + int fd = flowoplib_fdnum(threadflow, flowop); + + if (fd == -1) + return (-1); + + return (flowoplib_openfile_common(threadflow, flowop, fd)); +} + +/* + * Common file opening code for filesets. Uses the supplied + * file descriptor index to determine the tf_fd entry to use. + * If the entry is empty (0) and the fileset exists, fileset + * pick is called to select a fileset entry to use. The file + * specified in the filesetentry is opened, and the returned + * operating system file descriptor and a pointer to the + * filesetentry are stored in tf_fd[fd] and tf_fse[fd], + * respectively. Returns -1 on error, 0 on success. + */ +static int +flowoplib_openfile_common(threadflow_t *threadflow, flowop_t *flowop, int fd) +{ + filesetentry_t *file; + int tid = 0; + + /* + * If the flowop doesn't default to persistent fd + * then get unique thread ID for use by fileset_pick + */ + if (integer_isset(flowop->fo_rotatefd)) + tid = threadflow->tf_utid; + + if (threadflow->tf_fd[fd] != 0) { + filebench_log(LOG_ERROR, + "flowop %s attempted to open without closing on fd %d", + flowop->fo_name, fd); + return (-1); + } + + if (flowop->fo_fileset == NULL) { + filebench_log(LOG_ERROR, "flowop NULL file"); + return (-1); + } + + if ((file = fileset_pick(flowop->fo_fileset, + FILESET_PICKEXISTS, tid)) == NULL) { + filebench_log(LOG_ERROR, + "flowop %s failed to pick file from %s on fd %d", + flowop->fo_name, + flowop->fo_fileset->fs_name, fd); + return (-1); + } + + threadflow->tf_fse[fd] = file; + + flowop_beginop(threadflow, flowop); + threadflow->tf_fd[fd] = fileset_openfile(flowop->fo_fileset, + file, O_RDWR, 0666, flowoplib_fileattrs(flowop)); + flowop_endop(threadflow, flowop); + + if (threadflow->tf_fd[fd] < 0) { + filebench_log(LOG_ERROR, "failed to open file %s", + flowop->fo_name); + return (-1); + } + + filebench_log(LOG_DEBUG_SCRIPT, + "flowop %s: opened %s fd[%d] = %d", + flowop->fo_name, file->fse_path, fd, threadflow->tf_fd[fd]); + + return (0); +} + +/* + * Emulate create of a file. Uses the flowop's fdnumber to select + * tf_fd and tf_fse array locations to put the created file's file + * descriptor and filesetentry respectively. Uses fileset_pick() + * to select a specific filesetentry whose file does not currently + * exist for the file create operation. Then calls + * fileset_openfile() with the O_CREATE flag set to create the + * file. Returns -1 if the array index specified by fdnumber is + * already in use, the flowop has no associated fileset, or + * the create call fails. Returns 1 if a filesetentry with a + * nonexistent file cannot be found. Returns 0 on success. + */ +static int +flowoplib_createfile(threadflow_t *threadflow, flowop_t *flowop) +{ + filesetentry_t *file; + int fd = flowop->fo_fdnumber; + + if (threadflow->tf_fd[fd] != 0) { + filebench_log(LOG_ERROR, + "flowop %s attempted to create without closing on fd %d", + flowop->fo_name, fd); + return (-1); + } + + if (flowop->fo_fileset == NULL) { + filebench_log(LOG_ERROR, "flowop NULL file"); + return (-1); + } + + if ((file = fileset_pick(flowop->fo_fileset, + FILESET_PICKNOEXIST, 0)) == NULL) { + filebench_log(LOG_DEBUG_SCRIPT, "flowop %s failed to pick file", + flowop->fo_name); + return (1); + } + + threadflow->tf_fse[fd] = file; + + flowop_beginop(threadflow, flowop); + threadflow->tf_fd[fd] = fileset_openfile(flowop->fo_fileset, + file, O_RDWR | O_CREAT, 0666, flowoplib_fileattrs(flowop)); + flowop_endop(threadflow, flowop); + + if (threadflow->tf_fd[fd] < 0) { + filebench_log(LOG_ERROR, "failed to create file %s", + flowop->fo_name); + return (-1); + } + + filebench_log(LOG_DEBUG_SCRIPT, + "flowop %s: created %s fd[%d] = %d", + flowop->fo_name, file->fse_path, fd, threadflow->tf_fd[fd]); + + return (0); +} + +/* + * Emulates delete of a file. Picks an arbitrary filesetentry + * whose file exists and uses unlink() to delete it. Clears + * the FSE_EXISTS flag for the filesetentry. Returns -1 if the + * flowop has no associated fileset. Returns 1 if an appropriate + * filesetentry cannot be found, and 0 on success. + */ +static int +flowoplib_deletefile(threadflow_t *threadflow, flowop_t *flowop) +{ + filesetentry_t *file; + fileset_t *fileset; + char path[MAXPATHLEN]; + char *pathtmp; + + if (flowop->fo_fileset == NULL) { + filebench_log(LOG_ERROR, "flowop NULL file"); + return (-1); + } + + fileset = flowop->fo_fileset; + + if ((file = fileset_pick(flowop->fo_fileset, + FILESET_PICKEXISTS, 0)) == NULL) { + filebench_log(LOG_DEBUG_SCRIPT, "flowop %s failed to pick file", + flowop->fo_name); + return (1); + } + + *path = 0; + (void) strcpy(path, *fileset->fs_path); + (void) strcat(path, "/"); + (void) strcat(path, fileset->fs_name); + pathtmp = fileset_resolvepath(file); + (void) strcat(path, pathtmp); + free(pathtmp); + + flowop_beginop(threadflow, flowop); + (void) unlink(path); + flowop_endop(threadflow, flowop); + file->fse_flags &= ~FSE_EXISTS; + (void) ipc_mutex_unlock(&file->fse_lock); + + filebench_log(LOG_DEBUG_SCRIPT, "deleted file %s", file->fse_path); + + return (0); +} + +/* + * Emulates fsync of a file. Obtains the file descriptor index + * from the flowop, obtains the actual file descriptor from + * the threadflow's table, checks to be sure it is still an + * open file, then does an fsync operation on it. Returns -1 + * if the file no longer is open, 0 otherwise. + */ +static int +flowoplib_fsync(threadflow_t *threadflow, flowop_t *flowop) +{ + filesetentry_t *file; + int fd = flowop->fo_fdnumber; + + if (threadflow->tf_fd[fd] == 0) { + filebench_log(LOG_ERROR, + "flowop %s attempted to fsync a closed fd %d", + flowop->fo_name, fd); + return (-1); + } + + /* Measure time to fsync */ + flowop_beginop(threadflow, flowop); + (void) fsync(threadflow->tf_fd[fd]); + flowop_endop(threadflow, flowop); + + file = threadflow->tf_fse[fd]; + + filebench_log(LOG_DEBUG_SCRIPT, "fsync file %s", file->fse_path); + + return (0); +} + +/* + * Emulate fsync of an entire fileset. Search through the + * threadflow's file descriptor array, doing fsync() on each + * open file that belongs to the flowop's fileset. Always + * returns 0. + */ +static int +flowoplib_fsyncset(threadflow_t *threadflow, flowop_t *flowop) +{ + int fd; + + for (fd = 0; fd < THREADFLOW_MAXFD; fd++) { + filesetentry_t *file; + + /* Match the file set to fsync */ + if ((threadflow->tf_fse[fd] == NULL) || + (flowop->fo_fileset != threadflow->tf_fse[fd]->fse_fileset)) + continue; + + /* Measure time to fsync */ + flowop_beginop(threadflow, flowop); + (void) fsync(threadflow->tf_fd[fd]); + flowop_endop(threadflow, flowop); + + file = threadflow->tf_fse[fd]; + + filebench_log(LOG_DEBUG_SCRIPT, "fsync file %s", + file->fse_path); + } + + return (0); +} + +/* + * Emulate close of a file. Obtains the file descriptor index + * from the flowop, obtains the actual file descriptor from the + * threadflow's table, checks to be sure it is still an open + * file, then does a close operation on it. Then sets the + * threadflow file descriptor table entry to 0, and the file set + * entry pointer to NULL. Returns -1 if the file was not open, + * 0 otherwise. + */ +static int +flowoplib_closefile(threadflow_t *threadflow, flowop_t *flowop) +{ + filesetentry_t *file; + int fd = flowop->fo_fdnumber; + + if (threadflow->tf_fd[fd] == 0) { + filebench_log(LOG_ERROR, + "flowop %s attempted to close an already closed fd %d", + flowop->fo_name, fd); + return (-1); + } + + /* Measure time to close */ + flowop_beginop(threadflow, flowop); + (void) close(threadflow->tf_fd[fd]); + flowop_endop(threadflow, flowop); + + file = threadflow->tf_fse[fd]; + + threadflow->tf_fd[fd] = 0; + threadflow->tf_fse[fd] = NULL; + + filebench_log(LOG_DEBUG_SCRIPT, "closed file %s", file->fse_path); + + return (0); +} + +/* + * Emulate stat of a file. Picks an arbitrary filesetentry with + * an existing file from the flowop's fileset, then performs a + * stat() operation on it. Returns -1 if the flowop has no + * associated fileset. Returns 1 if an appropriate filesetentry + * cannot be found, and 0 on success. + */ +static int +flowoplib_statfile(threadflow_t *threadflow, flowop_t *flowop) +{ + filesetentry_t *file; + fileset_t *fileset; + char path[MAXPATHLEN]; + char *pathtmp; + + if (flowop->fo_fileset == NULL) { + filebench_log(LOG_ERROR, "flowop NULL file"); + return (-1); + } + + fileset = flowop->fo_fileset; + + if ((file = fileset_pick(flowop->fo_fileset, + FILESET_PICKEXISTS, 0)) == NULL) { + filebench_log(LOG_DEBUG_SCRIPT, "flowop %s failed to pick file", + flowop->fo_name); + return (1); + } + + *path = 0; + (void) strcpy(path, *fileset->fs_path); + (void) strcat(path, "/"); + (void) strcat(path, fileset->fs_name); + pathtmp = fileset_resolvepath(file); + (void) strcat(path, pathtmp); + free(pathtmp); + + flowop_beginop(threadflow, flowop); + flowop_endop(threadflow, flowop); + + (void) ipc_mutex_unlock(&file->fse_lock); + + return (0); +} + + +/* + * Additional reads and writes. Read and write whole files, write + * and append to files. Some of these work with both fileobjs and + * filesets, others only with filesets. The flowoplib_write routine + * writes from thread memory, while the others read or write using + * fo_buf memory. Note that both flowoplib_read() and + * flowoplib_aiowrite() use thread memory as well. + */ + + +/* + * Emulate a read of a whole file. The file must be open + * with file descriptor and filesetentry stored at the + * locations indexed by the flowop's fdnumber. It then seeks + * to the beginning of the associated file, and reads + * FILE_ALLOC_BLOCK bytes at a time until the end of the + * file. Returns -1 on error, 0 on success. + */ +static int +flowoplib_readwholefile(threadflow_t *threadflow, flowop_t *flowop) +{ + size_t memoffset; + long memsize, round; + off64_t bytes = 0; + int fd = flowop->fo_fdnumber; + int ret; + + if (threadflow->tf_fd[fd] == 0) { + filebench_log(LOG_ERROR, + "flowop %s attempted to read a closed fd %d", + flowop->fo_name, fd); + return (-1); + } + + if ((flowop->fo_buf == NULL) && + ((flowop->fo_buf = (char *)malloc(FILE_ALLOC_BLOCK)) == NULL)) + return (-1); + + if (threadflow->tf_fse[fd] == NULL) { + filebench_log(LOG_ERROR, "flowop %s: NULL file", + flowop->fo_name); + return (-1); + } + + memsize = *threadflow->tf_memsize; + round = *flowop->fo_iosize; + if (filebench_randomno(&memoffset, memsize, round) == -1) { + filebench_log(LOG_ERROR, + "tf_memsize smaller than IO size for thread %s", + flowop->fo_name); + return (-1); + } + + /* Measure time to read bytes */ + flowop_beginop(threadflow, flowop); + (void) lseek64(threadflow->tf_fd[fd], 0, SEEK_SET); + while ((ret = read(threadflow->tf_fd[fd], flowop->fo_buf, + FILE_ALLOC_BLOCK)) > 0) + bytes += ret; + + flowop_endop(threadflow, flowop); + + if (ret < 0) { + filebench_log(LOG_ERROR, + "Failed to read fd %d: %s", + fd, strerror(errno)); + return (-1); + } + + if (flowop->fo_iosize == NULL) + flowop->fo_iosize = integer_alloc(bytes); + *(flowop->fo_iosize) = bytes; + + return (0); +} + +/* + * Emulate a write to a file of size fo_iosize. Will write + * to a file from a fileset if the flowop's fo_fileset field + * specifies one or its fdnumber is non zero. Otherwise it + * will write to a fileobj file, if one exists. If the file + * is not currently open, the routine will attempt to open + * it. The flowop's fo_wss parameter will be used to set the + * maximum file size if it is non-zero, otherwise the + * filesetentry's fse_size will be used. A random memory + * buffer offset is calculated, and, if fo_random is TRUE, + * a random file offset is used for the write. Otherwise the + * write is to the next sequential location. Returns 1 on + * errors, 0 on success. + */ +static int +flowoplib_write(threadflow_t *threadflow, flowop_t *flowop) +{ + size_t memoffset; + vinteger_t wss; + long memsize, round; + int filedesc; + + if (flowop->fo_fileset || flowop->fo_fdnumber) { + int fd = flowoplib_fdnum(threadflow, flowop); + + if (fd == -1) + return (-1); + + if (threadflow->tf_fd[fd] == 0) { + (void) flowoplib_openfile_common(threadflow, + flowop, fd); + filebench_log(LOG_DEBUG_IMPL, "read opened file %s", + threadflow->tf_fse[fd]->fse_path); + } + filedesc = threadflow->tf_fd[fd]; + if (*flowop->fo_wss == 0) + wss = threadflow->tf_fse[fd]->fse_size; + else + wss = *flowop->fo_wss; + } else { + if (flowop->fo_file == NULL) { + filebench_log(LOG_ERROR, "flowop NULL file"); + return (-1); + } + if (flowop->fo_fd < 0) + flowop->fo_fd = fileobj_open(flowop->fo_file, + flowoplib_fileattrs(flowop)); + + if (flowop->fo_fd < 0) { + filebench_log(LOG_ERROR, "failed to open file %s", + flowop->fo_file->fo_name); + return (-1); + } + filedesc = flowop->fo_fd; + if (*flowop->fo_wss == 0) + wss = *flowop->fo_file->fo_size; + else + wss = *flowop->fo_wss; + } + + if ((flowop->fo_buf == NULL) && + ((flowop->fo_buf = (char *)malloc(FILE_ALLOC_BLOCK)) == NULL)) { + return (-1); + } + + if (*flowop->fo_iosize == 0) { + filebench_log(LOG_ERROR, "zero iosize for thread %s", + flowop->fo_name); + return (-1); + } + + memsize = *threadflow->tf_memsize; + round = *flowop->fo_iosize; + + /* Select memory offset for IO */ + if (filebench_randomno(&memoffset, memsize, round) == -1) { + filebench_log(LOG_ERROR, + "tf_memsize smaller than IO size for thread %s", + flowop->fo_name); + return (-1); + } + + if (*flowop->fo_random) { + uint64_t fileoffset; + + if (filebench_randomno64(&fileoffset, + wss, *flowop->fo_iosize) == -1) { + filebench_log(LOG_ERROR, + "file size smaller than IO size for thread %s", + flowop->fo_name); + return (-1); + } + flowop_beginop(threadflow, flowop); + if (pwrite64(filedesc, threadflow->tf_mem + memoffset, + *flowop->fo_iosize, (off64_t)fileoffset) == -1) { + filebench_log(LOG_ERROR, "write failed, " + "offset %lld memoffset %zd: %s", + fileoffset, memoffset, strerror(errno)); + flowop_endop(threadflow, flowop); + return (-1); + } + flowop_endop(threadflow, flowop); + } else { + flowop_beginop(threadflow, flowop); + if (write(filedesc, threadflow->tf_mem + memoffset, + *flowop->fo_iosize) == -1) { + filebench_log(LOG_ERROR, + "write failed, memoffset %zd: %s", + memoffset, strerror(errno)); + flowop_endop(threadflow, flowop); + return (-1); + } + flowop_endop(threadflow, flowop); + } + + return (0); +} + +/* + * Emulate a write of a whole file. The size of the file + * is taken from a filesetentry identified by fo_srcfdnumber, + * while the file descriptor used is identified by + * fo_fdnumber. Does multiple writes of FILE_ALLOC_BLOCK + * length until full file has been written. Returns -1 on + * error, 0 on success and sets flowop->fo_iosize to the + * number of bytes actually written. + */ +static int +flowoplib_writewholefile(threadflow_t *threadflow, flowop_t *flowop) +{ + size_t memoffset; + filesetentry_t *file; + int wsize; + off64_t seek; + off64_t bytes = 0; + long memsize, round; + int fd = flowop->fo_fdnumber; + int srcfd = flowop->fo_srcfdnumber; + int ret; + + if (threadflow->tf_fd[fd] == 0) { + filebench_log(LOG_ERROR, + "flowop %s attempted to write a closed fd %d", + flowop->fo_name, fd); + return (-1); + } + + if ((flowop->fo_buf == NULL) && + ((flowop->fo_buf = (char *)malloc(FILE_ALLOC_BLOCK)) == NULL)) { + return (-1); + } + + memsize = *threadflow->tf_memsize; + round = *flowop->fo_iosize; + if (filebench_randomno(&memoffset, memsize, round) == -1) { + filebench_log(LOG_ERROR, + "tf_memsize smaller than IO size for thread %s", + flowop->fo_name); + } + + file = threadflow->tf_fse[srcfd]; + if (((srcfd != 0) && (file == NULL)) || + ((file = threadflow->tf_fse[fd]) == NULL)) { + filebench_log(LOG_ERROR, "flowop %s: NULL file", + flowop->fo_name); + return (-1); + } + + wsize = MIN(file->fse_size, FILE_ALLOC_BLOCK); + + /* Measure time to write bytes */ + flowop_beginop(threadflow, flowop); + for (seek = 0; seek < file->fse_size; seek += FILE_ALLOC_BLOCK) { + ret = write(threadflow->tf_fd[fd], flowop->fo_buf, wsize); + if (ret != wsize) { + filebench_log(LOG_ERROR, + "Failed to write %d bytes on fd %d: %s", + threadflow->tf_fd[fd], fd, strerror(errno)); + flowop_endop(threadflow, flowop); + return (-1); + } + bytes += ret; + } + flowop_endop(threadflow, flowop); + + if (flowop->fo_iosize == NULL) + flowop->fo_iosize = integer_alloc(bytes); + *(flowop->fo_iosize) = bytes; + + return (0); +} + + +/* + * Emulate a fixed size append to a file. Will append data to + * a file chosen from a fileset if the flowop's fo_fileset + * field specifies one or if its fdnumber is non zero. + * Otherwise it will write to a fileobj file, if one exists. + * The flowop's fo_wss parameter will be used to set the + * maximum file size if it is non-zero, otherwise the + * filesetentry's fse_size will be used. A random memory + * buffer offset is calculated, then a logical seek to the + * end of file is done followed by a write of fo_iosize + * bytes. Writes are actually done from fo_buf, rather than + * tf_mem as is done with flowoplib_write(), and no check + * is made to see if fo_iosize exceeds the size of fo_buf. + * Returns -1 on error, 0 on success. + */ +static int +flowoplib_appendfile(threadflow_t *threadflow, flowop_t *flowop) +{ + size_t memoffset; + off64_t wsize; + long memsize; + int fd, filedesc; + /* LINTED E_FUNC_SET_NOT_USED */ + vinteger_t wss; + int ret; + + if (flowop->fo_fileset || flowop->fo_fdnumber) { + fd = flowoplib_fdnum(threadflow, flowop); + + if (fd == -1) + return (-1); + + if (threadflow->tf_fd[fd] == 0) { + (void) flowoplib_openfile_common(threadflow, + flowop, fd); + filebench_log(LOG_DEBUG_IMPL, "read opened file %s", + threadflow->tf_fse[fd]->fse_path); + } + filedesc = threadflow->tf_fd[fd]; + if (*flowop->fo_wss == 0) + wss = threadflow->tf_fse[fd]->fse_size; + else + wss = *flowop->fo_wss; + } else { + if (flowop->fo_file == NULL) { + filebench_log(LOG_ERROR, "flowop NULL file"); + return (-1); + } + if (flowop->fo_fd < 0) + flowop->fo_fd = fileobj_open(flowop->fo_file, + flowoplib_fileattrs(flowop)); + + if (flowop->fo_fd < 0) { + filebench_log(LOG_ERROR, "failed to open file %s", + flowop->fo_file->fo_name); + return (-1); + } + filedesc = flowop->fo_fd; + if (*flowop->fo_wss == 0) + wss = *flowop->fo_file->fo_size; + else + wss = *flowop->fo_wss; + } + /* XXX wss is not being used */ + + if ((flowop->fo_buf == NULL) && + ((flowop->fo_buf = (char *)malloc(FILE_ALLOC_BLOCK)) == NULL)) { + return (-1); + } + + memsize = *threadflow->tf_memsize; + wsize = *flowop->fo_iosize; + if (filebench_randomno(&memoffset, memsize, wsize) == -1) { + filebench_log(LOG_ERROR, + "tf_memsize smaller than IO size for thread %s", + flowop->fo_name); + return (-1); + } + + /* Measure time to write bytes */ + flowop_beginop(threadflow, flowop); + (void) lseek64(filedesc, 0, SEEK_END); + ret = write(filedesc, flowop->fo_buf, wsize); + if (ret != wsize) { + filebench_log(LOG_ERROR, + "Failed to write %d bytes on fd %d: %s", + wsize, fd, strerror(errno)); + flowop_endop(threadflow, flowop); + return (-1); + } + flowop_endop(threadflow, flowop); + + return (0); +} + +/* + * Emulate a random size append to a file. Will append data + * to a file chosen from a fileset if the flowop's fo_fileset + * field specifies one or if its fdnumber is non zero. Otherwise + * it will write to a fileobj file, if one exists. The flowop's + * fo_wss parameter will be used to set the maximum file size + * if it is non-zero, otherwise the filesetentry's fse_size + * will be used. A random transfer size (but at most fo_iosize + * bytes) and a random memory offset are calculated. A logical + * seek to the end of file is done, then writes of up to + * FILE_ALLOC_BLOCK in size are done until the full transfer + * size has been written. Writes are actually done from fo_buf, + * rather than tf_mem as is done with flowoplib_write(). + * Returns -1 on error, 0 on success. + */ +static int +flowoplib_appendfilerand(threadflow_t *threadflow, flowop_t *flowop) +{ + size_t memoffset; + uint64_t appendsize; + off64_t seek; + long memsize, round; + int fd, filedesc; + /* LINTED E_FUNC_SET_NOT_USED */ + vinteger_t wss; + + if (flowop->fo_fileset || flowop->fo_fdnumber) { + fd = flowoplib_fdnum(threadflow, flowop); + + if (fd == -1) + return (-1); + + if (threadflow->tf_fd[fd] == 0) { + (void) flowoplib_openfile_common(threadflow, + flowop, fd); + filebench_log(LOG_DEBUG_IMPL, "append opened file %s", + threadflow->tf_fse[fd]->fse_path); + } + filedesc = threadflow->tf_fd[fd]; + if (*flowop->fo_wss == 0) + wss = threadflow->tf_fse[fd]->fse_size; + else + wss = *flowop->fo_wss; + } else { + if (flowop->fo_file == NULL) { + filebench_log(LOG_ERROR, "flowop NULL file"); + return (-1); + } + if (flowop->fo_fd < 0) + flowop->fo_fd = fileobj_open(flowop->fo_file, + flowoplib_fileattrs(flowop)); + + if (flowop->fo_fd < 0) { + filebench_log(LOG_ERROR, "failed to open file %s", + flowop->fo_file->fo_name); + return (-1); + } + filedesc = flowop->fo_fd; + if (*flowop->fo_wss == 0) + wss = *flowop->fo_file->fo_size; + else + wss = *flowop->fo_wss; + } + /* XXX wss is not being used */ + + if ((flowop->fo_buf == NULL) && + ((flowop->fo_buf = (char *)malloc(FILE_ALLOC_BLOCK)) == NULL)) { + return (-1); + } + + memsize = *threadflow->tf_memsize; + round = *flowop->fo_iosize; + if (filebench_randomno(&memoffset, memsize, round) == -1) { + filebench_log(LOG_ERROR, "tf_memsize smaller than IO size" + "for thread %s", flowop->fo_name); + return (-1); + } + if (filebench_randomno64(&appendsize, *flowop->fo_iosize, 1LL) == -1) { + filebench_log(LOG_ERROR, "tf_memsize smaller than IO size" + "for thread %s", flowop->fo_name); + return (-1); + } + + /* Measure time to write bytes */ + flowop_beginop(threadflow, flowop); + for (seek = 0; seek < appendsize; seek += FILE_ALLOC_BLOCK) { + off64_t wsize; + int ret = 0; + + (void) lseek64(filedesc, 0, SEEK_END); + wsize = ((appendsize - seek) > FILE_ALLOC_BLOCK) ? + FILE_ALLOC_BLOCK : (appendsize - seek); + ret = write(filedesc, flowop->fo_buf, wsize); + if (ret != wsize) { + filebench_log(LOG_ERROR, + "Failed to write %d bytes on fd %d: %s", + wsize, fd, strerror(errno)); + flowop_endop(threadflow, flowop); + return (-1); + } + } + flowop_endop(threadflow, flowop); + + return (0); +} + + +/* + * Prints usage information for flowop operations. + */ +void +flowoplib_usage() +{ + (void) fprintf(stderr, + "flowop [openfile|createfile] name=<name>,fileset=<fname>\n"); + (void) fprintf(stderr, + " [,fd=<file desc num>]\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, + "flowop closefile name=<name>,fd=<file desc num>]\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, "flowop deletefile name=<name>\n"); + (void) fprintf(stderr, " [,fileset=<fname>]\n"); + (void) fprintf(stderr, + " [,fd=<file desc num>]\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, "flowop statfile name=<name>\n"); + (void) fprintf(stderr, " [,fileset=<fname>]\n"); + (void) fprintf(stderr, + " [,fd=<file desc num>]\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, + "flowop fsync name=<name>,fd=<file desc num>]\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, + "flowop fsyncset name=<name>,fileset=<fname>]\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, "flowop [write|read|aiowrite] name=<name>, \n"); + (void) fprintf(stderr, + " filename|fileset=<fname>,\n"); + (void) fprintf(stderr, " iosize=<size>\n"); + (void) fprintf(stderr, " [,directio]\n"); + (void) fprintf(stderr, " [,dsync]\n"); + (void) fprintf(stderr, " [,iters=<count>]\n"); + (void) fprintf(stderr, " [,random]\n"); + (void) fprintf(stderr, " [,opennext]\n"); + (void) fprintf(stderr, " [,workingset=<size>]\n"); + (void) fprintf(stderr, + "flowop [appendfile|appendfilerand] name=<name>, \n"); + (void) fprintf(stderr, + " filename|fileset=<fname>,\n"); + (void) fprintf(stderr, " iosize=<size>\n"); + (void) fprintf(stderr, " [,dsync]\n"); + (void) fprintf(stderr, " [,iters=<count>]\n"); + (void) fprintf(stderr, " [,workingset=<size>]\n"); + (void) fprintf(stderr, + "flowop [readwholefile|writewholefile] name=<name>, \n"); + (void) fprintf(stderr, + " filename|fileset=<fname>,\n"); + (void) fprintf(stderr, " iosize=<size>\n"); + (void) fprintf(stderr, " [,dsync]\n"); + (void) fprintf(stderr, " [,iters=<count>]\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, "flowop aiowait name=<name>,target=" + "<aiowrite-flowop>\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, "flowop sempost name=<name>," + "target=<semblock-flowop>,\n"); + (void) fprintf(stderr, + " value=<increment-to-post>\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, "flowop semblock name=<name>,value=" + "<decrement-to-receive>,\n"); + (void) fprintf(stderr, " highwater=" + "<inbound-queue-max>\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, "flowop block name=<name>\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, + "flowop wakeup name=<name>,target=<block-flowop>,\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, + "flowop hog name=<name>,value=<number-of-mem-ops>\n"); + (void) fprintf(stderr, + "flowop delay name=<name>,value=<number-of-seconds>\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, "flowop eventlimit name=<name>\n"); + (void) fprintf(stderr, "flowop bwlimit name=<name>,value=<mb/s>\n"); + (void) fprintf(stderr, "flowop iopslimit name=<name>,value=<iop/s>\n"); + (void) fprintf(stderr, + "flowop finishoncount name=<name>,value=<ops/s>\n"); + (void) fprintf(stderr, + "flowop finishonbytes name=<name>,value=<bytes>\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, "\n"); +} diff --git a/usr/src/cmd/filebench/common/gamma_dist.c b/usr/src/cmd/filebench/common/gamma_dist.c new file mode 100644 index 0000000000..06552cbb1e --- /dev/null +++ b/usr/src/cmd/filebench/common/gamma_dist.c @@ -0,0 +1,99 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include <stdlib.h> +#include <math.h> + +/* + * This is valid for 0 < a <= 1 + * + * From Knuth Volume 2, 3rd edition, pages 586 - 587. + */ +static double +gamma_dist_knuth_algG(double a) +{ + double p, U, V, X, q; + + p = M_E/(a + M_E); +G2: + U = drand48(); + do { + V = drand48(); + } while (V == 0); + + if (U < p) { + X = pow(V, 1/a); + /* q = e^(-X) */ + q = exp(-X); + } else { + X = 1 - log(V); + q = pow(X, a-1); + } + + /* + * X now has density g, and q = f(X)/cg(X) + */ + U = drand48(); + if (U >= q) + goto G2; + return (X); +} + +/* + * This is valid for a > 1 + * + * From Knuth Volume 2, 3rd edition, page 134. + */ +static double +gamma_dist_knuth_algA(double a) +{ + double U, Y, X, V; + +A1: + U = drand48(); + Y = tan(M_PI*U); + X = (sqrt((2*a) - 1) * Y) + a - 1; + + if (X <= 0) + goto A1; + + V = drand48(); + /* V > (1 + Y^2) * exp((a - 1) * log(X / (a - 1)) - sqrt(2a - 1) * Y) */ + if (V > ((1 + (Y*Y)) * exp((a-1) * log(X/(a-1)) - sqrt(2*a -1) * Y))) + goto A1; + + return (X); +} + +double +gamma_dist_knuth(double a, double b) +{ + if (a <= 1.0) + return (b * gamma_dist_knuth_algG(a)); + else + return (b * gamma_dist_knuth_algA(a)); +} diff --git a/usr/src/cmd/filebench/common/gamma_dist.h b/usr/src/cmd/filebench/common/gamma_dist.h new file mode 100644 index 0000000000..2773b626af --- /dev/null +++ b/usr/src/cmd/filebench/common/gamma_dist.h @@ -0,0 +1,41 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_GAMMA_DIST_H +#define _FB_GAMMA_DIST_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#ifdef __cplusplus +extern "C" { +#endif + +double gamma_dist_knuth(double a, double b); + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_GAMMA_DIST_H */ diff --git a/usr/src/cmd/filebench/common/ipc.c b/usr/src/cmd/filebench/common/ipc.c new file mode 100644 index 0000000000..beaf2b119a --- /dev/null +++ b/usr/src/cmd/filebench/common/ipc.c @@ -0,0 +1,818 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" + +#include <stdio.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/ipc.h> +#include <sys/sem.h> +#include <sys/errno.h> +#include <signal.h> +#include <pthread.h> +#include <sys/shm.h> +#include "filebench.h" + +/* IPC Hub and Simple memory allocator */ + +static int shmfd; +filebench_shm_t *filebench_shm = NULL; +static pthread_mutexattr_t *mutexattr = NULL; + +/* + * Interprocess Communication mechanisms. If multiple processes + * are used, filebench opens a shared file in memory mapped mode to hold + * a variety of global variables and data structures. If only using + * multiple threads, it just allocates a region of local memory. A + * region of interprocess shared memory and a set of shared semaphores + * are also created. Routines are provided to manage the creation, + * destruction, and allocation of these resoures. + */ + + +/* + * Locks a mutex and logs any errors. + */ +int +ipc_mutex_lock(pthread_mutex_t *mutex) +{ + int error; + + error = pthread_mutex_lock(mutex); + +#ifdef HAVE_ROBUST_MUTEX + if (error == EOWNERDEAD) { + if (pthread_mutex_consistent_np(mutex) != 0) { + filebench_log(LOG_FATAL, "mutex make consistent " + "failed: %s", strerror(error)); + return (-1); + } + return (0); + } +#endif /* HAVE_ROBUST_MUTEX */ + + if (error != 0) { + filebench_log(LOG_FATAL, "mutex lock failed: %s", + strerror(error)); + } + + return (error); +} + +/* + * Unlocks a mutex and logs any errors. + */ +int +ipc_mutex_unlock(pthread_mutex_t *mutex) +{ + int error; + + error = pthread_mutex_unlock(mutex); + +#ifdef HAVE_ROBUST_MUTEX + if (error == EOWNERDEAD) { + if (pthread_mutex_consistent_np(mutex) != 0) { + filebench_log(LOG_FATAL, "mutex make consistent " + "failed: %s", strerror(error)); + return (-1); + } + return (0); + } +#endif /* HAVE_ROBUST_MUTEX */ + + if (error != 0) { + filebench_log(LOG_FATAL, "mutex unlock failed: %s", + strerror(error)); + } + + return (error); +} + +/* + * On first invocation, allocates a mutex attributes structure + * and initializes it with appropriate attributes. In all cases, + * returns a pointer to the structure. + */ +pthread_mutexattr_t * +ipc_mutexattr(void) +{ +#ifdef USE_PROCESS_MODEL + if (mutexattr == NULL) { + if ((mutexattr = + malloc(sizeof (pthread_mutexattr_t))) == NULL) { + filebench_log(LOG_ERROR, "cannot alloc mutex attr"); + filebench_shutdown(1); + } +#ifdef HAVE_PROCSCOPE_PTHREADS + (void) pthread_mutexattr_init(mutexattr); + if (pthread_mutexattr_setpshared(mutexattr, + PTHREAD_PROCESS_SHARED) != 0) { + filebench_log(LOG_ERROR, "cannot set mutex attr " + "PROCESS_SHARED on this platform"); + filebench_shutdown(1); + } +#ifdef HAVE_PTHREAD_MUTEXATTR_SETPROTOCOL + if (pthread_mutexattr_setprotocol(mutexattr, + PTHREAD_PRIO_INHERIT) != 0) { + filebench_log(LOG_ERROR, "cannot set mutex attr " + "PTHREAD_PRIO_INHERIT on this platform"); + filebench_shutdown(1); + } +#endif /* HAVE_PTHREAD_MUTEXATTR_SETPROTOCOL */ +#endif /* HAVE_PROCSCOPE_PTHREADS */ +#ifdef HAVE_ROBUST_MUTEX + if (pthread_mutexattr_setrobust_np(mutexattr, + PTHREAD_MUTEX_ROBUST_NP) != 0) { + filebench_log(LOG_ERROR, "cannot set mutex attr " + "PTHREAD_MUTEX_ROBUST_NP on this platform"); + filebench_shutdown(1); + } + if (pthread_mutexattr_settype(mutexattr, + PTHREAD_MUTEX_ERRORCHECK) != 0) { + filebench_log(LOG_ERROR, "cannot set mutex attr " + "PTHREAD_MUTEX_ERRORCHECK on this platform"); + filebench_shutdown(1); + } +#endif /* HAVE_ROBUST_MUTEX */ + + } +#endif /* USE_PROCESS_MODEL */ + return (mutexattr); +} + +static pthread_condattr_t *condattr = NULL; + +/* + * On first invocation, allocates a condition variable attributes + * structure and initializes it with appropriate attributes. In + * all cases, returns a pointer to the structure. + */ +pthread_condattr_t * +ipc_condattr(void) +{ +#ifdef USE_PROCESS_MODEL + if (condattr == NULL) { + if ((condattr = malloc(sizeof (pthread_condattr_t))) == NULL) { + filebench_log(LOG_ERROR, "cannot alloc cond attr"); + filebench_shutdown(1); + } +#ifdef HAVE_PROCSCOPE_PTHREADS + (void) pthread_condattr_init(condattr); + if (pthread_condattr_setpshared(condattr, + PTHREAD_PROCESS_SHARED) != 0) { + filebench_log(LOG_ERROR, + "cannot set cond attr PROCESS_SHARED"); + filebench_shutdown(1); + } +#endif /* HAVE_PROCSCOPE_PTHREADS */ + } +#endif /* USE_PROCESS_MODEL */ + return (condattr); +} + +static pthread_rwlockattr_t *rwlockattr = NULL; + +/* + * On first invocation, allocates a readers/writers attributes + * structure and initializes it with appropriate attributes. + * In all cases, returns a pointer to the structure. + */ +static pthread_rwlockattr_t * +ipc_rwlockattr(void) +{ +#ifdef USE_PROCESS_MODEL + if (rwlockattr == NULL) { + if ((rwlockattr = + malloc(sizeof (pthread_rwlockattr_t))) == NULL) { + filebench_log(LOG_ERROR, "cannot alloc rwlock attr"); + filebench_shutdown(1); + } +#ifdef HAVE_PROCSCOPE_PTHREADS + (void) pthread_rwlockattr_init(rwlockattr); + if (pthread_rwlockattr_setpshared(rwlockattr, + PTHREAD_PROCESS_SHARED) != 0) { + filebench_log(LOG_ERROR, + "cannot set rwlock attr PROCESS_SHARED"); + filebench_shutdown(1); + } +#endif /* HAVE_PROCSCOPE_PTHREADS */ + } +#endif /* USE_PROCESS_MODEL */ + return (rwlockattr); +} + +char *shmpath = NULL; + +/* + * Calls semget() to get a set of shared system V semaphores. + */ +void +ipc_seminit(void) +{ + key_t key = filebench_shm->semkey; + + /* Already done? */ + if (filebench_shm->seminit) + return; + + if ((semget(key, FILEBENCH_NSEMS, IPC_CREAT | + S_IRUSR | S_IWUSR)) == -1) { + filebench_log(LOG_ERROR, + "could not create sysv semaphore set " + "(need to increase sems?): %s", + strerror(errno)); + exit(1); + } +} + +/* + * Initialize the Interprocess Communication system and its + * associated shared memory structure. It first creates a + * temporary file using either the mkstemp() function or the + * tempnam() and open() functions. If the process model is in + * use,it than sets the file large enough to hold the + * filebench_shm and an additional Megabyte. The file is then + * memory mapped. If the process model is not in use, it simply + * mallocs a region of sizeof (filebench_shm_t). + * + * Once the shared memory region / file is created, ipc_init + * initializes various locks pointers, and variables in the + * shared memory. It also uses ftok() to get a shared memory + * semaphore key for later use in allocating shared semaphores. + */ +void +ipc_init(void) +{ + filebench_shm_t *buf = malloc(MB); + key_t key; + caddr_t c1; + caddr_t c2; +#ifdef HAVE_SEM_RMID + int semid; +#endif + +#ifdef HAVE_MKSTEMP + shmpath = (char *)malloc(128); + (void) strcpy(shmpath, "/var/tmp/fbenchXXXXXX"); + shmfd = mkstemp(shmpath); +#else + shmfd = open(shmpath, O_CREAT | O_RDWR | O_TRUNC, 0666); + shmpath = tempnam("/var/tmp", "fbench"); +#endif /* HAVE_MKSTEMP */ + +#ifdef USE_PROCESS_MODEL + + if (shmfd < 0) { + filebench_log(LOG_FATAL, "Cannot open shm %s: %s", + shmpath, + strerror(errno)); + exit(1); + } + + (void) lseek(shmfd, sizeof (filebench_shm_t), SEEK_SET); + if (write(shmfd, buf, MB) != MB) { + filebench_log(LOG_FATAL, + "Cannot allocate shm: %s", strerror(errno)); + exit(1); + } + + /* LINTED E_BAD_PTR_CAST_ALIGN */ + if ((filebench_shm = (filebench_shm_t *)mmap((caddr_t)0, + sizeof (filebench_shm_t), PROT_READ | PROT_WRITE, MAP_SHARED, + shmfd, 0)) == NULL) { + filebench_log(LOG_FATAL, "Cannot mmap shm"); + exit(1); + } + +#else + if ((filebench_shm = + (filebench_shm_t *)malloc(sizeof (filebench_shm_t))) == NULL) { + filebench_log(LOG_FATAL, "Cannot malloc shm"); + exit(1); + } +#endif /* USE_PROCESS_MODEL */ + + c1 = (caddr_t)filebench_shm; + c2 = (caddr_t)&filebench_shm->marker; + + (void) memset(filebench_shm, 0, c2 - c1); + filebench_shm->epoch = gethrtime(); + filebench_shm->debug_level = 2; + filebench_shm->string_ptr = &filebench_shm->strings[0]; + filebench_shm->shm_ptr = (char *)filebench_shm->shm_addr; + filebench_shm->path_ptr = &filebench_shm->filesetpaths[0]; + + /* Setup mutexes for object lists */ + (void) pthread_mutex_init(&filebench_shm->fileobj_lock, + ipc_mutexattr()); + (void) pthread_mutex_init(&filebench_shm->fileset_lock, + ipc_mutexattr()); + (void) pthread_mutex_init(&filebench_shm->procflow_lock, + ipc_mutexattr()); + (void) pthread_mutex_init(&filebench_shm->threadflow_lock, + ipc_mutexattr()); + (void) pthread_mutex_init(&filebench_shm->flowop_lock, ipc_mutexattr()); + (void) pthread_mutex_init(&filebench_shm->msg_lock, ipc_mutexattr()); + (void) pthread_mutex_init(&filebench_shm->eventgen_lock, + ipc_mutexattr()); + (void) pthread_mutex_init(&filebench_shm->malloc_lock, ipc_mutexattr()); + (void) pthread_mutex_init(&filebench_shm->ism_lock, ipc_mutexattr()); + (void) pthread_cond_init(&filebench_shm->eventgen_cv, ipc_condattr()); + (void) pthread_rwlock_init(&filebench_shm->flowop_find_lock, + ipc_rwlockattr()); + (void) pthread_rwlock_init(&filebench_shm->run_lock, ipc_rwlockattr()); + (void) pthread_rwlock_rdlock(&filebench_shm->run_lock); + + (void) ipc_mutex_lock(&filebench_shm->ism_lock); + + /* Create semaphore */ + if ((key = ftok(shmpath, 1)) < 0) { + filebench_log(LOG_ERROR, "cannot create sem: %s", + strerror(errno)); + exit(1); + } + +#ifdef HAVE_SEM_RMID + if ((semid = semget(key, 0, 0)) != -1) + (void) semctl(semid, 0, IPC_RMID); +#endif + + filebench_shm->semkey = key; + filebench_shm->log_fd = -1; + filebench_shm->dump_fd = -1; + filebench_shm->eventgen_hz = 0; + filebench_shm->shm_id = -1; + + free(buf); +} + +/* + * If compiled to use process model, just unlinks the shmpath. + * Otherwise a no-op. + */ +void +ipc_cleanup(void) +{ +#ifdef USE_PROCESS_MODEL + (void) unlink(shmpath); +#endif /* USE_PROCESS_MODEL */ +} + +/* + * Attach to shared memory. Used by worker processes to open + * and mmap the shared memory region. If successful, it + * initializes the worker process' filebench_shm to point to + * the region and returns 0. Otherwise it returns -1. + */ +int +ipc_attach(caddr_t shmaddr) +{ + if ((shmfd = open(shmpath, O_RDWR, 0666)) < 0) { + filebench_log(LOG_ERROR, "Cannot open shm"); + return (-1); + } + + /* LINTED E_BAD_PTR_CAST_ALIGN */ + if ((filebench_shm = (filebench_shm_t *)mmap(shmaddr, + sizeof (filebench_shm_t), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, shmfd, 0)) == NULL) { + filebench_log(LOG_ERROR, "Cannot mmap shm"); + return (-1); + } + + filebench_log(LOG_DEBUG_IMPL, "addr = %zx", filebench_shm); + + return (0); +} + +static int filebench_sizes[] = { + FILEBENCH_NFILEOBJS, + FILEBENCH_NPROCFLOWS, + FILEBENCH_NTHREADFLOWS, + FILEBENCH_NFLOWOPS, + FILEBENCH_NVARS, + FILEBENCH_NVARS, + FILEBENCH_NVARS, + FILEBENCH_NFILESETS, + FILEBENCH_NFILESETENTRIES}; + +/* + * Allocates filebench objects from pre allocated region of + * shareable memory. The memory region is partitioned into sets + * of objects during initialization. This routine scans for + * the first unallocated object of type "type" in the set of + * available objects, and makes it as allocated. The routine + * returns a pointer to the object, or NULL if all objects have + * been allocated. + */ +void * +ipc_malloc(int type) +{ + int i; + int max = filebench_sizes[type]; + + (void) ipc_mutex_lock(&filebench_shm->malloc_lock); + + for (i = 0; i < max; i++) { + if (filebench_shm->bitmap[type][i] == 0) + break; + } + + if (i >= max) { + filebench_log(LOG_ERROR, "Out of shared memory (%d)!", type); + (void) ipc_mutex_unlock(&filebench_shm->malloc_lock); + return (NULL); + } + + filebench_shm->bitmap[type][i] = 1; + + switch (type) { + case FILEBENCH_FILEOBJ: + (void) memset((char *)&filebench_shm->fileobj[i], 0, + sizeof (fileobj_t)); + (void) ipc_mutex_unlock(&filebench_shm->malloc_lock); + return ((char *)&filebench_shm->fileobj[i]); + + case FILEBENCH_FILESET: + (void) memset((char *)&filebench_shm->fileset[i], 0, + sizeof (fileset_t)); + (void) ipc_mutex_unlock(&filebench_shm->malloc_lock); + return ((char *)&filebench_shm->fileset[i]); + + case FILEBENCH_FILESETENTRY: + (void) memset((char *)&filebench_shm->filesetentry[i], 0, + sizeof (filesetentry_t)); + (void) ipc_mutex_unlock(&filebench_shm->malloc_lock); + return ((char *)&filebench_shm->filesetentry[i]); + + case FILEBENCH_PROCFLOW: + (void) memset((char *)&filebench_shm->procflow[i], 0, + sizeof (procflow_t)); + (void) ipc_mutex_unlock(&filebench_shm->malloc_lock); + return ((char *)&filebench_shm->procflow[i]); + + case FILEBENCH_THREADFLOW: + (void) memset((char *)&filebench_shm->threadflow[i], 0, + sizeof (threadflow_t)); + (void) ipc_mutex_unlock(&filebench_shm->malloc_lock); + return ((char *)&filebench_shm->threadflow[i]); + + case FILEBENCH_FLOWOP: + (void) memset((char *)&filebench_shm->flowop[i], 0, + sizeof (flowop_t)); + (void) ipc_mutex_unlock(&filebench_shm->malloc_lock); + return ((char *)&filebench_shm->flowop[i]); + + case FILEBENCH_INTEGER: + filebench_shm->integer_ptrs[i] = NULL; + (void) ipc_mutex_unlock(&filebench_shm->malloc_lock); + return ((char *)&filebench_shm->integer_ptrs[i]); + + case FILEBENCH_STRING: + filebench_shm->string_ptrs[i] = NULL; + (void) ipc_mutex_unlock(&filebench_shm->malloc_lock); + return ((char *)&filebench_shm->string_ptrs[i]); + + case FILEBENCH_VARIABLE: + (void) memset((char *)&filebench_shm->var[i], 0, + sizeof (var_t)); + (void) ipc_mutex_unlock(&filebench_shm->malloc_lock); + return ((char *)&filebench_shm->var[i]); + } + + filebench_log(LOG_ERROR, "Attempt to ipc_malloc unknown type (%d)!", + type); + return (NULL); +} + +/* + * Frees a filebench object of type "type" at the location + * pointed to by "addr". It uses the type and address to + * calculate which object is being freed, and clears its + * allocation map entry. + */ +void +ipc_free(int type, char *addr) +{ + int item; + caddr_t base; + size_t offset; + size_t size; + + if (addr == NULL) { + filebench_log(LOG_ERROR, "Freeing type %d %zx", type, addr); + return; + } + + switch (type) { + case FILEBENCH_FILEOBJ: + base = (caddr_t)&filebench_shm->fileobj[0]; + size = sizeof (fileobj_t); + break; + + case FILEBENCH_FILESET: + base = (caddr_t)&filebench_shm->fileset[0]; + size = sizeof (fileset_t); + break; + + case FILEBENCH_FILESETENTRY: + base = (caddr_t)&filebench_shm->filesetentry[0]; + size = sizeof (filesetentry_t); + break; + + case FILEBENCH_PROCFLOW: + base = (caddr_t)&filebench_shm->procflow[0]; + size = sizeof (procflow_t); + break; + + case FILEBENCH_THREADFLOW: + base = (caddr_t)&filebench_shm->threadflow[0]; + size = sizeof (threadflow_t); + break; + + case FILEBENCH_FLOWOP: + base = (caddr_t)&filebench_shm->flowop[0]; + size = sizeof (flowop_t); + break; + + case FILEBENCH_INTEGER: + base = (caddr_t)&filebench_shm->integer_ptrs[0]; + size = sizeof (caddr_t); + break; + + case FILEBENCH_STRING: + base = (caddr_t)&filebench_shm->string_ptrs[0]; + size = sizeof (caddr_t); + break; + + case FILEBENCH_VARIABLE: + base = (caddr_t)&filebench_shm->var[0]; + size = sizeof (var_t); + break; + } + + offset = ((size_t)addr - (size_t)base); + item = offset / size; + + (void) ipc_mutex_lock(&filebench_shm->malloc_lock); + filebench_shm->bitmap[type][item] = 0; + (void) ipc_mutex_unlock(&filebench_shm->malloc_lock); +} + +/* + * Allocate a string from filebench string memory. The length + * of the allocated string is the same as the length of the + * supplied string "string", and the contents of string are + * copied to the newly allocated string. + */ +char * +ipc_stralloc(char *string) +{ + char *allocstr = filebench_shm->string_ptr; + + filebench_shm->string_ptr += strlen(string) + 1; + + if ((filebench_shm->string_ptr - &filebench_shm->strings[0]) > + FILEBENCH_STRINGMEMORY) { + filebench_log(LOG_ERROR, "Out of ipc string memory"); + return (NULL); + } + + (void) strncpy(allocstr, string, strlen(string)); + + return (allocstr); +} + +/* + * Allocate a path string from filebench path string memory. + * Specifically used for allocating fileset paths. The length + * of the allocated path string is the same as the length of + * the supplied path string "path", and the contents of path + * are copied to the newly allocated path string. Checks for + * out-of-path-string-memory condition and returns NULL if so. + * Otherwise it returns a pointer to the newly allocated path + * string. + */ +char * +ipc_pathalloc(char *path) +{ + char *allocpath = filebench_shm->path_ptr; + + filebench_shm->path_ptr += strlen(path) + 1; + + if ((filebench_shm->path_ptr - &filebench_shm->filesetpaths[0]) > + FILEBENCH_FILESETPATHMEMORY) { + filebench_log(LOG_ERROR, "Out of fileset path memory"); + return (NULL); + } + + (void) strncpy(allocpath, path, strlen(path)); + + return (allocpath); +} + +/* + * This is a limited functionality deallocator for path + * strings - it can only free all path strings at once, + * in order to avoid fragmentation. + */ +void +ipc_freepaths(void) +{ + filebench_shm->path_ptr = &filebench_shm->filesetpaths[0]; +} + +/* + * Allocates a semid from the table of semids for pre intialized + * semaphores. Searches for the first available semaphore, and + * sets the entry in the table to "1" to indicate allocation. + * Returns the allocated semid. Stops the run if all semaphores + * are already in use. + */ +int +ipc_semidalloc(void) +{ + int semid; + + for (semid = 0; filebench_shm->semids[semid] == 1; semid++) + ; + if (semid == FILEBENCH_NSEMS) { + filebench_log(LOG_ERROR, + "Out of semaphores, increase system tunable limit"); + filebench_shutdown(1); + } + filebench_shm->semids[semid] = 1; + return (semid); +} + +/* + * Frees up the supplied semid by seting its position in the + * allocation table to "0". + */ +void +ipc_semidfree(int semid) +{ + filebench_shm->semids[semid] = 0; +} + +/* + * Create a pool of shared memory to fit the per-thread + * allocations. Uses shmget() to create a shared memory region + * of size "size", attaches to it using shmat(), and stores + * the returned address of the region in filebench_shm->shm_addr. + * The pool is only created on the first call. The routine + * returns 0 if successful or the pool already exists, + * -1 otherwise. + */ +int +ipc_ismcreate(size_t size) +{ +#ifdef HAVE_SHM_SHARE_MMU + int flag = SHM_SHARE_MMU; +#else + int flag = 0; +#endif /* HAVE_SHM_SHARE_MMU */ + + /* Already done? */ + if (filebench_shm->shm_id != -1) + return (0); + + filebench_log(LOG_VERBOSE, + "Creating %zd bytes of ISM Shared Memory...", size); + + if ((filebench_shm->shm_id = + shmget(0, size, IPC_CREAT | 0666)) == -1) { + filebench_log(LOG_ERROR, + "Failed to create %zd bytes of ISM shared memory", size); + return (-1); + } + + if ((filebench_shm->shm_addr = (caddr_t)shmat(filebench_shm->shm_id, + 0, flag)) == (void *)-1) { + filebench_log(LOG_ERROR, + "Failed to attach %zd bytes of created ISM shared memory", + size); + return (-1); + } + + filebench_shm->shm_ptr = (char *)filebench_shm->shm_addr; + + filebench_log(LOG_VERBOSE, + "Allocated %zd bytes of ISM Shared Memory... at %zx", + size, filebench_shm->shm_addr); + + /* Locked until allocated to block allocs */ + (void) ipc_mutex_unlock(&filebench_shm->ism_lock); + + return (0); +} + +/* Per addr space ism */ +static int ism_attached = 0; + +/* + * Attach to interprocess shared memory. If already attached + * just return, otherwise use shmat() to attached to the region + * with ID of filebench_shm->shm_id. Returns -1 if shmat() + * fails, otherwise 0. + */ +static int +ipc_ismattach(void) +{ +#ifdef HAVE_SHM_SHARE_MMU + int flag = SHM_SHARE_MMU; +#else + int flag = 0; +#endif /* HAVE_SHM_SHARE_MMU */ + + + if (ism_attached) + return (0); + + /* Does it exist? */ + if (filebench_shm->shm_id == 999) + return (0); + + if (shmat(filebench_shm->shm_id, filebench_shm->shm_addr, + flag) == NULL) + return (-1); + + ism_attached = 1; + + return (0); +} + +/* + * Allocate from interprocess shared memory. Attaches to ism + * if necessary, then allocates "size" bytes, updates allocation + * information and returns a pointer to the allocated memory. + */ +/* + * XXX No check is made for out-of-memory condition + */ +char * +ipc_ismmalloc(size_t size) +{ + char *allocstr; + + (void) ipc_mutex_lock(&filebench_shm->ism_lock); + + /* Map in shared memory */ + (void) ipc_ismattach(); + + allocstr = filebench_shm->shm_ptr; + + filebench_shm->shm_ptr += size; + filebench_shm->shm_allocated += size; + + (void) ipc_mutex_unlock(&filebench_shm->ism_lock); + + return (allocstr); +} + +/* + * Deletes shared memory region and resets shared memory region + * information in filebench_shm. + */ +void +ipc_ismdelete(void) +{ + if (filebench_shm->shm_id == -1) + return; + + filebench_log(LOG_VERBOSE, "Deleting ISM..."); + + (void) ipc_mutex_lock(&filebench_shm->ism_lock); +#ifdef HAVE_SEM_RMID + (void) shmctl(filebench_shm->shm_id, IPC_RMID, 0); +#endif + filebench_shm->shm_ptr = (char *)filebench_shm->shm_addr; + filebench_shm->shm_id = -1; + filebench_shm->shm_allocated = 0; + (void) ipc_mutex_unlock(&filebench_shm->ism_lock); +} diff --git a/usr/src/cmd/filebench/common/ipc.h b/usr/src/cmd/filebench/common/ipc.h new file mode 100644 index 0000000000..be0c1ad228 --- /dev/null +++ b/usr/src/cmd/filebench/common/ipc.h @@ -0,0 +1,162 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_IPC_H +#define _FB_IPC_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" +#include <pthread.h> + +#include "procflow.h" +#include "threadflow.h" +#include "fileobj.h" +#include "fileset.h" +#include "flowop.h" +#include "filebench.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef USE_PROCESS_MODEL +#define FILEBENCH_MEMSIZE 4096 +#else +#define FILEBENCH_MEMSIZE 2048 +#endif /* USE_PROCESS_MODEL */ + +#define FILEBENCH_NFILEOBJS FILEBENCH_MEMSIZE +#define FILEBENCH_NFILESETS FILEBENCH_MEMSIZE +#define FILEBENCH_NFILESETENTRIES (1024 * 1024) +#define FILEBENCH_NPROCFLOWS FILEBENCH_MEMSIZE +#define FILEBENCH_NTHREADFLOWS 64 * FILEBENCH_MEMSIZE +#define FILEBENCH_NFLOWOPS 64 * FILEBENCH_MEMSIZE +#define FILEBENCH_NVARS FILEBENCH_MEMSIZE +#define FILEBENCH_FILESETPATHMEMORY FILEBENCH_NFILESETENTRIES*FSE_MAXPATHLEN +#define FILEBENCH_STRINGMEMORY FILEBENCH_NVARS * 128 +#define FILEBENCH_MAXBITMAP FILEBENCH_NFILESETENTRIES + +#define FILEBENCH_FILEOBJ 0 +#define FILEBENCH_PROCFLOW 1 +#define FILEBENCH_THREADFLOW 2 +#define FILEBENCH_FLOWOP 3 +#define FILEBENCH_INTEGER 4 +#define FILEBENCH_STRING 5 +#define FILEBENCH_VARIABLE 6 +#define FILEBENCH_FILESET 7 +#define FILEBENCH_FILESETENTRY 8 +#define FILEBENCH_TYPES 9 + +#define FILEBENCH_NSEMS 128 + +typedef struct filebench_shm { + pthread_mutex_t fileobj_lock; + pthread_mutex_t fileset_lock; + pthread_mutex_t procflow_lock; + pthread_mutex_t threadflow_lock; + pthread_mutex_t flowop_lock; + pthread_mutex_t msg_lock; + pthread_mutex_t malloc_lock; + pthread_mutex_t ism_lock; + pthread_rwlock_t run_lock; + pthread_rwlock_t flowop_find_lock; + + char *string_ptr; + char *path_ptr; + fileobj_t *filelist; + fileset_t *filesetlist; + flowop_t *flowoplist; + procflow_t *proclist; + var_t *var_list; + var_t *var_dyn_list; + int debug_level; + hrtime_t epoch; + hrtime_t starttime; + int bequiet; + key_t semkey; + int seminit; + int semid_seq; + int utid; + int log_fd; + int dump_fd; + char dump_filename[MAXPATHLEN]; + pthread_mutex_t eventgen_lock; + pthread_cond_t eventgen_cv; + int eventgen_hz; + int eventgen_q; + char fscriptname[1024]; + int shm_id; + size_t shm_required; + size_t shm_allocated; + caddr_t shm_addr; + char *shm_ptr; + int allrunning; + int f_abort; + + int marker; + + fileobj_t fileobj[FILEBENCH_NFILEOBJS]; + fileset_t fileset[FILEBENCH_NFILESETS]; + filesetentry_t filesetentry[FILEBENCH_NFILESETENTRIES]; + char filesetpaths[FILEBENCH_FILESETPATHMEMORY]; + procflow_t procflow[FILEBENCH_NPROCFLOWS]; + threadflow_t threadflow[FILEBENCH_NTHREADFLOWS]; + flowop_t flowop[FILEBENCH_NFLOWOPS]; + var_t var[FILEBENCH_NVARS]; + vinteger_t integer_ptrs[FILEBENCH_NVARS]; + char *string_ptrs[FILEBENCH_NVARS]; + char strings[FILEBENCH_STRINGMEMORY]; + char semids[FILEBENCH_NSEMS]; + int bitmap[FILEBENCH_TYPES][FILEBENCH_MAXBITMAP]; +} filebench_shm_t; + +extern char *shmpath; + +void ipc_init(void); +void *ipc_malloc(int type); +void ipc_free(int type, char *addr); +int ipc_attach(caddr_t shmaddr); +pthread_mutexattr_t *ipc_mutexattr(void); +pthread_condattr_t *ipc_condattr(void); +int ipc_semidalloc(void); +void ipc_semidfree(int semid); +char *ipc_stralloc(char *string); +char *ipc_pathalloc(char *string); +int ipc_mutex_lock(pthread_mutex_t *mutex); +int ipc_mutex_unlock(pthread_mutex_t *mutex); +void ipc_seminit(void); +char *ipc_ismmalloc(size_t size); +int ipc_ismcreate(size_t size); +void ipc_ismdelete(void); +void ipc_cleanup(void); + +extern filebench_shm_t *filebench_shm; + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_IPC_H */ diff --git a/usr/src/cmd/filebench/common/misc.c b/usr/src/cmd/filebench/common/misc.c new file mode 100644 index 0000000000..2d4281fe22 --- /dev/null +++ b/usr/src/cmd/filebench/common/misc.c @@ -0,0 +1,495 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include <stdio.h> +#include <fcntl.h> +#include <limits.h> +#include <time.h> +#include <libgen.h> +#include <unistd.h> +#include <strings.h> +#include "filebench.h" +#include "ipc.h" +#include "eventgen.h" +#include "utils.h" + +/* + * Routines to access high resolution system time, initialize and + * shutdown filebench, obtain system generated random numbers from + * "urandom", log filebench run progress and errors, and access system + * information strings. + */ + + +#if !defined(sun) && defined(USE_RDTSC) +/* + * Lets us use the rdtsc instruction to get highres time. + * Thanks to libmicro + */ +uint64_t cpu_hz = 0; + +/* + * Uses the rdtsc instruction to get high resolution (cpu + * clock ticks) time. Only used for non Sun compiles. + */ +__inline__ long long +rdtsc(void) +{ + unsigned long long x; + __asm__ volatile(".byte 0x0f, 0x31" : "=A" (x)); + return (x); +} + +/* + * Get high resolution time in nanoseconds. This is the version + * used when not compiled for Sun systems. It uses rdtsc call to + * get clock ticks and converts to nanoseconds + */ +uint64_t +gethrtime(void) +{ + uint64_t hrt; + + /* convert to nanosecs and return */ + hrt = 1000000000UL * rdtsc() / cpu_hz; + return (hrt); +} + +/* + * Gets CPU clock frequency in MHz from cpuinfo file. + * Converts to cpu_hz and stores in cpu_hz global uint64_t. + * Only used for non Sun compiles. + */ +static uint64_t +parse_cpu_hz(void) +{ + /* + * Parse the following from /proc/cpuinfo. + * cpu MHz : 2191.563 + */ + FILE *cpuinfo; + double hertz = -1; + uint64_t hz; + + if ((cpuinfo = fopen("/proc/cpuinfo", "r")) == NULL) { + filebench_log(LOG_ERROR, "open /proc/cpuinfo failed: %s", + strerror(errno)); + filebench_shutdown(1); + } + while (!feof(cpuinfo)) { + char buffer[80]; + + fgets(buffer, 80, cpuinfo); + if (strlen(buffer) == 0) continue; + if (strncasecmp(buffer, "cpu MHz", 7) == 0) { + char *token = strtok(buffer, ":"); + + if (token != NULL) { + token = strtok((char *)NULL, ":"); + hertz = strtod(token, NULL); + } + break; + } + } + printf("CPU Mhz %9.6f, sysconf:%ld\n", hertz, sysconf(_SC_CLK_TCK)); + hz = hertz * 1000000; + + return (hz); +} + +#elif !defined(sun) + +/* + * Get high resolution time in nanoseconds. This is the version + * used if compiled for Sun systems. It calls gettimeofday + * to get current time and converts it to nanoseconds. + */ +uint64_t +gethrtime(void) +{ + struct timeval tv; + uint64_t hrt; + + gettimeofday(&tv, NULL); + + hrt = (uint64_t)tv.tv_sec * 1000000000UL + + (uint64_t)tv.tv_usec * 1000UL; + return (hrt); +} +#endif + +static int urandomfd; + +/* + * Main filebench initialization. Opens the random number + * "device" file or shuts down the run if one is not found. + * Sets the cpu clock frequency variable or shuts down the + * run if one is not found. + */ +void +filebench_init(void) +{ + /* open the "urandom" random number device file */ + if ((urandomfd = open("/dev/urandom", O_RDONLY)) < 0) { + filebench_log(LOG_ERROR, "open /dev/urandom failed: %s", + strerror(errno)); + filebench_shutdown(1); + } +#if defined(USE_RDTSC) && (LINUX_PORT) + cpu_hz = parse_cpu_hz(); + if (cpu_hz <= 0) { + filebench_log(LOG_ERROR, "Error getting CPU Mhz: %s", + strerror(errno)); + filebench_shutdown(1); + } +#endif /* USE_RDTSC */ + +} + +/* + * Reads a 64 bit random number from the urandom "file". + * Shuts down the run if the read fails. Otherwise returns + * the random number after rounding it off by "round". + * Returns 0 on success, -1 on failure. + */ +int +filebench_randomno64(uint64_t *randp, uint64_t max, uint64_t round) +{ + uint64_t random; + + /* check for round value too large */ + if (max <= round) { + *randp = 0; + + /* if it just fits, its ok, otherwise error */ + if (max == round) + return (0); + else + return (-1); + } + + if (read(urandomfd, &random, + sizeof (uint64_t)) != sizeof (uint64_t)) { + filebench_log(LOG_ERROR, "read /dev/urandom failed: %s", + strerror(errno)); + filebench_shutdown(1); + } + + /* clip with max and optionally round */ + max -= round; + random = random / (FILEBENCH_RANDMAX64 / max); + if (round) { + random = random / round; + random *= round; + } + if (random > max) + random = max; + + *randp = random; + return (0); +} + + +/* + * Reads a 32 bit random number from the urandom "file". + * Shuts down the run if the read fails. Otherwise returns + * the random number after rounding it off by "round". + * Returns 0 on success, -1 on failure. + */ +int +filebench_randomno32(uint32_t *randp, uint32_t max, uint32_t round) +{ + uint32_t random; + + /* check for round value too large */ + if (max <= round) { + *randp = 0; + + /* if it just fits, its ok, otherwise error */ + if (max == round) + return (0); + else + return (-1); + } + + if (read(urandomfd, &random, + sizeof (uint32_t)) != sizeof (uint32_t)) { + filebench_log(LOG_ERROR, "read /dev/urandom failed: %s", + strerror(errno)); + filebench_shutdown(1); + } + + /* clip with max and optionally round */ + max -= round; + random = random / (FILEBENCH_RANDMAX32 / max); + if (round) { + random = random / round; + random *= round; + } + if (random > max) + random = max; + + *randp = random; + return (0); +} + +extern int lex_lineno; + +/* + * Writes a message consisting of information formated by + * "fmt" to the log file, dump file or stdout. The supplied + * "level" argument determines which file to write to and + * what other actions to take. The level LOG_LOG writes to + * the "log" file, and will open the file on the first + * invocation. The level LOG_DUMP writes to the "dump" file, + * and will open it on the first invocation. Other levels + * print to the stdout device, with the amount of information + * dependent on the error level and the current error level + * setting in filebench_shm->debug_level. + */ +void filebench_log +__V((int level, const char *fmt, ...)) +{ + va_list args; + hrtime_t now; + char line[131072]; + char buf[131072]; + + if (level == LOG_FATAL) + goto fatal; + + /* open logfile if not already open and writing to it */ + if ((level == LOG_LOG) && + (filebench_shm->log_fd < 0)) { + char path[MAXPATHLEN]; + char *s; + + (void) strcpy(path, filebench_shm->fscriptname); + if ((s = strstr(path, ".f"))) + *s = 0; + else + (void) strcpy(path, "filebench"); + + (void) strcat(path, ".csv"); + + filebench_shm->log_fd = + open(path, O_RDWR | O_CREAT | O_TRUNC, 0666); + } + + /* + * if logfile still not open, switch to LOG_ERROR level so + * it gets reported to stdout + */ + if ((level == LOG_LOG) && + (filebench_shm->log_fd < 0)) { + (void) snprintf(line, sizeof (line), "Open logfile failed: %s", + strerror(errno)); + level = LOG_ERROR; + } + + /* open dumpfile if not already open and writing to it */ + if ((level == LOG_DUMP) && + (*filebench_shm->dump_filename == 0)) + return; + + if ((level == LOG_DUMP) && + (filebench_shm->dump_fd < 0)) { + + filebench_shm->dump_fd = + open(filebench_shm->dump_filename, + O_RDWR | O_CREAT | O_TRUNC, 0666); + } + + if ((level == LOG_DUMP) && + (filebench_shm->dump_fd < 0)) { + (void) snprintf(line, sizeof (line), "Open logfile failed: %s", + strerror(errno)); + level = LOG_ERROR; + } + + /* Only log greater than debug setting */ + if ((level != LOG_DUMP) && (level != LOG_LOG) && + (level > filebench_shm->debug_level)) + return; + + now = gethrtime(); + +fatal: + +#ifdef __STDC__ + va_start(args, fmt); +#else + char *fmt; + va_start(args); + fmt = va_arg(args, char *); +#endif + + (void) vsprintf(line, fmt, args); + + va_end(args); + + if (level == LOG_FATAL) { + (void) fprintf(stdout, "%s\n", line); + return; + } + + /* Serialize messages to log */ + (void) ipc_mutex_lock(&filebench_shm->msg_lock); + + if (level == LOG_LOG) { + if (filebench_shm->log_fd > 0) { + (void) snprintf(buf, sizeof (buf), "%s\n", line); + (void) write(filebench_shm->log_fd, buf, strlen(buf)); + (void) fsync(filebench_shm->log_fd); + } + + } else if (level == LOG_DUMP) { + if (filebench_shm->dump_fd != -1) { + (void) snprintf(buf, sizeof (buf), "%s\n", line); + (void) write(filebench_shm->dump_fd, buf, strlen(buf)); + (void) fsync(filebench_shm->dump_fd); + } + + } else if (filebench_shm->debug_level > LOG_INFO) { + (void) fprintf(stdout, "%5ld: %4.3f: %s", + pid, (now - filebench_shm->epoch) / FSECS, + line); + + } else { + (void) fprintf(stdout, "%4.3f: %s", + (now - filebench_shm->epoch) / FSECS, + line); + } + + if (level == LOG_ERROR) { + (void) fprintf(stdout, " on line %d", lex_lineno); + } + + if ((level != LOG_LOG) && (level != LOG_DUMP)) { + (void) fprintf(stdout, "\n"); + (void) fflush(stdout); + } + + (void) ipc_mutex_unlock(&filebench_shm->msg_lock); +} + +/* + * Stops the run and exits filebench. If filebench is + * currently running a workload, calls procflow_shutdown() + * to stop the run. Also closes and deletes shared memory. + */ +void +filebench_shutdown(int error) { + filebench_log(LOG_DEBUG_IMPL, "Shutdown"); + (void) unlink("/tmp/filebench_shm"); + if (filebench_shm->allrunning) + procflow_shutdown(); + filebench_shm->f_abort = 1; + ipc_ismdelete(); + exit(error); +} + +/* + * Put the hostname in ${hostname}. The system supplied + * host name string is copied into an allocated string and + * the pointer to the string is placed in the supplied + * variable "var". If var->var_string already points to + * a string, the string is freed. The routine always + * returns zero (0). + */ +var_t * +host_var(var_t *var) +{ + char hoststr[128]; + + (void) gethostname(hoststr, 128); + if (var->var_string) + free(var->var_string); + var->var_string = fb_stralloc(hoststr); + return (0); +} + +/* + * Put the date string in ${date}. The system supplied date is + * copied into an allocated string and the pointer to the string + * is placed in the supplied var_t's var_string. If + * var->var_string already points to a string, the string + * is freed. The routine always returns a pointer to the + * supplied var_t. + */ +var_t * +date_var(var_t *var) +{ + char datestr[128]; +#ifdef HAVE_CFTIME + time_t t = time(NULL); +#else + struct tm t; +#endif + +#ifdef HAVE_CFTIME + cftime(datestr, "%y%m%d%H" "%M", &t); +#else + (void) strftime(datestr, sizeof (datestr), "%y%m%d%H %M", &t); +#endif + + if (var->var_string) + free(var->var_string); + var->var_string = fb_stralloc(datestr); + + return (var); +} + +extern char *fscriptname; + +/* + * Put the script name in ${script}. The path name of the script + * used with this filebench run trimmed of the trailing ".f" and + * all leading subdirectories. The remaining script name is + * copied into the var_string field of the supplied variable + * "var". The routine always returns a pointer to the supplied + * var_t. + */ +var_t * +script_var(var_t *var) +{ + char *scriptstr; + char *f = fb_stralloc(fscriptname); + + /* Trim the .f suffix */ + for (scriptstr = f + strlen(f) - 1; scriptstr != f; scriptstr--) { + if (*scriptstr == '.') { + *scriptstr = 0; + break; + } + } + + var->var_string = fb_stralloc(basename(f)); + free(f); + + return (var); +} diff --git a/usr/src/cmd/filebench/common/misc.h b/usr/src/cmd/filebench/common/misc.h new file mode 100644 index 0000000000..e330c0318f --- /dev/null +++ b/usr/src/cmd/filebench/common/misc.h @@ -0,0 +1,72 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_MISC_H +#define _FB_MISC_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" + +#include <stdio.h> +#ifdef HAVE_STDINT_H +#include <stdint.h> +#endif + +#include <sys/times.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#define DATE_VAR "date" +#define SCRIPT_VAR "scriptname" +#define HOST_VAR "hostname" + +#ifndef HAVE_HRTIME +uint64_t gethrtime(); +#define hrtime_t uint64_t +#endif +#define FSECS (double)1000000000.0 + +#define LOG_INFO 1 +#define LOG_VERBOSE 2 +#define LOG_DEBUG_SCRIPT 3 +#define LOG_DEBUG_IMPL 5 +#define LOG_DEBUG_NEVER 10 +#define LOG_LOG 1000 +#define LOG_DUMP 1001 +#define LOG_FATAL 999 +#define LOG_ERROR 0 + +var_t *date_var(var_t *var); +var_t *script_var(var_t *var); +var_t *host_var(var_t *var); + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_MISC_H */ diff --git a/usr/src/cmd/filebench/common/parser_gram.y b/usr/src/cmd/filebench/common/parser_gram.y new file mode 100644 index 0000000000..3ab8c72c2c --- /dev/null +++ b/usr/src/cmd/filebench/common/parser_gram.y @@ -0,0 +1,2793 @@ + +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +%{ +#pragma ident "%Z%%M% %I% %E% SMI" +%} + +%{ + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <signal.h> +#include <errno.h> +#include <sys/types.h> +#include <locale.h> +#include <sys/utsname.h> +#ifdef HAVE_STDINT_H +#include <stdint.h> +#endif +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/wait.h> +#ifdef HAVE_LIBTECLA +#include <libtecla.h> +#endif +#include "parsertypes.h" +#include "filebench.h" +#include "utils.h" +#include "stats.h" +#include "vars.h" +#include "eventgen.h" +#ifdef HAVE_LIBTECLA +#include "auto_comp.h" +#endif + +int dofile = FS_FALSE; +static const char cmdname[] = "filebench"; +static const char cmd_options[] = "pa:f:hi:s:m:"; +static void usage(int); + +static cmd_t *cmd = NULL; /* Command being processed */ +#ifdef HAVE_LIBTECLA +static GetLine *gl; /* GetLine resource object */ +#endif + +char *execname; +char *fscriptname; +int noproc = 0; +var_t *var_list = NULL; +pidlist_t *pidlist = NULL; +char *cwd = NULL; +FILE *parentscript = NULL; + +/* yacc externals */ +extern FILE *yyin; +extern int yydebug; +extern void yyerror(char *s); + +/* utilities */ +static void terminate(void); +static cmd_t *alloc_cmd(void); +static attr_t *alloc_attr(void); +static attr_t *get_attr(cmd_t *cmd, int64_t name); +static attr_t *get_attr_integer(cmd_t *cmd, int64_t name); +static attr_t *get_attr_bool(cmd_t *cmd, int64_t name); +static var_t *alloc_var(void); +static var_t *get_var(cmd_t *cmd, int64_t name); +static list_t *alloc_list(); + +/* Info Commands */ +static void parser_show(cmd_t *); +static void parser_list(cmd_t *); + +/* Define Commands */ +static void parser_proc_define(cmd_t *); +static void parser_thread_define(cmd_t *, procflow_t *, int instances); +static void parser_flowop_define(cmd_t *, threadflow_t *); +static void parser_file_define(cmd_t *); +static void parser_fileset_define(cmd_t *); + +/* Create Commands */ +static void parser_proc_create(cmd_t *); +static void parser_thread_create(cmd_t *); +static void parser_flowop_create(cmd_t *); +static void parser_file_create(cmd_t *); +static void parser_fileset_create(cmd_t *); + +/* Shutdown Commands */ +static void parser_proc_shutdown(cmd_t *); +static void parser_filebench_shutdown(cmd_t *cmd); + +/* Other Commands */ +static void parser_foreach_integer(cmd_t *cmd); +static void parser_foreach_string(cmd_t *cmd); +static void parser_sleep(cmd_t *cmd); +static void parser_sleep_variable(cmd_t *cmd); +static void parser_log(cmd_t *cmd); +static void parser_statscmd(cmd_t *cmd); +static void parser_statsdump(cmd_t *cmd); +static void parser_statsxmldump(cmd_t *cmd); +static void parser_echo(cmd_t *cmd); +static void parser_usage(cmd_t *cmd); +static void parser_vars(cmd_t *cmd); +static void parser_printvars(cmd_t *cmd); +static void parser_system(cmd_t *cmd); +static void parser_statssnap(cmd_t *cmd); +static void parser_directory(cmd_t *cmd); +static void parser_eventgen(cmd_t *cmd); +static void parser_run(cmd_t *cmd); +static void parser_run_variable(cmd_t *cmd); +static void parser_help(cmd_t *cmd); +static void arg_parse(const char *command); +static void parser_abort(int arg); + +%} + +%union { + int64_t ival; + uchar_t bval; + char * sval; + fs_u val; + cmd_t *cmd; + attr_t *attr; + list_t *list; +} + +%start commands + +%token FSC_LIST FSC_DEFINE FSC_EXEC FSC_QUIT FSC_DEBUG FSC_CREATE +%token FSC_SLEEP FSC_STATS FSC_FOREACH FSC_SET FSC_SHUTDOWN FSC_LOG +%token FSC_SYSTEM FSC_FLOWOP FSC_EVENTGEN FSC_ECHO FSC_LOAD FSC_RUN +%token FSC_USAGE FSC_HELP FSC_VARS +%token FSV_STRING FSV_VAL_INT FSV_VAL_BOOLEAN FSV_VARIABLE FSV_WHITESTRING +%token FST_INT FST_BOOLEAN +%token FSE_FILE FSE_PROC FSE_THREAD FSE_CLEAR FSE_ALL FSE_SNAP FSE_DUMP +%token FSE_DIRECTORY FSE_COMMAND FSE_FILESET FSE_XMLDUMP +%token FSK_SEPLST FSK_OPENLST FSK_CLOSELST FSK_ASSIGN FSK_IN FSK_QUOTE +%token FSK_DIRSEPLST +%token FSA_SIZE FSA_PREALLOC FSA_PARALLOC FSA_PATH FSA_REUSE +%token FSA_PROCESS FSA_MEMSIZE FSA_RATE FSA_CACHED +%token FSA_IOSIZE FSA_FILE FSA_WSS FSA_NAME FSA_RANDOM FSA_INSTANCES +%token FSA_DSYNC FSA_TARGET FSA_ITERS FSA_NICE FSA_VALUE FSA_BLOCKING +%token FSA_HIGHWATER FSA_DIRECTIO FSA_DIRWIDTH FSA_FD FSA_SRCFD FSA_ROTATEFD +%token FSA_NAMELENGTH FSA_FILESIZE FSA_ENTRIES FSA_FILESIZEGAMMA +%token FSA_DIRGAMMA FSA_USEISM + +%type <ival> FSV_VAL_INT +%type <bval> FSV_VAL_BOOLEAN +%type <sval> FSV_STRING +%type <sval> FSV_WHITESTRING +%type <sval> FSV_VARIABLE +%type <sval> FSK_ASSIGN + +%type <ival> FSC_LIST FSC_DEFINE FSC_SET FSC_LOAD FSC_RUN +%type <ival> FSE_FILE FSE_PROC FSE_THREAD FSE_CLEAR FSC_HELP + +%type <sval> name +%type <ival> entity +%type <val> value + +%type <cmd> command inner_commands load_command run_command +%type <cmd> list_command define_command debug_command create_command +%type <cmd> sleep_command stats_command set_command shutdown_command +%type <cmd> foreach_command log_command system_command flowop_command +%type <cmd> eventgen_command quit_command flowop_list thread_list +%type <cmd> thread echo_command usage_command help_command vars_command + +%type <attr> attr_op attr_ops +%type <attr> attr_value +%type <list> integer_seplist string_seplist string_list var_string_list var_string +%type <list> whitevar_string whitevar_string_list +%type <ival> attrs_define_file attrs_define_thread attrs_flowop attrs_define_fileset +%type <ival> attrs_define_proc attrs_eventgen +%type <ival> attr_name + +%% + +commands: commands command +{ + list_t *list = NULL; + list_t *list_end = NULL; + + if ($2->cmd != NULL) + $2->cmd($2); + + free($2); +} +| commands error +{ + if (dofile) + YYABORT; +} +|; + +inner_commands: command +{ + filebench_log(LOG_DEBUG_IMPL, "inner_command %zx", $1); + $$ = $1; +} +| inner_commands command +{ + cmd_t *list = NULL; + cmd_t *list_end = NULL; + + /* Find end of list */ + for (list = $1; list != NULL; + list = list->cmd_next) + list_end = list; + + list_end->cmd_next = $2; + + filebench_log(LOG_DEBUG_IMPL, + "inner_commands adding cmd %zx to list %zx", $2, $1); + + $$ = $1; +}; + +command: + define_command +| debug_command +| eventgen_command +| create_command +| echo_command +| usage_command +| vars_command +| foreach_command +| help_command +| list_command +| load_command +| log_command +| run_command +| set_command +| shutdown_command +| sleep_command +| stats_command +| system_command +| quit_command; + +foreach_command: FSC_FOREACH +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + filebench_log(LOG_DEBUG_IMPL, "foreach_command %zx", $$); +} +| foreach_command FSV_VARIABLE FSK_IN integer_seplist FSK_OPENLST inner_commands FSK_CLOSELST +{ + cmd_t *cmd, *inner_cmd; + list_t *list; + + $$ = $1; + $$->cmd_list = $6; + $$->cmd_tgt1 = $2; + $$->cmd_param_list = $4; + $$->cmd = parser_foreach_integer; + + for (list = $$->cmd_param_list; list != NULL; + list = list->list_next) { + for (inner_cmd = $$->cmd_list; + inner_cmd != NULL; + inner_cmd = inner_cmd->cmd_next) { + filebench_log(LOG_DEBUG_IMPL, + "packing foreach: %zx %s=%lld, cmd %zx", + $$, + $$->cmd_tgt1, + *list->list_integer, inner_cmd); + } + } +}| foreach_command FSV_VARIABLE FSK_IN string_seplist FSK_OPENLST inner_commands FSK_CLOSELST +{ + cmd_t *cmd, *inner_cmd; + list_t *list; + + $$ = $1; + $$->cmd_list = $6; + $$->cmd_tgt1 = $2; + $$->cmd_param_list = $4; + $$->cmd = parser_foreach_string; + + for (list = $$->cmd_param_list; list != NULL; + list = list->list_next) { + for (inner_cmd = $$->cmd_list; + inner_cmd != NULL; + inner_cmd = inner_cmd->cmd_next) { + filebench_log(LOG_DEBUG_IMPL, + "packing foreach: %zx %s=%s, cmd %zx", + $$, + $$->cmd_tgt1, + *list->list_string, inner_cmd); + } + } +}; + +integer_seplist: FSV_VAL_INT +{ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_integer = integer_alloc($1); +} +| integer_seplist FSK_SEPLST FSV_VAL_INT +{ + list_t *list = NULL; + list_t *list_end = NULL; + + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_integer = integer_alloc($3); + + /* Find end of list */ + for (list = $1; list != NULL; + list = list->list_next) + list_end = list; + list_end->list_next = $$; + $$ = $1; +}; + +string_seplist: FSK_QUOTE FSV_WHITESTRING FSK_QUOTE +{ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($2); +} +| string_seplist FSK_SEPLST FSK_QUOTE FSV_WHITESTRING FSK_QUOTE +{ + list_t *list = NULL; + list_t *list_end = NULL; + + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($4); + + /* Find end of list */ + for (list = $1; list != NULL; + list = list->list_next) + list_end = list; + list_end->list_next = $$; + $$ = $1; +}; + +eventgen_command: FSC_EVENTGEN +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = &parser_eventgen; +} +| eventgen_command attr_ops +{ + $1->cmd_attr_list = $2; +}; + +system_command: FSC_SYSTEM whitevar_string_list +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + + $$->cmd_param_list = $2; + $$->cmd = parser_system; +}; + +echo_command: FSC_ECHO whitevar_string_list +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + + $$->cmd_param_list = $2; + $$->cmd = parser_echo; +}; + +usage_command: FSC_USAGE whitevar_string_list +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + + $$->cmd_param_list = $2; + $$->cmd = parser_usage; +}; + +vars_command: FSC_VARS +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + + $$->cmd = parser_printvars; +}; + +string_list: FSV_VARIABLE +{ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($1); +} +| string_list FSK_SEPLST FSV_VARIABLE +{ + list_t *list = NULL; + list_t *list_end = NULL; + + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($3); + + /* Find end of list */ + for (list = $1; list != NULL; + list = list->list_next) + list_end = list; + list_end->list_next = $$; + $$ = $1; +}; + +var_string: FSV_VARIABLE +{ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($1); +} +| FSV_STRING +{ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($1); +}; + +var_string_list: var_string +{ + $$ = $1; +}| var_string FSV_STRING +{ + list_t *list = NULL; + list_t *list_end = NULL; + + /* Add string */ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($2); + + /* Find end of list */ + for (list = $1; list != NULL; + list = list->list_next) + list_end = list; + list_end->list_next = $$; + $$ = $1; + +}| var_string FSV_VARIABLE +{ + list_t *list = NULL; + list_t *list_end = NULL; + + /* Add variable */ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($2); + + /* Find end of list */ + for (list = $1; list != NULL; + list = list->list_next) + list_end = list; + list_end->list_next = $$; + $$ = $1; +} |var_string_list FSV_STRING +{ + list_t *list = NULL; + list_t *list_end = NULL; + + /* Add string */ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($2); + + /* Find end of list */ + for (list = $1; list != NULL; + list = list->list_next) + list_end = list; + list_end->list_next = $$; + $$ = $1; + +}| var_string_list FSV_VARIABLE +{ + list_t *list = NULL; + list_t *list_end = NULL; + + /* Add variable */ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($2); + + /* Find end of list */ + for (list = $1; list != NULL; + list = list->list_next) + list_end = list; + list_end->list_next = $$; + $$ = $1; +}; + +whitevar_string: FSK_QUOTE FSV_VARIABLE +{ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($2); +} +| FSK_QUOTE FSV_WHITESTRING +{ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($2); +}; + +whitevar_string_list: whitevar_string FSV_WHITESTRING +{ + list_t *list = NULL; + list_t *list_end = NULL; + + /* Add string */ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($2); + + /* Find end of list */ + for (list = $1; list != NULL; + list = list->list_next) + list_end = list; + list_end->list_next = $$; + $$ = $1; + +}| whitevar_string FSV_VARIABLE +{ + list_t *list = NULL; + list_t *list_end = NULL; + + /* Add variable */ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($2); + + /* Find end of list */ + for (list = $1; list != NULL; + list = list->list_next) + list_end = list; + list_end->list_next = $$; + $$ = $1; +} |whitevar_string_list FSV_WHITESTRING +{ + list_t *list = NULL; + list_t *list_end = NULL; + + /* Add string */ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($2); + + /* Find end of list */ + for (list = $1; list != NULL; + list = list->list_next) + list_end = list; + list_end->list_next = $$; + $$ = $1; + +}| whitevar_string_list FSV_VARIABLE +{ + list_t *list = NULL; + list_t *list_end = NULL; + + /* Add variable */ + if (($$ = alloc_list()) == NULL) + YYERROR; + + $$->list_string = string_alloc($2); + + /* Find end of list */ + for (list = $1; list != NULL; + list = list->list_next) + list_end = list; + list_end->list_next = $$; + $$ = $1; +}| whitevar_string_list FSK_QUOTE +{ + $$ = $1; +}| whitevar_string FSK_QUOTE +{ + $$ = $1; +}; + +list_command: FSC_LIST +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = &parser_list; +}; + +log_command: FSC_LOG whitevar_string_list +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = &parser_log; + $$->cmd_param_list = $2; +}; + +debug_command: FSC_DEBUG FSV_VAL_INT +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = NULL; + filebench_shm->debug_level = $2; + if (filebench_shm->debug_level > 9) + yydebug = 1; +}; + +set_command: FSC_SET FSV_VARIABLE FSK_ASSIGN FSV_VAL_INT +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + var_assign_integer($2, $4); + if (parentscript) { + $$->cmd_tgt1 = $2; + parser_vars($$); + } + $$->cmd = NULL; +} +| FSC_SET FSV_VARIABLE FSK_ASSIGN FSK_QUOTE FSV_WHITESTRING FSK_QUOTE +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + var_assign_string($2, $5); + if (parentscript) { + $$->cmd_tgt1 = $2; + parser_vars($$); + } + $$->cmd = NULL; +}| FSC_SET FSV_VARIABLE FSK_ASSIGN FSV_STRING +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + var_assign_string($2, $4); + if (parentscript) { + $$->cmd_tgt1 = $2; + parser_vars($$); + } + $$->cmd = NULL; +}| FSC_SET FSV_VARIABLE FSK_ASSIGN FSV_VARIABLE +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + var_assign_var($2, $4); + if (parentscript) { + $$->cmd_tgt1 = $2; + parser_vars($$); + } + $$->cmd = NULL; +}; + +stats_command: FSC_STATS FSE_SNAP +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = (void (*)(struct cmd *))&parser_statssnap; + break; + +} +| FSC_STATS FSE_CLEAR +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = (void (*)(struct cmd *))&stats_clear; + +} +| FSC_STATS FSE_DIRECTORY var_string_list +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd_param_list = $3; + $$->cmd = (void (*)(struct cmd *))&parser_directory; + +} +| FSC_STATS FSE_COMMAND whitevar_string_list +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + + $$->cmd_param_list = $3; + $$->cmd = parser_statscmd; + +}| FSC_STATS FSE_DUMP whitevar_string_list +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + + $$->cmd_param_list = $3; + $$->cmd = parser_statsdump; +}| FSC_STATS FSE_XMLDUMP whitevar_string_list +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + + $$->cmd_param_list = $3; + $$->cmd = parser_statsxmldump; +}; + +quit_command: FSC_QUIT +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = parser_filebench_shutdown; +}; + +flowop_list: flowop_command +{ + $$ = $1; +}| flowop_list flowop_command +{ + cmd_t *list = NULL; + cmd_t *list_end = NULL; + + /* Find end of list */ + for (list = $1; list != NULL; + list = list->cmd_next) + list_end = list; + + list_end->cmd_next = $2; + + filebench_log(LOG_DEBUG_IMPL, + "flowop_list adding cmd %zx to list %zx", $2, $1); + + $$ = $1; +}; + +thread: FSE_THREAD attr_ops FSK_OPENLST flowop_list FSK_CLOSELST +{ + /* + * Allocate a cmd node per thread, with a + * list of flowops attached to the cmd_list + */ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd_list = $4; + $$->cmd_attr_list = $2; +}; + +thread_list: thread +{ + $$ = $1; +}| thread_list thread +{ + cmd_t *list = NULL; + cmd_t *list_end = NULL; + + /* Find end of list */ + for (list = $1; list != NULL; + list = list->cmd_next) + list_end = list; + + list_end->cmd_next = $2; + + filebench_log(LOG_DEBUG_IMPL, + "thread_list adding cmd %zx to list %zx", $2, $1); + + $$ = $1; +}; + +define_command: FSC_DEFINE FSE_PROC attr_ops FSK_OPENLST thread_list FSK_CLOSELST +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = &parser_proc_define; + $$->cmd_list = $5; + $$->cmd_attr_list = $3; + +}| FSC_DEFINE FSE_FILE +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = &parser_file_define; +}| FSC_DEFINE FSE_FILESET +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = &parser_fileset_define; +} +| define_command attr_ops +{ + $1->cmd_attr_list = $2; +}; + +create_command: FSC_CREATE entity +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + switch ($2) { + case FSE_PROC: + $$->cmd = &parser_proc_create; + break; + case FSE_FILE: + $$->cmd = &parser_file_create; + break; + case FSE_FILESET: + $$->cmd = &parser_fileset_create; + break; + default: + filebench_log(LOG_ERROR, "unknown entity", $2); + YYERROR; + } + +}; + +shutdown_command: FSC_SHUTDOWN entity +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + switch ($2) { + case FSE_PROC: + $$->cmd = &parser_proc_shutdown; + break; + default: + filebench_log(LOG_ERROR, "unknown entity", $2); + YYERROR; + } + +}; + +sleep_command: FSC_SLEEP FSV_VAL_INT +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = parser_sleep; + $$->cmd_qty = $2; +} +| FSC_SLEEP FSV_VARIABLE +{ + vinteger_t *integer; + + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = parser_sleep_variable; + $$->cmd_tgt1 = fb_stralloc($2); +}; + +run_command: FSC_RUN FSV_VAL_INT +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = parser_run; + $$->cmd_qty = $2; +} +| FSC_RUN FSV_VARIABLE +{ + vinteger_t *integer; + + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = parser_run_variable; + $$->cmd_tgt1 = fb_stralloc($2); +} +| FSC_RUN +{ + vinteger_t *integer; + + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = parser_run; + $$->cmd_qty = 60UL; +}; + +help_command: FSC_HELP +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd = parser_help; +}; + +flowop_command: FSC_FLOWOP name +{ + if (($$ = alloc_cmd()) == NULL) + YYERROR; + $$->cmd_name = fb_stralloc($2); +} +| flowop_command attr_ops +{ + $1->cmd_attr_list = $2; +}; + +load_command: FSC_LOAD FSV_STRING +{ + FILE *newfile; + char loadfile[128]; + + if (($$ = alloc_cmd()) == NULL) + YYERROR; + + (void) strcpy(loadfile, $2); + (void) strcat(loadfile, ".f"); + + if ((newfile = fopen(loadfile, "r")) == NULL) { + (void) strcpy(loadfile, FILEBENCHDIR); + (void) strcat(loadfile, "/workloads/"); + (void) strcat(loadfile, $2); + (void) strcat(loadfile, ".f"); + if ((newfile = fopen(loadfile, "r")) == NULL) { + filebench_log(LOG_ERROR, "Cannot open %s", loadfile); + YYERROR; + } + } + + parentscript = yyin; + yyin = newfile; + yy_switchfileparent(yyin); +}; + +entity: FSE_PROC {$$ = FSE_PROC;} +| FSE_THREAD {$$ = FSE_THREAD;} +| FSE_FILESET {$$ = FSE_FILESET;} +| FSE_FILE {$$ = FSE_FILE;}; + +value: FSV_VAL_INT { $$.i = $1;} +| FSV_STRING { $$.s = $1;} +| FSV_VAL_BOOLEAN { $$.b = $1;}; + +name: FSV_STRING; + +attr_ops: attr_op +{ + $$ = $1; +} +| attr_ops FSK_SEPLST attr_op +{ + attr_t *attr = NULL; + attr_t *list_end = NULL; + + for (attr = $1; attr != NULL; + attr = attr->attr_next) + list_end = attr; /* Find end of list */ + + list_end->attr_next = $3; + + $$ = $1; +}; + +attr_op: attr_name FSK_ASSIGN attr_value +{ + $$ = $3; + $$->attr_name = $1; +} +| attr_name +{ + if (($$ = alloc_attr()) == NULL) + YYERROR; + $$->attr_name = $1; +} + +attr_name: attrs_define_file +|attrs_define_fileset +|attrs_define_thread +|attrs_define_proc +|attrs_flowop +|attrs_eventgen; + +attrs_define_proc: +FSA_NICE { $$ = FSA_NICE;} +|FSA_INSTANCES { $$ = FSA_INSTANCES;}; + +attrs_define_file: +FSA_SIZE { $$ = FSA_SIZE;} +| FSA_PATH { $$ = FSA_PATH;} +| FSA_REUSE { $$ = FSA_REUSE;} +| FSA_PREALLOC { $$ = FSA_PREALLOC;} +| FSA_PARALLOC { $$ = FSA_PARALLOC;}; + +attrs_define_fileset: +FSA_SIZE { $$ = FSA_SIZE;} +| FSA_PATH { $$ = FSA_PATH;} +| FSA_DIRWIDTH { $$ = FSA_DIRWIDTH;} +| FSA_PREALLOC { $$ = FSA_PREALLOC;} +| FSA_FILESIZEGAMMA { $$ = FSA_FILESIZEGAMMA;} +| FSA_DIRGAMMA { $$ = FSA_DIRGAMMA;} +| FSA_CACHED { $$ = FSA_CACHED;} +| FSA_ENTRIES { $$ = FSA_ENTRIES;}; + +attrs_define_thread: +FSA_PROCESS { $$ = FSA_PROCESS;} +|FSA_MEMSIZE { $$ = FSA_MEMSIZE;} +|FSA_USEISM { $$ = FSA_USEISM;} +|FSA_INSTANCES { $$ = FSA_INSTANCES;}; + +attrs_flowop: +FSA_WSS { $$ = FSA_WSS;} +|FSA_FILE { $$ = FSA_FILE;} +|FSA_NAME { $$ = FSA_NAME;} +|FSA_RANDOM { $$ = FSA_RANDOM;} +|FSA_FD { $$ = FSA_FD;} +|FSA_SRCFD { $$ = FSA_SRCFD;} +|FSA_ROTATEFD { $$ = FSA_ROTATEFD;} +|FSA_DSYNC { $$ = FSA_DSYNC;} +|FSA_DIRECTIO { $$ = FSA_DIRECTIO;} +|FSA_TARGET { $$ = FSA_TARGET;} +|FSA_ITERS { $$ = FSA_ITERS;} +|FSA_VALUE { $$ = FSA_VALUE;} +|FSA_BLOCKING { $$ = FSA_BLOCKING;} +|FSA_HIGHWATER { $$ = FSA_HIGHWATER;} +|FSA_IOSIZE { $$ = FSA_IOSIZE;}; + +attrs_eventgen: +FSA_RATE { $$ = FSA_RATE;}; + +attr_value: var_string_list { + if (($$ = alloc_attr()) == NULL) + YYERROR; + $$->attr_param_list = $1; +} | FSV_STRING +{ + if (($$ = alloc_attr()) == NULL) + YYERROR; + $$->attr_string = string_alloc($1); +} | FSV_VAL_INT { + if (($$ = alloc_attr()) == NULL) + YYERROR; + $$->attr_integer = integer_alloc($1); +} | FSV_VARIABLE { + if (($$ = alloc_attr()) == NULL) + YYERROR; + $$->attr_integer = var_ref_integer($1); + $$->attr_string = var_ref_string($1); +}; + +%% + +/* + * The following 'c' routines implement the various commands defined in the + * above yacc parser code. The yacc portion checks the syntax of the commands + * found in a workload file, or typed on interactive command lines, parsing + * the commands' parameters into lists. The lists are then passed in a cmd_t + * struct for each command to its related routine in the following section + * for actual execution. This section also includes a few utility routines + * and the main entry point for the program. + */ + +/* + * Entry point for filebench. Processes command line arguements. The -f + * option will read in a workload file (the full name and extension must + * must be given). The -a, -s, -m and -i options are used by worker process + * to receive their name, the base address of shared memory, its path, and + * the process' instance number, respectively. This information is supplied + * by the master process when it execs worker processes under the process + * model of execution. If the worker process arguments are passed then main + * will call the procflow_exec routine which creates worker threadflows and + * flowops and executes the procflow's portion of the workload model until + * completion. If worker process arguments are not passed to the process, + * then it becomes the master process for a filebench run. It initializes + * the various filebench components and either executes the supplied workload + * file, or enters interactive mode. + */ + +int +main(int argc, char *argv[]) +{ + int opt; + int docmd = FS_FALSE; + int instance; + char procname[128]; + caddr_t shmaddr; + char dir[MAXPATHLEN]; +#ifdef HAVE_SETRLIMIT + struct rlimit rlp; +#endif +#ifdef HAVE_LIBTECLA + char *line; +#else + char line[1024]; +#endif + char shmpathtmp[1024]; + +#ifdef HAVE_SETRLIMIT + /* Set resource limits */ + (void) getrlimit(RLIMIT_NOFILE, &rlp); + rlp.rlim_cur = rlp.rlim_max; + setrlimit(RLIMIT_NOFILE, &rlp); +#endif + + yydebug = 0; + execname = argv[0]; + *procname = 0; + cwd = getcwd(dir, MAXPATHLEN); + + while ((opt = getopt(argc, argv, cmd_options)) != (int)EOF) { + + switch (opt) { + case 'h': + usage(2); + break; + + case 'p': + noproc = 1; + break; + + case 'f': + if (optarg == NULL) + usage(1); + if ((yyin = fopen(optarg, "r")) == NULL) { + (void) fprintf(stderr, + "Cannot open file %s", optarg); + exit(1); + } + dofile = FS_TRUE; + fscriptname = optarg; + + break; + + case 'a': + if (optarg == NULL) + usage(1); + sscanf(optarg, "%s", &procname[0]); + break; + + case 's': + if (optarg == NULL) + usage(1); +#if defined(_LP64) || (__WORDSIZE == 64) + sscanf(optarg, "%llx", &shmaddr); +#else + sscanf(optarg, "%x", &shmaddr); +#endif + break; + + case 'm': + if (optarg == NULL) + usage(1); + sscanf(optarg, "%s", shmpathtmp); + shmpath = shmpathtmp; + break; + + case 'i': + if (optarg == NULL) + usage(1); + sscanf(optarg, "%d", &instance); + break; + + case '?': + default: + usage(1); + break; + } + } + +#ifdef USE_PROCESS_MODEL + if (!(*procname)) +#endif + printf("FileBench Version %s\n", FILEBENCH_VERSION); + filebench_init(); + +#ifdef USE_PROCESS_MODEL + if (*procname) { + pid = getpid(); + + if (ipc_attach(shmaddr) < 0) { + filebench_log(LOG_ERROR, "Cannot attach shm for %s", + procname); + exit(1); + } + + if (procflow_exec(procname, instance) < 0) { + filebench_log(LOG_ERROR, "Cannot startup process %s", + procname); + exit(1); + } + exit(1); + } +#endif + + pid = getpid(); + ipc_init(); + + if (fscriptname) + (void) strcpy(filebench_shm->fscriptname, fscriptname); + + flowop_init(); + stats_init(); + eventgen_init(); + + signal(SIGINT, parser_abort); + + if (dofile) + yyparse(); + else { +#ifdef HAVE_LIBTECLA + if ((gl = new_GetLine(MAX_LINE_LEN, MAX_CMD_HIST)) == NULL) { + filebench_log(LOG_ERROR, + "Failed to create GetLine object"); + filebench_shutdown(1); + } + + if (gl_customize_completion(gl, NULL, command_complete)) { + filebench_log(LOG_ERROR, + "Failed to register auto-completion function"); + filebench_shutdown(1); + } + + while (line = gl_get_line(gl, FILEBENCH_PROMPT, NULL, -1)) { + arg_parse(line); + yyparse(); + } + + del_GetLine(gl); +#else + while (!feof(stdin)) { + printf(FILEBENCH_PROMPT); + fflush(stdout); + if (fgets(line, sizeof (line), stdin) == NULL) { + if (errno == EINTR) + continue; + else + break; + } + arg_parse(line); + yyparse(); + } + printf("\n"); +#endif /* HAVE_LIBTECLA */ + } + + parser_filebench_shutdown((cmd_t *)0); + + return (0); +} + +/* + * arg_parse() puts the parser into command parsing mode. Create a tmpfile + * and instruct the parser to read instructions from this location by setting + * yyin to the value returned by tmpfile. Write the command into the file. + * Then seek back to to the start of the file so that the parser can read + * the instructions. + */ +static void +arg_parse(const char *command) +{ + if ((yyin = tmpfile()) == NULL) + filebench_log(LOG_FATAL, + "Cannot create tmpfile: %s", strerror(errno)); + + if (fwrite(command, strlen(command), 1, yyin) != 1) + filebench_log(LOG_FATAL, + "Cannot write tmpfile: %s", strerror(errno)); + + if (fseek(yyin, 0, SEEK_SET) != 0) + filebench_log(LOG_FATAL, + "Cannot seek tmpfile: %s", strerror(errno)); +} + +/* + * Converts a list of var_strings or ordinary strings to a single ordinary + * string. It returns a pointer to the string (in malloc'd memory) if found, + * or NULL otherwise. + */ +char * +parser_list2string(list_t *list) +{ + list_t *l; + char *string; + char *tmp; + vinteger_t *integer; + + if ((string = malloc(MAXPATHLEN)) == NULL) { + filebench_log(LOG_ERROR, "Failed to allocate memory"); + return (NULL); + } + + *string = 0; + + + /* Format args */ + for (l = list; l != NULL; + l = l->list_next) { + filebench_log(LOG_DEBUG_SCRIPT, + "converting string '%s'", *l->list_string); + + if ((tmp = var_to_string(*l->list_string)) != NULL) { + (void) strcat(string, tmp); + free(tmp); + } else { + (void) strcat(string, *l->list_string); + } + } + return (string); +} + +/* + * If the list just contains a single string starting with '$', then find + * or create the named var and return the var's var_string component. + * Otherwise, convert the list to a string, and allocate a var_string + * containing a copy of that string. On failure either returns NULL + * or shuts down the run. + */ +var_string_t +parser_list2varstring(list_t *list) +{ + /* Special case - variable name */ + if ((list->list_next == NULL) && (*(*list->list_string) == '$')) + return (var_ref_string(*list->list_string)); + + return (string_alloc(parser_list2string(list))); +} + +/* + * Looks for the var named in list_string of the first element of the + * supplied list. If found, returns the var_integer portion of the var. + * If the var is not found, cannot be allocated, the supplied list is + * NULL, or the list_string filed is empty, returns NULL. + */ +var_integer_t +parser_list2integer(list_t *list) +{ + var_integer_t v; + + if (list && (*(list->list_string) != NULL)) { + v = var_ref_integer(*(list->list_string)); + return (v); + } + + return (NULL); +} + +/* + * Sets the event generator rate from the attribute supplied with the + * command. If the attribute doesn't exist the routine does nothing. + */ +static void +parser_eventgen(cmd_t *cmd) +{ + attr_t *attr; + vinteger_t rate; + + /* Get the rate from attribute */ + if (attr = get_attr_integer(cmd, FSA_RATE)) { + if (attr->attr_integer) { + filebench_log(LOG_VERBOSE, + "Eventgen: %lld per second", + *attr->attr_integer); + eventgen_setrate(*attr->attr_integer); + } + } + +} + +/* + * Assigns the designated integer variable successive values from the + * supplied comma seperated integer list. After each successive integer + * assignment, it executes the bracket enclosed list of commands. For + * example, repeated runs of a workload with increasing io sizes can + * be done using the following command line: + * foreach $iosize in 2k, 4k, 8k {run 60} + */ +static void +parser_foreach_integer(cmd_t *cmd) +{ + list_t *list = cmd->cmd_param_list; + cmd_t *inner_cmd; + + for (; list != NULL; list = list->list_next) { + var_assign_integer(cmd->cmd_tgt1, *list->list_integer); + filebench_log(LOG_VERBOSE, "Iterating %s=%lld", + cmd->cmd_tgt1, + *list->list_integer); + for (inner_cmd = cmd->cmd_list; inner_cmd != NULL; + inner_cmd = inner_cmd->cmd_next) { + inner_cmd->cmd(inner_cmd); + } + } +} + +/* + * Similar to parser_foreach_integer(), except takes a list of strings after + * the "in" token. For example, to run twice using a different directory, + * perhaps using a different filesystem, the following command line + * could be used: + * foreach $dir in "/ufs_top/fbt", "/zfs_top/fbt" {run 60) + */ +static void +parser_foreach_string(cmd_t *cmd) +{ + list_t *list = cmd->cmd_param_list; + cmd_t *inner_cmd; + + for (; list != NULL; list = list->list_next) { + var_assign_string(cmd->cmd_tgt1, *list->list_string); + filebench_log(LOG_VERBOSE, "Iterating %s=%s", + cmd->cmd_tgt1, + *list->list_string); + for (inner_cmd = cmd->cmd_list; inner_cmd != NULL; + inner_cmd = inner_cmd->cmd_next) { + inner_cmd->cmd(inner_cmd); + } + } +} + +/* + * Lists the file name, path name and file size for all defined file objects. + */ +static void +parser_list(cmd_t *cmd) +{ + (void) fileobj_iter(fileobj_print); +} + +/* + * Calls procflow_define() to allocate "instances" number of procflow(s) + * (processes) with the supplied name. The default number of instances is + * one. An optional priority level attribute can be supplied and is stored in + * pf_nice. Finally the routine loops through the list of inner commands, if + * any, which are defines for threadflows, and passes them one at a time to + * parser_thread_define() to allocate threadflow entities for the process(es). + */ +static void +parser_proc_define(cmd_t *cmd) +{ + procflow_t *procflow, template; + char *name; + attr_t *attr; + var_integer_t instances = integer_alloc(1); + cmd_t *inner_cmd; + + /* Get the name of the process */ + if (attr = get_attr(cmd, FSA_NAME)) { + name = *attr->attr_string; + } else { + filebench_log(LOG_ERROR, + "define proc: proc specifies no name"); + filebench_shutdown(1); + } + + /* Get the memory size from attribute */ + if (attr = get_attr_integer(cmd, FSA_INSTANCES)) { + filebench_log(LOG_DEBUG_IMPL, + "Setting instances = %lld", + *attr->attr_integer); + instances = attr->attr_integer; + } + + if ((procflow = procflow_define(name, NULL, instances)) == NULL) { + filebench_log(LOG_ERROR, + "Failed to instantiate %d %s process(es)\n", + instances, name); + filebench_shutdown(1); + } + + /* Get the pri from attribute */ + if (attr = get_attr_integer(cmd, FSA_NICE)) { + filebench_log(LOG_DEBUG_IMPL, "Setting pri = %lld", + *attr->attr_integer); + procflow->pf_nice = attr->attr_integer; + } else + procflow->pf_nice = integer_alloc(0); + + + /* Create the list of threads for this process */ + for (inner_cmd = cmd->cmd_list; inner_cmd != NULL; + inner_cmd = inner_cmd->cmd_next) { + parser_thread_define(inner_cmd, procflow, *instances); + } +} + +/* + * Calls threadflow_define() to allocate "instances" number of threadflow(s) + * (threads) with the supplied name. The default number of instances is + * one. Two other optional attributes may be supplied, one to set the memory + * size, stored in tf_memsize, and to select the use of Interprocess Shared + * Memory, which sets the THREADFLOW_USEISM flag in tf_attrs. Finally + * the routine loops through the list of inner commands, if any, which are + * defines for flowops, and passes them one at a time to + * parser_flowop_define() to allocate flowop entities for the threadflows. + */ +static void +parser_thread_define(cmd_t *cmd, procflow_t *procflow, int procinstances) +{ + threadflow_t *threadflow, template; + attr_t *attr; + var_integer_t instances = integer_alloc(1); + cmd_t *inner_cmd; + char *name; + + memset(&template, 0, sizeof (threadflow_t)); + + /* Get the name of the thread */ + if (attr = get_attr(cmd, FSA_NAME)) { + name = *attr->attr_string; + } else { + filebench_log(LOG_ERROR, + "define thread: thread in process %s specifies no name", + procflow->pf_name); + filebench_shutdown(1); + } + + /* Get the number of instances from attribute */ + if (attr = get_attr_integer(cmd, FSA_INSTANCES)) { + filebench_log(LOG_DEBUG_IMPL, + "define thread: Setting instances = %lld", + *attr->attr_integer); + instances = attr->attr_integer; + } + + /* Get the memory size from attribute */ + if (attr = get_attr_integer(cmd, FSA_MEMSIZE)) { + filebench_log(LOG_DEBUG_IMPL, + "define thread: Setting memsize = %lld", + *attr->attr_integer); + template.tf_memsize = attr->attr_integer; + } else + template.tf_memsize = integer_alloc(0); + + if ((threadflow = threadflow_define(procflow, name, + &template, instances)) == NULL) { + filebench_log(LOG_ERROR, + "define thread: Failed to instantiate thread\n"); + filebench_shutdown(1); + } + + /* Use ISM Memory? */ + if (attr = get_attr(cmd, FSA_USEISM)) { + threadflow->tf_attrs |= THREADFLOW_USEISM; + } + + /* Create the list of flowops */ + for (inner_cmd = cmd->cmd_list; inner_cmd != NULL; + inner_cmd = inner_cmd->cmd_next) { + parser_flowop_define(inner_cmd, threadflow); + } +} + +/* + * Calls flowop_define() to allocate a flowop with the supplied name. + * The allocated flowop inherits attributes from a base flowop of the + * same type. If the new flowop has a file or fileset attribute specified, + * it must specify a defined fileobj or fileset or an error will be logged. + * The new flowop may also have the following attributes set by + * the program: + * - file size (fo_iosize) + * - working set size (fo_wss) + * - do random io (fo_random) + * - do synchronous io (fo_dsync) + * - perform each operation multiple times before advancing (fo_iter) + * - target name (fo_targetname) + * - An integer value (fo_value) + * - a file descriptor (fo_fd) + * - specify to rotate file descriptors (fo_rotatefd) + * - a source fd (fo_srcfdnumber) + * - specify a blocking operation (fo_blocking) + * - specify a highwater mark (fo_highwater) + * + * After all the supplied attributes are stored in their respective locations + * in the flowop object, the flowop's init function is called. No errors are + * returned, but the filebench run will be terminated if the flowtype is not + * specified, a name for the new flowop is not supplied, the flowop_define + * call fails, or a file or fileset name is supplied but the corresponding + * fileobj or fileset cannot be located. + */ +static void +parser_flowop_define(cmd_t *cmd, threadflow_t *thread) +{ + flowop_t *flowop, *flowop_type; + fileobj_t *fileobj; + char *type = (char *)cmd->cmd_name; + char *name; + attr_t *attr; + + /* Get the inherited flowop */ + flowop_type = flowop_find(type); + if (flowop_type == NULL) { + filebench_log(LOG_ERROR, + "define flowop: flowop type %s not found", + type); + filebench_shutdown(1); + } + + /* Get the name of the flowop */ + if (attr = get_attr(cmd, FSA_NAME)) { + name = *attr->attr_string; + } else { + filebench_log(LOG_ERROR, + "define flowop: flowop %s specifies no name", + flowop_type->fo_name); + filebench_shutdown(1); + } + + if ((flowop = flowop_define(thread, name, + flowop_type, FLOW_MASTER, 0)) == NULL) { + filebench_log(LOG_ERROR, + "define flowop: Failed to instantiate flowop %s\n", + cmd->cmd_name); + filebench_shutdown(1); + } + + /* Get the filename from attribute */ + if (attr = get_attr(cmd, FSA_FILE)) { + flowop->fo_file = fileobj_find(*attr->attr_string); + flowop->fo_fileset = fileset_find(*attr->attr_string); + + if ((flowop->fo_file == NULL) && + (flowop->fo_fileset == NULL)) { + filebench_log(LOG_ERROR, + "define flowop: file %s not found", + *attr->attr_string); + filebench_shutdown(1); + } + } + + /* Get the iosize of the op */ + if (attr = get_attr_integer(cmd, FSA_IOSIZE)) + flowop->fo_iosize = attr->attr_integer; + else + flowop->fo_iosize = integer_alloc(0); + + /* Get the working set size of the op */ + if (attr = get_attr_integer(cmd, FSA_WSS)) + flowop->fo_wss = attr->attr_integer; + else + flowop->fo_wss = integer_alloc(0); + + /* Random I/O? */ + if (attr = get_attr_bool(cmd, FSA_RANDOM)) + flowop->fo_random = attr->attr_integer; + else + flowop->fo_random = integer_alloc(0); + + /* Sync I/O? */ + if (attr = get_attr_bool(cmd, FSA_DSYNC)) + flowop->fo_dsync = attr->attr_integer; + else + flowop->fo_dsync = integer_alloc(0); + + /* Iterations */ + if (attr = get_attr_integer(cmd, FSA_ITERS)) + flowop->fo_iters = attr->attr_integer; + else + flowop->fo_iters = integer_alloc(1); + + + /* Target, for wakeup etc */ + if (attr = get_attr(cmd, FSA_TARGET)) + (void) strcpy(flowop->fo_targetname, *attr->attr_string); + + /* Value */ + if (attr = get_attr_integer(cmd, FSA_VALUE)) + flowop->fo_value = attr->attr_integer; + else + flowop->fo_value = integer_alloc(0); + + /* FD */ + if (attr = get_attr_integer(cmd, FSA_FD)) + flowop->fo_fdnumber = *attr->attr_integer; + + /* Rotatefd? */ + if (attr = get_attr_bool(cmd, FSA_ROTATEFD)) + flowop->fo_rotatefd = attr->attr_integer; + else + flowop->fo_rotatefd = integer_alloc(0); + + /* SRC FD, for copies etc... */ + if (attr = get_attr_integer(cmd, FSA_SRCFD)) + flowop->fo_srcfdnumber = *attr->attr_integer; + + /* Blocking operation? */ + if (attr = get_attr_bool(cmd, FSA_BLOCKING)) + flowop->fo_blocking = attr->attr_integer; + else + flowop->fo_blocking = integer_alloc(0); + + /* Blocking operation? */ + if (attr = get_attr_bool(cmd, FSA_DIRECTIO)) + flowop->fo_directio = attr->attr_integer; + else + flowop->fo_directio = integer_alloc(0); + + /* Highwater mark */ + if (attr = get_attr_integer(cmd, FSA_HIGHWATER)) + flowop->fo_highwater = attr->attr_integer; + else + flowop->fo_highwater = integer_alloc(1); +} + +/* + * Calls fileobj_define() to allocate a fileobj with the supplied name + * and initializes the fileobj's pathname attribute, fo_path and fo_create, + * and optionally the fo_prealloc, fo_paralloc, fo_reuse, fo_cached, + * and fo_size attributes. + */ +static void +parser_file_define(cmd_t *cmd) +{ + fileobj_t *fileobj; + char *name; + attr_t *attr; + var_string_t pathname; + + /* Get the name of the file */ + if (attr = get_attr(cmd, FSA_NAME)) { + name = *attr->attr_string; + } else { + filebench_log(LOG_ERROR, + "define file: file specifies no name"); + return; + } + + if ((fileobj = fileobj_define(name)) == NULL) { + filebench_log(LOG_ERROR, + "define file: failed to instantiate file %s\n", + cmd->cmd_name); + return; + } + + /* Get the pathname from attribute */ + if ((attr = get_attr(cmd, FSA_PATH)) == NULL) { + filebench_log(LOG_ERROR, + "define file: no pathname specified"); + return; + } + + /* Expand variables in pathname */ + if ((pathname = parser_list2varstring(attr->attr_param_list)) + == NULL) { + filebench_log(LOG_ERROR, "Cannot interpret path"); + return; + } + + fileobj->fo_path = pathname; + + /* For now, all files are pre-created */ + fileobj->fo_create = integer_alloc(1); + + /* Should we preallocate? */ + if (attr = get_attr_bool(cmd, FSA_PREALLOC)) { + fileobj->fo_prealloc = attr->attr_integer; + } else + fileobj->fo_prealloc = integer_alloc(0); + + /* Should we prealloc in parallel? */ + if (attr = get_attr_bool(cmd, FSA_PARALLOC)) { + fileobj->fo_paralloc = attr->attr_integer; + } else + fileobj->fo_paralloc = integer_alloc(0); + + /* Should we reuse the existing file? */ + if (attr = get_attr_bool(cmd, FSA_REUSE)) { + fileobj->fo_reuse = attr->attr_integer; + } else + fileobj->fo_reuse = integer_alloc(0); + + /* Should we leave in cache? */ + if (attr = get_attr_bool(cmd, FSA_CACHED)) { + fileobj->fo_cached = attr->attr_integer; + } else + fileobj->fo_cached = integer_alloc(0); + + /* Get the size of the file */ + if (attr = get_attr_integer(cmd, FSA_SIZE)) { + fileobj->fo_size = attr->attr_integer; + } else + fileobj->fo_size = integer_alloc(0); + +} + +/* + * Calls fileset_define() to allocate a fileset with the supplied name and + * initializes the fileset's pathname attribute, and optionally the fs_cached, + * fs_reuse, fs_preallocpercent, fs_prealloc, fs_entries, fs_dirwidth, + * fs_size, fs_dirgamma, and fs_sizegamma attributes. + */ +static void +parser_fileset_define(cmd_t *cmd) +{ + fileset_t *fileset; + char *name; + attr_t *attr; + var_string_t pathname; + + /* Get the name of the file */ + if (attr = get_attr(cmd, FSA_NAME)) { + name = *attr->attr_string; + } else { + filebench_log(LOG_ERROR, + "define file: file specifies no name"); + return; + } + + if ((fileset = fileset_define(name)) == NULL) { + filebench_log(LOG_ERROR, + "define file: failed to instantiate file %s\n", + cmd->cmd_name); + return; + } + + /* Get the pathname from attribute */ + if ((attr = get_attr(cmd, FSA_PATH)) == NULL) { + filebench_log(LOG_ERROR, "define file: no pathname specified"); + return; + } + + /* Expand variables in pathname */ + if ((pathname = parser_list2varstring(attr->attr_param_list)) == NULL) { + filebench_log(LOG_ERROR, "Cannot interpret path"); + return; + } + + fileset->fs_path = pathname; + + /* Should we leave in cache? */ + if (attr = get_attr_bool(cmd, FSA_CACHED)) { + fileset->fs_cached = attr->attr_integer; + } else + fileset->fs_cached = integer_alloc(0); + + /* Should we reuse the existing file? */ + if (attr = get_attr_bool(cmd, FSA_REUSE)) { + fileset->fs_reuse = attr->attr_integer; + } else + fileset->fs_reuse = integer_alloc(0); + + /* How much should we prealloc */ + if ((attr = get_attr_integer(cmd, FSA_PREALLOC)) && + attr->attr_integer) { + fileset->fs_preallocpercent = attr->attr_integer; + } else if (attr && !attr->attr_integer) { + fileset->fs_preallocpercent = integer_alloc(100); + } else { + fileset->fs_preallocpercent = integer_alloc(0); + } + + /* Should we preallocate? */ + if (attr = get_attr_bool(cmd, FSA_PREALLOC)) { + fileset->fs_prealloc = attr->attr_integer; + } else + fileset->fs_prealloc = integer_alloc(0); + + /* Get the size of the fileset */ + if (attr = get_attr_integer(cmd, FSA_ENTRIES)) { + fileset->fs_entries = attr->attr_integer; + } else { + filebench_log(LOG_ERROR, "Fileset has zero entries"); + fileset->fs_entries = integer_alloc(0); + } + + /* Get the mean dir width of the fileset */ + if (attr = get_attr_integer(cmd, FSA_DIRWIDTH)) { + fileset->fs_dirwidth = attr->attr_integer; + } else { + filebench_log(LOG_ERROR, "Fileset has zero directory width"); + fileset->fs_dirwidth = integer_alloc(0); + } + + /* Get the mean or absolute size of the file */ + if (attr = get_attr_integer(cmd, FSA_SIZE)) { + fileset->fs_size = attr->attr_integer; + } else + fileset->fs_size = integer_alloc(0); + + /* Get the gamma value for dir width distributions */ + if (attr = get_attr_integer(cmd, FSA_DIRGAMMA)) { + fileset->fs_dirgamma = attr->attr_integer; + } else + fileset->fs_dirgamma = integer_alloc(1500); + + /* Get the gamma value for dir width distributions */ + if (attr = get_attr_integer(cmd, FSA_FILESIZEGAMMA)) { + fileset->fs_sizegamma = attr->attr_integer; + } else + fileset->fs_sizegamma = integer_alloc(1500); +} + +/* + * Creates and starts all defined procflow processes. The call to + * procflow_init() results in creation of the requested number of + * process instances for each previously defined procflow. The + * child processes exec() a new instance of filebench, passing it + * the instance number and address of the shared memory region. + * The child processes will then create their threads and flowops. + * The routine then unlocks the run_lock to allow all the processes' + * threads to start and waits for all of them to begin execution. + * Finally, it records the start time and resets the event generation + * system. + */ +static void +parser_proc_create(cmd_t *cmd) +{ + if (procflow_init() != 0) { + filebench_log(LOG_ERROR, "Failed to create processes\n"); + filebench_shutdown(1); + } + + /* Release the read lock, allowing threads to start */ + (void) pthread_rwlock_unlock(&filebench_shm->run_lock); + + /* Wait for all threads to start */ + if (procflow_allstarted() != 0) { + filebench_log(LOG_ERROR, "Could not start run"); + return; + } + + + if (filebench_shm->shm_required && + (ipc_ismcreate(filebench_shm->shm_required) < 0)) { + filebench_log(LOG_ERROR, "Could not allocate shared memory"); + return; + } + + filebench_shm->starttime = gethrtime(); + eventgen_reset(); +} + +/* + * Calls fileobj_init() to create and optionally pre fill files + * for all fileobjs on the master list of fileobjs (filelist). + * If errors are encountered, calls filebench_shutdown() + * to exit filebench. + */ +static void +parser_file_create(cmd_t *cmd) +{ + fileobj_t *fileobj; + + if (fileobj_init() != 0) { + filebench_log(LOG_ERROR, "Failed to create files"); + filebench_shutdown(1); + } +} + +/* + * Calls fileset_createset() to populate all filesets and create all + * associated, initially existant, files and subdirectories. + * If errors are encountered, calls filebench_shutdown() + * to exit filebench. + */ +static void +parser_fileset_create(cmd_t *cmd) +{ + fileset_t *fileset; + + if (fileset_createset(NULL) != 0) { + filebench_log(LOG_ERROR, "Failed to create filesets"); + filebench_shutdown(1); + } +} + +/* + * Shuts down all processes and their associated threads. When finished + * it deletes interprocess shared memory and resets the event generator. + * It does not exit the filebench program though. + */ +static void +parser_proc_shutdown(cmd_t *cmd) +{ + filebench_log(LOG_INFO, "Shutting down processes"); + procflow_shutdown(); + if (filebench_shm->shm_required) + ipc_ismdelete(); + eventgen_reset(); +} + +/* + * Ends filebench run after first destoring any interprocess + * shared memory. The call to filebench_shutdown() + * also causes filebench to exit. + */ +static void +parser_filebench_shutdown(cmd_t *cmd) +{ + ipc_cleanup(); + filebench_shutdown(1); +} + +/* + * Sleeps for cmd->cmd_qty seconds, one second at a time. + */ +static void +parser_sleep(cmd_t *cmd) +{ + int sleeptime; + + /* check for startup errors */ + if (filebench_shm->f_abort) + return; + + sleeptime = cmd->cmd_qty; + filebench_log(LOG_INFO, "Running..."); + while (sleeptime) { + (void) sleep(1); + sleeptime--; + if (filebench_shm->f_abort) + break; + } + filebench_log(LOG_INFO, "Run took %lld seconds...", + cmd->cmd_qty - sleeptime); +} + +/* + * Do a file bench run. Calls routines to create file sets, files, and + * processes. It resets the statistics counters, then sleeps for the runtime + * passed as an argument to it on the command line in 1 second increments. + * When it is finished sleeping, it collects a snapshot of the statistics + * and ends the run. + */ +static void +parser_run(cmd_t *cmd) +{ + int runtime; + + runtime = cmd->cmd_qty; + parser_fileset_create(cmd); + parser_file_create(cmd); + parser_proc_create(cmd); + + /* check for startup errors */ + if (filebench_shm->f_abort) + return; + + filebench_log(LOG_INFO, "Running..."); + stats_clear(); + while (runtime) { + (void) sleep(1); + runtime--; + if (filebench_shm->f_abort) + break; + } + filebench_log(LOG_INFO, "Run took %lld seconds...", + cmd->cmd_qty - runtime); + parser_statssnap(cmd); + parser_proc_shutdown(cmd); +} + +/* + * Similar to parser_run, but gets the sleep time from a variable + * whose name is supplied as an argument to the command. + */ +static void +parser_run_variable(cmd_t *cmd) +{ + vinteger_t *integer = var_ref_integer(cmd->cmd_tgt1); + int runtime; + + if (integer == NULL) { + filebench_log(LOG_ERROR, "Unknown variable %s", + cmd->cmd_tgt1); + return; + } + + runtime = *integer; + + /* check for startup errors */ + if (filebench_shm->f_abort) + return; + + filebench_log(LOG_INFO, "Running..."); + stats_clear(); + while (runtime) { + (void) sleep(1); + runtime--; + if (filebench_shm->f_abort) + break; + } + filebench_log(LOG_INFO, "Run took %lld seconds...", + *integer - runtime); + parser_statssnap(cmd); +} + +char *usagestr = NULL; + +/* + * Prints usage string if defined, else just a message requesting load of a + * personality. + */ +static void +parser_help(cmd_t *cmd) +{ + int runtime; + + if (usagestr) { + filebench_log(LOG_INFO, "%s", usagestr); + } else { + filebench_log(LOG_INFO, + "load <personality> (ls " + "/usr/benchmarks/filebench/workloads for list)"); + } +} + +char *varstr = NULL; + +/* + * Prints the string of all var definitions, if there is one. + */ +static void +parser_printvars(cmd_t *cmd) +{ + int runtime; + char *str, *c; + + if (varstr) { + str = strdup(varstr); + for (c = str; *c != '\0'; c++) { + if ((char)*c == '$') + *c = ' '; + } + filebench_log(LOG_INFO, "%s", str); + free(str); + } +} + +/* + * Used by the SET command to add a var and default value string to the + * varstr string. It allocates a new, larger varstr string, copies the + * old contents of varstr into it, then adds the new var string on the end. + */ +static void +parser_vars(cmd_t *cmd) +{ + char *string = cmd->cmd_tgt1; + char *newvars; + + if (string == NULL) + return; + + if (dofile) + return; + + if (varstr == NULL) { + newvars = malloc(strlen(string) + 2); + *newvars = 0; + } else { + newvars = malloc(strlen(varstr) + strlen(string) + 2); + (void) strcpy(newvars, varstr); + } + (void) strcat(newvars, string); + (void) strcat(newvars, " "); + + if (varstr) + free(varstr); + + varstr = newvars; +} + +/* + * Same as parser_sleep, except the sleep time is obtained from a variable + * whose name is passed to it as an argument on the command line. + */ +static void +parser_sleep_variable(cmd_t *cmd) +{ + vinteger_t *integer = var_ref_integer(cmd->cmd_tgt1); + int sleeptime; + + if (integer == NULL) { + filebench_log(LOG_ERROR, "Unknown variable %s", + cmd->cmd_tgt1); + return; + } + + sleeptime = *integer; + + /* check for startup errors */ + if (filebench_shm->f_abort) + return; + + filebench_log(LOG_INFO, "Running..."); + while (sleeptime) { + (void) sleep(1); + sleeptime--; + if (filebench_shm->f_abort) + break; + } + filebench_log(LOG_INFO, "Run took %lld seconds...", + *integer - sleeptime); +} + +/* + * Parser log prints the values of a list of variables to the log file. + * The list of variables is placed on the command line, separated + * by comas and the entire list is enclosed in quotes. + * For example, if $dir contains "/export/home/tmp" and $filesize = 1048576, + * then typing: log "$dir, $filesize" prints: log /export/home/tmp, 1048576 + */ +static void +parser_log(cmd_t *cmd) +{ + char *string; + + if (cmd->cmd_param_list == NULL) + return; + + string = parser_list2string(cmd->cmd_param_list); + + if (string == NULL) + return; + + filebench_log(LOG_VERBOSE, "log %s", string); + filebench_log(LOG_LOG, "%s", string); +} + +/* + * Implements the stats directory command. changes the directory for + * dumping statistics to supplied directory path. For example: + * stats directory /tmp + * changes the stats directory to "/tmp". + */ +static void +parser_directory(cmd_t *cmd) +{ + char newdir[MAXPATHLEN]; + char *dir; + + if ((dir = parser_list2string(cmd->cmd_param_list)) == NULL) { + filebench_log(LOG_ERROR, "Cannot interpret directory"); + return; + } + + *newdir = 0; + /* Change dir relative to cwd if path not fully qualified */ + if (*dir != '/') { + (void) strcat(newdir, cwd); + (void) strcat(newdir, "/"); + } + (void) strcat(newdir, dir); + (void) mkdir(newdir, 0755); + filebench_log(LOG_VERBOSE, "Change dir to %s", newdir); + chdir(newdir); + free(dir); +} + +#define PIPE_PARENT 1 +#define PIPE_CHILD 0 + +/* + * Runs the quoted unix command as a background process. Intended for + * running statistics gathering utilities such as mpstat while the filebench + * workload is running. Also records the pid's of the background processes + * so that parser_statssnap() can terminate them when the run completes. + */ +static void +parser_statscmd(cmd_t *cmd) +{ + char *string; + pid_t pid; + pidlist_t *pidlistent; + int pipe_fd[2]; + int newstdout; + + if (cmd->cmd_param_list == NULL) + return; + + string = parser_list2string(cmd->cmd_param_list); + + if (string == NULL) + return; + + if ((pipe(pipe_fd)) < 0) { + filebench_log(LOG_ERROR, "statscmd pipe failed"); + return; + } + +#ifdef HAVE_FORK1 + if ((pid = fork1()) < 0) { + filebench_log(LOG_ERROR, "statscmd fork failed"); + return; + } +#elif HAVE_FORK + if ((pid = fork()) < 0) { + filebench_log(LOG_ERROR, "statscmd fork failed"); + return; + } +#else + Crash! - Need code to deal with no fork1! +#endif /* HAVE_FORK1 */ + + if (pid == 0) { + + setsid(); + + filebench_log(LOG_VERBOSE, + "Backgrounding %s", string); + /* + * Child + * - close stdout + * - dup to create new stdout + * - close pipe fds + */ + (void) close(1); + + if ((newstdout = dup(pipe_fd[PIPE_CHILD])) < 0) { + filebench_log(LOG_ERROR, + "statscmd dup failed: %s", + strerror(errno)); + } + + (void) close(pipe_fd[PIPE_PARENT]); + (void) close(pipe_fd[PIPE_CHILD]); + + if (system(string) < 0) { + filebench_log(LOG_ERROR, + "statscmd exec failed: %s", + strerror(errno)); + } + /* Failed! */ + exit(1); + + } else { + + /* Record pid in pidlist for subsequent reaping by stats snap */ + if ((pidlistent = (pidlist_t *)malloc(sizeof (pidlist_t))) + == NULL) { + filebench_log(LOG_ERROR, "pidlistent malloc failed"); + return; + } + + pidlistent->pl_pid = pid; + pidlistent->pl_fd = pipe_fd[PIPE_PARENT]; + (void) close(pipe_fd[PIPE_CHILD]); + + /* Add fileobj to global list */ + if (pidlist == NULL) { + pidlist = pidlistent; + pidlistent->pl_next = NULL; + } else { + pidlistent->pl_next = pidlist; + pidlist = pidlistent; + } + } +} + +/* + * Launches a shell to run the unix command supplied in the argument. + * The command should be enclosed in quotes, as in: + * system "rm xyz" + * which would run the "rm" utility to delete the file "xyz". + */ +static void +parser_system(cmd_t *cmd) +{ + char *string; + + if (cmd->cmd_param_list == NULL) + return; + + string = parser_list2string(cmd->cmd_param_list); + + if (string == NULL) + return; + + filebench_log(LOG_VERBOSE, + "Running '%s'", string); + + if (system(string) < 0) { + filebench_log(LOG_ERROR, + "system exec failed: %s", + strerror(errno)); + } + free(string); +} + +/* + * Echos string supplied with command to the log. + */ +static void +parser_echo(cmd_t *cmd) +{ + char *string; + + if (cmd->cmd_param_list == NULL) + return; + + string = parser_list2string(cmd->cmd_param_list); + + if (string == NULL) + return; + + filebench_log(LOG_INFO, "%s", string); +} + + +/* + * Adds the string supplied as the argument to the usage command + * to the end of the string printed by the help command. + */ +static void +parser_usage(cmd_t *cmd) +{ + char *string; + char *newusage; + + if (cmd->cmd_param_list == NULL) + return; + + string = parser_list2string(cmd->cmd_param_list); + + if (string == NULL) + return; + + if (dofile) + return; + + if (usagestr == NULL) { + newusage = malloc(strlen(string) + 2); + *newusage = 0; + } else { + newusage = malloc(strlen(usagestr) + strlen(string) + 2); + (void) strcpy(newusage, usagestr); + } + (void) strcat(newusage, "\n"); + (void) strcat(newusage, string); + + if (usagestr) + free(usagestr); + + usagestr = newusage; + + filebench_log(LOG_INFO, "%s", string); +} + +/* + * Updates the global dump filename with the filename supplied + * as the command's argument. Then dumps the statistics of each + * worker flowop into the dump file, followed by a summary of + * overall totals. + */ +static void +parser_statsdump(cmd_t *cmd) +{ + char *string; + + if (cmd->cmd_param_list == NULL) + return; + + string = parser_list2string(cmd->cmd_param_list); + + if (string == NULL) + return; + + filebench_log(LOG_VERBOSE, + "Stats dump to file '%s'", string); + + stats_dump(string); + + free(string); +} + +/* + * Same as parser_statsdump, but in xml format. + */ +static void +parser_statsxmldump(cmd_t *cmd) +{ + char *string; + + if (cmd->cmd_param_list == NULL) + return; + + string = parser_list2string(cmd->cmd_param_list); + + if (string == NULL) + return; + + filebench_log(LOG_VERBOSE, + "Stats dump to file '%s'", string); + + stats_xmldump(string); + + free(string); +} + +/* + * Kills off background statistics collection processes, then takes a snapshot + * of the filebench run's collected statistics using stats_snap() from + * stats.c. + */ +static void +parser_statssnap(cmd_t *cmd) +{ + pidlist_t *pidlistent; + int stat; + pid_t pid; + + for (pidlistent = pidlist; pidlistent != NULL; + pidlistent = pidlistent->pl_next) { + filebench_log(LOG_VERBOSE, "Killing session %d for pid %d", + getsid(pidlistent->pl_pid), + pidlistent->pl_pid); + if (pidlistent->pl_fd) + (void) close(pidlistent->pl_fd); +#ifdef HAVE_SIGSEND + sigsend(P_SID, getsid(pidlistent->pl_pid), SIGTERM); +#else + (void) kill(-1, SIGTERM); +#endif + + /* Close pipe */ + if (pidlistent->pl_fd) + (void) close(pidlistent->pl_fd); + + /* Wait for cmd and all its children */ + while ((pid = waitpid(pidlistent->pl_pid * -1, &stat, 0)) > 0) + filebench_log(LOG_DEBUG_IMPL, + "Waited for pid %lld", pid); + } + + for (pidlistent = pidlist; pidlistent != NULL; + pidlistent = pidlistent->pl_next) { + free(pidlistent); + } + + pidlist = NULL; + stats_snap(); +} + +/* + * Shutdown filebench. + */ +static void +parser_abort(int arg) +{ + (void) sigignore(SIGINT); + filebench_log(LOG_INFO, "Aborting..."); + filebench_shutdown(1); +} + +/* + * alloc_cmd() allocates the required resources for a cmd_t. On failure, a + * filebench_log is issued and NULL is returned. + */ +static cmd_t * +alloc_cmd(void) +{ + cmd_t *cmd; + + if ((cmd = malloc(sizeof (cmd_t))) == NULL) { + filebench_log(LOG_ERROR, "Alloc cmd failed"); + return (NULL); + } + + (void) memset(cmd, 0, sizeof (cmd_t)); + + return (cmd); +} + +/* + * Frees the resources of a cmd_t and then the cmd_t "cmd" itself. + */ +static void +free_cmd(cmd_t *cmd) +{ + free((void *)cmd->cmd_tgt1); + free((void *)cmd->cmd_tgt2); + free(cmd); +} + +/* + * Allocates an attr_t structure and zeros it. Returns NULL on failure, or + * a pointer to the attr_t. + */ +static attr_t * +alloc_attr() +{ + attr_t *attr; + + if ((attr = malloc(sizeof (attr_t))) == NULL) { + return (NULL); + } + + (void) memset(attr, 0, sizeof (attr_t)); + return (attr); +} + +/* + * Searches the attribute list for the command for the named attribute type. + * The attribute list is created by the parser from the list of attributes + * supplied with certain commands, such as the define and flowop commands. + * Returns a pointer to the attribute structure if the named attribute is + * found, otherwise returns NULL. If the attribute includes a parameter list, + * the list is converted to a string and stored in the attr_string field of + * the returned attr_t struct. + */ +static attr_t * +get_attr(cmd_t *cmd, int64_t name) +{ + attr_t *attr; + attr_t *rtn = NULL; + char *string; + + for (attr = cmd->cmd_attr_list; attr != NULL; + attr = attr->attr_next) { + filebench_log(LOG_DEBUG_IMPL, + "attr %d = %d %llx?", + attr->attr_name, + name, + attr->attr_integer); + + if (attr->attr_name == name) + rtn = attr; + } + + if (rtn == NULL) + return (NULL); + + if (rtn->attr_param_list) { + filebench_log(LOG_DEBUG_SCRIPT, "attr is param list"); + string = parser_list2string(rtn->attr_param_list); + if (string != NULL) { + rtn->attr_string = string_alloc(string); + filebench_log(LOG_DEBUG_SCRIPT, + "attr string %s", string); + } + } + + return (rtn); +} + +/* + * Similar to get_attr, but converts the parameter string supplied with the + * named attribute to an integer and stores the integer in the attr_integer + * portion of the returned attr_t struct. + */ +static attr_t * +get_attr_integer(cmd_t *cmd, int64_t name) +{ + attr_t *attr; + attr_t *rtn = NULL; + + for (attr = cmd->cmd_attr_list; attr != NULL; + attr = attr->attr_next) { + if (attr->attr_name == name) + rtn = attr; + } + + if (rtn == NULL) + return (NULL); + + if (rtn->attr_param_list) { + rtn->attr_integer = parser_list2integer(rtn->attr_param_list); + } + + return (rtn); +} + +/* + * Similar to get_attr, but converts the parameter string supplied with the + * named attribute to an integer and stores the integer in the attr_integer + * portion of the returned attr_t struct. If no parameter string is supplied + * then it defaults to TRUE (1). + */ +static attr_t * +get_attr_bool(cmd_t *cmd, int64_t name) +{ + attr_t *attr; + attr_t *rtn = NULL; + + for (attr = cmd->cmd_attr_list; attr != NULL; + attr = attr->attr_next) { + if (attr->attr_name == name) + rtn = attr; + } + + if (rtn == NULL) + return (NULL); + + if (rtn->attr_param_list) { + rtn->attr_integer = parser_list2integer(rtn->attr_param_list); + } else if (rtn->attr_integer == 0) { + rtn->attr_integer = integer_alloc(1); + } + + return (rtn); +} + +/* + * Allocates memory for a list_t structure, initializes it to zero, and + * returns a pointer to it. On failure, returns NULL. + */ +static list_t * +alloc_list() +{ + list_t *list; + + if ((list = malloc(sizeof (list_t))) == NULL) { + return (NULL); + } + + (void) memset(list, 0, sizeof (list_t)); + return (list); +} + + +#define USAGE1 \ +"Usage:\n" \ +"%s: interpret f script and generate file workload\n" \ +"Options:\n" \ +" [-h] Display verbose help\n" \ +" [-p] Disable opening /proc to set uacct to enable truss\n" + +#define PARSER_CMDS \ +"create [files|filesets|processes]\n" \ +"stats [clear|snap]\n" \ +"stats command \"shell command $var1,$var2...\"\n" \ +"stats directory <directory>\n" \ +"sleep <sleep-value>\n" \ +"quit\n\n" \ +"Variables:\n" \ +"set $var = value\n" \ +" $var - regular variables\n" \ +" ${var} - internal special variables\n" \ +" $(var) - environment variables\n\n" + +#define PARSER_EXAMPLE \ +"Example:\n\n" \ +"#!/usr/bin/filebench -f\n" \ +"\n" \ +"define file name=bigfile,path=bigfile,size=1g,prealloc,reuse\n" \ +"define process name=randomizer\n" \ +"{\n" \ +" thread random-thread procname=randomizer\n" \ +" {\n" \ +" flowop read name=random-read,filename=bigfile,iosize=16k,random\n" \ +" }\n" \ +"}\n" \ +"create files\n" \ +"create processes\n" \ +"stats clear\n" \ +"sleep 30\n" \ +"stats snap\n" + +/* + * usage() display brief or verbose help for the filebench(1) command. + */ +static void +usage(int help) +{ + if (help >= 1) + (void) fprintf(stderr, USAGE1, cmdname); + if (help >= 2) { + + (void) fprintf(stderr, + "\n'f' language definition:\n\n"); + fileobj_usage(); + fileset_usage(); + procflow_usage(); + threadflow_usage(); + flowoplib_usage(); + eventgen_usage(); + (void) fprintf(stderr, PARSER_CMDS); + (void) fprintf(stderr, PARSER_EXAMPLE); + } + exit(E_USAGE); +} + +int +yywrap() +{ + char buf[1024]; + + if (parentscript) { + yyin = parentscript; + yy_switchfilescript(yyin); + parentscript = NULL; + return (0); + } else + return (1); +} diff --git a/usr/src/cmd/filebench/common/parser_lex.l b/usr/src/cmd/filebench/common/parser_lex.l new file mode 100644 index 0000000000..586179774e --- /dev/null +++ b/usr/src/cmd/filebench/common/parser_lex.l @@ -0,0 +1,301 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +%{ +#pragma ident "%Z%%M% %I% %E% SMI" +%} + +%{ + +#include <stdlib.h> +#include <sys/types.h> +#include <assert.h> +#include <string.h> +#include <errno.h> +#ifdef HAVE_STDINT_H +#include <stdint.h> +#endif +#include "parsertypes.h" +#include "utils.h" +#include "filebench.h" + +#include "parser_gram.h" + +int lex_lineno = 1; /* line-number for error reporting */ +extern void yyerror(char *s); +extern int dofile; /* are we processing a file? */ +%} + +%s WHITESTRINGSTATE + +%a 50000 +%p 50000 +%o 50000 +%n 5000 + +%% + +\n { lex_lineno++; } + +<INITIAL>[ \t]+ ; + +<INITIAL>#.* ; + +eventgen { return FSC_EVENTGEN; } +create { return FSC_CREATE; } +define { return FSC_DEFINE; } +debug { return FSC_DEBUG; } +echo { return FSC_ECHO; } +exit { return FSC_QUIT; } +foreach { return FSC_FOREACH; } +flowop { return FSC_FLOWOP; } +help { return FSC_HELP; } +list { return FSC_LIST; } +load { return FSC_LOAD; } +log { return FSC_LOG; } +run { return FSC_RUN; } +set { return FSC_SET; } +shutdown { return FSC_SHUTDOWN; } +sleep { return FSC_SLEEP; } +stats { return FSC_STATS; } +system { return FSC_SYSTEM; } +usage { return FSC_USAGE; } +vars { return FSC_VARS; } +quit { return FSC_QUIT; } + +file[s]* { return FSE_FILE; } +fileset[s]* { return FSE_FILESET; } +directory { return FSE_DIRECTORY; } +command { return FSE_COMMAND; } +process[es]* { return FSE_PROC; } +thread { return FSE_THREAD; } +clear { return FSE_CLEAR; } +snap { return FSE_SNAP; } +dump { return FSE_DUMP; } +xmldump { return FSE_XMLDUMP; } +all { return FSE_ALL; } + +cached { return FSA_CACHED; } +dirwidth { return FSA_DIRWIDTH; } +dirgamma { return FSA_DIRGAMMA; } +namelength { return FSA_NAMELENGTH; } +filesize { return FSA_FILESIZE; } +filesizegamma { return FSA_FILESIZEGAMMA; } +directio { return FSA_DIRECTIO; } +dsync { return FSA_DSYNC; } +fd { return FSA_FD; } +srcfd { return FSA_SRCFD; } +opennext { return FSA_ROTATEFD; } +filename { return FSA_FILE; } +filesetname { return FSA_FILE; } +instances { return FSA_INSTANCES;} +iosize { return FSA_IOSIZE; } +iters { return FSA_ITERS;} +memsize { return FSA_MEMSIZE; } +name { return FSA_NAME;} +nice { return FSA_NICE;} +entries { return FSA_ENTRIES;} +prealloc { return FSA_PREALLOC; } +paralloc { return FSA_PARALLOC; } +reuse { return FSA_REUSE; } +path { return FSA_PATH; } +procname { return FSA_PROCESS; } +random { return FSA_RANDOM;} +rate { return FSA_RATE;} +size { return FSA_SIZE; } +target { return FSA_TARGET;} +useism { return FSA_USEISM;} +value { return FSA_VALUE;} +workingset { return FSA_WSS; } +blocking { return FSA_BLOCKING; } +highwater { return FSA_HIGHWATER; } + +<INITIAL>\" { + BEGIN WHITESTRINGSTATE; + return FSK_QUOTE; + } + +<WHITESTRINGSTATE>\" { + BEGIN INITIAL; + return FSK_QUOTE; + } + +<WHITESTRINGSTATE>[^$^\\^"][^$^"]*[^\\^$^"] { + if ((yylval.sval = strdup(yytext)) == NULL) { + yyerror("Out of memory"); + filebench_shutdown(E_ERROR); + } + return FSV_WHITESTRING; + } + +<WHITESTRINGSTATE>\\n { + yylval.sval = "\n"; + return FSV_WHITESTRING; + } + + +<WHITESTRINGSTATE>\\$[^"^$^\\]+ { + if ((yylval.sval = strdup(yytext + 1)) == NULL) { + yyerror("Out of memory"); + filebench_shutdown(E_ERROR); + } + return FSV_WHITESTRING; + } + +<WHITESTRINGSTATE>[^$^\\^"] { + if ((yylval.sval = strdup(yytext)) == NULL) { + yyerror("Out of memory"); + filebench_shutdown(E_ERROR); + } + return FSV_WHITESTRING; + } + + +<INITIAL>\{ { return FSK_OPENLST; } +<INITIAL>\} { return FSK_CLOSELST; } +<INITIAL>= { return FSK_ASSIGN; } +<INITIAL>\, { return FSK_SEPLST; } +<INITIAL>in { return FSK_IN; } + +<INITIAL>[0-9]+ { + errno = 0; + yylval.ival = strtoll(yytext, NULL, 10); + if (errno == EINVAL || errno == ERANGE) { + (void) filebench_log(LOG_ERROR, + "Invalid I value '%s':%s", yytext, + strerror(errno)); + } + return FSV_VAL_INT; +} + +<INITIAL>[0-9]+k { + errno = 0; + yylval.ival = KB * strtoll(yytext, NULL, 10); + if (errno == EINVAL || errno == ERANGE) { + (void) filebench_log(LOG_ERROR, + "Invalid I value '%s':%s", yytext, + strerror(errno)); + } + return FSV_VAL_INT; +} + +<INITIAL>[0-9]+m { + errno = 0; + yylval.ival = MB * strtoll(yytext, NULL, 10); + if (errno == EINVAL || errno == ERANGE) { + (void) filebench_log(LOG_ERROR, + "Invalid I value '%s':%s", yytext, + strerror(errno)); + } + return FSV_VAL_INT; +} + +<INITIAL>[0-9]+g { + errno = 0; + yylval.ival = GB * strtoll(yytext, NULL, 10); + if (errno == EINVAL || errno == ERANGE) { + (void) filebench_log(LOG_ERROR, + "Invalid I value '%s':%s", yytext, + strerror(errno)); + } + return FSV_VAL_INT; +} + +<INITIAL>true|false { + if (strcmp(yytext, "true") == 0) + yylval.bval = 1; + else + yylval.bval = 0; + return FSV_VAL_BOOLEAN; + } + + + +$[({A-Za-z][A-Za-z0-9._]*[A-Za-z0-9][)}]* { + if ((yylval.sval = strdup(yytext)) == NULL) { + yyerror("Out of memory"); + filebench_shutdown(E_ERROR); + } + return FSV_VARIABLE; + } + +<INITIAL>[/A-Za-z-][/A-Za-z0-9._-]* { + if ((yylval.sval = strdup(yytext)) == NULL) { + yyerror("Out of memory"); + filebench_shutdown(E_ERROR); + } + return FSV_STRING; + } + +. { + yyerror("Illegal character"); + } + +%% + +void +yyerror(char *s) +{ + if (dofile == FS_TRUE) { + if (yytext[0] == '\0') { + filebench_log(LOG_ERROR, + "%s, token expected", + s); + return; + } + (void) filebench_log(LOG_ERROR, + "%s at '%s'", + s, + yytext); + } else { + if (yytext[0] == '\0') { + (void) filebench_log(LOG_ERROR, + "%s, token expected", s); + return; + } + (void) filebench_log(LOG_ERROR, "%s at '%s'", s, yytext); + } +} + +struct yy_buffer_state *parent; +struct yy_buffer_state *script; + +int +yy_switchfileparent(FILE *file) +{ + script = YY_CURRENT_BUFFER; + parent = (struct yy_buffer_state *)yy_create_buffer(yyin, 128); + yy_switch_to_buffer(parent); + return (0); +} + +int +yy_switchfilescript(FILE *file) +{ + yy_switch_to_buffer(script); + return (0); +} + diff --git a/usr/src/cmd/filebench/common/parsertypes.h b/usr/src/cmd/filebench/common/parsertypes.h new file mode 100644 index 0000000000..a7ceaca379 --- /dev/null +++ b/usr/src/cmd/filebench/common/parsertypes.h @@ -0,0 +1,99 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_PARSERTYPES_H +#define _FB_PARSERTYPES_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" +#include "vars.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef sun +typedef unsigned char uchar_t; +#endif + +#define E_ERROR 1 +#define E_USAGE 2 +#define FS_FALSE 0 +#define FS_TRUE 1 + +#define FSE_SYSTEM 1 + +typedef struct list { + struct list *list_next; + var_string_t list_string; + var_integer_t list_integer; +} list_t; + +typedef struct attr { + int attr_name; + struct attr *attr_next; + var_string_t attr_string; + var_integer_t attr_integer; + list_t *attr_param_list; +} attr_t; + +typedef struct cmd { + void (*cmd)(struct cmd *); + char *cmd_name; + char *cmd_tgt1; + char *cmd_tgt2; + char *cmd_tgt3; + char *cmd_tgt4; + char *thread_name; + uint64_t cmd_qty; + struct cmd *cmd_list; + struct cmd *cmd_next; + attr_t *cmd_attr_list; + list_t *cmd_param_list; + list_t *cmd_param_list2; +} cmd_t; + +typedef union { + int64_t i; + uchar_t b; + char *s; +} fs_u; + +typedef struct pidlist { + struct pidlist *pl_next; + int pl_fd; + pid_t pl_pid; +} pidlist_t; + +typedef void (*cmdfunc)(cmd_t *); +int yy_switchfileparent(FILE *file); +int yy_switchfilescript(FILE *file); + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_PARSERTYPES_H */ diff --git a/usr/src/cmd/filebench/common/procflow.c b/usr/src/cmd/filebench/common/procflow.c new file mode 100644 index 0000000000..0fd2700f0c --- /dev/null +++ b/usr/src/cmd/filebench/common/procflow.c @@ -0,0 +1,699 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include <signal.h> +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/wait.h> + +#include "procflow.h" +#include "filebench.h" +#include "flowop.h" +#include "ipc.h" + +pid_t pid; +static int procflow_delete_wait_cnt = 0; + +static procflow_t *procflow_define_common(procflow_t **list, char *name, + procflow_t *inherit, int instance); + +#ifdef USE_PROCESS_MODEL + +static enum create_n_wait { + CNW_DONE, + CNW_ERROR +} cnw_wait; + +static pthread_cond_t procflow_procs_created; + +#endif /* USE_PROCESS_MODEL */ + + +/* + * Procflows are filebench entities which manage processes. Each + * worker procflow spawns a separate filebench process, with attributes + * inherited from a FLOW_MASTER procflow created during f model language + * parsing. This section contains routines to define, create, control, + * and delete procflows. + * + * Each process defined in the f model creates a FLOW_MASTER + * procflow which encapsulates the defined attributes, and threads of + * the f process, including the number of instances to create. At + * runtime, a worker procflow instance with an associated filebench + * process is created, which runs until told to quite by the original + * filebench process or is specifically deleted. + */ + + +/* + * Prints a summary of the syntax for setting procflow parameters. + */ +void +procflow_usage(void) +{ + (void) fprintf(stderr, + "define process name=<name>[,instances=<count>]\n"); + (void) fprintf(stderr, "{\n"); + (void) fprintf(stderr, " thread ...\n"); + (void) fprintf(stderr, " thread ...\n"); + (void) fprintf(stderr, " thread ...\n"); + (void) fprintf(stderr, "}\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, "\n"); +} + +/* + * If filebench has been compiled to support multiple processes + * (USE_PROCESS_MODEL defined), this routine forks a child + * process and uses either system() or exec() to start up a new + * instance of filebench, passing it the procflow name, instance + * number and shared memory region address. + * If USE_PROCESS_MODEL is NOT defined, then the routine + * just creates a child thread which begins executing + * threadflow_init() for the specified procflow. + */ +static int +procflow_createproc(procflow_t *procflow) +{ + char instance[128]; + char shmaddr[128]; + char procname[128]; + pid_t pid; + +#ifdef USE_PROCESS_MODEL + + (void) snprintf(instance, sizeof (instance), "%d", + procflow->pf_instance); + (void) snprintf(procname, sizeof (procname), "%s", procflow->pf_name); +#if defined(_LP64) || (__WORDSIZE == 64) + (void) snprintf(shmaddr, sizeof (shmaddr), "%llx", filebench_shm); +#else + (void) snprintf(shmaddr, sizeof (shmaddr), "%x", filebench_shm); +#endif + filebench_log(LOG_DEBUG_IMPL, "creating process %s", + procflow->pf_name); + + procflow->pf_running = 0; + +#ifdef HAVE_FORK1 + if ((pid = fork1()) < 0) { + filebench_log(LOG_ERROR, + "procflow_createproc fork failed: %s", + strerror(errno)); + return (-1); + } +#else + if ((pid = fork()) < 0) { + filebench_log(LOG_ERROR, + "procflow_createproc fork failed: %s", + strerror(errno)); + return (-1); + } +#endif /* HAVE_FORK1 */ + + if (pid == 0) { +#ifdef USE_SYSTEM + char syscmd[1024]; +#endif + + (void) sigignore(SIGINT); + filebench_log(LOG_DEBUG_SCRIPT, + "Starting %s-%d", procflow->pf_name, + procflow->pf_instance); + /* Child */ + +#ifdef USE_SYSTEM + (void) snprintf(syscmd, sizeof (syscmd), "%s -a %s -i %s -s %s", + execname, + procname, + instance, + shmaddr); + if (system(syscmd) < 0) { + filebench_log(LOG_ERROR, + "procflow exec proc failed: %s", + strerror(errno)); + filebench_shutdown(1); + } + +#else + if (execl(execname, procname, "-a", procname, "-i", + instance, "-s", shmaddr, "-m", shmpath, NULL) < 0) { + filebench_log(LOG_ERROR, + "procflow exec proc failed: %s", + strerror(errno)); + filebench_shutdown(1); + } +#endif + exit(1); + } else { + procflow->pf_pid = pid; + } +#else + procflow->pf_running = 1; + if (pthread_create(&procflow->pf_tid, NULL, + (void *(*)(void*))threadflow_init, procflow) != 0) { + filebench_log(LOG_ERROR, "proc-thread create failed"); + procflow->pf_running = 0; + } +#endif + filebench_log(LOG_DEBUG_IMPL, "procflow_createproc created pid %d", + pid); + + return (0); +} + +/* + * Find a procflow of name "name" and instance "instance" on the + * master procflow list, filebench_shm->proclist. Locks the list + * and scans through it searching for a procflow with matching + * name and instance number. If found returns a pointer to the + * procflow, otherwise returns NULL. + */ +static procflow_t * +procflow_find(char *name, int instance) +{ + procflow_t *procflow = filebench_shm->proclist; + + filebench_log(LOG_DEBUG_IMPL, "Find: (%s-%d) proclist = %zx", + name, instance, procflow); + + (void) ipc_mutex_lock(&filebench_shm->procflow_lock); + + while (procflow) { + filebench_log(LOG_DEBUG_IMPL, "Find: (%s-%d) == (%s-%d)", + name, instance, + procflow->pf_name, + procflow->pf_instance); + if ((strcmp(name, procflow->pf_name) == 0) && + (instance == procflow->pf_instance)) { + + (void) ipc_mutex_unlock(&filebench_shm->procflow_lock); + + return (procflow); + } + procflow = procflow->pf_next; + } + + (void) ipc_mutex_unlock(&filebench_shm->procflow_lock); + + return (NULL); +} + +static int +procflow_create_all_procs(void) +{ + procflow_t *procflow = filebench_shm->proclist; + int ret = 0; + + while (procflow) { + int i; + + filebench_log(LOG_INFO, "Starting %lld %s instances", + *(procflow->pf_instances), procflow->pf_name); + + /* Create instances of procflow */ + for (i = 0; (i < *procflow->pf_instances) && (ret == 0); i++) { + procflow_t *newproc; + + /* Create processes */ + newproc = + procflow_define_common(&filebench_shm->proclist, + procflow->pf_name, procflow, i + 1); + if (newproc == NULL) + ret = -1; + else + ret = procflow_createproc(newproc); + } + + if (ret != 0) + break; + + procflow = procflow->pf_next; + } + + return (ret); +} + +#ifdef USE_PROCESS_MODEL +/* + * Used to start up threads on a child process, when filebench is + * compiled to support multiple processes. Uses the name string + * and instance number passed to the child to find the previously + * created procflow entity. Then uses nice() to reduce the + * process' priority by at least 10. A call is then made to + * threadflow_init() which creates and runs the process' threads + * and flowops to completion. When threadflow_init() returns, + * a call to exit() terminates the child process. + */ +int +procflow_exec(char *name, int instance) +{ + procflow_t *procflow; + int proc_nice; +#ifdef HAVE_SETRLIMIT + struct rlimit rlp; +#endif + + filebench_log(LOG_DEBUG_IMPL, + "procflow_execproc %s-%d", + name, instance); + + if ((procflow = procflow_find(name, instance)) == NULL) { + filebench_log(LOG_ERROR, + "procflow_exec could not find %s-%d", + name, instance); + return (-1); + } + procflow->pf_pid = pid; + + filebench_log(LOG_DEBUG_IMPL, + "Started up %s pid %d", procflow->pf_name, pid); + + filebench_log(LOG_DEBUG_IMPL, + "nice = %llx", procflow->pf_nice); + + proc_nice = *procflow->pf_nice; + filebench_log(LOG_DEBUG_IMPL, "Setting pri of %s-%d to %d", + name, instance, nice(proc_nice + 10)); + + procflow->pf_running = 1; + +#ifdef HAVE_SETRLIMIT + /* Get resource limits */ + (void) getrlimit(RLIMIT_NOFILE, &rlp); + filebench_log(LOG_DEBUG_SCRIPT, "%d file descriptors", rlp.rlim_cur); +#endif + + if (threadflow_init(procflow) < 0) { + filebench_log(LOG_ERROR, + "Failed to start threads for %s pid %d", + procflow->pf_name, pid); + procflow->pf_running = 0; + exit(1); + } + filebench_log(LOG_DEBUG_IMPL, "procflow_createproc exiting..."); + procflow->pf_running = 0; + exit(0); + + return (0); +} + + +/* + * A special thread from which worker (child) processes are created, and + * which then waits for worker processes to die. If they die unexpectedly, + * that is not a simple exit(0), then report an error and terminate the + * run. + */ +/* ARGSUSED */ +static void * +procflow_createnwait(void *nothing) +{ + (void) ipc_mutex_lock(&filebench_shm->procflow_lock); + + if (procflow_create_all_procs() == 0) + cnw_wait = CNW_DONE; + else + cnw_wait = CNW_ERROR; + + if (pthread_cond_signal(&procflow_procs_created) != 0) + exit(1); + + (void) ipc_mutex_unlock(&filebench_shm->procflow_lock); + + /* CONSTCOND */ + while (1) { + siginfo_t status; + + /* wait for any child process to exit */ + if (waitid(P_ALL, 0, &status, WEXITED) != 0) + pthread_exit(0); + + /* if normal shutdown in progress, just quit */ + if (filebench_shm->f_abort) + pthread_exit(0); + + if (status.si_code == CLD_EXITED) { + /* A process called exit(); check returned status */ + if (status.si_status != 0) { + filebench_log(LOG_ERROR, + "Unexpected Process termination; exiting", + status.si_status); + filebench_shutdown(1); + } + } else { + /* A process quit because of some fatal error */ + filebench_log(LOG_ERROR, + "Unexpected Process termination Code %d, Errno %d", + status.si_code, status.si_errno); + filebench_shutdown(1); + } + + if (filebench_shm->allrunning == 0) + pthread_exit(0); + } + /* NOTREACHED */ + return (NULL); +} +#endif /* USE_PROCESS_MODEL */ + +/* + * Iterates through proclist, the master list of procflows, + * creating the number of instances of each procflow specified + * by its pf_instance attribute. Returns 0 on success, or -1 + * times the number of procflow instances that were not + * successfully created. + */ +int +procflow_init(void) +{ + procflow_t *procflow = filebench_shm->proclist; + pthread_t tid; + int ret = 0; + + filebench_log(LOG_DEBUG_IMPL, + "procflow_init %s, %lld", + procflow->pf_name, *(procflow->pf_instances)); + +#ifdef USE_PROCESS_MODEL + if ((ret = pthread_cond_init(&procflow_procs_created, NULL)) != 0) + return (ret); + + if ((pthread_create(&tid, NULL, procflow_createnwait, NULL)) != 0) + return (ret); + + (void) ipc_mutex_lock(&filebench_shm->procflow_lock); + + if ((ret = pthread_cond_wait(&procflow_procs_created, + &filebench_shm->procflow_lock)) != 0) + return (ret); + + if (cnw_wait == CNW_ERROR) + ret = -1; + +#else /* USE_PROCESS_MODEL */ + (void) ipc_mutex_lock(&filebench_shm->procflow_lock); + + ret = procflow_create_all_procs(); +#endif /* USE_PROCESS_MODEL */ + + (void) ipc_mutex_unlock(&filebench_shm->procflow_lock); + + return (ret); +} + +#ifdef USE_PROCESS_MODEL +/* + * Waits for child processes to finish and returns their exit + * status. Used by procflow_delete() when the process model is + * enabled to wait for a deleted process to exit. + */ +static void +procflow_wait(pid_t pid) +{ + pid_t wpid; + int stat; + + (void) waitpid(pid, &stat, 0); + while ((wpid = waitpid(getpid() * -1, &stat, WNOHANG)) > 0) + filebench_log(LOG_DEBUG_IMPL, "Waited for pid %lld", wpid); +} +#endif + +/* + * Deletes the designated procflow and all its threadflows except + * for FLOW_MASTER ones. Waits 10 seconds if the procflow is still + * running, then kills the associated process. Finally it frees the + * procflow entity. filebench_shm->procflow_lock must be held on entry. + * + * If the designated procflow is not found on the list it returns -1 and + * the procflow is not deleted. Otherwise it returns 0. + */ +static int +procflow_delete(procflow_t *procflow) +{ + procflow_t *entry; + + threadflow_delete_all(&procflow->pf_threads); + + filebench_log(LOG_DEBUG_SCRIPT, + "Deleted proc: (%s-%d) pid %d", + procflow->pf_name, + procflow->pf_instance, + procflow->pf_pid); + + while (procflow->pf_running == 1) { + filebench_log(LOG_DEBUG_SCRIPT, + "Waiting for process %s-%d %d", + procflow->pf_name, + procflow->pf_instance, + procflow->pf_pid); + + if (procflow_delete_wait_cnt < 10) { + (void) ipc_mutex_unlock(&filebench_shm->procflow_lock); + (void) sleep(1); + (void) ipc_mutex_lock(&filebench_shm->procflow_lock); + procflow_delete_wait_cnt++; + continue; + } +#ifdef USE_PROCESS_MODEL + (void) kill(procflow->pf_pid, SIGKILL); + filebench_log(LOG_DEBUG_SCRIPT, + "Had to kill process %s-%d %d!", + procflow->pf_name, + procflow->pf_instance, + procflow->pf_pid); + procflow->pf_running = 0; +#endif + } + +#ifdef USE_PROCESS_MODEL + procflow_wait(procflow->pf_pid); +#endif + /* remove entry from proclist */ + entry = filebench_shm->proclist; + + /* unlink procflow entity from proclist */ + if (entry == procflow) { + /* at head of list */ + filebench_shm->proclist = procflow->pf_next; + } else { + /* search list for procflow */ + while (entry && entry->pf_next != procflow) + entry = entry->pf_next; + + /* if entity found, unlink it */ + if (entry == NULL) + return (-1); + else + entry->pf_next = procflow->pf_next; + } + + /* free up the procflow entity */ + ipc_free(FILEBENCH_PROCFLOW, (char *)procflow); + return (0); +} + + +/* + * Waits till all threadflows are started, or a timeout occurs. + * Checks through the list of procflows, waiting up to 30 + * seconds for each one to set its pf_running flag to 1. If not + * set after 30 seconds, continues on to the next procflow + * anyway after logging the fact. Once pf_running is set + * to 1 for a given procflow or the timeout is reached, + * threadflow_allstarted() is called to start the threads. + * Returns 0 (OK), unless filebench_shm->f_abort is signaled, + * in which case it returns -1. + */ +int +procflow_allstarted() +{ + procflow_t *procflow = filebench_shm->proclist; + int ret = 0; + + (void) ipc_mutex_lock(&filebench_shm->procflow_lock); + + (void) sleep(1); + + while (procflow) { + int waits; + + if (procflow->pf_instance && + (procflow->pf_instance == FLOW_MASTER)) { + procflow = procflow->pf_next; + continue; + } + + waits = 10; + while (waits && procflow->pf_running == 0) { + (void) ipc_mutex_unlock(&filebench_shm->procflow_lock); + if (filebench_shm->f_abort == 1) + return (-1); + + if (waits < 3) + filebench_log(LOG_INFO, + "Waiting for process %s-%d %d", + procflow->pf_name, + procflow->pf_instance, + procflow->pf_pid); + + (void) sleep(3); + waits--; + (void) ipc_mutex_lock(&filebench_shm->procflow_lock); + } + + if (waits == 0) + filebench_log(LOG_INFO, "Failed to start process %s-%d", + procflow->pf_name, + procflow->pf_instance); + + threadflow_allstarted(procflow->pf_pid, procflow->pf_threads); + + procflow = procflow->pf_next; + } + + filebench_shm->allrunning = 1; + (void) ipc_mutex_unlock(&filebench_shm->procflow_lock); + + + return (ret); +} + + +/* + * Sets the f_abort flag and clears the allrunning flag to stop + * all the flowop execution threads from running. Iterates + * through the procflow list and deletes all procflows except + * for the FLOW_MASTER procflow. Resets the f_abort flag when + * finished. + */ +void +procflow_shutdown(void) +{ + procflow_t *procflow = filebench_shm->proclist; + + (void) ipc_mutex_lock(&filebench_shm->procflow_lock); + filebench_shm->allrunning = 0; + filebench_shm->f_abort = 1; + procflow_delete_wait_cnt = 0; + + while (procflow) { + if (procflow->pf_instance && + (procflow->pf_instance == FLOW_MASTER)) { + procflow = procflow->pf_next; + continue; + } + filebench_log(LOG_DEBUG_IMPL, "Deleting process %s-%d %d", + procflow->pf_name, + procflow->pf_instance, + procflow->pf_pid); + (void) procflow_delete(procflow); + procflow = procflow->pf_next; + } + + filebench_shm->f_abort = 0; + + (void) ipc_mutex_unlock(&filebench_shm->procflow_lock); +} + + +/* + * Create an in-memory process object. Allocates a procflow + * entity, initialized from the "inherit" procflow if supplied. + * The name and instance number are set from the supplied name + * and instance number and the procflow is added to the head of + * the master procflow list. Returns pointer to the allocated + * procflow, or NULL if a name isn't supplied or the procflow + * entity cannot be allocated. + * + * The calling routine must hold the filebench_shm->procflow_lock. + */ +static procflow_t * +procflow_define_common(procflow_t **list, char *name, + procflow_t *inherit, int instance) +{ + procflow_t *procflow; + + if (name == NULL) + return (NULL); + + procflow = (procflow_t *)ipc_malloc(FILEBENCH_PROCFLOW); + + if (procflow == NULL) + return (NULL); + + if (inherit) + (void) memcpy(procflow, inherit, sizeof (procflow_t)); + else + (void) memset(procflow, 0, sizeof (procflow_t)); + + procflow->pf_instance = instance; + (void) strcpy(procflow->pf_name, name); + + filebench_log(LOG_DEBUG_IMPL, "defining process %s-%d", name, instance); + + filebench_log(LOG_DEBUG_IMPL, "process %s-%d proclist %zx", + name, instance, filebench_shm->proclist); + /* Add procflow to list, lock is being held already */ + if (*list == NULL) { + *list = procflow; + procflow->pf_next = NULL; + } else { + procflow->pf_next = *list; + *list = procflow; + } + filebench_log(LOG_DEBUG_IMPL, "process %s-%d proclist %zx", + name, instance, filebench_shm->proclist); + + return (procflow); +} + +/* + * Create an in-memory process object as described by the syntax. + * Acquires the filebench_shm->procflow_lock and calls + * procflow_define_common() to create and initialize a + * FLOW_MASTER procflow entity from the optional "inherit" + * procflow with the given name and configured for "instances" + * number of worker procflows. Currently only called from + * parser_proc_define(). + */ +procflow_t * +procflow_define(char *name, procflow_t *inherit, var_integer_t instances) +{ + procflow_t *procflow; + + (void) ipc_mutex_lock(&filebench_shm->procflow_lock); + + procflow = procflow_define_common(&filebench_shm->proclist, + name, inherit, FLOW_MASTER); + procflow->pf_instances = instances; + + (void) ipc_mutex_unlock(&filebench_shm->procflow_lock); + + return (procflow); +} diff --git a/usr/src/cmd/filebench/common/procflow.h b/usr/src/cmd/filebench/common/procflow.h new file mode 100644 index 0000000000..7b8134be65 --- /dev/null +++ b/usr/src/cmd/filebench/common/procflow.h @@ -0,0 +1,66 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_PROCFLOW_H +#define _FB_PROCFLOW_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" + +#include "vars.h" +#include "stats.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct procflow { + char pf_name[128]; + int pf_instance; + var_integer_t pf_instances; + int pf_running; + struct procflow *pf_next; + pid_t pf_pid; + pthread_t pf_tid; + struct threadflow *pf_threads; + int pf_attrs; + var_integer_t pf_nice; + flowstat_t pf_stats; +} procflow_t; + +procflow_t *procflow_define(char *name, procflow_t *inherit, + var_integer_t instances); +int procflow_init(void); +void procflow_shutdown(void); +int procflow_exec(char *name, int instance); +void procflow_usage(void); +int procflow_allstarted(void); + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_PROCFLOW_H */ diff --git a/usr/src/cmd/filebench/common/stats.c b/usr/src/cmd/filebench/common/stats.c new file mode 100644 index 0000000000..985f387b77 --- /dev/null +++ b/usr/src/cmd/filebench/common/stats.c @@ -0,0 +1,752 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" + +#include <stdio.h> +#include <fcntl.h> +#include <sys/types.h> + +#ifdef HAVE_SYSINFO +#include <sys/sysinfo.h> +#endif + +#ifdef HAVE_LIBKSTAT +#include <kstat.h> +#include <sys/cpuvar.h> +#endif /* HAVE_LIBKSTAT */ + +#include <stdarg.h> + +#include "filebench.h" +#include "flowop.h" +#include "vars.h" +#include "stats.h" + +/* + * A set of routines for collecting and dumping various filebench + * run statistics. + */ + +/* Global statistics */ +static flowstat_t *globalstats = NULL; + +static hrtime_t stats_cputime = 0; + +#ifdef HAVE_LIBKSTAT +static kstat_ctl_t *kstatp = NULL; +static kstat_t *sysinfo_ksp = NULL; + +/* + * Uses the kstat library or, if it is not available, the /proc/stat file + * to obtain cpu statistics. Collects statistics for each cpu, initializes + * a local pointer to the sysinfo kstat, and returns the sum of user and + * kernel time for all the cpus. + */ +static vinteger_t +kstats_read_cpu(void) +{ + int ncpus; + kstat_t **cpu_stat_list = NULL; + ulong_t cputime_states[CPU_STATES]; + hrtime_t cputime; + int i; + + kstat_t *ksp; + + if (kstatp == NULL) { + if ((kstatp = kstat_open()) == (kstat_ctl_t *)NULL) { + filebench_log(LOG_ERROR, "Cannot read kstats"); + return (-1); + } + } + + /* + * Per-CPU statistics + */ + + ncpus = 0; + for (ksp = kstatp->kc_chain; ksp; ksp = ksp->ks_next) + if (strncmp(ksp->ks_name, "cpu_stat", 8) == 0) + ncpus++; + + if ((cpu_stat_list = + (kstat_t **)malloc(ncpus * sizeof (kstat_t *))) == NULL) { + filebench_log(LOG_ERROR, "malloc failed"); + return (-1); + } + + ncpus = 0; + for (ksp = kstatp->kc_chain; ksp; ksp = ksp->ks_next) + if (strncmp(ksp->ks_name, "cpu_stat", 8) == 0 && + kstat_read(kstatp, ksp, NULL) != -1) + cpu_stat_list[ncpus++] = ksp; + + if (ncpus == 0) { + filebench_log(LOG_ERROR, + "kstats can't find any cpu statistics"); + return (0); + } + + if (sysinfo_ksp == NULL) + sysinfo_ksp = kstat_lookup(kstatp, "unix", 0, "sysinfo"); + + /* Sum across all CPUs */ + (void) memset(&cputime_states, 0, sizeof (cputime_states)); + for (i = 0; i < ncpus; i++) { + cpu_stat_t cpu_stats; + int j; + + (void) kstat_read(kstatp, cpu_stat_list[i], + (void *) &cpu_stats); + for (j = 0; j < CPU_STATES; j++) + cputime_states[j] += cpu_stats.cpu_sysinfo.cpu[j]; + } + + cputime = cputime_states[CPU_KERNEL] + cputime_states[CPU_USER]; + + return (10000000LL * cputime); +} +#else /* HAVE_LIBKSTAT */ +#ifdef HAVE_PROC_STAT +static FILE *statfd = 0; +vinteger_t +kstats_read_cpu(void) +{ + /* + * Linux provides system wide statistics in /proc/stat + * The entry for cpu is + * cpu 1636 67 1392 208671 5407 20 12 + * cpu0 626 8 997 104476 2499 7 7 + * cpu1 1010 58 395 104195 2907 13 5 + * + * The number of jiffies (1/100ths of a second) that the + * system spent in user mode, user mode with low priority + * (nice), system mode, and the idle task, respectively. + */ + unsigned int user, nice, system; + char cpu[128]; /* placeholder to read "cpu" */ + if (statfd == 0) { + statfd = fopen("/proc/stat", "r"); + if (statfd < 0) { + filebench_log(LOG_ERROR, "Cannot open /proc/stat"); + return (-1); + } + } + if (fscanf(statfd, "%s %u %u %u", cpu, &user, &nice, &system) < 0) { + filebench_log(LOG_ERROR, "Cannot read /proc/stat"); + return (-1); + } + /* convert jiffies to nanosecs */ + return ((user+nice+system)*1000000); +} + +#else /* HAVE_PROC_STAT */ +vinteger_t +kstats_read_cpu(void) +{ + return (0); +} +#endif +#endif /* HAVE_LIBKSTAT */ + +/* + * Returns the net cpu time used since the beginning of the run. + * Just calls kstat_read_cpu() and subtracts stats_cputime which + * is set at the beginning of the filebench run. + */ +static hrtime_t +kstats_read_cpu_relative(void) +{ + hrtime_t cputime; + + cputime = kstats_read_cpu(); + return (cputime - stats_cputime); +} + +/* + * IO Overhead CPU is the amount of CPU that is incurred running + * the benchmark infrastructure. + * + * It is computed as the sum of micro-state cpu time for each + * thread around the op being tested. + * + * Overhead time is computed for each flow. + * + * System overhead is computed as the overhead for I/O flows + * plus all other time running non-io related flowops + * + */ + +/* + * Computes and returns the overhead CPU time attibutable to + * IO type flowops. + */ +static hrtime_t +io_stats_ohead(void) +{ + flowstat_t *iostat = &globalstats[FLOW_TYPE_IO]; + flowstat_t *aiostat = &globalstats[FLOW_TYPE_AIO]; + flowstat_t *glstat = &globalstats[FLOW_TYPE_GLOBAL]; + + filebench_log(LOG_DEBUG_NEVER, + "Computing overhead as %lld + %lld - %lld - %lld", + glstat->fs_mstate[FLOW_MSTATE_OHEAD], + glstat->fs_mstate[FLOW_MSTATE_CPU], + iostat->fs_mstate[FLOW_MSTATE_CPU], + aiostat->fs_mstate[FLOW_MSTATE_CPU]); + + return ((glstat->fs_mstate[FLOW_MSTATE_OHEAD] + + glstat->fs_mstate[FLOW_MSTATE_CPU] - + iostat->fs_mstate[FLOW_MSTATE_CPU] - + aiostat->fs_mstate[FLOW_MSTATE_CPU])); +} + +/* + * Returns the total overhead CPU time. + */ +static hrtime_t +gl_stats_ohead(void) +{ + flowstat_t *glstat = &globalstats[FLOW_TYPE_GLOBAL]; + + return (glstat->fs_mstate[FLOW_MSTATE_OHEAD]); +} + +/* + * Places the value represented by "name" into the var_integer field of the + * supplied var_t. Compares the supplied "name" with a set of predefined + * names and calculates the value from the appropriate globalstats field(s). + */ +var_t * +stats_findvar(var_t *var, char *name) +{ + flowstat_t *iostat = &globalstats[FLOW_TYPE_IO]; + flowstat_t *aiostat = &globalstats[FLOW_TYPE_AIO]; + flowstat_t *glstat = &globalstats[FLOW_TYPE_GLOBAL]; + + filebench_log(LOG_DEBUG_IMPL, "reading stats %s", name); + + if (globalstats == NULL) + globalstats = malloc(FLOW_TYPES * sizeof (flowstat_t)); + + if (strcmp(name, "iocount") == 0) { + var->var_integer = iostat->fs_count + + aiostat->fs_count; + filebench_log(LOG_DEBUG_IMPL, "reading stats %s = %lld", + name, var->var_integer); + return (var); + } + + if (strcmp(name, "iorate") == 0) { + /* LINTED E_ASSIGMENT_CAUSE_LOSS_PREC */ + var->var_integer = (iostat->fs_count + aiostat->fs_count) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS); + return (var); + } + + + if (strcmp(name, "ioreadrate") == 0) { + /* LINTED E_ASSIGMENT_CAUSE_LOSS_PREC */ + var->var_integer = (iostat->fs_rcount + aiostat->fs_rcount) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS); + return (var); + } + + + if (strcmp(name, "iowriterate") == 0) { + /* LINTED E_ASSIGMENT_CAUSE_LOSS_PREC */ + var->var_integer = (iostat->fs_wcount + aiostat->fs_wcount) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS); + return (var); + } + + + if (strcmp(name, "iobandwidth") == 0) { + /* LINTED E_ASSIGMENT_CAUSE_LOSS_PREC */ + var->var_integer = + ((iostat->fs_bytes + aiostat->fs_bytes) / (1024 * 1024)) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS); + return (var); + } + + if (strcmp(name, "iolatency") == 0) { + var->var_integer = iostat->fs_count ? + iostat->fs_mstate[FLOW_MSTATE_LAT] / + (iostat->fs_count * 1000UL) : 0; + return (var); + } + + if (strcmp(name, "iocpu") == 0) { + var->var_integer = (iostat->fs_count + aiostat->fs_count) ? + (iostat->fs_mstate[FLOW_MSTATE_CPU] + + aiostat->fs_mstate[FLOW_MSTATE_CPU]) / ((iostat->fs_count + + aiostat->fs_count) * 1000UL) : 0; + return (var); + } + + + if (strcmp(name, "oheadcpu") == 0) { + var->var_integer = (iostat->fs_count + aiostat->fs_count) ? + io_stats_ohead() / ((iostat->fs_count + + aiostat->fs_count) * 1000UL) : 0; + return (var); + } + + if (strcmp(name, "iowait") == 0) { + var->var_integer = iostat->fs_count ? + iostat->fs_mstate[FLOW_MSTATE_WAIT] / + (iostat->fs_count * 1000UL) : 0; + return (var); + } + + if (strcmp(name, "syscpu") == 0) { + /* LINTED E_ASSIGMENT_CAUSE_LOSS_PREC */ + var->var_integer = glstat->fs_syscpu / 1000.0; + return (var); + } + + if (strcmp(name, "iocpusys") == 0) { + var->var_integer = (iostat->fs_count + aiostat->fs_count) ? + iostat->fs_syscpu / ((iostat->fs_count + + aiostat->fs_count) * 1000UL) : 0; + + return (var); + } + + filebench_log(LOG_DEBUG_IMPL, + "error reading stats %s", name); + + return (NULL); +} + +/* + * Initializes the static variable "stats_cputime" with the + * current cpu time, for use by kstats_read_cpu_relative. + */ +void +stats_init(void) +{ +#if defined(HAVE_LIBKSTAT) || defined(LINUX_PORT) + stats_cputime = kstats_read_cpu(); +#else + stats_cputime = 0; +#endif /* HAVE_LIBKSTAT */ +} + +/* + * Add a flowstat b to a, leave sum in a. + */ +static void +stats_add(flowstat_t *a, flowstat_t *b) +{ + int i; + + a->fs_count += b->fs_count; + a->fs_rcount += b->fs_rcount; + a->fs_wcount += b->fs_wcount; + a->fs_bytes += b->fs_bytes; + a->fs_rbytes += b->fs_rbytes; + a->fs_wbytes += b->fs_wbytes; + + for (i = 0; i < FLOW_MSTATES; i++) + a->fs_mstate[i] += b->fs_mstate[i]; +} + +/* + * Takes a "snapshot" of the global statistics. Actually, it calculates + * them from the local statistics maintained by each flowop. + * First the routine pauses filebench, then rolls the statistics for + * each flowop into its associated FLOW_MASTER flowop. + * Next all the FLOW_MASTER flowops' statistics are written + * to the log file followed by the global totals. Then filebench + * operation is allowed to resume. + */ +void +stats_snap(void) +{ + flowstat_t *iostat = &globalstats[FLOW_TYPE_IO]; + flowstat_t *aiostat = &globalstats[FLOW_TYPE_AIO]; + flowstat_t *glstat = &globalstats[FLOW_TYPE_GLOBAL]; + hrtime_t cputime; + flowop_t *flowop; + char *str; + + if (globalstats == NULL) { + filebench_log(LOG_ERROR, + "'stats snap' called before 'stats clear'"); + return; + } + + globalstats->fs_etime = gethrtime(); + + filebench_log(LOG_DEBUG_SCRIPT, "Stats period = %ds", + (globalstats->fs_etime - globalstats->fs_stime) / 1000000000); + + /* Freeze statistics during update */ + filebench_shm->bequiet = 1; + + flowop = filebench_shm->flowoplist; + while (flowop) { + flowop_t *flowop_master; + + if (flowop->fo_instance == FLOW_MASTER) { + flowop = flowop->fo_next; + continue; + } + + flowop_master = flowop_find_one(flowop->fo_name, + FLOW_MASTER); + + /* Roll up per-flowop into global stats */ + stats_add(&globalstats[flowop->fo_type], + &flowop->fo_stats); + stats_add(&globalstats[FLOW_TYPE_GLOBAL], + &flowop->fo_stats); + + if (flowop_master && IS_FLOW_ACTIVE(flowop)) { + flowop_master->fo_stats.fs_active++; + } + + if (flowop_master) { + /* Roll up per-flow stats into master */ + flowop_master->fo_stats.fs_children++; + stats_add(&flowop_master->fo_stats, &flowop->fo_stats); + } else { + filebench_log(LOG_DEBUG_NEVER, + "flowop_stats could not find %s", + flowop->fo_name); + } + + filebench_log(LOG_DEBUG_SCRIPT, + "flowop %-20s-%4d - %5d ops, %5.1lf, ops/s %5.1lfmb/s " + "%8.3fms/op", + flowop->fo_name, + flowop->fo_instance, + flowop->fo_stats.fs_count, + flowop->fo_stats.fs_count / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS), + (flowop->fo_stats.fs_bytes / (1024 * 1024)) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS), + flowop->fo_stats.fs_count ? + flowop->fo_stats.fs_mstate[FLOW_MSTATE_LAT] / + (flowop->fo_stats.fs_count * 1000000.0) : 0); + + flowop = flowop->fo_next; + + } + +#if defined(HAVE_LIBKSTAT) || defined(LINUX_PORT) + cputime = kstats_read_cpu_relative(); +#endif /* HAVE_LIBKSTAT */ + + filebench_log(LOG_DEBUG_IMPL, + "cputime = %lld, ohead = %lld", + cputime / 1000000000, + io_stats_ohead() / 1000000000); + iostat->fs_syscpu = + (cputime > io_stats_ohead()) ? + (cputime - io_stats_ohead()) : 0; + glstat->fs_syscpu = + (cputime > gl_stats_ohead()) ? + (cputime - gl_stats_ohead()) : 0; + + + flowop = filebench_shm->flowoplist; + str = malloc(1048576); + *str = NULL; + (void) strcpy(str, "Per-Operation Breakdown\n"); + while (flowop) { + char line[1024]; + + if (flowop->fo_instance != FLOW_MASTER) { + flowop = flowop->fo_next; + continue; + } + + (void) snprintf(line, sizeof (line), "%-20s %8.0lfops/s " + "%5.1lfmb/s %8.1fms/op %8.0fus/op-cpu\n", + flowop->fo_name, + flowop->fo_stats.fs_count / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS), + (flowop->fo_stats.fs_bytes / (1024 * 1024)) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS), + flowop->fo_stats.fs_count ? + flowop->fo_stats.fs_mstate[FLOW_MSTATE_LAT] / + (flowop->fo_stats.fs_count * 1000000.0) : 0, + flowop->fo_stats.fs_count ? + flowop->fo_stats.fs_mstate[FLOW_MSTATE_CPU] / + (flowop->fo_stats.fs_count * 1000.0) : 0); + (void) strcat(str, line); + + flowop = flowop->fo_next; + } + + filebench_log(LOG_INFO, "%s", str); + free(str); + + filebench_log(LOG_INFO, + "\nIO Summary: %5d ops %5.1lf ops/s, (%0.0lf/%0.0lf r/w) " + "%5.1lfmb/s, %6.0fus cpu/op, %5.1fms latency", + iostat->fs_count + aiostat->fs_count, + (iostat->fs_count + aiostat->fs_count) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS), + (iostat->fs_rcount + aiostat->fs_rcount) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS), + (iostat->fs_wcount + aiostat->fs_wcount) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS), + ((iostat->fs_bytes + aiostat->fs_bytes) / (1024 * 1024)) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS), + (iostat->fs_rcount + iostat->fs_wcount + + aiostat->fs_rcount + aiostat->fs_wcount) ? + (iostat->fs_syscpu / 1000.0) / + (iostat->fs_rcount + iostat->fs_wcount + + aiostat->fs_rcount + aiostat->fs_wcount) : 0, + (iostat->fs_rcount + iostat->fs_wcount) ? + iostat->fs_mstate[FLOW_MSTATE_LAT] / + ((iostat->fs_rcount + iostat->fs_wcount) * 1000000.0) : 0); + + + filebench_shm->bequiet = 0; +} + +/* + * Dumps the per-operation statistics and global statistics to the dump file. + */ +void +stats_dump(char *filename) +{ + flowstat_t *iostat = &globalstats[FLOW_TYPE_IO]; + flowstat_t *aiostat = &globalstats[FLOW_TYPE_AIO]; + flowop_t *flowop; + + (void) strcpy(filebench_shm->dump_filename, filename); + + filebench_log(LOG_INFO, "in statsdump %s", filename); + + if (filebench_shm->dump_fd > 0) { + (void) close(filebench_shm->dump_fd); + filebench_shm->dump_fd = -1; + } + + filebench_log(LOG_DUMP, "Flowop totals:"); + + flowop = filebench_shm->flowoplist; + while (flowop) { + + if (flowop->fo_instance != FLOW_MASTER) { + flowop = flowop->fo_next; + continue; + } + + filebench_log(LOG_DUMP, + "%-20s %8.0lfops/s %5.1lfmb/s " + "%8.1fms/op %8.0fus/op-cpu", + flowop->fo_name, + flowop->fo_stats.fs_count / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS), + (flowop->fo_stats.fs_bytes / (1024 * 1024)) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS), + flowop->fo_stats.fs_count ? + flowop->fo_stats.fs_mstate[FLOW_MSTATE_LAT] / + (flowop->fo_stats.fs_count * 1000000.0) : 0, + flowop->fo_stats.fs_count ? + flowop->fo_stats.fs_mstate[FLOW_MSTATE_CPU] / + (flowop->fo_stats.fs_count * 1000.0) : 0); + + flowop = flowop->fo_next; + } + + filebench_log(LOG_DUMP, ""); + filebench_log(LOG_DUMP, + "IO Summary: %8d ops %8.1lf ops/s, %8.0lf/%0.0lf r/w" + "%8.1lfmb/s, %8.0fuscpu/op", + + iostat->fs_count + aiostat->fs_count, + (iostat->fs_count + aiostat->fs_count) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS), + + (iostat->fs_rcount + aiostat->fs_rcount) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS), + + (iostat->fs_wcount + aiostat->fs_wcount) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS), + + ((iostat->fs_bytes + aiostat->fs_bytes) / (1024 * 1024)) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS), + + (iostat->fs_rcount + iostat->fs_wcount + + aiostat->fs_rcount + aiostat->fs_wcount) ? + (iostat->fs_syscpu / 1000.0) / + (iostat->fs_rcount + iostat->fs_wcount + + aiostat->fs_rcount + aiostat->fs_wcount) : 0); +} + +/* + * Same as stats_dump, but in xml format. + */ +void +stats_xmldump(char *filename) +{ + flowstat_t *iostat = &globalstats[FLOW_TYPE_IO]; + flowstat_t *aiostat = &globalstats[FLOW_TYPE_AIO]; + flowop_t *flowop; + + (void) strcpy(filebench_shm->dump_filename, filename); + + if (filebench_shm->dump_fd > 0) { + (void) close(filebench_shm->dump_fd); + filebench_shm->dump_fd = -1; + } + + filebench_log(LOG_DUMP, "<stat_doc name=\"Filebench Workload\">"); + filebench_log(LOG_DUMP, "<stat_group name=\"Flowop totals\">"); + filebench_log(LOG_DUMP, "<cell_list>"); + + flowop = filebench_shm->flowoplist; + while (flowop) { + if (flowop->fo_instance != FLOW_MASTER) { + flowop = flowop->fo_next; + continue; + } + + filebench_log(LOG_DUMP, "<cell>%0.0lf</cell>", + flowop->fo_stats.fs_count / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS)); + filebench_log(LOG_DUMP, "<cell>%0.1lf</cell>", + (flowop->fo_stats.fs_bytes / (1024 * 1024)) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS)); + filebench_log(LOG_DUMP, "<cell>%0.1lf</cell>", + flowop->fo_stats.fs_count ? + flowop->fo_stats.fs_mstate[FLOW_MSTATE_LAT] / + (flowop->fo_stats.fs_count * 1000000.0) : 0); + filebench_log(LOG_DUMP, "<cell>%0.0lf</cell>", + flowop->fo_stats.fs_count ? + flowop->fo_stats.fs_mstate[FLOW_MSTATE_CPU] / + (flowop->fo_stats.fs_count * 1000.0) : 0); + + flowop = flowop->fo_next; + } + filebench_log(LOG_DUMP, "</cell_list>"); + + filebench_log(LOG_DUMP, "<dim_list>"); + filebench_log(LOG_DUMP, "<dim>"); + filebench_log(LOG_DUMP, "<dimval>Operations/sec</dimval>"); + filebench_log(LOG_DUMP, "<dimval>MB/sec</dimval>"); + filebench_log(LOG_DUMP, "<dimval>Latency (ms per operation)</dimval>"); + filebench_log(LOG_DUMP, "<dimval>CPU (us per operation)</dimval>"); + filebench_log(LOG_DUMP, "</dim>"); + + filebench_log(LOG_DUMP, "<dim>"); + flowop = filebench_shm->flowoplist; + while (flowop) { + if (flowop->fo_instance != FLOW_MASTER) { + flowop = flowop->fo_next; + continue; + } + filebench_log(LOG_DUMP, "<dimval>%s</dimval>", flowop->fo_name); + flowop = flowop->fo_next; + } + filebench_log(LOG_DUMP, "</dim>"); + filebench_log(LOG_DUMP, "</dim_list>"); + filebench_log(LOG_DUMP, "</stat_group>"); + + filebench_log(LOG_DUMP, "<stat_group name=\"IO Summary\">"); + filebench_log(LOG_DUMP, "<cell_list>"); + filebench_log(LOG_DUMP, "<cell>%0d</cell>", + iostat->fs_count + aiostat->fs_count); + filebench_log(LOG_DUMP, "<cell>%0.1lf</cell>", + (iostat->fs_count + aiostat->fs_count) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS)); + filebench_log(LOG_DUMP, "<cell>%0.0lf</cell>", + (iostat->fs_rcount + aiostat->fs_rcount) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS)); + filebench_log(LOG_DUMP, "<cell>%0.0lf</cell>", + (iostat->fs_wcount + aiostat->fs_wcount) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS)); + filebench_log(LOG_DUMP, "<cell>%0.1lf</cell>", + ((iostat->fs_bytes + aiostat->fs_bytes) / (1024 * 1024)) / + ((globalstats->fs_etime - globalstats->fs_stime) / FSECS)); + filebench_log(LOG_DUMP, "<cell>%0.0f</cell>", + (iostat->fs_rcount + iostat->fs_wcount + aiostat->fs_rcount + + aiostat->fs_wcount) ? (iostat->fs_syscpu / 1000.0) / + (iostat->fs_rcount + iostat->fs_wcount + aiostat->fs_rcount + + aiostat->fs_wcount) : 0); + filebench_log(LOG_DUMP, "</cell_list>"); + + filebench_log(LOG_DUMP, "<dim_list>"); + filebench_log(LOG_DUMP, "<dim>"); + filebench_log(LOG_DUMP, "<dimval>Operations</dimval>"); + filebench_log(LOG_DUMP, "<dimval>Operations/sec</dimval>"); + filebench_log(LOG_DUMP, "<dimval>Reads</dimval>"); + filebench_log(LOG_DUMP, "<dimval>Writes</dimval>"); + filebench_log(LOG_DUMP, "<dimval>MB/sec</dimval>"); + filebench_log(LOG_DUMP, "<dimval>CPU (us per operation)</dimval>"); + filebench_log(LOG_DUMP, "</dim>"); + + filebench_log(LOG_DUMP, "<dim>"); + filebench_log(LOG_DUMP, "<dimval>IO Summary</dimval>"); + filebench_log(LOG_DUMP, "</dim>"); + filebench_log(LOG_DUMP, "</dim_list>"); + filebench_log(LOG_DUMP, "</stat_group>"); + filebench_log(LOG_DUMP, "</stat_doc>"); +} + +/* + * Clears all the statistics variables (fo_stats) for every defined flowop. + * It also creates a global flowstat table if one doesn't already exist and + * clears it. + */ +void +stats_clear(void) +{ + flowop_t *flowop; + +#ifdef HAVE_LIBKSTAT + stats_cputime = kstats_read_cpu(); +#else + stats_cputime = 0; +#endif /* HAVE_LIBKSTAT */ + + if (globalstats == NULL) + globalstats = malloc(FLOW_TYPES * sizeof (flowstat_t)); + + (void) memset(globalstats, 0, FLOW_TYPES * sizeof (flowstat_t)); + + flowop = filebench_shm->flowoplist; + + while (flowop) { + filebench_log(LOG_DEBUG_IMPL, "Clearing stats for %s-%d", + flowop->fo_name, + flowop->fo_instance); + (void) memset(&flowop->fo_stats, 0, sizeof (flowstat_t)); + flowop = flowop->fo_next; + } + + (void) memset(globalstats, 0, sizeof (flowstat_t)); + globalstats->fs_stime = gethrtime(); +} diff --git a/usr/src/cmd/filebench/common/stats.h b/usr/src/cmd/filebench/common/stats.h new file mode 100644 index 0000000000..917b3a584f --- /dev/null +++ b/usr/src/cmd/filebench/common/stats.h @@ -0,0 +1,86 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_STATS_H +#define _FB_STATS_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" +#ifdef HAVE_STDINT_H +#include <stdint.h> +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +var_t *stats_findvar(var_t *var, char *name); +void stats_init(void); +void stats_clear(void); +void stats_snap(void); +void stats_dump(char *filename); +void stats_xmldump(char *filename); + +#ifndef HAVE_HRTIME +/* typedef uint64_t hrtime_t; */ +#define hrtime_t uint64_t +#endif + +#define STATS_VAR "stats." + +#define FLOW_MSTATES 4 +#define FLOW_MSTATE_LAT 0 /* Total service time of op */ +#define FLOW_MSTATE_CPU 1 /* On-cpu time of op */ +#define FLOW_MSTATE_WAIT 2 /* Wait-time, excluding waiting for CPU */ +#define FLOW_MSTATE_OHEAD 3 /* overhead time, around op */ + +typedef struct flowstats { + int fs_children; /* Number of contributors */ + int fs_active; /* Number of active contributors */ + int fs_count; /* Number of ops */ + uint64_t fs_rbytes; /* Number of bytes */ + uint64_t fs_wbytes; /* Number of bytes */ + uint64_t fs_bytes; /* Number of bytes */ + uint64_t fs_rcount; /* Number of ops */ + uint64_t fs_wcount; /* Number of ops */ + hrtime_t fs_stime; /* Time stats for flow started */ + hrtime_t fs_etime; /* Time stats for flow ended */ + hrtime_t fs_mstate[FLOW_MSTATES]; /* Microstate breakdown */ + hrtime_t fs_syscpu; /* System wide cpu, global only */ +} flowstat_t; + + +#define IS_FLOW_IOP(x) (x->fo_stats.fs_rcount + x->fo_stats.fs_wcount) +#define STAT_IOPS(x) ((x->fs_rcount) + (x->fs_wcount)) +#define IS_FLOW_ACTIVE(x) (x->fo_stats.fs_count) +#define STAT_CPUTIME(x) (x->fs_cpu_op) +#define STAT_OHEADTIME(x) (x->fs_cpu_ohead) + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_STATS_H */ diff --git a/usr/src/cmd/filebench/common/threadflow.c b/usr/src/cmd/filebench/common/threadflow.c new file mode 100644 index 0000000000..9e6c99a92e --- /dev/null +++ b/usr/src/cmd/filebench/common/threadflow.c @@ -0,0 +1,482 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" +#include <pthread.h> +#ifdef HAVE_LWPS +#include <sys/lwp.h> +#endif +#include <signal.h> +#include "threadflow.h" +#include "filebench.h" +#include "flowop.h" +#include "ipc.h" + +static threadflow_t *threadflow_define_common(procflow_t *procflow, + char *name, threadflow_t *inherit, int instance); + +/* + * Threadflows are filebench entities which manage operating system + * threads. Each worker threadflow spawns a separate filebench thread, + * with attributes inherited from a FLOW_MASTER threadflow created during + * f model language parsing. This section contains routines to define, + * create, control, and delete threadflows. + * + * Each thread defined in the f model creates a FLOW_MASTER + * threadflow which encapsulates the defined attributes and flowops of + * the f language thread, including the number of instances to create. + * At runtime, a worker threadflow instance with an associated filebench + * thread is created, which runs until told to quit or is specifically + * deleted. + */ + + +/* + * Prints information about threadflow syntax. + */ +void +threadflow_usage(void) +{ + (void) fprintf(stderr, " thread name=<name>[,instances=<count>]\n"); + (void) fprintf(stderr, "\n"); + (void) fprintf(stderr, " {\n"); + (void) fprintf(stderr, " flowop ...\n"); + (void) fprintf(stderr, " flowop ...\n"); + (void) fprintf(stderr, " flowop ...\n"); + (void) fprintf(stderr, " }\n"); + (void) fprintf(stderr, "\n"); +} + +/* + * Creates a thread for the supplied threadflow. If interprocess + * shared memory is desired, then increments the amount of shared + * memory needed by the amount specified in the threadflow's + * tf_memsize parameter. The thread starts in routine + * flowop_start() with a poineter to the threadflow supplied + * as the argument. + */ +static int +threadflow_createthread(threadflow_t *threadflow) +{ + int fp = 0; + + filebench_log(LOG_DEBUG_SCRIPT, "Creating thread %s, memory = %ld", + threadflow->tf_name, + *threadflow->tf_memsize); + + if (threadflow->tf_attrs & THREADFLOW_USEISM) + filebench_shm->shm_required += (*threadflow->tf_memsize); + + if (pthread_create(&threadflow->tf_tid, NULL, + (void *(*)(void*))flowop_start, threadflow) != 0) { + filebench_log(LOG_ERROR, "thread create failed"); + filebench_shutdown(1); + } + + /* XXX */ + return (fp < 0); +} + +#ifndef USE_PROCESS_MODEL +static procflow_t *my_procflow; + +/* + * Terminates (exits) all the threads of the procflow (process). + * The procflow is determined from a process private pointer + * initialized by threadflow_init(). + */ +/* ARGSUSED */ +static void +threadflow_cancel(int arg1) +{ + threadflow_t *threadflow = my_procflow->pf_threads; + +#ifdef HAVE_LWPS + filebench_log(LOG_DEBUG_IMPL, "Thread signal handler on tid %d", + _lwp_self()); +#endif + + my_procflow->pf_running = 0; + exit(0); + + while (threadflow) { + if (threadflow->tf_tid) { + (void) pthread_cancel(threadflow->tf_tid); + filebench_log(LOG_DEBUG_IMPL, "Thread %d cancelled...", + threadflow->tf_tid); + } + threadflow = threadflow->tf_next; + } +} +#endif /* USE_PROCESS_MODEL */ + +/* + * Creates threads for the threadflows associated with a procflow. + * The routine iterates through the list of threadflows in the + * supplied procflow's pf_threads list. For each threadflow on + * the list, it defines tf_instances number of cloned + * threadflows, and then calls threadflow_createthread() for + * each to create and start the actual operating system thread. + * Note that each of the newly defined threadflows will be linked + * into the procflows threadflow list, but at the head of the + * list, so they will not become part of the supplied set. After + * all the threads have been created, threadflow_init enters + * a join loop for all the threads in the newly defined + * threadflows. Once all the created threads have exited, + * threadflow_init will return 0. If errors are encountered, it + * will return a non zero value. + */ +int +threadflow_init(procflow_t *procflow) +{ + threadflow_t *threadflow = procflow->pf_threads; + int ret = 0; + + (void) ipc_mutex_lock(&filebench_shm->threadflow_lock); +#ifndef USE_PROCESS_MODEL + my_procflow = procflow; + + (void) signal(SIGUSR1, threadflow_cancel); +#endif + while (threadflow) { + threadflow_t *newthread; + int i; + + filebench_log(LOG_VERBOSE, + "Starting %lld %s threads", + *(threadflow->tf_instances), + threadflow->tf_name); + + for (i = 1; i < *threadflow->tf_instances; i++) { + /* Create threads */ + newthread = + threadflow_define_common(procflow, + threadflow->tf_name, threadflow, i + 1); + if (newthread == NULL) + return (-1); + ret += threadflow_createthread(newthread); + } + + newthread = threadflow_define_common(procflow, + threadflow->tf_name, + threadflow, 1); + + if (newthread == NULL) + return (-1); + + /* Create threads */ + ret += threadflow_createthread(newthread); + + threadflow = threadflow->tf_next; + } + + threadflow = procflow->pf_threads; + + (void) ipc_mutex_unlock(&filebench_shm->threadflow_lock); + + while (threadflow) { + void *status; + + if (threadflow->tf_tid) + (void) pthread_join(threadflow->tf_tid, &status); + + ret += *(int *)status; + threadflow = threadflow->tf_next; + } + + procflow->pf_running = 0; + + return (ret); +} + +/* + * Tells the threadflow's thread to stop and optionally signals + * its associated process to end the thread. + */ +static void +threadflow_kill(threadflow_t *threadflow) +{ + /* Tell thread to finish */ + threadflow->tf_abort = 1; + +#ifdef USE_PROCESS_MODEL +#ifdef HAVE_SIGSEND + (void) sigsend(P_PID, threadflow->tf_process->pf_pid, SIGUSR1); +#else + (void) kill(threadflow->tf_process->pf_pid, SIGUSR1); +#endif +#else /* USE_PROCESS_MODEL */ + threadflow->tf_process->pf_running = 0; +#endif /* USE_PROCESS_MODEL */ +} + +/* + * Deletes the specified threadflow from the specified threadflow + * list after first terminating the threadflow's thread, deleting + * the threadflow's flowops, and finally freeing the threadflow + * entity. It also subtracts the threadflow's shared memory + * requirements from the total amount required, shm_required. If + * the specified threadflow is found, returns 0, otherwise + * returns -1. + */ +static int +threadflow_delete(threadflow_t **threadlist, threadflow_t *threadflow) +{ + threadflow_t *entry = *threadlist; + + filebench_log(LOG_DEBUG_IMPL, "Deleting thread: (%s-%d)", + threadflow->tf_name, + threadflow->tf_instance); + + if (threadflow->tf_attrs & THREADFLOW_USEISM) { + filebench_shm->shm_required -= (*threadflow->tf_memsize); + } + + if (threadflow == *threadlist) { + /* First on list */ + filebench_log(LOG_DEBUG_IMPL, "Deleted thread: (%s-%d)", + threadflow->tf_name, + threadflow->tf_instance); + + threadflow_kill(threadflow); + flowop_delete_all(&threadflow->tf_ops); + *threadlist = threadflow->tf_next; + ipc_free(FILEBENCH_THREADFLOW, (char *)threadflow); + return (0); + } + + while (entry->tf_next) { + filebench_log(LOG_DEBUG_IMPL, + "Delete thread: (%s-%d) == (%s-%d)", + entry->tf_next->tf_name, + entry->tf_next->tf_instance, + threadflow->tf_name, + threadflow->tf_instance); + + if (threadflow == entry->tf_next) { + /* Delete */ + filebench_log(LOG_DEBUG_IMPL, + "Deleted thread: (%s-%d)", + entry->tf_next->tf_name, + entry->tf_next->tf_instance); + threadflow_kill(entry->tf_next); + flowop_delete_all(&entry->tf_next->tf_ops); + ipc_free(FILEBENCH_THREADFLOW, (char *)threadflow); + entry->tf_next = entry->tf_next->tf_next; + return (0); + } + entry = entry->tf_next; + } + + return (-1); +} + +/* + * Given a pointer to the thread list of a procflow, cycles + * through all the threadflows on the list, deleting each one + * except the FLOW_MASTER. + */ +void +threadflow_delete_all(threadflow_t **threadlist) +{ + threadflow_t *threadflow = *threadlist; + + (void) ipc_mutex_lock(&filebench_shm->threadflow_lock); + + filebench_log(LOG_DEBUG_IMPL, "Deleting all threads"); + + while (threadflow) { + if (threadflow->tf_instance && + (threadflow->tf_instance == FLOW_MASTER)) { + threadflow = threadflow->tf_next; + continue; + } + (void) threadflow_delete(threadlist, threadflow); + threadflow = threadflow->tf_next; + } + + (void) ipc_mutex_unlock(&filebench_shm->threadflow_lock); +} + +/* + * Waits till all threadflows are started, or a timeout occurs. + * Checks through the list of threadflows, waiting up to 10 + * seconds for each one to set its tf_running flag to 1. If not + * set after 10 seconds, continues on to the next threadflow + * anyway. + */ +void +threadflow_allstarted(pid_t pid, threadflow_t *threadflow) +{ + (void) ipc_mutex_lock(&filebench_shm->threadflow_lock); + + while (threadflow) { + int waits; + + if ((threadflow->tf_instance == 0) || + (threadflow->tf_instance == FLOW_MASTER)) { + threadflow = threadflow->tf_next; + continue; + } + + filebench_log(LOG_DEBUG_IMPL, "Checking pid %d thread %s-%d", + pid, + threadflow->tf_name, + threadflow->tf_instance); + + waits = 10; + while (waits && threadflow->tf_running == 0) { + (void) ipc_mutex_unlock( + &filebench_shm->threadflow_lock); + if (waits < 3) + filebench_log(LOG_INFO, + "Waiting for pid %d thread %s-%d", + pid, + threadflow->tf_name, + threadflow->tf_instance); + + (void) sleep(1); + (void) ipc_mutex_lock(&filebench_shm->threadflow_lock); + waits--; + } + + threadflow = threadflow->tf_next; + } + + (void) ipc_mutex_unlock(&filebench_shm->threadflow_lock); +} + +/* + * Create an in-memory thread object linked to a parent procflow. + * A threadflow entity is allocated from shared memory and + * initialized from the "inherit" threadflow if supplied, + * otherwise to zeros. The threadflow is assigned a unique + * thread id, the supplied instance number, the supplied name + * and added to the procflow's pf_thread list. If no name is + * supplied or the threadflow can't be allocated, NULL is + * returned Otherwise a pointer to the newly allocated threadflow + * is returned. + * + * The filebench_shm->threadflow_lock must be held by the caller. + */ +static threadflow_t * +threadflow_define_common(procflow_t *procflow, char *name, + threadflow_t *inherit, int instance) +{ + threadflow_t *threadflow; + threadflow_t **threadlistp = &procflow->pf_threads; + + if (name == NULL) + return (NULL); + + threadflow = (threadflow_t *)ipc_malloc(FILEBENCH_THREADFLOW); + + if (threadflow == NULL) + return (NULL); + + if (inherit) + (void) memcpy(threadflow, inherit, sizeof (threadflow_t)); + else + (void) memset(threadflow, 0, sizeof (threadflow_t)); + + threadflow->tf_utid = ++filebench_shm->utid; + + threadflow->tf_instance = instance; + (void) strcpy(threadflow->tf_name, name); + threadflow->tf_process = procflow; + + filebench_log(LOG_DEBUG_IMPL, "Defining thread %s-%d", + name, instance); + + /* Add threadflow to list */ + if (*threadlistp == NULL) { + *threadlistp = threadflow; + threadflow->tf_next = NULL; + } else { + threadflow->tf_next = *threadlistp; + *threadlistp = threadflow; + } + + return (threadflow); +} + +/* + * Create an in memory FLOW_MASTER thread object as described + * by the syntax. Acquire the filebench_shm->threadflow_lock and + * call threadflow_define_common() to create a threadflow entity. + * Set the number of instances to create at runtime, + * tf_instances, to "instances". Return the threadflow pointer + * returned by the threadflow_define_common call. + */ +threadflow_t * +threadflow_define(procflow_t *procflow, char *name, + threadflow_t *inherit, var_integer_t instances) +{ + threadflow_t *threadflow; + + (void) ipc_mutex_lock(&filebench_shm->threadflow_lock); + + if ((threadflow = threadflow_define_common(procflow, name, + inherit, FLOW_MASTER)) == NULL) + return (NULL); + + threadflow->tf_instances = instances; + + (void) ipc_mutex_unlock(&filebench_shm->threadflow_lock); + + return (threadflow); +} + + +/* + * Searches the provided threadflow list for the named threadflow. + * A pointer to the threadflow is returned, or NULL if threadflow + * is not found. + */ +threadflow_t * +threadflow_find(threadflow_t *threadlist, char *name) +{ + threadflow_t *threadflow = threadlist; + + (void) ipc_mutex_lock(&filebench_shm->threadflow_lock); + + while (threadflow) { + if (strcmp(name, threadflow->tf_name) == 0) { + + (void) ipc_mutex_unlock( + &filebench_shm->threadflow_lock); + + return (threadflow); + } + threadflow = threadflow->tf_next; + } + + (void) ipc_mutex_unlock(&filebench_shm->threadflow_lock); + + + return (NULL); +} diff --git a/usr/src/cmd/filebench/common/threadflow.h b/usr/src/cmd/filebench/common/threadflow.h new file mode 100644 index 0000000000..eeb5331cce --- /dev/null +++ b/usr/src/cmd/filebench/common/threadflow.h @@ -0,0 +1,120 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_THREADFLOW_H +#define _FB_THREADFLOW_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" +#include <pthread.h> + +#ifndef HAVE_CADDR_T1 +/* typedef char *caddr_t; */ +#endif + +#ifdef LINUX_PORT +#include <linux/types.h> /* for "caddr_t" */ +#endif + + +#ifdef HAVE_AIO +#include <aio.h> +#endif +#ifdef HAVE_PROCFS +#include <procfs.h> +#endif +#include "vars.h" +#include "procflow.h" +#include "fileset.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define AL_READ 1 +#define AL_WRITE 2 + + +#ifdef HAVE_AIO +typedef struct aiolist { + int al_type; + struct aiolist *al_next; + struct aiolist *al_worknext; + struct aiocb64 al_aiocb; +} aiolist_t; +#endif + +#define THREADFLOW_MAXFD 128 +#define THREADFLOW_USEISM 0x1 + +typedef struct threadflow { + char tf_name[128]; /* Name */ + int tf_attrs; /* Attributes */ + int tf_instance; /* Instance number */ + int tf_running; /* Thread running indicator */ + int tf_abort; /* Shutdown thread */ + int tf_utid; /* Unique id for thread */ + struct procflow *tf_process; /* Back pointer to process */ + pthread_t tf_tid; /* Thread id */ + pthread_mutex_t tf_lock; /* Mutex around threadflow */ + var_integer_t tf_instances; /* Number of instances for this flow */ + struct threadflow *tf_next; /* Next on proc list */ + struct flowop *tf_ops; /* Flowop list */ + caddr_t tf_mem; /* Private Memory */ + var_integer_t tf_memsize; /* Private Memory size */ + int tf_fd[THREADFLOW_MAXFD + 1]; /* Thread local fd's */ + filesetentry_t *tf_fse[THREADFLOW_MAXFD + 1]; /* Thread local files */ + int tf_fdrotor; /* Rotating fd within set */ + flowstat_t tf_stats; /* Thread statistics */ + hrtime_t tf_stime; /* Start time of op */ +#ifdef HAVE_PROCFS + struct prusage tf_susage; /* Resource usage snapshot, start */ + struct prusage tf_eusage; /* Resource usage snapshot, end */ +#endif + int tf_lwpusagefd; /* /proc lwp usage fd */ +#ifdef HAVE_AIO + aiolist_t *tf_aiolist; /* List of async I/Os */ +#endif + +} threadflow_t; + +/* Thread attrs */ +#define THREADFLOW_DEFAULTMEM 1024*1024LL; + +threadflow_t *threadflow_define(procflow_t *, char *name, threadflow_t *inherit, + var_integer_t instances); +threadflow_t *threadflow_find(threadflow_t *, char *); +int threadflow_init(procflow_t *); +void flowop_start(threadflow_t *threadflow); +void threadflow_usage(void); +void threadflow_allstarted(pid_t pid, threadflow_t *threadflow); +void threadflow_delete_all(threadflow_t **threadlist); + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_THREADFLOW_H */ diff --git a/usr/src/cmd/filebench/common/utils.c b/usr/src/cmd/filebench/common/utils.c new file mode 100644 index 0000000000..3683a5f25c --- /dev/null +++ b/usr/src/cmd/filebench/common/utils.c @@ -0,0 +1,63 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include <limits.h> +#include <string.h> +#include <stdlib.h> +#include <stdarg.h> +#include <stdio.h> +#include <errno.h> +#ifdef HAVE_STDINT_H +#include <stdint.h> +#endif +#include "utils.h" +#include "parsertypes.h" + +/* + * For now, just two routines: one to allocate a string in shared + * memory, and one to get the final file or directory name from a + * supplied pathname. + * + */ + + +/* + * Allocates space for a new string of the same length as + * the supplied string "str". Copies the old string into + * the new string and returns a pointer to the new string. + * Returns NULL if memory allocation for the new string fails. + */ +char * +fb_stralloc(char *str) +{ + char *newstr; + + if ((newstr = malloc(strlen(str) + 1)) == NULL) + return (NULL); + (void) strcpy(newstr, str); + return (newstr); +} diff --git a/usr/src/cmd/filebench/common/utils.h b/usr/src/cmd/filebench/common/utils.h new file mode 100644 index 0000000000..d2a8b655a3 --- /dev/null +++ b/usr/src/cmd/filebench/common/utils.h @@ -0,0 +1,48 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_UTILS_H +#define _FB_UTILS_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" + +#include <sys/types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#define E_ERROR 1 /* Exit status for error */ +#define E_USAGE 2 /* Exit status for usage error */ + +extern char *fb_stralloc(char *str); + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_UTILS_H */ diff --git a/usr/src/cmd/filebench/common/vars.c b/usr/src/cmd/filebench/common/vars.c new file mode 100644 index 0000000000..6145547cff --- /dev/null +++ b/usr/src/cmd/filebench/common/vars.c @@ -0,0 +1,547 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <errno.h> + +#include "vars.h" +#include "misc.h" +#include "utils.h" +#include "stats.h" +#include "eventgen.h" +#include "filebench.h" + +static var_t *var_find_dynamic(char *name); + +/* + * The filebench variables consist of var_integers (var_integer_t) + * and var_strings (var_string_t), which are pointers to integers and + * strings respectively, and vars (var_t), which are named, typed + * entities which contain either an integer or a string and can be + * placed on a linked list. All three of these objects are allocated + * from interprocess shared memory space. + * + * The routines in this module are used to allocate, locate, and + * manipulate the var_integers, var_strings, and vars. Routines are + * also included to convert between the component strings and integers + * of vars, and var_strings and var_integers. + */ + +/* + * Returns the int pointed to by the supplied var_integer_t "v". + */ +int +integer_isset(var_integer_t v) +{ + if (v == NULL) + return (0); + + return (*v); +} + +/* + * Allocates a var_integer_t from ipc memory space and + * pre-loads it with the vinteger_t "integer". Returns + * the var_integer_t on success, NULL on failure. + */ +var_integer_t +integer_alloc(vinteger_t integer) +{ + var_integer_t rtn; + + if ((rtn = (vinteger_t *)ipc_malloc(FILEBENCH_INTEGER)) == NULL) { + filebench_log(LOG_ERROR, "Alloc integer failed"); + return (NULL); + } + + *rtn = integer; + + filebench_log(LOG_DEBUG_IMPL, "Alloc integer %lld", integer); + + return (rtn); +} + +/* + * Allocates a string pointer in interprocess shared memory, + * then allocates and initializes a piece of shared string memory, + * putting the pointer to it into the just allocated string + * pointer location. The routine returns a pointer to the + * string pointer location or returns NULL on error. + */ +var_string_t +string_alloc(char *string) +{ + char **rtn; + + if ((rtn = (char **)ipc_malloc(FILEBENCH_STRING)) == NULL) { + filebench_log(LOG_ERROR, "Alloc string failed"); + return (NULL); + } + + *rtn = ipc_stralloc(string); + + filebench_log(LOG_DEBUG_IMPL, + "Alloc string %s ptr %zx", + string, rtn); + + return (rtn); +} + + +/* + * Allocates a var (var_t) from interprocess shared memory. + * Places the allocated var on the end of the globally shared + * var_list. Finally, the routine allocates a string containing + * a copy of the supplied "name" string. If any allocations + * fails, returns NULL, otherwise it returns a pointer to the + * newly allocated var. + */ +static var_t * +var_alloc(char *name) +{ + var_t *var = NULL; + var_t *prev = NULL; + var_t *newvar; + + if ((newvar = (var_t *)ipc_malloc(FILEBENCH_VARIABLE)) == NULL) { + filebench_log(LOG_ERROR, "Out of memory for variables"); + return (NULL); + } + (void) memset(newvar, 0, sizeof (newvar)); + + for (var = filebench_shm->var_list; var != NULL; var = var->var_next) + prev = var; /* Find end of list */ + if (prev != NULL) + prev->var_next = newvar; + else + filebench_shm->var_list = newvar; + + if ((newvar->var_name = ipc_stralloc(name)) == NULL) { + filebench_log(LOG_ERROR, "Out of memory for variables"); + return (NULL); + } + + return (newvar); +} + +/* + * Allocates a var (var_t) from interprocess shared memory. + * Places the allocated var on the end of the globally shared + * var_dyn_list. Finally, the routine allocates a string + * containing a copy of the supplied "name" string. If any + * allocations fails, returns NULL, otherwise it returns a + * pointer to the newly allocated var. + */ +static var_t * +var_alloc_dynamic(char *name) +{ + var_t *var = NULL; + var_t *prev = NULL; + var_t *newvar; + + if ((newvar = (var_t *)ipc_malloc(FILEBENCH_VARIABLE)) == NULL) { + filebench_log(LOG_ERROR, "Out of memory for variables"); + return (NULL); + } + (void) memset(newvar, 0, sizeof (newvar)); + + for (var = filebench_shm->var_dyn_list; var != NULL; + var = var->var_next) + prev = var; /* Find end of list */ + if (prev != NULL) + prev->var_next = newvar; + else + filebench_shm->var_dyn_list = newvar; + + if ((newvar->var_name = ipc_stralloc(name)) == NULL) { + filebench_log(LOG_ERROR, "Out of memory for variables"); + return (NULL); + } + + return (newvar); +} + +/* + * Searches for var_t with name "name" in the master var_list. + * If successful, returns a pointer to the var_t, otherwise + * returns NULL. + */ +static var_t * +var_find(char *name) +{ + var_t *var; + + for (var = filebench_shm->var_list; var != NULL; var = var->var_next) { + if (strcmp(var->var_name, name) == 0) + return (var); + } + + return (NULL); +} + +/* + * Searches for the named var, and, if found, sets its + * var_integer's value to that of the supplied integer. + * If not found, the routine allocates a new var and sets + * its var_integers's value to that of the supplied + * integer. If the named var cannot be found or allocated + * the routine returns -1, otherwise it returns 0. + */ +int +var_assign_integer(char *name, vinteger_t integer) +{ + var_t *var; + + name += 1; + + if ((var = var_find(name)) == NULL) + var = var_alloc(name); + + if (var == NULL) { + filebench_log(LOG_ERROR, "Cannot assign variable %s", + name); + return (-1); + } + + var->var_integer = integer; + + filebench_log(LOG_DEBUG_SCRIPT, "Assign integer %s=%lld", + name, integer); + + return (0); +} + +/* + * Searches for the named var, and if found returns a pointer + * to the var's var_integer. If not found, attempts to allocate + * a var named "name" and returns a pointer to it's (zeroed) + * var_integer. If the var cannot be found or allocated, an + * error is logged and the run is terminated. + */ +vinteger_t * +var_ref_integer(char *name) +{ + var_t *var; + + name += 1; + + if ((var = var_find(name)) == NULL) + var = var_find_dynamic(name); + + if (var == NULL) + var = var_alloc(name); + + if (var == NULL) { + filebench_log(LOG_ERROR, "Invalid variable $%s", + name); + filebench_shutdown(1); + } + + return (&var->var_integer); + +} + +/* + * Searches for the named var, and if found copies the var_string, + * if it exists, or a decimal number string representation of + * var_integer, into a malloc'd bit of memory using fb_stralloc(). + * Returns a pointer to the created string, or NULL on failure. + */ +char * +var_to_string(char *name) +{ + var_t *var; + char tmp[128]; + + name += 1; + + if ((var = var_find(name)) == NULL) + var = var_find_dynamic(name); + + if (var == NULL) + return (NULL); + + if (var->var_string) + return (fb_stralloc(var->var_string)); + + (void) snprintf(tmp, sizeof (tmp), "%lld", var->var_integer); + + return (fb_stralloc(tmp)); +} + +/* + * Searches for the named var, and if found returns the value, + * of var_integer. If the var is not found, or the var_integer's + * value is 0, logs an error and returns 0. + */ +vinteger_t +var_to_integer(char *name) +{ + var_t *var; + + name += 1; + + if ((var = var_find(name)) == NULL) + var = var_find_dynamic(name); + + if ((var != NULL) && (var->var_integer)) + return (var->var_integer); + + filebench_log(LOG_ERROR, + "Variable %s referenced before set", name); + + return (0); +} + +/* + * Searches for the var named "name", and if not found + * allocates it. The then extracts the var_string from + * the var named "string" and copies it into the var_string + * of the var "name", after first allocating a piece of + * interprocess shared string memory. If the var "name" + * cannot be found or allocated, or the var "string" cannot + * be found, the routine returns -1, otherwise it returns 0. + */ +int +var_assign_var(char *name, char *string) +{ + var_t *var; + var_string_t str; + + name += 1; + + if ((var = var_find(name)) == NULL) + var = var_alloc(name); + + if (var == NULL) { + filebench_log(LOG_ERROR, "Cannot assign variable %s", + name); + return (-1); + } + + if ((str = var_ref_string(string)) == NULL) + return (-1); + + if ((var->var_string = ipc_stralloc(*str)) == NULL) { + filebench_log(LOG_ERROR, "Cannot assign variable %s", + name); + return (-1); + } + filebench_log(LOG_VERBOSE, "Assign string %s=%s", name, string); + return (0); +} + +/* + * Like var_assign_integer, only this routine copies the + * supplied "string" into the var named "name". If the var + * named "name" cannot be found then it is first allocated + * before the copy. Space for the string in the var comes + * from interprocess shared memory. If the var "name" + * cannot be found or allocated, or the memory for the + * var_string copy of "string" cannot be allocated, the + * routine returns -1, otherwise it returns 0. + */ +int +var_assign_string(char *name, char *string) +{ + var_t *var; + + name += 1; + + if ((var = var_find(name)) == NULL) + var = var_alloc(name); + + if (var == NULL) { + filebench_log(LOG_ERROR, "Cannot assign variable %s", + name); + return (-1); + } + + if ((var->var_string = ipc_stralloc(string)) == NULL) { + filebench_log(LOG_ERROR, "Cannot assign variable %s", + name); + return (-1); + } + + filebench_log(LOG_DEBUG_SCRIPT, "Assign string %s=%s", name, string); + + return (0); +} + +/* + * Searches for the named var, and if found returns a pointer + * to the var's var_string. If not found, attempts to allocate + * a var named "name" and returns a pointer to it's (empty) + * var_string. If the var cannot be found or allocated, an + * error is logged and the run is terminated. + */ +char ** +var_ref_string(char *name) +{ + var_t *var; + + name += 1; + + if ((var = var_find(name)) == NULL) + var = var_find_dynamic(name); + + if (var == NULL) + var = var_alloc(name); + + if (var == NULL) { + filebench_log(LOG_ERROR, "Cannot reference variable %s", + name); + filebench_shutdown(1); + } + + return (&var->var_string); +} + +/* + * Implements a simple path name like scheme for finding values + * to place in certain specially named vars. The first part of + * the name is interpreted as a category of either: stats, + * eventgen, date, script, or host var. If a match is found, + * the appropriate routine is called to fill in the requested + * value in the provided var_t, and a pointer to the supplied + * var_t is returned. If the requested value is not found, NULL + * is returned. + */ +static var_t * +var_find_internal(var_t *var) +{ + char *n = fb_stralloc(var->var_name); + char *name = n; + var_t *rtn = NULL; + + name++; + if (name[strlen(name) - 1] != '}') + return (NULL); + name[strlen(name) - 1] = 0; + + if (strncmp(name, STATS_VAR, strlen(STATS_VAR)) == 0) + rtn = stats_findvar(var, name + strlen(STATS_VAR)); + + if (strcmp(name, EVENTGEN_VAR) == 0) + rtn = eventgen_ratevar(var); + + if (strcmp(name, DATE_VAR) == 0) + rtn = date_var(var); + + if (strcmp(name, SCRIPT_VAR) == 0) + rtn = script_var(var); + + if (strcmp(name, HOST_VAR) == 0) + rtn = host_var(var); + + free(n); + + return (rtn); +} + +/* + * Calls the C library routine getenv() to obtain the value + * for the environment variable specified by var->var_name. + * If found, the value string is returned in var->var_string. + * If the requested value is not found, NULL is returned. + */ +static var_t * +var_find_environment(var_t *var) +{ + char *n = fb_stralloc(var->var_name); + char *name = n; + + name++; + if (name[strlen(name) - 1] != ')') + return (NULL); + name[strlen(name) - 1] = 0; + + if ((var->var_string = getenv(name)) != NULL) { + free(n); + return (var); + } else { + free(n); + return (NULL); + } +} + +/* + * Look up special variables. The "name" argument is used to find + * the desired special var and fill it with an appropriate string + * value. Looks for an already allocated var of the same name on + * the var_dyn_list. If not found a new dynamic var is allocated. + * if the name begins with '{', it is an internal variable, and + * var_find_internal() is called. If the name begins with '(' it + * is an environment varable, and var_find_environment() is + * called. On success, a pointer to the var_t is returned, + * otherwise, NULL is returned. + */ +static var_t * +var_find_dynamic(char *name) +{ + var_t *var = NULL; + var_t *v = filebench_shm->var_dyn_list; + var_t *rtn; + + /* + * Lookup a reference to the var handle for this + * special var + */ + for (v = filebench_shm->var_dyn_list; v != NULL; v = v->var_next) { + if (strcmp(v->var_name, name) == 0) { + var = v; + break; + } + } + + if (var == NULL) + var = var_alloc_dynamic(name); + + /* Internal system control variable */ + if (*name == '{') { + rtn = var_find_internal(var); + if (rtn == NULL) + filebench_log(LOG_ERROR, + "Cannot find internal variable %s", + var->var_name); + return (rtn); + } + + /* Lookup variable in environment */ + if (*name == '(') { + rtn = var_find_environment(var); + if (rtn == NULL) + filebench_log(LOG_ERROR, + "Cannot find environment variable %s", + var->var_name); + return (rtn); + } + + return (NULL); +} diff --git a/usr/src/cmd/filebench/common/vars.h b/usr/src/cmd/filebench/common/vars.h new file mode 100644 index 0000000000..7b394a0ec8 --- /dev/null +++ b/usr/src/cmd/filebench/common/vars.h @@ -0,0 +1,72 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FB_VARS_H +#define _FB_VARS_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include "config.h" + +#include <stdio.h> +#include <sys/types.h> +#ifdef HAVE_STDINT_H +#include <stdint.h> +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef uint64_t vinteger_t; +typedef vinteger_t *var_integer_t; +typedef char **var_string_t; + +typedef struct var { + char *var_name; + int var_type; + struct var *var_next; + char *var_string; + vinteger_t var_integer; +} var_t; + +#define VAR_TYPE_DYNAMIC 1 + +vinteger_t *integer_alloc(vinteger_t integer); +char **string_alloc(char *string); +int var_assign_integer(char *name, vinteger_t integer); +vinteger_t *var_ref_integer(char *name); +int var_assign_string(char *name, char *string); +int var_assign_var(char *name, char *string); +char **var_ref_string(char *name); +char *var_to_string(char *name); +vinteger_t var_to_integer(char *name); +int integer_isset(var_integer_t); + +#ifdef __cplusplus +} +#endif + +#endif /* _FB_VARS_H */ diff --git a/usr/src/cmd/filebench/config/Makefile b/usr/src/cmd/filebench/config/Makefile new file mode 100644 index 0000000000..462bee5394 --- /dev/null +++ b/usr/src/cmd/filebench/config/Makefile @@ -0,0 +1,50 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +.KEEP_STATE: + +include ../../Makefile.cmd + +CONFIGS = fileio.prof filemacro.prof filemicro.prof generic.func \ +seqread.prof randomread.prof +ROOTUSRBENCHDIR = $(ROOT)/usr/benchmarks +ROOTUSRBENCHFBCONFIGDIR = $(ROOTUSRBENCHDIR)/filebench/config +FBCONFIGS = $(CONFIGS:%=$(ROOTUSRBENCHFBCONFIGDIR)/%) + +FILEMODE= 0444 + +all clobber clean lint: + +$(ROOTUSRBENCHDIR): + $(INS.dir) + +$(ROOTUSRBENCHFBCONFIGDIR): + $(INS.dir) + +$(ROOTUSRBENCHFBCONFIGDIR)/%:% + $(INS.file) + +install: $(ROOTUSRBENCHDIR) .WAIT $(ROOTUSRBENCHFBCONFIGDIR) .WAIT $(FBCONFIGS) diff --git a/usr/src/cmd/filebench/config/fileio.prof b/usr/src/cmd/filebench/config/fileio.prof new file mode 100644 index 0000000000..a5ac590228 --- /dev/null +++ b/usr/src/cmd/filebench/config/fileio.prof @@ -0,0 +1,125 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +DEFAULTS { + runtime = 120; + dir = /tmp; + stats = /tmp; + filesystem = tmpfs; + description = "fileio tmpfs"; + filesize = 10g; +} + +CONFIG randomread2k { + function = generic; + personality = randomread; + nthreads = 16; + iosize = 2k; +} + +CONFIG randomread8k { + function = generic; + personality = randomread; + nthreads = 16; + iosize = 8k; +} + +CONFIG randomread1m { + function = generic; + personality = randomread; + nthreads = 16; + iosize = 1m; +} + +CONFIG randomwrite2k { + function = generic; + personality = randomwrite; + nthreads = 16; + iosize = 2k; +} + +CONFIG randomwrite8k { + function = generic; + personality = randomwrite; + nthreads = 16; + iosize = 8k; +} + +CONFIG randomwrite1m { + function = generic; + personality = randomwrite; + nthreads = 16; + iosize = 1m; +} + +CONFIG singlestreamread1m { + function = generic; + personality = singlestreamread; + iosize = 1m; +} + +CONFIG singlestreamreaddirect1m { + function = generic; + personality = singlestreamreaddirect; + iosize = 1m; +} + +CONFIG singlestreamwrite1m { + function = generic; + personality = singlestreamwrite; + iosize = 1m; +} + +CONFIG singlestreamwritedirect1m { + function = generic; + personality = singlestreamwritedirect; + iosize = 1m; +} + +CONFIG multistreamread1m { + function = generic; + personality = multistreamread; + iosize = 1m; +} + +CONFIG multistreamreaddirect1m { + function = generic; + personality = multistreamreaddirect; + iosize = 1m; +} + +CONFIG multistreamwrite1m { + function = generic; + personality = multistreamwrite; + iosize = 1m; +} + +CONFIG multistreamwritedirect1m { + function = generic; + personality = multistreamwritedirect; + iosize = 1m; +} + diff --git a/usr/src/cmd/filebench/config/filemacro.prof b/usr/src/cmd/filebench/config/filemacro.prof new file mode 100644 index 0000000000..d03841981f --- /dev/null +++ b/usr/src/cmd/filebench/config/filemacro.prof @@ -0,0 +1,129 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +DEFAULTS { + runtime = 120; + dir = /tmp; + stats = /tmp; + filesystem = tmpfs; + description = "filemacro tmpfs"; +} + +CONFIG fileserver { + function = generic; + personality = fileserver; + nfiles = 20000; + meandirwidth = 20; + filesize = 2k; + nthreads = 100; + meaniosize = 16k; +} + +CONFIG varmail { + personality = varmail; + function = generic; + nfiles = 20000; + meandirwidth = 1000000; + filesize = 1k; + nthreads = 16; + meaniosize = 16k; +} + +CONFIG webproxy { + personality = webproxy; + function = generic; + nfiles = 20000; + meandirwidth = 1000000; + filesize = 1k; + nthreads = 100; + meaniosize = 16k; +} + +CONFIG webserver { + personality = webserver; + function = generic; + nfiles = 20000; + meandirwidth = 20; + filesize = 1k; + nthreads = 100; +} + +CONFIG large_db_oltp_2k_cached { + personality = oltp; + function = generic; + cached = 1; + directio = 0; + iosize = 2k; + nshadows = 200; + ndbwriters = 10; + usermode = 20000; + filesize = 1g; + memperthread = 1m; + workingset = 0; +} + +CONFIG large_db_oltp_2k_uncached { + personality = oltp; + function = generic; + cached = 0; + directio = 1; + iosize = 2k; + nshadows = 200; + ndbwriters = 10; + usermode = 20000; + filesize = 1g; + memperthread = 1m; + workingset = 0; +} + +CONFIG large_db_oltp_8k_cached { + personality = oltp; + function = generic; + cached = 1; + directio = 0; + iosize = 8k; + nshadows = 200; + ndbwriters = 10; + usermode = 20000; + filesize = 1g; + memperthread = 1m; + workingset = 0; +} + +CONFIG large_db_oltp_8k_uncached { + personality = oltp; + function = generic; + cached = 0; + directio = 1; + iosize = 8k; + nshadows = 200; + ndbwriters = 10; + usermode = 20000; + filesize = 1g; + memperthread = 1m; + workingset = 0; +} + diff --git a/usr/src/cmd/filebench/config/filemicro.prof b/usr/src/cmd/filebench/config/filemicro.prof new file mode 100644 index 0000000000..b498eff9b4 --- /dev/null +++ b/usr/src/cmd/filebench/config/filemicro.prof @@ -0,0 +1,187 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +DEFAULTS { + runtime = 3600; + dir = /tmp; + stats = /tmp; + filesystem = tmpfs; + description = "filemicro tmpfs"; +} + +CONFIG createfiles { + personality = createfiles; + function = generic; + nfiles = 1000; + dirwidth = 20; + filesize = 16k; + nthreads = 16; +} + +CONFIG copyfiles { + function = generic; + personality = copyfiles; + filesize = 16k; + nthreads = 16; + nfiles = 10000; + dirwidth = 20; +} + +CONFIG deletefiles { + function = generic; + personality = deletefiles; + filesize = 16k; + nthreads = 16; + nfiles = 50000; + meandirwidth = 20; +} + +CONFIG createandalloc128k { + function = generic; + personality = filemicro_create; + nthreads = 1; + iosize = 1m; + count = 64; + nfiles = 1000; +} + +CONFIG createallocsync { + function = generic; + personality = filemicro_create; + nthreads = 1; + iosize = 1m; + count = 1k; + sync = 1; +} + +CONFIG createallocfsync { + function = generic; + personality = filemicro_writefsync; + nthreads = 1; +} + +CONFIG createallocappend { + function = generic; + personality = filemicro_createrand; + nthreads = 1; +} + +CONFIG randread2k { + function = generic; + personality = filemicro_rread; + cached = 0; + iosize = 2k; +} + +CONFIG randread2kcached { + function = generic; + personality = filemicro_rread; + cached = 1; + iosize = 2k; +} + +CONFIG randwrite2ksync { + function = generic; + personality = filemicro_rwrite; + iosize = 2k; + nthreads = 1; + sync = 1; +} + +CONFIG randwrite2ksync4thread { + function = generic; + personality = filemicro_rwrite; + iosize = 2k; + nthreads = 4; + sync = 1; +} + +CONFIG randwrite8kfsynccached { + function = generic; + personality = filemicro_rwritefsync; + iosize = 8k; + nthreads = 1; + cached = 1; +} + +CONFIG seqread32k { + function = generic; + personality = filemicro_seqread; + iosize = 32k; + nthreads = 1; + cached = 0; + filesize = 100m; +} + +CONFIG seqread32kcached { + function = generic; + personality = filemicro_seqread; + iosize = 32k; + nthreads = 1; + cached = 1; + filesize = 100m; +} + +CONFIG seqwrite32k { + function = generic; + personality = filemicro_seqwrite; + iosize = 32k; + count = 32k; + nthreads = 1; + cached = 0; + sync = 0; +} + +CONFIG seqwrite32kdsync { + function = generic; + personality = filemicro_seqwrite; + iosize = 32k; + count = 32k; + nthreads = 1; + cached = 0; + sync = 1; +} + +CONFIG seqwriterand8k { + function = generic; + personality = filemicro_seqwriterand; + iosize = 8k; + count = 128k; + nthreads = 1; + cached = 0; + sync = 0; +} + +CONFIG unlink1g { + function = generic; + personality = filemicro_delete; + nthreads = 1; + filesize = 1g; + nfiles = 10; +} + + + diff --git a/usr/src/cmd/filebench/config/generic.func b/usr/src/cmd/filebench/config/generic.func new file mode 100644 index 0000000000..b0ae54d402 --- /dev/null +++ b/usr/src/cmd/filebench/config/generic.func @@ -0,0 +1,69 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +sub pre_run { + # Initialize filebench to appropriate personality, create files + # and processes + op_init(); + + # The op_load command automatically creates files + op_load(conf_reqval("personality")); + + # Flush the FS cache + op_command("system \"" . get_FILEBENCH() . "/scripts/fs_flush " . conf_reqval("filesystem") . " " . conf_reqval("dir") . "\""); + + # Initialise statistics and argument arrays + @ext_stats=(); + @file_stats=(); + @arg_stats=(); +} + +sub post_run { + my $statsbase = get_STATSBASE(); + # Shutdown processes and quit filebench + op_quit(); + # Create a html summary of the run + system ("cd $statsbase; " . get_FILEBENCH() . "/scripts/filebench_compare $statsbase") +} + +sub bm_run { + my $runtime = conf_reqval("runtime"); + my $fs = get_CONFNAME(); + + # The following array must not contain empty values ! This causes the + # statistics scripts to miss arguments ! + # Clear, run the benchmark, snap statistics + # This command will also run external statistics (supplied in an array) + # if desired + # Statistics automatically dumped into directory matching stats + # profile variable + # <stats>/<hostname>-<date-time>/<personality> + + # create processes and start run, then collect statistics + op_stats($runtime,"stats.$fs",@ext_stats,@file_stats,@arg_stats); +} + +1; diff --git a/usr/src/cmd/filebench/config/randomread.prof b/usr/src/cmd/filebench/config/randomread.prof new file mode 100644 index 0000000000..c224064268 --- /dev/null +++ b/usr/src/cmd/filebench/config/randomread.prof @@ -0,0 +1,41 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +DEFAULTS { + runtime = 60; + dir = /tmp; + stats = /tmp; + filesystem = tmpfs; + description = "randomread tmpfs"; +} + +CONFIG randomread2k { + function = generic; + personality = randomread; + filesize = 160m; + iosize = 2k; + nthreads = 1; +} diff --git a/usr/src/cmd/filebench/config/seqread.prof b/usr/src/cmd/filebench/config/seqread.prof new file mode 100644 index 0000000000..17b87e0410 --- /dev/null +++ b/usr/src/cmd/filebench/config/seqread.prof @@ -0,0 +1,41 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +DEFAULTS { + runtime = 120; + dir = /tmp; + stats = /tmp; + filesystem = tmpfs; + description = "seqread tmpfs"; +} + +CONFIG seqread1m { + function = generic; + personality = multistreamread; + filesize = 40m; + iosize = 1m; + nthreads = 1; +} diff --git a/usr/src/cmd/filebench/fbscript/Makefile b/usr/src/cmd/filebench/fbscript/Makefile new file mode 100644 index 0000000000..2b874e8c25 --- /dev/null +++ b/usr/src/cmd/filebench/fbscript/Makefile @@ -0,0 +1,53 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +.KEEP_STATE: +.SUFFIXES: .pl + +include ../../Makefile.cmd + +SCRIPT = filebench +ROOTUSRBENCHDIR = $(ROOT)/usr/benchmarks +ROOTUSRBENCHFBBINDIR = $(ROOTUSRBENCHDIR)/filebench/bin +FBSCRIPT = $(SCRIPT:%=$(ROOTUSRBENCHFBBINDIR)/%) + +.pl: + $(RM) $@ + $(CAT) $< > $@ + $(CHMOD) +x $@ + +all clean lint: + +clobber: + $(RM) $(SCRIPT) + +$(ROOTUSRBENCHDIR) $(ROOTUSRBENCHFBBINDIR): + $(INS.dir) + +$(ROOTUSRBENCHFBBINDIR)/%: % + $(INS.file) + +install: $(ROOTUSRBENCHDIR) .WAIT $(ROOTUSRBENCHFBBINDIR) .WAIT $(FBSCRIPT) diff --git a/usr/src/cmd/filebench/fbscript/filebench.pl b/usr/src/cmd/filebench/fbscript/filebench.pl new file mode 100755 index 0000000000..1ec7a698a3 --- /dev/null +++ b/usr/src/cmd/filebench/fbscript/filebench.pl @@ -0,0 +1,608 @@ +#!/usr/bin/perl +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" +# + +use POSIX; + +my $QUIT = 0; +my $USE_XANADU = 0; +my $TIMEOUT = 60; +my $FILEBENCH = "/usr/benchmarks/filebench"; +my $PROG = "/usr/benchmarks/filebench/bin/go_filebench"; +my $FSCRIPT; +my $SCRIPT_NO; +my @CONFLIST; +my %DEFDATA = (); +my %CONFDATA = (); +my %STATSHASH = (); +@ext_stats=(); +@file_stats=(); +@arg_stats=(); +@pid_arr=(); + +# The following if test caters for running benchpoint from an alternative path +#if (-r $ENV{"FILEBENCH") { +# $FILEBENCH = $ENV{"FILEBENCH"}; +#} + +############################################################################## +## Configuration hash data operations +############################################################################## + +# This sub allows a function program to extract the base directory for filebench +sub get_FILEBENCH { + return ($FILEBENCH); +} + +sub get_STATSBASE { + return ($STATSBASE); +} + +sub get_CONFNAME { + return ($CONFNAME); +} + +sub get_CONFNAME { + return ($CONFNAME); +} + +sub conf_getval { + my ($key) = shift; + return ("@{$CONFDATA{$key}}"); +} + +sub conf_reqval { + my ($key) = shift; + + if (exists($CONFDATA{$key})) { + return ("@{$CONFDATA{$key}}"); + } + print "ERROR: required key \"$key\" missing from configuration\n"; + exit(1); +} + +sub conf_exists { + my ($key) = shift; + if (exists($CONFDATA{$key})) { + return (1); + } + return (0); +} + +sub conf_hash { + return(%CONFDATA); +} + +############################################################################## +## Filebench Operations +############################################################################## + +sub op_init { +} + +sub op_load { + my ($workload) = shift; + my $scriptname = conf_reqval("statsdir") . "/thisrun.f"; + if($workload ne '') { + open (FSCRIPT, ">$scriptname"); + chmod (0755, $scriptname); + print FSCRIPT "#!$PROG -f\n\n"; + # Load the df + print FSCRIPT "load $workload\n"; + # Load the user defined defaults + op_load_defaults(); + + # Create the associated files and filesets + print FSCRIPT "create files\n"; + + print FSCRIPT "create filesets\n"; + + } + $SCRIPT_NO = 1; + return(0); +} + +sub op_set { + my ($var, $val) = @_; + if($var eq 'debug') { + print FSCRIPT "debug $val\n"; + } elsif($var ne '') { + print FSCRIPT "set \$$var=$val\n"; + } + return(0); +} + +sub op_eventrate { + my ($eventrate) = shift; + if ($eventrate ne '') { + print FSCRIPT "eventgen rate=$eventrate\n"; + return(0); + } +} + +sub op_run { + my ($time) = shift; + print FSCRIPT "run $time\n"; + return(0); +} + +sub op_sleep { + my ($time) = shift; + print FSCRIPT "sleep $time\n"; + return(0); +} + +sub op_msg { + my ($msg) = shift; + print FSCRIPT "echo \"$msg\"\n"; + return(0); +} + +sub op_quit { + if($QUIT) { + # Shutdown the appropriate processes + print FSCRIPT "shutdown processes\n"; + + # Quit filebench + print FSCRIPT "quit\n"; + close(FSCRIPT); + print "Running " . conf_reqval("statsdir") . "/thisrun.f\n"; + system (conf_reqval("statsdir") . "/thisrun.f"); + } else { + print STDOUT "ERROR: pre-mature call to op_quit\n"; + } +} + +sub op_statsdir { + print FSCRIPT "stats directory ".conf_reqval("statsdir")."\n"; + return(0); +} + +sub op_indiv_vars { + my ($ivar) = shift; + print FSCRIPT "echo \"\$$ivar\"\n"; + my ($imatch, $ierr, $ibefore, $iafter) = &expect(FSCRIPT, + $TIMEOUT, "filebench>"); + + $ibefore =~ /(.*): (.*): (-*\d+)/; + $imatch = $3; + $imatch =~ s/^\s+//; + chomp($imatch); + return($imatch); +} + +sub op_indiv_stats { + my ($var) = shift; + print FSCRIPT "echo \"\${stats.$var}\"\n"; + my ($match, $err, $before, $after) = &expect(FSCRIPT, + $TIMEOUT, "filebench>"); + + $before =~ /(.*): (.*): (-*\d+)/; + $match = $3; + $match =~ s/^\s+//; + chomp($match); + return($match); +} + +sub op_stats { + my ($time) = shift; + my ($statsfile) = shift; + + # Create the associated processes and start them running + print FSCRIPT "create processes\n"; + + if (($time ne '') && ($statsfile ne '')) { + # Clear the current statistics buffers + print FSCRIPT "stats clear\n"; + + # Start external statistics collection (if any) + # Note all statistics arrays MUST be the same length ! + if (@ext_stats != ()) { + if (($#ext_stats == $#file_stats) && ($#ext_stats == $#arg_stats)) { + $script = conf_reqval("statsdir") . "/stats$SCRIPT_NO.sh"; + open (RUNSCRIPT, ">$script"); + chmod (0755, $script); + print FSCRIPT "system \"$script\"\n"; + $SCRIPT_NO++; + $index=0; + foreach my $ext (@ext_stats) { + print RUNSCRIPT "$FILEBENCH/scripts/collect_$ext $ext $file_stats[$index] "; + print RUNSCRIPT conf_reqval("statsdir"); + print RUNSCRIPT " $time $FILEBENCH $arg_stats[$index] &\n"; + $index++; + } + } + } + close(RUNSCRIPT); + + # Sleep for the run time + print FSCRIPT "sleep $time\n"; + + # Snap the statistics + print FSCRIPT "stats snap\n"; + + # Dump the statistics to a raw file - out required due to filename constraint + print FSCRIPT "stats dump \"$statsfile.out\"\n"; + + # Statistics reaping occurs here + if (@ext_stats != ()) { + if (($#ext_stats == $#file_stats) && ($#ext_stats == $#arg_stats)) { + $script = conf_reqval("statsdir") . "/stats$SCRIPT_NO.sh"; + open (RUNSCRIPT, ">$script"); + chmod (0755, $script); + print FSCRIPT "system \"$script\"\n"; + $SCRIPT_NO++; + foreach my $ext (@ext_stats) { + print RUNSCRIPT "$FILEBENCH/scripts/kill_stats $ext &\n"; + } + close(RUNSCRIPT); + } + } + + # Dump the statistics to a Xanadu compatible XML file + if ($USE_XANADU) { + op_xmlstats($statsfile); + + $script = conf_reqval("statsdir") . "/stats$SCRIPT_NO.pl"; + open (RUNSCRIPT, ">$script"); + chmod (0755, $script); + print FSCRIPT "system \"$script\"\n"; + $SCRIPT_NO++; + + # The following loop adds the benchpoint run parameters and statistics into the filebench XML file + # We capture the meta data from the start of the filebench xml file + print RUNSCRIPT "#!/usr/bin/perl\n"; + print RUNSCRIPT "\$phase=1;\n"; + print RUNSCRIPT "open(STATSFILE,\"<".conf_reqval("statsdir")."/$statsfile.xml\");\n"; + print RUNSCRIPT "open(OSTATSFILE,\">".conf_reqval("statsdir")."/$statsfile.new.xml\");\n"; + print RUNSCRIPT "while (<STATSFILE>) {\n"; + print RUNSCRIPT "\t\$temp=\$_;\n"; + print RUNSCRIPT "\tif ((!((/.*meta.*/) || (/.*stat_doc.*/))) && (\$phase == 1)) {\n"; + print RUNSCRIPT "\t\topen(XMLFILE,\"<".conf_reqval("statsdir")."/$statsfile.config.xml\");\n"; + print RUNSCRIPT "\t\twhile (<XMLFILE>) {\n"; + print RUNSCRIPT "\t\t\tprint OSTATSFILE \$_;\n"; + print RUNSCRIPT "\t\t}\n"; + print RUNSCRIPT "\t\tclose(XMLFILE);\n"; + print RUNSCRIPT "\t\t\$phase++;\n"; + print RUNSCRIPT "\t}\n"; + print RUNSCRIPT "\tprint OSTATSFILE \$temp;\n"; + print RUNSCRIPT "}\n"; + print RUNSCRIPT "close(STATSFILE);\n"; + print RUNSCRIPT "close(OSTATSFILE);\n"; + print RUNSCRIPT "unlink(\"".conf_reqval("statsdir")."/$statsfile.xml\");\n"; + print RUNSCRIPT "unlink(\"".conf_reqval("statsdir")."/$statsfile.config.xml\");\n"; + print RUNSCRIPT "system(\"mv ".conf_reqval("statsdir")."/$statsfile.new.xml ".conf_reqval("statsdir")."/$statsfile.xml\");\n"; + + $script = conf_reqval("statsdir") . "/stats$SCRIPT_NO.sh"; + open (RUNSCRIPT, ">$script"); + chmod (0755, $script); + print FSCRIPT "system \"$script\"\n"; + $SCRIPT_NO++; + + print RUNSCRIPT "mkdir ".conf_reqval("statsdir")."/xml\n"; + print RUNSCRIPT "mkdir ".conf_reqval("statsdir")."/html\n"; + + print RUNSCRIPT "mv ".conf_reqval("statsdir")."/$statsfile.xml ".conf_reqval("statsdir")."/xml/$statsfile.xml\n"; + + # Process XML file using Xanadu 2 + print RUNSCRIPT "$FILEBENCH/xanadu/scripts/xanadu import ".conf_reqval("statsdir")." ".conf_reqval("statsdir")."/xml ".conf_reqval("function")."-".conf_reqval("statsdir")."\n"; + print RUNSCRIPT "$FILEBENCH/xanadu/scripts/xanadu export ".conf_reqval("statsdir")."/xml ".conf_reqval("statsdir")."/html\n"; + close(RUNSCRIPT); + } + } + return(0); +} + +sub op_xmlstats { + my ($statsfile) = shift; + if($statsfile ne '') { + print FSCRIPT "stats xmldump \"$statsfile.xml\"\n"; + + # The following loop adds the benchpoint run parameters and statistics into a temporary XML file + open(OSTATSFILE,">".conf_reqval("statsdir")."/$statsfile.config.xml"); + %CONFHASH = conf_hash(); + # There is no test for whether CONFHASH contains no keys + # The following two lines is to obtain the stats run directory name for xanadu meta data + print OSTATSFILE "<meta name=\"RunId\" value=\"".conf_reqval("function")."-".conf_reqval("statsdir")."\"/>\n"; + print OSTATSFILE "<stat_group name=\"Benchpoint Configuration\">\n"; + print OSTATSFILE "<cell_list>\n"; + foreach $k (keys(%CONFHASH)) { + print OSTATSFILE "<cell>@{ $CONFHASH{$k} }</cell>\n"; + } + print OSTATSFILE "</cell_list>\n"; + print OSTATSFILE "<dim_list>\n"; + print OSTATSFILE "<dim>\n"; + print OSTATSFILE "<dimval>Value</dimval>\n"; + print OSTATSFILE "</dim>\n"; + print OSTATSFILE "<dim>\n"; + foreach $k (keys(%CONFHASH)) { + print OSTATSFILE "<dimval>$k</dimval>\n"; + } + print OSTATSFILE "</dim>\n"; + print OSTATSFILE "</dim_list>\n"; + print OSTATSFILE "</stat_group>\n"; + close(OSTATSFILE); + + return(0); + } + return(1); +} + +sub op_command { + my ($command) = shift; + if($command ne '') { + print FSCRIPT "$command\n"; + } + return(0); +} + +sub op_statshash { + op_indiv_stats("iocount"); + $STATSHASH{"iocount"} = op_indiv_stats("iocount"); + $STATSHASH{"iorate"} = op_indiv_stats("iorate"); + $STATSHASH{"ioreadrate"} = op_indiv_stats("ioreadrate"); + $STATSHASH{"iowriterate"} = op_indiv_stats("iowriterate"); + $STATSHASH{"iobandwidth"} = op_indiv_stats("iobandwidth"); + $STATSHASH{"iolatency"} = op_indiv_stats("iolatency"); + $STATSHASH{"iocpu"} = op_indiv_stats("iocpu"); + $STATSHASH{"oheadcpu"} = op_indiv_stats("oheadcpu"); + $STATSHASH{"iowait"} = op_indiv_stats("iowait"); + $STATSHASH{"syscpu"} = op_indiv_stats("syscpu"); + $STATSHASH{"iocpusys"} = op_indiv_stats("iocpusys"); + return(%STATSHASH); +} + +sub op_load_defaults { +# The following code causes an intermittent bug - may be fixed at a later date +# Prevents the capture of filebench default parameters +# print FSCRIPT "vars\n"; +# my ($match, $err, $before, $after) = &expect(FSCRIPT, +# $TIMEOUT, "filebench>"); +# chomp($before); +# $before =~ /(.*): (.*): (.*)/; +# $match = $3; +# my @vars = split(/ /, $match); +# my $value = ""; +# # Cater for the default filebench commands +# foreach my $var (@vars) { +# if (!conf_exists($var)) { +# $var =~ s/ //g; +# if ($var ne '') { +# $value = op_indiv_vars($var); +# push(@{ $CONFDATA{$var} }, $value); +# } +# } +# } + + # Cater for the user defined defaults +# foreach my $var (@vars) { + + + # Cater for the user defined defaults + foreach $var (keys(%CONFDATA)) { + if (conf_exists($var)) { + $var =~ s/ //g; + my $val = conf_getval($var); + op_set($var, $val); + } + } +} + +############################################################################## +## Local functions +############################################################################## + +sub parse_profile { + my ($profile) = shift; + my ($config_section, $default_section); + + open(CFILE, "$profile") or + die "ERROR: couldn't open profile"; + + while(<CFILE>) { + my ($line) = $_; + chomp($line); + $line =~ s/^\s+//; # Get rid of spaces + + if($line =~ /^#/ or $line eq "") { + } else { + if($line =~ /}/) { + if($default_section == 1) { + $default_section = 0; + } + if($config_section == 1) { + $config_section = 0; + } + } elsif($default_section) { + $line =~ /(.+) = (.+);/; + my $opt = $1; + my $val = $2; + chomp($opt); + chomp($val); + my @vals = (); + # Check to see if this needs to be a list + if($val =~ /,/) { + push(@vals, $+) while $val =~ + m{"([^\"\\]*(?:\\.[^\"\\]*)*)",? | ([^,]+),? | , }gx; + push(@vals, undef) if substr($val, -1,1) eq ','; + @{ $DEFDATA{$opt} } = @vals; + } else { + @{CONFDATA{$opt}} = (); + push(@{ $DEFDATA{$opt} }, $val); + } + } else { + if($line =~ /^CONFIG /) { + my $config = $line; + $config =~ s/CONFIG[ ]+(.+) {/$1/; + push(@CONFLIST, $config); + $config_section = 1; + } elsif($line =~ /DEFAULTS {/) { + $default_section = 1; + } + } + } + } +} + + +# +# Parse the configuration file +# +sub parse_config { + my ($config) = shift; + + my $config_section = 0; + + print "parsing profile for config: $config\n"; + + # Howdy + seek(CFILE, 0, 0); + + while(<CFILE>) { + # Read in the line and chomp...munch...chomp + my ($line) = $_; + chomp($line); + $line =~ s/^\s+//; # Get rid of spaces + + # look for our stuff + if ($line =~ /CONFIG $config /) { + $config_section = 1; + } + + if($line =~ /}/) { + $config_section = 0; + } + + # Skip until our config is found + next if (!$config_section); + + next if ($line =~ /^#/ or $line eq ""); + + $line =~ /(.+) = (.+);/; + my $opt = $1; + my $val = $2; + chomp($opt); + chomp($val); + my @vals = (); + # Check to see if this needs to be a list + if($val =~ /,/) { + push(@vals, $+) while $val =~ + m{"([^\"\\]*(?:\\.[^\"\\]*)*)",? | ([^,]+),? | , }gx; + push(@vals, undef) if substr($val, -1,1) eq ','; + @{ $CONFDATA{$opt} } = @vals; + } else { + @{CONFDATA{$opt}} = (); + push(@{ $CONFDATA{$opt} }, $val); + } + } + + # Bye, bye + #close(CFILE) or die "ERROR: config file closing difficulties"; +} + +sub print_usage +{ + print "Usage:\n\tfilebench <profile name>\n\tfilebench -c <stat_dir> ...\n"; +} + +############################################################################## +## Main program +############################################################################## + +## Make sure arguments are okay +$numargs = $#ARGV + 1; + +if($numargs < 1) { + print_usage(); + exit(2); +} + +if($ARGV[0] eq "-c") { + if($numargs < 2) { + print_usage(); + exit(2); + } + shift(ARGV); + exec("$FILEBENCH/scripts/filebench_compare", @ARGV); +} + +$PROFILENAME = $ARGV[0]; +$PROFILE = $PROFILENAME; +$PROFILE =~ s/.*\/(.+)$/$1/; +parse_profile("$PROFILENAME.prof"); + +%CONFDATA = (); +%CONFDATA = %DEFDATA; + +# Setup the statistics base directory +$STATSBASE = conf_reqval("stats"); +my $filesystem = conf_reqval("filesystem"); +my $hostname = `hostname`; +chomp($hostname); +$STATSBASE = $STATSBASE . "/$hostname-$filesystem-$PROFILENAME-"; +my $timestamp = strftime "%b_%e_%Y-%Hh_%Mm_%Ss", localtime; +$timestamp =~ s/ //; +$STATSBASE = $STATSBASE . $timestamp; + +foreach $CONFNAME (@CONFLIST) { + %CONFDATA = (); + %CONFDATA = %DEFDATA; + parse_config("$CONFNAME"); + my $function = conf_reqval("function"); + if (-f "$function.func") { + require "$function.func"; + } else { + require "$FILEBENCH/config/$function.func"; + } + $QUIT = 0; + + # Setup the statistics directory + $statsdir = $STATSBASE . "/" . $CONFNAME; + push(@{ $CONFDATA{"statsdir"} }, $statsdir); + system("mkdir -p $statsdir"); + + # The following function is taken from the user's function file + pre_run(); + + # Leave a log of the run info + open (RUNLOG, ">$STATSBASE/thisrun.prof"); + print RUNLOG "# " . conf_reqval("description") . "\n"; + close (RUNLOG); + system ("cat $PROFILENAME.prof >>$STATSBASE/thisrun.prof"); + + # Set the global statistics directory for this run + op_statsdir(); + + # The following function is taken from the user's function file + bm_run(); + + $QUIT = 1; + + # The following function is taken from the user's function file + post_run(); + print "\n"; +} diff --git a/usr/src/cmd/filebench/i386/Makefile b/usr/src/cmd/filebench/i386/Makefile new file mode 100644 index 0000000000..8b50a9fe5e --- /dev/null +++ b/usr/src/cmd/filebench/i386/Makefile @@ -0,0 +1,38 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +#ident "%Z%%M% %I% %E% SMI" + +include ../Makefile.com + +FBBINDIRi386 = $(ROOTFBBINDIR)/i386 +PROGi386 = $(PROG:%=$(FBBINDIRi386)/%) + +$(FBBINDIRi386): + $(INS.dir) + +$(FBBINDIRi386)/%: % + $(INS.file) + +install: all .WAIT $(FBBINDIRi386) .WAIT $(PROGi386) diff --git a/usr/src/cmd/filebench/scripts/Makefile b/usr/src/cmd/filebench/scripts/Makefile new file mode 100644 index 0000000000..09837abc82 --- /dev/null +++ b/usr/src/cmd/filebench/scripts/Makefile @@ -0,0 +1,53 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +.KEEP_STATE: +.SUFFIXES: .pl + +include ../../Makefile.cmd + +SCRIPTS = filebench_compare fs_flush +ROOTUSRBENCHDIR = $(ROOT)/usr/benchmarks +ROOTUSRBENCHFBSCRIPTSDIR = $(ROOTUSRBENCHDIR)/filebench/scripts +FBSCRIPTS = $(SCRIPTS:%=$(ROOTUSRBENCHFBSCRIPTSDIR)/%) + +.pl: + $(RM) $@ + $(CAT) $< > $@ + $(CHMOD) +x $@ + +all clean lint: + +clobber: + $(RM) $(SCRIPTS) + +$(ROOTUSRBENCHDIR) $(ROOTUSRBENCHFBSCRIPTSDIR): + $(INS.dir) + +$(ROOTUSRBENCHFBSCRIPTSDIR)/%: % + $(INS.file) + +install: $(ROOTUSRBENCHDIR) .WAIT $(ROOTUSRBENCHFBSCRIPTSDIR) .WAIT $(FBSCRIPTS) diff --git a/usr/src/cmd/filebench/scripts/filebench_compare.pl b/usr/src/cmd/filebench/scripts/filebench_compare.pl new file mode 100755 index 0000000000..9b4801f135 --- /dev/null +++ b/usr/src/cmd/filebench/scripts/filebench_compare.pl @@ -0,0 +1,244 @@ +#!/usr/bin/perl +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" +# + +# +# Compare filebench results +# +# Usage: filebench_summary <dir1> <dir2> ... +# + +use CGI ':standard'; + +$maxiopsoverall = 0; +$maxiopsrate = 0; +$maxbandwidth = 0; + +# +# Create html and text +# +open (HTML, ">index.html"); +print HTML start_html(-title=>'Filebench'); +print HTML "<body>"; +# +# Print aggregate flowop stats +# +foreach $dir (@ARGV) { + + printf ("Generating html for $dir\n"); + open (PROFILE, "<$dir/thisrun.prof"); + $description = <PROFILE>; + $description =~ s/.*"(.+)".*/$1/; + + $files = `ls $dir/stats.*.out $dir/*/stats.*.out 2>/dev/null`; + foreach $file (split(/\n/, $files)) { + print "file = $file\n"; + ($prefix, $workload, $fstype) = split(/\./, $file); + $dataset = $dir; + $dataset =~ s/.*\/(.+)$/$1/; + $dataset =~ s/\/$//; + $desc{$dataset} = "$description"; +# print "dataset = $dataset\n"; +# print "workload = $workload\n"; +# print "fstype = $fstype\n"; + open (STATS, $file); + $tmp = <STATS>; + while (<STATS>) { + ($flowop, $ops, $bandwidth, $latency, $cpu, $wait, $seconds) = split(/[ \t]+/, $_); + + if (/^$/) { + $tmp = <STATS>; + ($fluff, $opcnt, $ops, $reads, $writes, $bandwidth, + $cpu) = split(/[ \tA-z:\/,]+/, $tmp); + $ops{$workload, $dataset} = $ops; + last; + } + + $ops =~ s/ops\/s//; + $bandwidth =~ s/mb\/s//; + $latency =~ s/ms\/op//; + $cpu =~ s/us\/op//; + + # Collapse shadow reads into single metric + if ($flowop =~ /shadowread/) { + $flowop = "shadow-read"; + } + + # Collapse database writes into single metric + if ($flowop =~ /db.*write/) { + $flowop = "db-write"; + } + + # Collapse database writes into single metric + if ($flowop =~ /db.*write/) { + $flowop = "db-write"; + } + + $datasets{$dataset} = $dataset; + $workloads{$workload} = $workload; + + $flowops{$flowop} = $flowop; + $wkl_flowops{$flowop, $workload} = $flowop; + $wkl_workload{$flowop, $workload} = $workload; + $flow_ops{$flowop, $workload, $dataset} += $ops; + $flow_bandwidth{$flowop, $workload, $dataset} += $bandwidth; + $flow_latency{$flowop, $workload, $dataset} += $latency; + $flow_cpu{$flowop, $workload, $dataset} += $cpu; + + $bandwidth{$workload, $dataset} += $bandwidth; + $latency{$workload, $dataset} += $latency; + $cpu{$workload, $dataset} += $cpu; + + $flowopcnt{$flowop, $workload, $dataset}++; + $workloadcnt{$workload, $dataset}++; + } + close(STATS); + } +} + +# HTML IOPS +print HTML h1('Throughput breakdown (ops per second)'); +print HTML "<table border=1>"; +print HTML "<b><td>Workload</td>"; +foreach $dataset (sort {$a cmp $b}(keys %datasets)) { + print HTML "<td>$desc{$dataset}</td>"; +} +print HTML "</b></tr>"; + +foreach $workload (sort (keys %workloads)) { + print HTML "<b><tr><td>$workload</td>"; + $last = 0; + foreach $dataset (sort {$a cmp $b}(keys %datasets)) { + $color = "white"; + $this = $ops{$workload, $dataset}; + if ($last && ($this - $last) < ($last * -0.1)) { + $color = "red"; + } + if ($last && ($this - $last) > ($last * 0.1)) { + $color = "green"; + } + printf HTML ("<td>%d</td\n", $this); + $last = $ops{$workload, $dataset}; + } + print HTML "</b></tr>"; +} +print HTML "</table>"; + +# HTML Bandwidth +print HTML h1('Bandwidth breakdown (MB/s)'); +print HTML "<table border=1>"; +print HTML "<td>Workload</td>"; +foreach $dataset (sort {$a cmp $b}(keys %datasets)) { + print HTML "<td>$desc{$dataset}</td>"; +} +print HTML "</tr>"; +foreach $workload (sort (keys %workloads)) { + $bandwidth = 0; + foreach $dataset (sort {$a cmp $b}(keys %datasets)) { + $bandwidth += $bandwidth{$workload, $dataset}; + } + next if ($bandwidth == 0); + print HTML "<tr><td>$workload</td>"; + foreach $dataset (sort {$a cmp $b}(keys %datasets)) { + printf HTML ("<td>%d</td>\n", $bandwidth{$workload, $dataset}); + } + print HTML "</tr>"; +} +print HTML "</table>"; + +# HTML Latency +print HTML h1('Latency breakdown (ms per op)'); +print HTML "<table border=1>"; +print HTML "<td>Workload</td>"; +foreach $dataset (sort {$a cmp $b}(keys %datasets)) { + print HTML "<td>$desc{$dataset}</td>"; +} + +print HTML "</tr>"; +foreach $workload (sort (keys %workloads)) { + print HTML "<tr><td>$workload</td>"; + foreach $dataset (sort {$a cmp $b}(keys %datasets)) { + if ( $workloadcnt{$workload, $dataset}) { + printf HTML ("<td>%.1f</td>", $latency{$workload, + $dataset} / $workloadcnt{$workload, $dataset}); + } else { + printf HTML ("<td></td>"); + } + } + print HTML "</tr>"; + foreach $flowop (keys %wkl_flowops) { + next if ("$wkl_workload{$flowop}" ne "$workload"); + print HTML "<tr><td><i>__$wkl_flowops{$flowop}</td>"; + foreach $dataset (sort {$a cmp $b}(keys %datasets)) { + if ( $flowopcnt{$flowop, $dataset}) { + printf HTML ("<td>%.1f</td>\n", $flow_latency{$flowop, + $dataset} / $flowopcnt{$flowop, $dataset}); + } else { + printf HTML ("<td></td>"); + } + } + print HTML "</i></tr>"; + } +} +print HTML "</table>"; + +# HTML Efficiency +print HTML h1('Efficiency breakdown (Code path length in uS/op)'); +print HTML "<table border=1>"; +print HTML "<td>Workload</td>"; +foreach $dataset (sort {$a cmp $b}(keys %datasets)) { + print HTML "<td>$desc{$dataset}</td>"; +} +print HTML "</tr>"; +foreach $workload (sort (keys %workloads)) { + print HTML "<tr><td>$workload</td>"; + foreach $dataset (sort {$a cmp $b}(keys %datasets)) { + if ($workloadcnt{$workload, $dataset}) { + printf HTML ("<td>%d</td>", $cpu{$workload, $dataset} + / $workloadcnt{$workload, $dataset}); + } else { + printf HTML ("<td></td>"); + } + } + print HTML "</tr>"; + foreach $flowop (keys %wkl_flowops) { + next if ("$wkl_workload{$flowop}" ne "$workload"); + print HTML "<tr><td><i>__$wkl_flowops{$flowop}</td>"; + foreach $dataset (sort {$a cmp $b}(keys %datasets)) { + if ($flowopcnt{$flowop, $dataset}) { + printf HTML ("<td>%d</td>\n", $flow_cpu{$flowop, + $dataset} / $flowopcnt{$flowop, $dataset}); + } else { + printf HTML ("<td></td>"); + } + } + print HTML "</i></tr>"; + } +} +print HTML "</table>"; + +end_html(); diff --git a/usr/src/cmd/filebench/scripts/fs_flush.pl b/usr/src/cmd/filebench/scripts/fs_flush.pl new file mode 100755 index 0000000000..5eb71bfd01 --- /dev/null +++ b/usr/src/cmd/filebench/scripts/fs_flush.pl @@ -0,0 +1,95 @@ +#!/usr/bin/perl +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +# +# Put commands in here to flush the file system cache after +# file set creation but prior to steady state +# +# For most file systems, filebench already handles fs cache flushing +# For ZFS, it needs some help, so this script does +# "zpool export <poolname>" then "zpool import <poolname>" +# + +$fs = $ARGV[0]; +$dir = $ARGV[1]; + +# +# if not zfs, inform user and exit. +# +if (($fs =~ m/^zfs$/) != 1) { + print "filesystem type is: $fs, no action required, so exiting\n"; + exit(0); +} + +# +# It is zfs. Find name of pool to export/import from supplied +# directory path name $dir +# +# Example: +# # zfs list -H +# tank 164K 24.0G 19K /tank +# tank/a 18K 24.0G 18K /tank/a +# tank/b 18K 24.0G 18K /wombat +# # +# # ./fs_flush zfs /wombat +# 'zpool export tank' +# 'zpool import tank' +# # +# + +# Get a list of zfs file systems mounted locally +@zlist = `/usr/sbin/zfs list -H`; + +# +# Compare the list with the supplied directory path name +# +chomp @zlist; +foreach ( @zlist ) { + # + # For listed zfs file systems, extract the root and + # mount point information + # + my $zline = $_; + ($root, $b, $c, $d, $mntpnt) = split /\t/, $zline, 5; + + # See if the supplied directory path includes this mount point + if ($dir =~/^$mntpnt/) { + + # + # We have a winner! The root name up to the + # first forward slash is the pool name + # + ($pool) = split /\//, $root; + + # Do the cache flushing + print "'zpool export $pool'\n"; + system("zpool export $pool"); + print "'zpool import $pool'\n"; + system("zpool import $pool"); + exit(0); + } +} diff --git a/usr/src/cmd/filebench/sparcv9/Makefile b/usr/src/cmd/filebench/sparcv9/Makefile new file mode 100644 index 0000000000..b1d03827ba --- /dev/null +++ b/usr/src/cmd/filebench/sparcv9/Makefile @@ -0,0 +1,38 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +#ident "%Z%%M% %I% %E% SMI" + +include ../Makefile.com + +FBBINDIRsparcv9 = $(ROOTFBBINDIR)/sparcv9 +PROGsparcv9 = $(PROG:%=$(FBBINDIRsparcv9)/%) + +$(FBBINDIRsparcv9): + $(INS.dir) + +$(FBBINDIRsparcv9)/%: % + $(INS.file) + +install: all .WAIT $(FBBINDIRsparcv9) .WAIT $(PROGsparcv9) diff --git a/usr/src/cmd/filebench/workloads/Makefile b/usr/src/cmd/filebench/workloads/Makefile new file mode 100644 index 0000000000..76e7dab8a5 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/Makefile @@ -0,0 +1,84 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +.KEEP_STATE: + +include ../../Makefile.cmd + +WORKLOADS = \ + bringover.f \ + copyfiles.f \ + createfiles.f \ + deletefiles.f \ + filemicro_create.f \ + filemicro_createfiles.f \ + filemicro_createrand.f \ + filemicro_delete.f \ + filemicro_rread.f \ + filemicro_rwrite.f \ + filemicro_rwritedsync.f \ + filemicro_rwritefsync.f \ + filemicro_seqread.f \ + filemicro_seqwrite.f \ + filemicro_seqwriterand.f \ + filemicro_writefsync.f \ + fileserver.f \ + mongo.f \ + multistreamread.f \ + multistreamreaddirect.f \ + multistreamwrite.f \ + multistreamwritedirect.f \ + oltp.f \ + randomread.f \ + randomrw.f \ + randomwrite.f \ + singlestreamread.f \ + singlestreamreaddirect.f \ + singlestreamwrite.f \ + singlestreamwritedirect.f \ + tpcso.f \ + varmail.f \ + webproxy.f \ + webserver.f + +ROOTUSRBENCHDIR = $(ROOT)/usr/benchmarks +ROOTUSRBENCHFBWORKLOADSDIR = $(ROOTUSRBENCHDIR)/filebench/workloads +FBWORKLOADS = $(WORKLOADS:%=$(ROOTUSRBENCHFBWORKLOADSDIR)/%) + +FILEMODE= 0444 + +all clobber clean lint: + +$(ROOTUSRBENCHDIR): + $(INS.dir) + +$(ROOTUSRBENCHFBWORKLOADSDIR): + $(INS.dir) + +$(ROOTUSRBENCHFBWORKLOADSDIR)/%: % + $(INS.file) + +install: $(ROOTUSRBENCHDIR) .WAIT $(ROOTUSRBENCHFBWORKLOADSDIR) .WAIT $(FBWORKLOADS) diff --git a/usr/src/cmd/filebench/workloads/bringover.f b/usr/src/cmd/filebench/workloads/bringover.f new file mode 100644 index 0000000000..58f94d68d3 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/bringover.f @@ -0,0 +1,56 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + + +set $dir=/tmp +set $nfiles=1000 +set $dirwidth=20 +set $filesize=16k +set $nthreads=1 + +define fileset name=srcfiles,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc +define fileset name=destfiles,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth + +define process name=filereader,instances=1 +{ + thread name=filereaderthread,memsize=10m,instances=$nthreads + { + flowop openfile name=openfile1,filesetname=srcfiles,fd=1 + flowop readwholefile name=readfile1,fd=1 + flowop createfile name=createfile2,filesetname=destfiles,fd=2 + flowop writewholefile name=writefile2,filesetname=destfiles,fd=2,srcfd=1 + flowop closefile name=closefile1,fd=1 + flowop closefile name=closefile2,fd=2 + } +} + +echo "Bringover Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nfiles=<value> defaults to $nfiles" +usage " set \$dirwidth=<value> defaults to $dirwidth" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/copyfiles.f b/usr/src/cmd/filebench/workloads/copyfiles.f new file mode 100644 index 0000000000..d4ed82266f --- /dev/null +++ b/usr/src/cmd/filebench/workloads/copyfiles.f @@ -0,0 +1,55 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nfiles=1000 +set $dirwidth=20 +set $filesize=16k +set $nthreads=1 + +define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc=100 +define fileset name=destfiles,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth + +define process name=filereader,instances=1 +{ + thread name=filereaderthread,memsize=10m,instances=$nthreads + { + flowop openfile name=openfile1,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile1,fd=1 + flowop createfile name=createfile2,filesetname=destfiles,fd=2 + flowop writewholefile name=writefile2,filesetname=destfiles,fd=2,srcfd=1 + flowop closefile name=closefile1,fd=1 + flowop closefile name=closefile2,fd=2 + } +} + +echo "CopyFiles Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nfiles=<value> defaults to $nfiles" +usage " set \$dirwidth=<value> defaults to $dirwidth" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/createfiles.f b/usr/src/cmd/filebench/workloads/createfiles.f new file mode 100644 index 0000000000..945362394c --- /dev/null +++ b/usr/src/cmd/filebench/workloads/createfiles.f @@ -0,0 +1,54 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nfiles=50000 +set $meandirwidth=100 +set $filesize=16k +set $nthreads=16 + +define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$meandirwidth + +define process name=filecreate,instances=1 +{ + thread name=filecreatethread,memsize=10m,instances=$nthreads + { + flowop createfile name=createfile1,filesetname=bigfileset,fd=1 + flowop writewholefile name=writefile1,filesetname=bigfileset,fd=1 + flowop closefile name=closefile1,fd=1 + flowop opslimit name=limit + } +} + +echo "Createfiles Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nfiles=<value> defaults to $nfiles" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$meandirwidth=<size> defaults to $meandirwidth" +usage "(sets mean dir width and dir depth is calculated as log (width, nfiles)" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/deletefiles.f b/usr/src/cmd/filebench/workloads/deletefiles.f new file mode 100644 index 0000000000..7a3502bcf4 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/deletefiles.f @@ -0,0 +1,52 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nfiles=50000 +set $meandirwidth=100 +set $filesize=16k +set $nthreads=16 + +define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$meandirwidth,prealloc=100 + +define process name=filedelete,instances=1 +{ + thread name=filedeletethread,memsize=10m,instances=$nthreads + { + flowop deletefile name=deletefile1,filesetname=bigfileset,fd=1 + flowop opslimit name=limit + } +} + +echo "Deletefiles Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nfiles=<value> defaults to $nfiles" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$meandirwidth=<size> defaults to $meandirwidth" +usage "(sets mean dir width and dir depth is calculated as log (width, nfiles)" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/filemicro_create.f b/usr/src/cmd/filebench/workloads/filemicro_create.f new file mode 100644 index 0000000000..2c4a247f36 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/filemicro_create.f @@ -0,0 +1,51 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nthreads=1 +set $iosize=1m +set $count=1024 +set $sync=0 + +define fileset name=bigfileset,path=$dir,size=0,entries=128,dirwidth=1024,prealloc=100 + +define process name=filecreater,instances=1 +{ + thread name=filecreaterthread,memsize=10m,instances=$nthreads + { + flowop appendfile name=append-file,filesetname=bigfileset,dsync=$sync,iosize=$iosize,fd=1 + flowop finishoncount name=finish,value=$count + } +} + +echo "FileMicro-Create Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$iosize=<size> defaults to $iosize" +usage " set \$count=<value> defaults to $count" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$sync=<bool> defaults to $sync" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/filemicro_createfiles.f b/usr/src/cmd/filebench/workloads/filemicro_createfiles.f new file mode 100644 index 0000000000..0b0d2ac158 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/filemicro_createfiles.f @@ -0,0 +1,54 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nfiles=20000 +set $meandirwidth=100000 +set $filesize=1k +set $nthreads=1 +set $count=5000 + +define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$meandirwidth,prealloc=50 + +define process name=filecreate,instances=1 +{ + thread name=filecreatethread,memsize=10m,instances=$nthreads + { + flowop createfile name=createfile1,filesetname=bigfileset,fd=1 + flowop writewholefile name=writefile1,filesetname=bigfileset,fd=1 + flowop closefile name=closefile1,fd=1 + flowop finishoncount name=finish,value=$count + } +} + +echo "FileMicro-Createfiles Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nfiles=<value> defaults to $nfiles" +usage " set \$count=<value> defaults to $count" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/filemicro_createrand.f b/usr/src/cmd/filebench/workloads/filemicro_createrand.f new file mode 100644 index 0000000000..01e8cb40ac --- /dev/null +++ b/usr/src/cmd/filebench/workloads/filemicro_createrand.f @@ -0,0 +1,60 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +# 3- open() and allocation of a 1GB file with write() +# of size picked uniformly in [1K,8K] range and issuing +# fsync() every 10MB. +# fm_create8k + + +set $dir=/tmp +set $nthreads=1 +set $iosize=1m +set $count=1280 +set $bytes=1g +set $sync=0 + +define fileset name=bigfileset,path=$dir,size=0,entries=128,dirwidth=1024,prealloc=100 + +define process name=filecreater,instances=1 +{ + thread name=filecreaterthread,memsize=10m,instances=$nthreads + { + flowop appendfilerand name=append-file,filesetname=bigfileset,dsync=$sync,iosize=$iosize,fd=1 + flowop fsync name=sync,fd=1 + flowop finishonbytes name=finish,value=$bytes + } +} + +echo "FileMicro-CreateRand Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$iosize=<size> defaults to $iosize" +usage " set \$bytes=<value> defaults to $bytes" +usage " set \$count=<value> defaults to $count" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$sync=<bool> defaults to $sync" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/filemicro_delete.f b/usr/src/cmd/filebench/workloads/filemicro_delete.f new file mode 100644 index 0000000000..ec988b2b53 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/filemicro_delete.f @@ -0,0 +1,57 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +# delete 1000 files + +set $dir=/tmp +set $nfiles=50000 +set $meandirwidth=100 +set $filesize=16k +set $nthreads=16 +set $count=1000 + +define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$meandirwidth,prealloc=100 + +define process name=filedelete,instances=1 +{ + thread name=filedeletethread,memsize=10m,instances=$nthreads + { + flowop deletefile name=deletefile1,filesetname=bigfileset,fd=1 + flowop opslimit name=limit + flowop finishoncount name=finish,value=$count + } +} + +echo "FileMicro-Delete Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nfiles=<value> defaults to $nfiles" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$count=<value> defaults to $count" +usage " set \$meandirwidth=<size> defaults to $meandirwidth" +usage "(sets mean dir width and dir depth is calculated as log (width, nfiles)" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/filemicro_rread.f b/usr/src/cmd/filebench/workloads/filemicro_rread.f new file mode 100644 index 0000000000..9a3b44599a --- /dev/null +++ b/usr/src/cmd/filebench/workloads/filemicro_rread.f @@ -0,0 +1,52 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nthreads=1 +set $iosize=2k +set $bytes=128m +set $iters=1 +set $filesize=1g + +define file name=bigfile1,path=$dir,size=$filesize,prealloc,reuse,cached=$cached + +define process name=filereader,instances=1 +{ + thread name=filereaderthread,memsize=10m,instances=$nthreads + { + flowop read name=write-file,filesetname=bigfile1,random,iosize=$iosize,iters=$iters + flowop finishonbytes name=finish,value=$bytes + } +} + +echo "FileMicro-ReadRand Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$iosize=<size> defaults to $iosize" +usage " set \$bytes=<value> defaults to $bytes" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/filemicro_rwrite.f b/usr/src/cmd/filebench/workloads/filemicro_rwrite.f new file mode 100644 index 0000000000..40ac003bff --- /dev/null +++ b/usr/src/cmd/filebench/workloads/filemicro_rwrite.f @@ -0,0 +1,57 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +# 10- 128M worth of 2K write(2) (to random offset), open with +# O_DSYNC, uncached. + +set $dir=/tmp +set $nthreads=1 +set $iosize=2k +set $bytes=128m +set $iters=1 +set $cached=0 +set $filesize=1g + +define file name=bigfile1,path=$dir,size=$filesize,prealloc,reuse,cached=$cached + +define process name=filewriter,instances=1 +{ + thread name=filewriterthread,memsize=10m,instances=$nthreads + { + flowop write name=write-file,filename=bigfile1,random,dsync,iosize=$iosize,iters=$iters + flowop finishonbytes name=finish,value=$bytes + } +} + +echo "FileMicro-WriteRandDsync Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$iosize=<size> defaults to $iosize" +usage " set \$bytes=<value> defaults to $bytes" +usage " set \$cached=<value> defaults to $cached" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/filemicro_rwritedsync.f b/usr/src/cmd/filebench/workloads/filemicro_rwritedsync.f new file mode 100644 index 0000000000..75d30b0c94 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/filemicro_rwritedsync.f @@ -0,0 +1,55 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +# 128M worth of random 8K-aligned write(2) to a 1G file; followed by fsync(); cached. + +set $dir=/tmp +set $nthreads=1 +set $iosize=2k +set $count=65536 +set $filesize=1g +set $iters=1 + +define fileset name=bigfile,path=$dir,size=$filesize,entries=$nthreads,dirwidth=1024,prealloc + +define process name=filewriter,instances=1 +{ + thread name=filewriterthread,memsize=10m,instances=$nthreads + { + flowop write name=write-file,filesetname=bigfile,random,dsync,iosize=$iosize,fd=1,iters=$iters + flowop finishoncount name=finish,value=$count + } +} + +echo "FileMicro-WriteRandFsync Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$iosize=<size> defaults to $iosize" +usage " set \$count=<value> defaults to $count" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$iters=<value> defaults to $iters" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/filemicro_rwritefsync.f b/usr/src/cmd/filebench/workloads/filemicro_rwritefsync.f new file mode 100644 index 0000000000..ab4c2ab1dc --- /dev/null +++ b/usr/src/cmd/filebench/workloads/filemicro_rwritefsync.f @@ -0,0 +1,56 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +# 128M worth of random 8K-aligned write(2) to a 1G file; followed by fsync(); cached. + +set $dir=/tmp +set $nthreads=1 +set $iosize=8k +set $count=1 +set $iters=16384 +set $filesize=1g +set $cached=0 + +define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nthreads,dirwidth=1024,prealloc=100,cached=$cached + +define process name=filewriter,instances=1 +{ + thread name=filewriterthread,memsize=10m,instances=$nthreads + { + flowop write name=write-file,filesetname=bigfileset,random,iosize=$iosize,fd=1,iters=$iters + flowop fsync name=sync-file,fd=1 + flowop finishoncount name=finish,value=$count + } +} + +echo "FileMicro-WriteRandFsync Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$iosize=<size> defaults to $iosize" +usage " set \$count=<value> defaults to $count" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$cached=<value> defaults to $cached" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/filemicro_seqread.f b/usr/src/cmd/filebench/workloads/filemicro_seqread.f new file mode 100644 index 0000000000..09f99daab5 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/filemicro_seqread.f @@ -0,0 +1,53 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +# 4- Sequential read(32K) of a 1G file, cached +# 5- Sequential read(32K) of a 1G file, uncached + +set $dir=/tmp +set $nthreads=1 +set $iosize=1m +set $filesize=1g +set $cached=0 + +define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nthreads,dirwidth=1024,prealloc=100,cached=$cached + +define process name=filereader,instances=1 +{ + thread name=filereaderthread,memsize=10m,instances=$nthreads + { + flowop read name=append-file,filesetname=bigfileset,iosize=$iosize,fd=1 + } +} + +echo "FileMicro-SeqRead Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$iosize=<size> defaults to $iosize" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$cached=<bool> defaults to $cached" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/filemicro_seqwrite.f b/usr/src/cmd/filebench/workloads/filemicro_seqwrite.f new file mode 100644 index 0000000000..588dcb78be --- /dev/null +++ b/usr/src/cmd/filebench/workloads/filemicro_seqwrite.f @@ -0,0 +1,53 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +# 4- Sequential write(32K) of a 1G file, cached +# 5- Sequential write(32K) of a 1G file, uncached + +set $dir=/tmp +set $nthreads=1 +set $iosize=1m +set $cached=0 + +define fileset name=bigfileset,path=$dir,size=0,entries=$nthreads,dirwidth=1024,prealloc=100,cached=$cached + +define process name=filewriter,instances=1 +{ + thread name=filewriterthread,memsize=10m,instances=$nthreads + { + flowop appendfile name=write-file,dsync=$sync,filesetname=bigfileset,iosize=$iosize,fd=1,iters=$count + flowop closefile name=close,fd=1 + flowop finishoncount name=finish,value=1 + } +} + +echo "FileMicro-SeqWrite Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$iosize=<size> defaults to $iosize" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$cached=<bool> defaults to $cached" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/filemicro_seqwriterand.f b/usr/src/cmd/filebench/workloads/filemicro_seqwriterand.f new file mode 100644 index 0000000000..fe53c09f87 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/filemicro_seqwriterand.f @@ -0,0 +1,59 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +# 7- Sequential write() of a 1G file, size picked uniformly in +# the [1K,8K] range, followed by close(), cached. + + + +set $dir=/tmp +set $nthreads=1 +set $iosize=8k +set $cached=0 +set $sync=0 +set $count=128k + +define fileset name=bigfileset,path=$dir,size=0,entries=$nthreads,dirwidth=1024,prealloc=100,cached=$cached + +define process name=filewriter,instances=1 +{ + thread name=filewriterthread,memsize=10m,instances=$nthreads + { + flowop appendfilerand name=write-file,dsync=$sync,filesetname=bigfileset,iosize=$iosize,fd=1,iters=$count + flowop closefile name=close,fd=1 + flowop finishoncount name=finish,value=1 + } +} + +echo "FileMicro-SeqWriteRand Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$iosize=<size> defaults to $iosize" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$cached=<bool> defaults to $cached" +usage " set \$count=<bool> defaults to $cached" +usage " set \$sync=<bool> defaults to $cached" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/filemicro_writefsync.f b/usr/src/cmd/filebench/workloads/filemicro_writefsync.f new file mode 100644 index 0000000000..93cd4cb74f --- /dev/null +++ b/usr/src/cmd/filebench/workloads/filemicro_writefsync.f @@ -0,0 +1,55 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +# 3- open() and allocation of a 1GB file with write() +# of size picked uniformly in [1K,8K] range and issuing +# fsync() every 10MB. + +set $dir=/tmp +set $nthreads=1 +set $iosize=8k +set $count=128k +set $iters=1250 + +define fileset name=bigfileset,path=$dir,size=0,entries=128,dirwidth=1024,prealloc=100 + +define process name=filecreater,instances=1 +{ + thread name=filecreaterthread,memsize=10m,instances=$nthreads + { + flowop appendfile name=append-file,filesetname=bigfileset,iosize=$iosize,fd=1,iters=$iters + flowop fsync name=sync-file,fd=1 + flowop finishoncount name=finish,value=$count + } +} + +echo "FileMicro-WriteFsync Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$iosize=<size> defaults to $iosize" +usage " set \$count=<value> defaults to $count" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/fileserver.f b/usr/src/cmd/filebench/workloads/fileserver.f new file mode 100644 index 0000000000..6926db70b7 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/fileserver.f @@ -0,0 +1,60 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nfiles=1000 +set $meandirwidth=20 +set $filesize=128k +set $nthreads=100 +set $meaniosize=16k + +define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$meandirwidth,prealloc + +define process name=filereader,instances=1 +{ + thread name=filereaderthread,memsize=10m,instances=$nthreads + { + flowop openfile name=openfile1,filesetname=bigfileset,fd=1 + flowop appendfilerand name=appendfilerand1,iosize=$meaniosize,fd=1 + flowop closefile name=closefile1,fd=1 + flowop openfile name=openfile2,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile1,fd=1 + flowop closefile name=closefile2,fd=1 + flowop deletefile name=deletefile1,filesetname=bigfileset + flowop statfile name=statfile1,filesetname=bigfileset + } +} + +echo "FileServer Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nfiles=<value> defaults to $nfiles" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$meaniosize=<value> defaults to $meaniosize" +usage " set \$meandirwidth=<size> defaults to $meandirwidth" +usage "(sets mean dir width and dir depth is calculated as log (width, nfiles)" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/mongo.f b/usr/src/cmd/filebench/workloads/mongo.f new file mode 100644 index 0000000000..9a5e33a87f --- /dev/null +++ b/usr/src/cmd/filebench/workloads/mongo.f @@ -0,0 +1,58 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nfiles=1000 +set $dirwidth=20 +set $filesize=16k +set $nthreads=1 +set $meaniosize=16k + +define fileset name=postset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc +define fileset name=postsetdel,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc + +define process name=filereader,instances=1 +{ + thread name=filereaderthread,memsize=10m,instances=$nthreads + { + flowop openfile name=openfile1,filesetname=postset,fd=1 + flowop appendfilerand name=appendfilerand1,iosize=$meaniosize,fd=1 + flowop closefile name=closefile1,fd=1 + flowop openfile name=openfile2,filesetname=postset,fd=1 + flowop readwholefile name=readfile1,fd=1 + flowop closefile name=closefile2,fd=1 + flowop deletefile name=deletefile1,filesetname=postsetdel + } +} + +echo "Mongo-like Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nfiles=<value> defaults to $nfiles" +usage " set \$dirwidth=<value> defaults to $dirwidth" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$meaniosize=<value> defaults to $meaniosize" +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/multistreamread.f b/usr/src/cmd/filebench/workloads/multistreamread.f new file mode 100644 index 0000000000..66dd7d54d9 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/multistreamread.f @@ -0,0 +1,70 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $filesize=1g +set $nthreads=1 +set $iosize=1m +set $directio=0 + +define file name=largefile1,path=$dir,size=$filesize,prealloc,reuse +define file name=largefile2,path=$dir,size=$filesize,prealloc,reuse +define file name=largefile3,path=$dir,size=$filesize,prealloc,reuse +define file name=largefile4,path=$dir,size=$filesize,prealloc,reuse + +define process name=seqread,instances=1 +{ + thread name=seqread1,memsize=10m,instances=$nthreads + { + flowop read name=seqread1,filename=largefile1,iosize=$iosize,directio=$directio + flowop bwlimit name=limit + } + thread name=seqread2,memsize=10m,instances=$nthreads + { + flowop read name=seqread2,filename=largefile2,iosize=$iosize,directio=$directio + flowop bwlimit name=limit + } + thread name=seqread3,memsize=10m,instances=$nthreads + { + flowop read name=seqread3,filename=largefile3,iosize=$iosize,directio=$directio + flowop bwlimit name=limit + } + thread name=seqread4,memsize=10m,instances=$nthreads + { + flowop read name=seqread4,filename=largefile4,iosize=$iosize,directio=$directio + flowop bwlimit name=limit + } +} + +echo "Multi Stream Read Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$iosize=<value> defaults to $iosize" +usage " set \$directio=<bool> defaults to $directio" +usage " " +usage " run runtime (e.g. run 60)" + diff --git a/usr/src/cmd/filebench/workloads/multistreamreaddirect.f b/usr/src/cmd/filebench/workloads/multistreamreaddirect.f new file mode 100644 index 0000000000..36c5c31546 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/multistreamreaddirect.f @@ -0,0 +1,68 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nthreads=1 +set $filesize=1g +set $iosize=1m + +define file name=largefile1,path=$dir,size=$filesize,prealloc,reuse +define file name=largefile2,path=$dir,size=$filesize,prealloc,reuse +define file name=largefile3,path=$dir,size=$filesize,prealloc,reuse +define file name=largefile4,path=$dir,size=$filesize,prealloc,reuse + +define process name=seqread,instances=1 +{ + thread name=seqread1,memsize=10m,instances=$nthreads + { + flowop read name=seqread1,filename=largefile1,iosize=$iosize,directio + flowop bwlimit name=limit + } + thread name=seqread2,memsize=10m,instances=$nthreads + { + flowop read name=seqread2,filename=largefile2,iosize=$iosize,directio + flowop bwlimit name=limit + } + thread name=seqread3,memsize=10m,instances=$nthreads + { + flowop read name=seqread3,filename=largefile3,iosize=$iosize,directio + flowop bwlimit name=limit + } + thread name=seqread4,memsize=10m,instances=$nthreads + { + flowop read name=seqread4,filename=largefile4,iosize=$iosize,directio + flowop bwlimit name=limit + } +} + +echo "Multi Stream Read Direct Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$iosize=<value> defaults to $iosize" +usage " " +usage " run runtime (e.g. run 60)" + diff --git a/usr/src/cmd/filebench/workloads/multistreamwrite.f b/usr/src/cmd/filebench/workloads/multistreamwrite.f new file mode 100644 index 0000000000..af536ce9ff --- /dev/null +++ b/usr/src/cmd/filebench/workloads/multistreamwrite.f @@ -0,0 +1,70 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $filesize=1g +set $nthreads=1 +set $iosize=1m +set $directio=0 + +define file name=largefile1,path=$dir,size=$filesize,prealloc,reuse +define file name=largefile2,path=$dir,size=$filesize,prealloc,reuse +define file name=largefile3,path=$dir,size=$filesize,prealloc,reuse +define file name=largefile4,path=$dir,size=$filesize,prealloc,reuse + +define process name=seqwrite,instances=1 +{ + thread name=seqwrite1,memsize=10m,instances=$nthreads + { + flowop write name=seqwrite1,filename=largefile1,iosize=$iosize,directio=$directio + flowop bwlimit name=limit + } + thread name=seqwrite2,memsize=10m,instances=$nthreads + { + flowop write name=seqwrite2,filename=largefile2,iosize=$iosize,directio=$directio + flowop bwlimit name=limit + } + thread name=seqwrite3,memsize=10m,instances=$nthreads + { + flowop write name=seqwrite3,filename=largefile3,iosize=$iosize,directio=$directio + flowop bwlimit name=limit + } + thread name=seqwrite4,memsize=10m,instances=$nthreads + { + flowop write name=seqwrite4,filename=largefile4,iosize=$iosize,directio=$directio + flowop bwlimit name=limit + } +} + +echo "Multi Stream Write Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$iosize=<value> defaults to $iosize" +usage " set \$directio=<bool> defaults to $directio" +usage " " +usage " run runtime (e.g. run 60)" + diff --git a/usr/src/cmd/filebench/workloads/multistreamwritedirect.f b/usr/src/cmd/filebench/workloads/multistreamwritedirect.f new file mode 100644 index 0000000000..8ba21ab7f5 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/multistreamwritedirect.f @@ -0,0 +1,67 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $filesize=1g +set $nthreads=1 +set $iosize=1m + +define file name=largefile1,path=$dir,size=$filesize,prealloc,reuse +define file name=largefile2,path=$dir,size=$filesize,prealloc,reuse +define file name=largefile3,path=$dir,size=$filesize,prealloc,reuse +define file name=largefile4,path=$dir,size=$filesize,prealloc,reuse + +define process name=seqwrite,instances=1 +{ + thread name=seqwrite1,memsize=10m,instances=$nthreads + { + flowop write name=seqwrite1,filename=largefile1,iosize=$iosize,directio + flowop bwlimit name=limit + } + thread name=seqwrite2,memsize=10m,instances=$nthreads + { + flowop write name=seqwrite2,filename=largefile2,iosize=$iosize,directio + flowop bwlimit name=limit + } + thread name=seqwrite3,memsize=10m,instances=$nthreads + { + flowop write name=seqwrite3,filename=largefile3,iosize=$iosize,directio + flowop bwlimit name=limit + } + thread name=seqwrite4,memsize=10m,instances=$nthreads + { + flowop write name=seqwrite4,filename=largefile4,iosize=$iosize,directio + flowop bwlimit name=limit + } +} + +echo "Multi Stream Write Direct Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$iosize=<value> defaults to $iosize" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/oltp.f b/usr/src/cmd/filebench/workloads/oltp.f new file mode 100644 index 0000000000..bfdadb6157 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/oltp.f @@ -0,0 +1,111 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +# $iosize - iosize for database block access +# $dir - directory for datafiles +# $nshadows - number of shadow processes +# $ndbwriters - number of database writers +# +set $dir=/tmp +set $runtime=30 +set $iosize=2k +set $nshadows=200 +set $ndbwriters=10 +set $usermode=200000 +set $filesize=10m +set $memperthread=1m +set $workingset=0 +set $cached=0 +set $logfilesize=10m +set $nfiles=10 +set $nlogfiles=1 +set $directio=0 + +# Define a datafile and logfile +define fileset name=datafiles,path=$dir,size=$filesize,filesizegamma=0,entries=$nfiles,dirwidth=1024,prealloc=100,cached=$cached,reuse +define fileset name=logfile,path=$dir,size=$logfilesize,filesizegamma=0,entries=$nlogfiles,dirwidth=1024,prealloc=100,cached=$cached,reuse + +define process name=lgwr,instances=1 +{ + thread name=lgwr,memsize=$memperthread,useism + { + flowop aiowrite name=lg-write,filesetname=logfile, + iosize=256k,random,directio=$directio + flowop aiowait name=lg-aiowait + flowop semblock name=lg-block,value=3200,highwater=1000 + } +} + +# Define database writer processes +define process name=dbwr,instances=$ndbwriters +{ + thread name=dbwr,memsize=$memperthread,useism + { + flowop aiowrite name=dbwrite-a,filesetname=datafiles, + iosize=$iosize,workingset=$workingset,random,iters=100,opennext,directio=$directio + flowop hog name=dbwr-hog,value=10000 + flowop semblock name=dbwr-block,value=1000,highwater=2000 + flowop aiowait name=dbwr-aiowait + } +} + + +define process name=shadow,instances=$nshadows +{ + thread name=shadow,memsize=$memperthread,useism + { + flowop read name=shadowread,filesetname=datafiles, + iosize=$iosize,workingset=$workingset,random,opennext,directio=$directio + flowop hog name=shadowhog,value=$usermode + flowop sempost name=shadow-post-lg,value=1,target=lg-block,blocking + flowop sempost name=shadow-post-dbwr,value=1,target=dbwr-block,blocking + flowop eventlimit name=random-rate + } +} + +echo "OLTP Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " " +usage " set \$filesize=<size> defaults to $filesize, n.b. there are ten files of this size" +usage " " +usage " set \$logfilesize=<size> defaults to $logfilesize, n.b. there is one file of this size" +usage " " +usage " set \$iosize=<value> defaults to $iosize, typically 2k or 8k" +usage " " +usage " set \$cached=<bool> defaults to $cached" +usage " " +usage " set \$memperthread=<value> defaults to $memperthread" +usage " " +usage " set \$directio=<value> defaults to $directio" +usage " " +usage " run runtime (e.g. run 60)" +usage " " +usage "Note - total filesize should be at least 2x physical memory size for conforming test)" +usage " i.e. if physmem = 4G, set filesize to 4G * 2 / 10, or 800m" +usage " " +usage "Note - this workload needs at least 512MB of of memory" +usage " " + diff --git a/usr/src/cmd/filebench/workloads/randomread.f b/usr/src/cmd/filebench/workloads/randomread.f new file mode 100644 index 0000000000..f528ed9044 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/randomread.f @@ -0,0 +1,52 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nthreads=1 +set $iosize=8k +set $filesize=1m +set $workingset=0 +set $directio=0 + +define file name=largefile1,path=$dir,size=$filesize,prealloc,reuse,paralloc + +define process name=rand-read,instances=1 +{ + thread name=rand-thread,memsize=5m,instances=$nthreads + { + flowop read name=rand-read1,filename=largefile1,iosize=$iosize,random,workingset=$workingset,directio=$directio + flowop eventlimit name=rand-rate + } +} + +echo "Random Read Version 2.0 IO personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$iosize=<value> defaults to $iosize" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$workingset=<value> defaults to $workingset" +usage " set \$directio=<bool> defaults to $directio" +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/randomrw.f b/usr/src/cmd/filebench/workloads/randomrw.f new file mode 100644 index 0000000000..c2251bfbf0 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/randomrw.f @@ -0,0 +1,57 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nthreads=1 +set $iosize=8k +set $filesize=1m +set $workingset=0 +set $directio=0 + +define file name=largefile1,path=$dir,size=$filesize,prealloc,reuse,paralloc + +define process name=rand-rw,instances=1 +{ + thread name=rand-r-thread,memsize=5m,instances=$nthreads + { + flowop read name=rand-read1,filename=largefile1,iosize=$iosize,random,workingset=$workingset,directio=$directio + flowop eventlimit name=rand-rate + } + thread name=rand-w-thread,memsize=5m,instances=$nthreads + { + flowop write name=rand-write1,filename=largefile1,iosize=$iosize,random,workingset=$workingset,directio=$directio + flowop eventlimit name=rand-rate + } +} + +echo "Random RW Version 2.0 IO personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$iosize=<value> defaults to $iosize" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$workingset=<value> defaults to $workingset" +usage " set \$directio=<bool> defaults to $directio" +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/randomwrite.f b/usr/src/cmd/filebench/workloads/randomwrite.f new file mode 100644 index 0000000000..15d3323940 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/randomwrite.f @@ -0,0 +1,52 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nthreads=1 +set $iosize=8k +set $filesize=1m +set $workingset=0 +set $directio=0 + +define file name=largefile1,path=$dir,size=$filesize,prealloc,reuse,paralloc + +define process name=rand-write,instances=1 +{ + thread name=rand-thread,memsize=5m,instances=$nthreads + { + flowop write name=rand-write1,filename=largefile1,iosize=$iosize,random,workingset=$workingset,directio=$directio + flowop eventlimit name=rand-rate + } +} + +echo "Random Write Version 2.0 IO personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$iosize=<value> defaults to $iosize" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$workingset=<value> defaults to $workingset" +usage " set \$directio=<bool> defaults to $directio" +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/singlestreamread.f b/usr/src/cmd/filebench/workloads/singlestreamread.f new file mode 100644 index 0000000000..a5852dc7dc --- /dev/null +++ b/usr/src/cmd/filebench/workloads/singlestreamread.f @@ -0,0 +1,53 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $filesize=5g +set $nthreads=1 +set $iosize=1m +set $directio=0 + +define file name=largefile1,path=$dir,size=$filesize,prealloc,reuse + +define process name=seqread,instances=1 +{ + thread name=seqread,memsize=10m,instances=$nthreads + { + flowop read name=seqread,filename=largefile1,iosize=$iosize,directio=$directio + flowop bwlimit name=limit + } +} + +echo "Single Stream Read Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$iosize=<value> defaults to $iosize" +usage " set \$directio=<bool> defaults to $directio" +usage " " +usage "This workload needs $filesize of disk space by default" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/singlestreamreaddirect.f b/usr/src/cmd/filebench/workloads/singlestreamreaddirect.f new file mode 100644 index 0000000000..abd44e12d8 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/singlestreamreaddirect.f @@ -0,0 +1,51 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $filesize=5g +set $nthreads=1 +set $iosize=1m + +define file name=largefile1,path=$dir,size=$filesize,prealloc,reuse + +define process name=seqread,instances=1 +{ + thread name=seqread,memsize=10m,instances=$nthreads + { + flowop read name=seqread,filename=largefile1,iosize=$iosize,directio + flowop bwlimit name=limit + } +} + +echo "Single Stream Read Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$iosize=<value> defaults to $iosize" +usage " " +usage "This workload needs $filesize of disk space by default" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/singlestreamwrite.f b/usr/src/cmd/filebench/workloads/singlestreamwrite.f new file mode 100644 index 0000000000..485ef7b5e6 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/singlestreamwrite.f @@ -0,0 +1,50 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nthreads=1 +set $iosize=1m +set $directio=0 + +define file name=largefile1,path=$dir + +define process name=seqwrite,instances=1 +{ + thread name=seqwrite,memsize=10m,instances=$nthreads + { + flowop write name=seqwrite,filename=largefile1,iosize=$iosize,directio=$directio + flowop bwlimit name=limit + } +} + +echo "Single Stream Write Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$iosize=<value> defaults to $iosize" +usage " set \$directio=<value> defaults to $directio" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/singlestreamwritedirect.f b/usr/src/cmd/filebench/workloads/singlestreamwritedirect.f new file mode 100644 index 0000000000..22fcf9aa4b --- /dev/null +++ b/usr/src/cmd/filebench/workloads/singlestreamwritedirect.f @@ -0,0 +1,48 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nthreads=1 +set $iosize=1m + +define file name=largefile1,path=$dir + +define process name=seqwrite,instances=1 +{ + thread name=seqwrite,memsize=10m,instances=$nthreads + { + flowop write name=seqwrite,filename=largefile1,iosize=$iosize,directio + flowop bwlimit name=limit + } +} + +echo "Single Stream Write Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$iosize=<value> defaults to $iosize" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/tpcso.f b/usr/src/cmd/filebench/workloads/tpcso.f new file mode 100644 index 0000000000..2690730a69 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/tpcso.f @@ -0,0 +1,264 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +# $iosize - iosize for database block access +# $dir - directory for datafiles +# $nshadows - number of shadow processes +# $ndbwriters - number of database writers + +set $dir=/tmp +set $runtime=30 +set $iosize=2k +set $nshadows=200 +set $ndbwriters=10 +set $usermode=20000 +set $memperthread=1m + +debug 1 + +# Define a datafile and logfile +define file name=aux.df,path=$dir,size=251m,reuse,prealloc,paralloc +define file name=control_001,path=$dir,size=2m,reuse,prealloc,paralloc +define file name=cust_0_0,path=$dir,size=6704m,reuse,prealloc,paralloc +define file name=cust_0_1,path=$dir,size=6704m,reuse,prealloc,paralloc +define file name=cust_0_2,path=$dir,size=6704m,reuse,prealloc,paralloc +define file name=cust_0_3,path=$dir,size=6704m,reuse,prealloc,paralloc +define file name=dist_0_0,path=$dir,size=31m,reuse,prealloc,paralloc +define file name=hist_0_0,path=$dir,size=3002m,reuse,prealloc,paralloc +define file name=icust1_0_0,path=$dir,size=4943m,reuse,prealloc,paralloc +define file name=icust2_0_0,path=$dir,size=4943m,reuse,prealloc,paralloc +define file name=idist_0_0,path=$dir,size=11m,reuse,prealloc,paralloc +define file name=iitem_0_0,path=$dir,size=11m,reuse,prealloc,paralloc +define file name=iordr2_0_0,path=$dir,size=1651m,reuse,prealloc,paralloc +define file name=istok_0_0,path=$dir,size=2262m,reuse,prealloc,paralloc +define file name=item_0_0,path=$dir,size=21m,reuse,prealloc,paralloc +define file name=iware_0_0,path=$dir,size=11m,reuse,prealloc,paralloc +define file name=nord_0_0,path=$dir,size=561m,reuse,prealloc,paralloc +define file name=ordr_0_0,path=$dir,size=44301m,reuse,prealloc,paralloc +define file name=roll1,path=$dir,size=2001m,reuse,prealloc,paralloc +define file name=sp_0,path=$dir,size=1001m,reuse,prealloc,paralloc +define file name=stok_0_0,path=$dir,size=8052m,reuse,prealloc,paralloc +define file name=stok_0_1,path=$dir,size=8052m,reuse,prealloc,paralloc +define file name=stok_0_2,path=$dir,size=8052m,reuse,prealloc,paralloc +define file name=stok_0_3,path=$dir,size=8052m,reuse,prealloc,paralloc +define file name=stok_0_4,path=$dir,size=8052m,reuse,prealloc,paralloc +define file name=system_1,path=$dir,size=401m,reuse,prealloc,paralloc +define file name=temp_0_0,path=$dir,size=4943m,reuse,prealloc,paralloc +define file name=temp_0_1,path=$dir,size=4943m,reuse,prealloc,paralloc +define file name=ware_0_0,path=$dir,size=11m,reuse,prealloc,paralloc +define file name=log_1_1,path=$dir,size=1021m,reuse,prealloc,paralloc + +# Define database writer processes +define process name=dbwr,instances=$ndbwriters +{ + thread name=dbwr,memsize=$memperthread,useism + { + flowop aiowrite name=dbaiowrite-aux.df,filename=aux.df, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-control_001,filename=control_001, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-cust_0_0,filename=cust_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-cust_0_1,filename=cust_0_1, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-cust_0_2,filename=cust_0_2, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-cust_0_3,filename=cust_0_3, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-dist_0_0,filename=dist_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-hist_0_0,filename=hist_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-icust1_0_0,filename=icust1_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-icust2_0_0,filename=icust2_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-idist_0_0,filename=idist_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-iitem_0_0,filename=iitem_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-iordr2_0_0,filename=iordr2_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-istok_0_0,filename=istok_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-item_0_0,filename=item_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-iware_0_0,filename=iware_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-nord_0_0,filename=nord_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-ordr_0_0,filename=ordr_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-roll1,filename=roll1, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-sp_0,filename=sp_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-stok_0_0,filename=stok_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-stok_0_1,filename=stok_0_1, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-stok_0_2,filename=stok_0_2, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-stok_0_3,filename=stok_0_3, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-stok_0_4,filename=stok_0_4, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-system_1,filename=system_1, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-temp_0_0,filename=temp_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-temp_0_1,filename=temp_0_1, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop aiowrite name=dbaiowrite-ware_0_0,filename=ware_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio,iters=10 + flowop hog name=dbwr-hog,value=10000 + flowop semblock name=dbwr-block,value=100,highwater=10000 + flowop aiowait name=dbwr-aiowait + } +} + +define process name=lgwr,instances=1 +{ + thread name=lgwr,memsize=$memperthread,useism + { + flowop write name=lg-write,filename=log_1_1, + iosize=256k,workingset=1g,random,dsync,directio +# flowop delay name=lg-delay,value=1 + flowop semblock name=lg-block,value=320,highwater=1000 + } +} + +define process name=shadow,instances=$nshadows +{ + thread name=shadow,memsize=$memperthread,useism + { + flowop read name=shadowread-aux.df,filename=aux.df, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-control_001,filename=control_001, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-cust_0_0,filename=cust_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-cust_0_1,filename=cust_0_1, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-cust_0_2,filename=cust_0_2, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-cust_0_3,filename=cust_0_3, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-dist_0_0,filename=dist_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-hist_0_0,filename=hist_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-icust1_0_0,filename=icust1_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-icust2_0_0,filename=icust2_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-idist_0_0,filename=idist_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-iitem_0_0,filename=iitem_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-iordr2_0_0,filename=iordr2_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-istok_0_0,filename=istok_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-item_0_0,filename=item_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-iware_0_0,filename=iware_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-nord_0_0,filename=nord_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-ordr_0_0,filename=ordr_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-roll1,filename=roll1, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-sp_0,filename=sp_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-stok_0_0,filename=stok_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-stok_0_1,filename=stok_0_1, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-stok_0_2,filename=stok_0_2, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-stok_0_3,filename=stok_0_3, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-stok_0_4,filename=stok_0_4, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-system_1,filename=system_1, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-temp_0_0,filename=temp_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-temp_0_1,filename=temp_0_1, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-ware_0_0,filename=ware_0_0, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop read name=shadowread-log_1_1,filename=log_1_1, + iosize=$iosize,workingset=10g,random,dsync,directio + flowop hog name=shadowhog,value=$usermode + flowop sempost name=shadow-post-lg,value=1,target=lg-block,blocking + flowop sempost name=shadow-post-dbwr,value=1,target=dbwr-block,blocking + flowop eventlimit name=random-rate + } +} + +echo "Tpcso Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " " +usage " set \$iosize=<value> defaults to $iosize, typically 2k or 8k" +usage " " +usage " set \$memperthread=<value> defaults to $memperthread, there are 211 threads" +usage " " +usage " run runtime (e.g. run 60)" +usage " " +usage "Note - this workload needs at least 512MB of of memory" +usage " " diff --git a/usr/src/cmd/filebench/workloads/varmail.f b/usr/src/cmd/filebench/workloads/varmail.f new file mode 100644 index 0000000000..f28973727d --- /dev/null +++ b/usr/src/cmd/filebench/workloads/varmail.f @@ -0,0 +1,68 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nfiles=1000 +set $meandirwidth=1000000 +set $filesize=16k +set $nthreads=16 +set $meaniosize=16k + +define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$meandirwidth,prealloc=80 + +define process name=filereader,instances=1 +{ + thread name=filereaderthread,memsize=10m,instances=$nthreads + { + flowop deletefile name=deletefile1,filesetname=bigfileset + flowop createfile name=createfile2,filesetname=bigfileset,fd=1 + flowop appendfilerand name=appendfilerand2,iosize=$meaniosize,fd=1 + flowop fsync name=fsyncfile2,fd=1 + flowop closefile name=closefile2,fd=1 + flowop openfile name=openfile3,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile3,fd=1 + flowop appendfilerand name=appendfilerand3,iosize=$meaniosize,fd=1 + flowop fsync name=fsyncfile3,fd=1 + flowop closefile name=closefile3,fd=1 + flowop openfile name=openfile4,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile4,fd=1 + flowop closefile name=closefile4,fd=1 + } +} + +echo "Varmail Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nfiles=<value> defaults to $nfiles" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$meaniosize=<value> defaults to $meaniosize" +usage " set \$meandirwidth=<size> defaults to $meandirwidth" +usage "(sets mean dir width and dir depth is calculated as log (width, nfiles)" +usage " dirdepth therefore defaults to dir depth of 1 as in postmark" +usage " set $meandir lower to increase depth beyond 1 if desired)" +usage " " +usage " run runtime (e.g. run 60)" + diff --git a/usr/src/cmd/filebench/workloads/webproxy.f b/usr/src/cmd/filebench/workloads/webproxy.f new file mode 100644 index 0000000000..6aad98e220 --- /dev/null +++ b/usr/src/cmd/filebench/workloads/webproxy.f @@ -0,0 +1,74 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nfiles=1000 +set $meandirwidth=1000000 +set $filesize=16k +set $nthreads=100 +set $meaniosize=16k + +define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$meandirwidth,prealloc=80 + +define process name=proxycache,instances=1 +{ + thread name=proxycache,memsize=10m,instances=$nthreads + { + flowop deletefile name=deletefile1,filesetname=bigfileset + flowop createfile name=createfile1,filesetname=bigfileset,fd=1 + flowop appendfilerand name=appendfilerand1,iosize=$meaniosize,fd=1 + flowop closefile name=closefile1,fd=1 + flowop openfile name=openfile2,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile2,fd=1 + flowop closefile name=closefile2,fd=1 + flowop openfile name=openfile3,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile3,fd=1 + flowop closefile name=closefile3,fd=1 + flowop openfile name=openfile4,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile4,fd=1 + flowop closefile name=closefile4,fd=1 + flowop openfile name=openfile5,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile5,fd=1 + flowop closefile name=closefile5,fd=1 + flowop openfile name=openfile6,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile6,fd=1 + flowop closefile name=closefile6,fd=1 + flowop opslimit name=limit + } +} + +echo "Proxy Cache Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nfiles=<value> defaults to $nfiles" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " set \$meaniosize=<value> defaults to $meaniosize" +usage " set \$meandirwidth=<size> defaults to $meandirwidth" +usage "(sets mean dir width and dir depth is calculated as log (width, nfiles)" +usage " dirdepth therefore defaults to dir depth of 1 as in postmark" +usage " set $meandir lower to increase depth beyond 1 if desired)" +usage " " +usage " run runtime (e.g. run 60)" diff --git a/usr/src/cmd/filebench/workloads/webserver.f b/usr/src/cmd/filebench/workloads/webserver.f new file mode 100644 index 0000000000..c42f329b4d --- /dev/null +++ b/usr/src/cmd/filebench/workloads/webserver.f @@ -0,0 +1,80 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" + +set $dir=/tmp +set $nfiles=1000 +set $dirwidth=20 +set $filesize=16k +set $nthreads=100 + +define fileset name=bigfileset,path=$dir,size=$filesize,entries=$nfiles,dirwidth=$dirwidth,prealloc=100 +define fileset name=logfiles,path=$dir,size=$filesize,entries=1,dirwidth=$dirwidth,prealloc + +define process name=filereader,instances=1 +{ + thread name=filereaderthread,memsize=10m,instances=$nthreads + { + flowop openfile name=openfile1,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile1,fd=1 + flowop closefile name=closefile1,fd=1 + flowop openfile name=openfile2,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile2,fd=1 + flowop closefile name=closefile2,fd=1 + flowop openfile name=openfile3,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile3,fd=1 + flowop closefile name=closefile3,fd=1 + flowop openfile name=openfile4,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile4,fd=1 + flowop closefile name=closefile4,fd=1 + flowop openfile name=openfile5,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile5,fd=1 + flowop closefile name=closefile5,fd=1 + flowop openfile name=openfile6,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile6,fd=1 + flowop closefile name=closefile6,fd=1 + flowop openfile name=openfile7,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile7,fd=1 + flowop closefile name=closefile7,fd=1 + flowop openfile name=openfile8,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile8,fd=1 + flowop closefile name=closefile8,fd=1 + flowop openfile name=openfile9,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile9,fd=1 + flowop closefile name=closefile9,fd=1 + flowop openfile name=openfile10,filesetname=bigfileset,fd=1 + flowop readwholefile name=readfile10,fd=1 + flowop closefile name=closefile10,fd=1 + flowop appendfilerand name=appendlog,filesetname=logfiles,iosize=16k,fd=2 + } +} + +echo "Webserver Version 2.0 personality successfully loaded" +usage "Usage: set \$dir=<dir>" +usage " set \$filesize=<size> defaults to $filesize" +usage " set \$nfiles=<value> defaults to $nfiles" +usage " set \$dirwidth=<value> defaults to $dirwidth" +usage " set \$nthreads=<value> defaults to $nthreads" +usage " run runtime (e.g. run 60)" diff --git a/usr/src/pkgdefs/Makefile b/usr/src/pkgdefs/Makefile index 055643b3bb..1ef0fae3e8 100644 --- a/usr/src/pkgdefs/Makefile +++ b/usr/src/pkgdefs/Makefile @@ -206,6 +206,7 @@ COMMON_SUBDIRS= \ SUNWdtrp \ SUNWdtrt \ SUNWesu \ + SUNWfilebench \ SUNWfmd \ SUNWfmdr \ SUNWfss \ diff --git a/usr/src/pkgdefs/SUNWfilebench/Makefile b/usr/src/pkgdefs/SUNWfilebench/Makefile new file mode 100644 index 0000000000..c2da7ba6f3 --- /dev/null +++ b/usr/src/pkgdefs/SUNWfilebench/Makefile @@ -0,0 +1,37 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" +# + +include ../Makefile.com + +.KEEP_STATE: + +all: $(FILES) depend + +install: all pkg + +include ../Makefile.targ diff --git a/usr/src/pkgdefs/SUNWfilebench/depend b/usr/src/pkgdefs/SUNWfilebench/depend new file mode 100644 index 0000000000..759ad37f76 --- /dev/null +++ b/usr/src/pkgdefs/SUNWfilebench/depend @@ -0,0 +1,51 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" +# +# This package information file defines software dependencies associated +# with the pkg. You can define three types of pkg dependencies with this file: +# P indicates a prerequisite for installation +# I indicates an incompatible package +# R indicates a reverse dependency +# <pkg.abbr> see pkginfo(4), PKG parameter +# <name> see pkginfo(4), NAME parameter +# <version> see pkginfo(4), VERSION parameter +# <arch> see pkginfo(4), ARCH parameter +# <type> <pkg.abbr> <name> +# (<arch>)<version> +# (<arch>)<version> +# ... +# <type> <pkg.abbr> <name> +# ... +# + +P SUNWcar Core Architecture, (Root) +P SUNWcakr Core Solaris Kernel Architecture (Root) +P SUNWkvm Core Architecture, (Kvm) +P SUNWcsr Core Solaris, (Root) +P SUNWcsu Core Solaris, (Usr) +P SUNWcsl Core Solaris Libraries +P SUNWperl584core Perl 5.8.4 (core) diff --git a/usr/src/pkgdefs/SUNWfilebench/pkginfo.tmpl b/usr/src/pkgdefs/SUNWfilebench/pkginfo.tmpl new file mode 100644 index 0000000000..18af01ad22 --- /dev/null +++ b/usr/src/pkgdefs/SUNWfilebench/pkginfo.tmpl @@ -0,0 +1,52 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" +# + +# +# This required package information file describes characteristics of the +# package, such as package abbreviation, full package name, package version, +# and package architecture. +# +PKG="SUNWfilebench" +NAME="FileBench" +ARCH="ISA" +VERSION="ONVERS,REV=0.0.0" +SUNW_PRODNAME="SunOS" +SUNW_PRODVERS="RELEASE/VERSION" +SUNW_PKGTYPE="usr" +MAXINST="1000" +CATEGORY="system" +DESC="FileBench Commands, Workloads, Scripts, and Config Files" +VENDOR="Sun Microsystems, Inc." +HOTLINE="Please contact your local service provider" +EMAIL="" +CLASSES="none" +BASEDIR=/ +SUNW_PKGVERS="1.0" +SUNW_PKG_ALLZONES="true" +SUNW_PKG_HOLLOW="false" +SUNW_PKG_THISZONE="false" diff --git a/usr/src/pkgdefs/SUNWfilebench/prototype_com b/usr/src/pkgdefs/SUNWfilebench/prototype_com new file mode 100644 index 0000000000..d349b1e491 --- /dev/null +++ b/usr/src/pkgdefs/SUNWfilebench/prototype_com @@ -0,0 +1,87 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" +# + +# packaging files +i copyright +i depend +i pkginfo + +# +# SUNWfilebench +# +d none usr 755 root sys +d none usr/benchmarks 755 root bin +d none usr/benchmarks/filebench 755 root bin +d none usr/benchmarks/filebench/bin 755 root bin +f none usr/benchmarks/filebench/bin/filebench 555 root bin +l none usr/benchmarks/filebench/bin/go_filebench=../../../../../usr/lib/isaexec +d none usr/benchmarks/filebench/config 755 root bin +f none usr/benchmarks/filebench/config/fileio.prof 444 root bin +f none usr/benchmarks/filebench/config/filemacro.prof 444 root bin +f none usr/benchmarks/filebench/config/filemicro.prof 444 root bin +f none usr/benchmarks/filebench/config/generic.func 444 root bin +f none usr/benchmarks/filebench/config/seqread.prof 444 root bin +f none usr/benchmarks/filebench/config/randomread.prof 444 root bin +d none usr/benchmarks/filebench/scripts 755 root bin +f none usr/benchmarks/filebench/scripts/filebench_compare 555 root bin +f none usr/benchmarks/filebench/scripts/fs_flush 555 root bin +d none usr/benchmarks/filebench/workloads 755 root bin +f none usr/benchmarks/filebench/workloads/bringover.f 444 root bin +f none usr/benchmarks/filebench/workloads/copyfiles.f 444 root bin +f none usr/benchmarks/filebench/workloads/createfiles.f 444 root bin +f none usr/benchmarks/filebench/workloads/deletefiles.f 444 root bin +f none usr/benchmarks/filebench/workloads/filemicro_create.f 444 root bin +f none usr/benchmarks/filebench/workloads/filemicro_createfiles.f 444 root bin +f none usr/benchmarks/filebench/workloads/filemicro_createrand.f 444 root bin +f none usr/benchmarks/filebench/workloads/filemicro_delete.f 444 root bin +f none usr/benchmarks/filebench/workloads/filemicro_rread.f 444 root bin +f none usr/benchmarks/filebench/workloads/filemicro_rwrite.f 444 root bin +f none usr/benchmarks/filebench/workloads/filemicro_rwritedsync.f 444 root bin +f none usr/benchmarks/filebench/workloads/filemicro_rwritefsync.f 444 root bin +f none usr/benchmarks/filebench/workloads/filemicro_seqread.f 444 root bin +f none usr/benchmarks/filebench/workloads/filemicro_seqwrite.f 444 root bin +f none usr/benchmarks/filebench/workloads/filemicro_seqwriterand.f 444 root bin +f none usr/benchmarks/filebench/workloads/filemicro_writefsync.f 444 root bin +f none usr/benchmarks/filebench/workloads/fileserver.f 444 root bin +f none usr/benchmarks/filebench/workloads/mongo.f 444 root bin +f none usr/benchmarks/filebench/workloads/multistreamread.f 444 root bin +f none usr/benchmarks/filebench/workloads/multistreamreaddirect.f 444 root bin +f none usr/benchmarks/filebench/workloads/multistreamwrite.f 444 root bin +f none usr/benchmarks/filebench/workloads/multistreamwritedirect.f 444 root bin +f none usr/benchmarks/filebench/workloads/oltp.f 444 root bin +f none usr/benchmarks/filebench/workloads/randomread.f 444 root bin +f none usr/benchmarks/filebench/workloads/randomrw.f 444 root bin +f none usr/benchmarks/filebench/workloads/randomwrite.f 444 root bin +f none usr/benchmarks/filebench/workloads/singlestreamread.f 444 root bin +f none usr/benchmarks/filebench/workloads/singlestreamreaddirect.f 444 root bin +f none usr/benchmarks/filebench/workloads/singlestreamwrite.f 444 root bin +f none usr/benchmarks/filebench/workloads/singlestreamwritedirect.f 444 root bin +f none usr/benchmarks/filebench/workloads/tpcso.f 444 root bin +f none usr/benchmarks/filebench/workloads/varmail.f 444 root bin +f none usr/benchmarks/filebench/workloads/webproxy.f 444 root bin +f none usr/benchmarks/filebench/workloads/webserver.f 444 root bin diff --git a/usr/src/pkgdefs/SUNWfilebench/prototype_i386 b/usr/src/pkgdefs/SUNWfilebench/prototype_i386 new file mode 100644 index 0000000000..d2fe366a49 --- /dev/null +++ b/usr/src/pkgdefs/SUNWfilebench/prototype_i386 @@ -0,0 +1,37 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" +# + +!include prototype_com + +# +# SUNWfilebench +# +d none usr/benchmarks/filebench/bin/i386 755 root bin +d none usr/benchmarks/filebench/bin/amd64 755 root bin +f none usr/benchmarks/filebench/bin/i386/go_filebench 555 root bin +f none usr/benchmarks/filebench/bin/amd64/go_filebench 555 root bin diff --git a/usr/src/pkgdefs/SUNWfilebench/prototype_sparc b/usr/src/pkgdefs/SUNWfilebench/prototype_sparc new file mode 100644 index 0000000000..d0c79da5bc --- /dev/null +++ b/usr/src/pkgdefs/SUNWfilebench/prototype_sparc @@ -0,0 +1,35 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# ident "%Z%%M% %I% %E% SMI" +# + +!include prototype_com + +# +# SUNWfilebench +# +d none usr/benchmarks/filebench/bin/sparcv9 755 root bin +f none usr/benchmarks/filebench/bin/sparcv9/go_filebench 555 root bin |