Imported Upstream version 4.6.3upstream/4.6.3

16 files changed, 338 insertions, 60 deletions

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 42ef32e..0045f00 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,6 +1,6 @@
 if ENABLE_TESTBENCH
 TESTRUNS = rt_init rscript
-check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq
+check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq randomgen
 TESTS = $(TESTRUNS) cfg.sh \
 	validation-run.sh \
 	imtcp-multiport.sh \
@@ -40,6 +40,10 @@ TESTS += omod-if-array.sh \
 	 fieldtest.sh
 endif
 
+if ENABLE_EXTENDED_TESTS
+TESTS += random.sh 
+endif
+
 check_JAVA = DiagTalker.java
 
 endif # if ENABLE_TESTBENCH
@@ -109,6 +113,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/parse3.conf \
 	   testsuites/reallife.parse3 \
 	   testsuites/parse-nodate.conf \
+	   testsuites/samples.parse-nodate \
 	   testsuites/parse_invld_regex.conf \
 	   testsuites/samples.parse_invld_regex \
 	   testsuites/parse-3164-buggyday.conf \
@@ -180,6 +185,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/gzipwr_large_dynfile.conf \
 	   complex1.sh \
 	   testsuites/complex1.conf \
+	   random.sh \
+	   testsuites/random.conf \
 	   dynfile_invld_async.sh \
 	   dynfile_invld_sync.sh \
 	   dynfile_cachemiss.sh \
@@ -203,6 +210,9 @@ chkseq_SOURCES = chkseq.c
 tcpflood_SOURCES = tcpflood.c
 tcpflood_LDADD = $(SOL_LIBS)
 
+randomgen_SOURCES = randomgen.c
+randomgen_LDADD = $(SOL_LIBS)
+
 nettester_SOURCES = nettester.c getline.c
 nettester_LDADD = $(SOL_LIBS)
 
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 2f8b12d..a792be3 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -35,7 +35,8 @@ build_triplet = @build@
 host_triplet = @host@
 @ENABLE_TESTBENCH_TRUE@check_PROGRAMS = $(am__EXEEXT_1) \
 @ENABLE_TESTBENCH_TRUE@	ourtail$(EXEEXT) nettester$(EXEEXT) \
-@ENABLE_TESTBENCH_TRUE@	tcpflood$(EXEEXT) chkseq$(EXEEXT)
+@ENABLE_TESTBENCH_TRUE@	tcpflood$(EXEEXT) chkseq$(EXEEXT) \
+@ENABLE_TESTBENCH_TRUE@	randomgen$(EXEEXT)
 @ENABLE_TESTBENCH_TRUE@TESTS = $(am__EXEEXT_1) cfg.sh \
 @ENABLE_TESTBENCH_TRUE@	validation-run.sh imtcp-multiport.sh \
 @ENABLE_TESTBENCH_TRUE@	diskqueue.sh diskqueue-fsync.sh \
@@ -50,7 +51,7 @@ host_triplet = @host@
 @ENABLE_TESTBENCH_TRUE@	dynfile_invld_sync.sh \
 @ENABLE_TESTBENCH_TRUE@	dynfile_invalid2.sh complex1.sh \
 @ENABLE_TESTBENCH_TRUE@	queue-persist.sh pipeaction.sh \
-@ENABLE_TESTBENCH_TRUE@	$(am__append_1)
+@ENABLE_TESTBENCH_TRUE@	$(am__append_1) $(am__append_2)
 @ENABLE_OMSTDOUT_TRUE@@ENABLE_TESTBENCH_TRUE@am__append_1 = omod-if-array.sh \
 @ENABLE_OMSTDOUT_TRUE@@ENABLE_TESTBENCH_TRUE@	 proprepltest.sh \
 @ENABLE_OMSTDOUT_TRUE@@ENABLE_TESTBENCH_TRUE@	 parsertest.sh \
@@ -60,6 +61,7 @@ host_triplet = @host@
 @ENABLE_OMSTDOUT_TRUE@@ENABLE_TESTBENCH_TRUE@	 threadingmqaq.sh \
 @ENABLE_OMSTDOUT_TRUE@@ENABLE_TESTBENCH_TRUE@	 fieldtest.sh
 
+@ENABLE_EXTENDED_TESTS_TRUE@@ENABLE_TESTBENCH_TRUE@am__append_2 = random.sh 
 subdir = tests
 DIST_COMMON = README $(srcdir)/Makefile.am $(srcdir)/Makefile.in \
 	INSTALL
@@ -87,6 +89,9 @@ nettester_DEPENDENCIES = $(am__DEPENDENCIES_1)
 am_ourtail_OBJECTS = ourtail.$(OBJEXT)
 ourtail_OBJECTS = $(am_ourtail_OBJECTS)
 ourtail_LDADD = $(LDADD)
+am_randomgen_OBJECTS = randomgen.$(OBJEXT)
+randomgen_OBJECTS = $(am_randomgen_OBJECTS)
+randomgen_DEPENDENCIES = $(am__DEPENDENCIES_1)
 am__objects_1 = rscript-runtime-dummy.$(OBJEXT)
 am_rscript_OBJECTS = rscript-rscript.$(OBJEXT) \
 	rscript-getline.$(OBJEXT) $(am__objects_1)
@@ -121,10 +126,11 @@ LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
 	--mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
 	$(LDFLAGS) -o $@
 SOURCES = $(chkseq_SOURCES) $(nettester_SOURCES) $(ourtail_SOURCES) \
-	$(rscript_SOURCES) $(rt_init_SOURCES) $(tcpflood_SOURCES)
-DIST_SOURCES = $(chkseq_SOURCES) $(nettester_SOURCES) \
-	$(ourtail_SOURCES) $(rscript_SOURCES) $(rt_init_SOURCES) \
+	$(randomgen_SOURCES) $(rscript_SOURCES) $(rt_init_SOURCES) \
 	$(tcpflood_SOURCES)
+DIST_SOURCES = $(chkseq_SOURCES) $(nettester_SOURCES) \
+	$(ourtail_SOURCES) $(randomgen_SOURCES) $(rscript_SOURCES) \
+	$(rt_init_SOURCES) $(tcpflood_SOURCES)
 JAVAC = javac
 CLASSPATH_ENV = CLASSPATH=$(JAVAROOT):$(srcdir)/$(JAVAROOT):$$CLASSPATH
 JAVAROOT = $(top_builddir)
@@ -349,6 +355,7 @@ EXTRA_DIST = 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/parse3.conf \
 	   testsuites/reallife.parse3 \
 	   testsuites/parse-nodate.conf \
+	   testsuites/samples.parse-nodate \
 	   testsuites/parse_invld_regex.conf \
 	   testsuites/samples.parse_invld_regex \
 	   testsuites/parse-3164-buggyday.conf \
@@ -420,6 +427,8 @@ EXTRA_DIST = 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/gzipwr_large_dynfile.conf \
 	   complex1.sh \
 	   testsuites/complex1.conf \
+	   random.sh \
+	   testsuites/random.conf \
 	   dynfile_invld_async.sh \
 	   dynfile_invld_sync.sh \
 	   dynfile_cachemiss.sh \
@@ -441,6 +450,8 @@ ourtail_SOURCES = ourtail.c
 chkseq_SOURCES = chkseq.c
 tcpflood_SOURCES = tcpflood.c
 tcpflood_LDADD = $(SOL_LIBS)
+randomgen_SOURCES = randomgen.c
+randomgen_LDADD = $(SOL_LIBS)
 nettester_SOURCES = nettester.c getline.c
 nettester_LDADD = $(SOL_LIBS)
 rt_init_SOURCES = rt-init.c $(test_files)
@@ -503,6 +514,9 @@ nettester$(EXEEXT): $(nettester_OBJECTS) $(nettester_DEPENDENCIES)
 ourtail$(EXEEXT): $(ourtail_OBJECTS) $(ourtail_DEPENDENCIES) 
 	@rm -f ourtail$(EXEEXT)
 	$(LINK) $(ourtail_OBJECTS) $(ourtail_LDADD) $(LIBS)
+randomgen$(EXEEXT): $(randomgen_OBJECTS) $(randomgen_DEPENDENCIES) 
+	@rm -f randomgen$(EXEEXT)
+	$(LINK) $(randomgen_OBJECTS) $(randomgen_LDADD) $(LIBS)
 rscript$(EXEEXT): $(rscript_OBJECTS) $(rscript_DEPENDENCIES) 
 	@rm -f rscript$(EXEEXT)
 	$(rscript_LINK) $(rscript_OBJECTS) $(rscript_LDADD) $(LIBS)
@@ -523,6 +537,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/getline.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/nettester.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ourtail.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/randomgen.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rscript-getline.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rscript-rscript.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rscript-runtime-dummy.Po@am__quote@
diff --git a/tests/complex1.sh b/tests/complex1.sh
index b5dc2c9..7f3cd99 100755
--- a/tests/complex1.sh
+++ b/tests/complex1.sh
@@ -7,8 +7,8 @@ echo ===========================================================================
 echo TEST: \[complex1.sh\]: complex test with gzip and multiple action queues
 source $srcdir/diag.sh init
 # uncomment for debugging support:
-export RSYSLOG_DEBUG="debug nostdout"
-export RSYSLOG_DEBUGLOG="log"
+#export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUGLOG="log"
 source $srcdir/diag.sh startup complex1.conf
 # send 30,000 messages of 400 bytes plus header max, via three dest ports
 source $srcdir/diag.sh tcpflood -m40000 -rd400 -P129 -f5 -n3 -c15 -i1
diff --git a/tests/diag.sh b/tests/diag.sh
index 8f268a5..51ad5f6 100755
--- a/tests/diag.sh
+++ b/tests/diag.sh
@@ -17,10 +17,10 @@ case $1 in
 		cp $srcdir/testsuites/diag-common.conf diag-common.conf
 		cp $srcdir/testsuites/diag-common2.conf diag-common2.conf
 		rm -f rsyslog.action.*.include
-		rm -f rsyslogd.started work-*.conf
+		rm -f rsyslogd.started work-*.conf rsyslog.random.data
 		rm -f rsyslogd2.started work-*.conf
 		rm -f work rsyslog.out.log rsyslog.out.log.save # common work files
-		rm -f rsyslog.out.*.log
+		rm -f rsyslog.out.*.log work-presort
 		rm -rf test-spool
 		rm -f core.* vgcore.*
 		mkdir test-spool
@@ -28,7 +28,7 @@ case $1 in
    'exit')	rm -f rsyslogd.started work-*.conf diag-common.conf
    		rm -f rsyslogd2.started diag-common2.conf rsyslog.action.*.include
 		rm -f work rsyslog.out.log rsyslog.out.log.save # common work files
-		rm -f rsyslog.out.*.log
+		rm -f rsyslog.out.*.log rsyslog.random.data work-presort
 		rm -rf test-spool
 		;;
    'startup')   # start rsyslogd with default params. $2 is the config file name to use
diff --git a/tests/dynfile_invalid2.sh b/tests/dynfile_invalid2.sh
index b6fb394..cb3ef51 100755
--- a/tests/dynfile_invalid2.sh
+++ b/tests/dynfile_invalid2.sh
@@ -9,8 +9,8 @@ echo ===========================================================================
 echo TEST: \[dynfile_invalid2.sh\]: test open fail for dynafiles
 source $srcdir/diag.sh init
 # uncomment for debugging support:
-export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
-export RSYSLOG_DEBUGLOG="log"
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
 source $srcdir/diag.sh startup dynfile_invalid2.conf
 # we send handcrafted message. We have a dynafile cache of 4, and now send one message
 # each to fill up the cache.
diff --git a/tests/gzipwr_large_dynfile.sh b/tests/gzipwr_large_dynfile.sh
index ebd4c55..73d4479 100755
--- a/tests/gzipwr_large_dynfile.sh
+++ b/tests/gzipwr_large_dynfile.sh
@@ -18,8 +18,8 @@ echo ===========================================================================
 echo TEST: \[gzipwr_large_dynfile.sh\]: test for gzip file writing for large message sets
 source $srcdir/diag.sh init
 # uncomment for debugging support:
-export RSYSLOG_DEBUG="debug nostdout"
-export RSYSLOG_DEBUGLOG="log"
+#export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUGLOG="log"
 source $srcdir/diag.sh startup gzipwr_large_dynfile.conf
 # send 4000 messages of 10.000bytes plus header max, randomized
 source $srcdir/diag.sh tcpflood -m4000 -r -d10000 -P129 -f5
diff --git a/tests/random.sh b/tests/random.sh
new file mode 100755
index 0000000..d1f392d
--- /dev/null
+++ b/tests/random.sh
@@ -0,0 +1,20 @@
+# Test if rsyslog survives sending truely random data to it...
+#
+# added 2010-04-01 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[random.sh\]: testing random data
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup random.conf
+# generate random data
+./randomgen -f rsyslog.random.data -s 100000
+ls -l rsyslog.random.data
+source $srcdir/diag.sh tcpflood -B -I rsyslog.random.data -c5 -C10
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+# we do not check anything yet, the point is if rsyslog survived ;)
+# TODO: check for exit message, but we'll notice an abort anyhow, so not that important
+source $srcdir/diag.sh exit
diff --git a/tests/randomgen.c b/tests/randomgen.c
new file mode 100644
index 0000000..9ba5695
--- /dev/null
+++ b/tests/randomgen.c
@@ -0,0 +1,130 @@
+/* generates random data for later use in test cases. Of course,
+ * we could generate random data during the testcase itself, but
+ * the core idea is that we record the random data so that we have
+ * a chance to reproduce a problem should it occur. IMHO this 
+ * provides the best compromise, by a) having randomness but
+ * b) knowing what was used during the test.
+ *
+ * Params
+ * -f	output file name (stdout if not given)
+ * -s	size of test data, plain number is size in k, 1MB default
+ * -u   uses /dev/urandom instead of libc random number generator
+ *      (when available). Note that this is usually much slower.
+ *
+ * Part of the testbench for rsyslog.
+ *
+ * Copyright 2010 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ */
+#include "config.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <signal.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <netinet/in.h>
+
+#define EXIT_FAILURE 1
+
+static char *fileName = NULL;		/* name of output file */
+static int tryUseURandom = 0;		/* try to use /dev/urandom? */
+static long long fileSize = 1024*1024;	/* file size in K, 1MB default */
+
+
+/* generate the random file. This code really can be improved (e.g. read /dev/urandom
+ * when available)
+ */
+static inline void
+genFile()
+{
+	long i;
+	FILE *fp;
+	FILE *rfp = NULL;
+
+	if(fileName == NULL) {
+		fp = stdout;
+	} else {
+		if((fp = fopen(fileName, "w")) == NULL) {
+			perror(fileName);
+		}
+	}
+
+	/* try to use /dev/urandom, if available */
+	if(tryUseURandom)
+		rfp = fopen("/dev/urandom", "r");
+
+	if(rfp == NULL) {
+		/* fallback, use libc random number generator */
+		for(i = 0 ; i < fileSize ; ++i) {
+			if(fputc((char) rand(), fp) == EOF) {
+				perror(fileName);
+				exit(1);
+			}
+		}
+	} else {
+		/* use /dev/urandom */
+		printf("using /dev/urandom");
+		for(i = 0 ; i < fileSize ; ++i) {
+			if(fputc(fgetc(rfp), fp) == EOF) {
+				perror(fileName);
+				exit(1);
+			}
+		}
+	}
+
+	if(fileName != NULL)
+		fclose(fp);
+}
+
+
+/* Run the test.
+ * rgerhards, 2009-04-03
+ */
+int main(int argc, char *argv[])
+{
+	int ret = 0;
+	int opt;
+
+	srand(time(NULL));	/* seed is good enough for our needs */
+
+	while((opt = getopt(argc, argv, "f:s:u")) != -1) {
+		switch (opt) {
+		case 'f':	fileName = optarg;
+				break;
+		case 's':	fileSize = atol(optarg) * 1024;
+				break;
+		case 'u':	tryUseURandom = 1;
+				break;
+		default:	printf("invalid option '%c' or value missing - terminating...\n", opt);
+				exit (1);
+				break;
+		}
+	}
+
+	printf("generating random data file '%s' of %ldkb - may take a short while...\n",
+		fileName, (long) (fileSize / 1024));
+	genFile();
+
+	exit(ret);
+}
diff --git a/tests/tcpflood.c b/tests/tcpflood.c
index 32bf959..d8c3f03 100644
--- a/tests/tcpflood.c
+++ b/tests/tcpflood.c
@@ -20,6 +20,14 @@
  *      one field to the right. Zero (default) disables this functionality.
  * -M   the message to be sent. Disables all message format options, as
  *      only that exact same message is sent.
+ * -I   read specified input file, do NOT generate own test data. The test
+ *      completes when eof is reached.
+ * -B   The specified file (-I) is binary. No data processing is done by
+ *      tcpflood. If multiple connections are specified, data is read in
+ *      chunks and spread across the connections without taking any record
+ *      delemiters into account.
+ * -C	when input from a file is read, this file is transmitted -C times
+ *      (C like cycle, running out of meaningful option switches ;))
  *
  * Part of the testbench for rsyslog.
  *
@@ -75,6 +83,10 @@ static int *sockArray;  /* array of sockets to use */
 static int msgNum = 0;	/* initial message number to start with */
 static int bShowProgress = 1; /* show progress messages */
 static char *MsgToSend = NULL; /* if non-null, this is the actual message to send */
+static int bBinaryFile = 0;	/* is -I file binary */
+static char *dataFile = NULL;	/* name of data file, if NULL, generate own data */
+static int numFileIterations = 1;/* how often is file data to be sent? */
+FILE *dataFP = NULL;		/* file pointer for data file, if used */
 
 
 /* open a single tcp connection
@@ -166,8 +178,8 @@ int openConnections(void)
 void closeConnections(void)
 {
 	int i;
-	char msgBuf[128];
 	size_t lenMsg;
+	char msgBuf[128];
 
 	if(bShowProgress)
 		write(1, "      close connections", sizeof("      close connections")-1);
@@ -186,6 +198,62 @@ void closeConnections(void)
 }
 
 
+/* generate the message to be sent according to program command line parameters.
+ * this has been moved to its own function as we now have various different ways
+ * of constructing test messages. -- rgerhards, 2010-03-31
+ */
+static inline void
+genMsg(char *buf, size_t maxBuf, int *pLenBuf)
+{
+	int edLen; /* actual extra data length to use */
+	char extraData[MAX_EXTRADATA_LEN + 1];
+	char dynFileIDBuf[128] = "";
+	static int numMsgsGen = 0;
+	int done;
+
+	if(dataFP != NULL) {
+		/* get message from file */
+		do {
+			done = 1;
+			*pLenBuf = fread(buf, 1, 1024, dataFP);
+			if(feof(dataFP)) {
+				if(--numFileIterations > 0)  {
+					rewind(dataFP);
+					done = 0; /* need new iteration */
+				} else {
+					*pLenBuf = 0;
+					goto finalize_it;
+				}
+			}
+		} while(!done); /* Attention: do..while()! */
+	} else if(MsgToSend == NULL) {
+		if(dynFileIDs > 0) {
+			snprintf(dynFileIDBuf, maxBuf, "%d:", rand() % dynFileIDs);
+		}
+		if(extraDataLen == 0) {
+			*pLenBuf = snprintf(buf, maxBuf, "<%s>Mar  1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:\n",
+					       msgPRI, dynFileIDBuf, msgNum);
+		} else {
+			if(bRandomizeExtraData)
+				edLen = ((long) rand() + extraDataLen) % extraDataLen + 1;
+			else
+				edLen = extraDataLen;
+			memset(extraData, 'X', edLen);
+			extraData[edLen] = '\0';
+			*pLenBuf = snprintf(buf, maxBuf, "<%s>Mar  1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:%d:%s\n",
+					       msgPRI, dynFileIDBuf, msgNum, edLen, extraData);
+		}
+	} else {
+		/* use fixed message format from command line */
+		*pLenBuf = snprintf(buf, maxBuf, "%s\n", MsgToSend);
+	}
+
+	if(numMsgsGen++ >= numMsgsToSend)
+		*pLenBuf = 0; /* indicate end of run */
+
+finalize_it: ;
+}
+
 /* send messages to the tcp connections we keep open. We use
  * a very basic format that helps identify the message
  * (via msgnum:<number>: e.g. msgnum:00000001:). This format is suitable
@@ -196,52 +264,42 @@ void closeConnections(void)
  */
 int sendMessages(void)
 {
-	int i;
+	int i = 0;
 	int socknum;
 	int lenBuf;
 	int lenSend;
-	int edLen; /* actual extra data length to use */
-	char dynFileIDBuf[128] = "";
+	char *statusText;
 	char buf[MAX_EXTRADATA_LEN + 1024];
-	char extraData[MAX_EXTRADATA_LEN + 1];
 
-	printf("Sending %d messages.\n", numMsgsToSend);
+	if(dataFile == NULL) {
+		printf("Sending %d messages.\n", numMsgsToSend);
+		statusText = "messages";
+	} else {
+		printf("Sending file '%s' %d times.\n", dataFile, numFileIterations);
+		statusText = "kb";
+	}
 	if(bShowProgress)
-		printf("\r%8.8d messages sent", 0);
-	for(i = 0 ; i < numMsgsToSend ; ++i) {
+		printf("\r%8.8d %s sent", 0, statusText);
+	while(1) { /* broken inside loop! */
 		if(i < numConnections)
 			socknum = i;
 		else if(i >= numMsgsToSend - numConnections)
 			socknum = i - (numMsgsToSend - numConnections);
-		else
-			socknum = rand() % numConnections;
-		if(MsgToSend == NULL) {
-			if(dynFileIDs > 0) {
-				sprintf(dynFileIDBuf, "%d:", rand() % dynFileIDs);
-			}
-			if(extraDataLen == 0) {
-				lenBuf = sprintf(buf, "<%s>Mar  1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:\n",
-						       msgPRI, dynFileIDBuf, msgNum);
-			} else {
-				if(bRandomizeExtraData)
-					edLen = ((long) rand() + extraDataLen) % extraDataLen + 1;
-				else
-					edLen = extraDataLen;
-				memset(extraData, 'X', edLen);
-				extraData[edLen] = '\0';
-				lenBuf = sprintf(buf, "<%s>Mar  1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:%d:%s\n",
-						       msgPRI, dynFileIDBuf, msgNum, edLen, extraData);
-			}
-		} else {
-			/* use fixed message format from command line */
-			lenBuf = sprintf(buf, "%s\n", MsgToSend);
+		else {
+			int rnd = rand();
+			//socknum = rand() % numConnections;
+			socknum = rnd % numConnections;
 		}
+		genMsg(buf, sizeof(buf), &lenBuf); /* generate the message to send according to params */
+		if(lenBuf == 0)
+			break; /* end of processing! */
 		lenSend = send(sockArray[socknum], buf, lenBuf, 0);
 		if(lenSend != lenBuf) {
 			printf("\r%5.5d\n", i);
 			fflush(stdout);
 			perror("send test data");
-			printf("send() failed at socket %d, index %d, msgNum %d\n", socknum, i, msgNum);
+			printf("send() failed at socket %d, index %d, msgNum %d\n",
+				sockArray[socknum], i, msgNum);
 			fflush(stderr);
 			return(1);
 		}
@@ -250,8 +308,9 @@ int sendMessages(void)
 				printf("\r%8.8d", i);
 		}
 		++msgNum;
+		++i;
 	}
-	printf("\r%8.8d messages sent\n", i);
+	printf("\r%8.8d %s sent\n", i, statusText);
 
 	return 0;
 }
@@ -335,7 +394,7 @@ int main(int argc, char *argv[])
 	if(!isatty(1))
 		bShowProgress = 0;
 
-	while((opt = getopt(argc, argv, "f:t:p:c:m:i:P:d:n:M:r")) != -1) {
+	while((opt = getopt(argc, argv, "f:t:p:c:C:m:i:I:P:d:n:M:rB")) != -1) {
 		switch (opt) {
 		case 't':	targetIP = optarg;
 				break;
@@ -345,6 +404,8 @@ int main(int argc, char *argv[])
 				break;
 		case 'c':	numConnections = atoi(optarg);
 				break;
+		case 'C':	numFileIterations = atoi(optarg);
+				break;
 		case 'm':	numMsgsToSend = atoi(optarg);
 				break;
 		case 'i':	msgNum = atoi(optarg);
@@ -364,12 +425,27 @@ int main(int argc, char *argv[])
 				break;
 		case 'M':	MsgToSend = optarg;
 				break;
+		case 'I':	dataFile = optarg;
+				/* in this mode, we do not know the num messages to send, so
+				 * we set a (high) number to keep the code happy.
+				 */
+				numMsgsToSend = 1000000;
+				break;
+		case 'B':	bBinaryFile = 1;
+				break;
 		default:	printf("invalid option '%c' or value missing - terminating...\n", opt);
 				exit (1);
 				break;
 		}
 	}
 
+	if(dataFile != NULL) {
+		if((dataFP = fopen(dataFile, "r")) == NULL) {
+			perror(dataFile);
+			exit(1);
+		}
+	}
+
 	if(openConnections() != 0) {
 		printf("error opening connections\n");
 		exit(1);
diff --git a/tests/testsuites/parse3.conf b/tests/testsuites/parse3.conf
index d5cf77d..8a3cb31 100644
--- a/tests/testsuites/parse3.conf
+++ b/tests/testsuites/parse3.conf
@@ -1,8 +1,10 @@
+# note: we need to strip off the TZ designator in the rfc3339 timestamp
+# as this test otherwise fails in different timezones!
 $ModLoad ../plugins/omstdout/.libs/omstdout
 $IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
 
 $ErrorMessagesToStderr off
 
 # use a special format that we can easily parse in expect
-$Template output,"%timereported:1:$:date-rfc3339,csv%, %hostname:::csv%, %programname:::csv%, %syslogtag:R,ERE,0,BLANK:[0-9]+--end:csv%, %syslogseverity:::csv%, %msg:::drop-last-lf,csv%\n" 
+$Template output,"%timereported:1:19:date-rfc3339,csv%, %hostname:::csv%, %programname:::csv%, %syslogtag:R,ERE,0,BLANK:[0-9]+--end:csv%, %syslogseverity:::csv%, %msg:::drop-last-lf,csv%\n" 
 *.* :omstdout:;output
diff --git a/tests/testsuites/parse_invld_regex.conf b/tests/testsuites/parse_invld_regex.conf
index 736aae5..d18a2b3 100644
--- a/tests/testsuites/parse_invld_regex.conf
+++ b/tests/testsuites/parse_invld_regex.conf
@@ -1,8 +1,10 @@
+# note: we need to strip off the TZ designator in the rfc3339 timestamp
+# as this test otherwise fails in different timezones!
 $ModLoad ../plugins/omstdout/.libs/omstdout
 $IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
 
 $ErrorMessagesToStderr off
 
 # use a special format that we can easily parse in expect
-$Template output,"%timereported:1:$:date-rfc3339,csv%, %hostname:::csv%, %programname:::csv%, %syslogtag:R,ERE,0,BLANK:[0-9+--end:csv%, %syslogseverity:::csv%, %msg:::drop-last-lf,csv%\n" 
+$Template output,"%timereported:1:19:date-rfc3339,csv%, %hostname:::csv%, %programname:::csv%, %syslogtag:R,ERE,0,BLANK:[0-9+--end:csv%, %syslogseverity:::csv%, %msg:::drop-last-lf,csv%\n" 
 *.* :omstdout:;output
diff --git a/tests/testsuites/random.conf b/tests/testsuites/random.conf
new file mode 100644
index 0000000..a7079df
--- /dev/null
+++ b/tests/testsuites/random.conf
@@ -0,0 +1,13 @@
+# we write to /dev/null, as we have no chance to verify the output
+# in any case. What we really check is that rsyslogd does not
+# segfault or otherwise abort.
+# rgerhards, 2010-04-01
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%rawmsg%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+*.* /dev/null
diff --git a/tests/testsuites/reallife.parse3 b/tests/testsuites/reallife.parse3
index 465635b..dad3f56 100644
--- a/tests/testsuites/reallife.parse3
+++ b/tests/testsuites/reallife.parse3
@@ -9,7 +9,7 @@
 # documentation by IANA.
 # rgerhards, 2009-10-19
 <175>Oct 16 2009 23:47:31 hostname tag This is a message
-"2009-10-16T23:47:31+01:00", "hostname", "tag", "", "7", " This is a message"
+"2009-10-16T23:47:31", "hostname", "tag", "", "7", " This is a message"
 #
 <175>Oct 16 2009 23:47:31 hostname tag[1234] This is a message
-"2009-10-16T23:47:31+01:00", "hostname", "tag", "1234", "7", " This is a message"
+"2009-10-16T23:47:31", "hostname", "tag", "1234", "7", " This is a message"
diff --git a/tests/testsuites/samples.parse-nodate b/tests/testsuites/samples.parse-nodate
new file mode 100644
index 0000000..7f16181
--- /dev/null
+++ b/tests/testsuites/samples.parse-nodate
@@ -0,0 +1,6 @@
+<27>xapi: [error|xen3|15|Guest liveness monitor D:bca30ab3f1c1|master_connection] Connection to master died. I will continue to retry indefinitely (supressing future logging of this message)
+27,daemon,err,localhost,xapi,xapi:, [error|xen3|15|Guest liveness monitor D:bca30ab3f1c1|master_connection] Connection to master died. I will continue to retry indefinitely (supressing future logging of this message)
+# a message with just text (as permitted by rfc 3164)
+# it is questionable if the current sample result is really correct as of 3164!
+This is a message!
+13,user,notice,This,is,is, a message!
diff --git a/tests/testsuites/samples.parse_invld_regex b/tests/testsuites/samples.parse_invld_regex
index 9ac2c3a..0d0e4ce 100644
--- a/tests/testsuites/samples.parse_invld_regex
+++ b/tests/testsuites/samples.parse_invld_regex
@@ -13,4 +13,4 @@
 # configured format.
 # rgerhards, 2010-02-08
 <175>Feb 08 2008 23:47:31 hostname tag This is a message
-"2008-02-08T23:47:31+01:00", "hostname", "tag", **NO MATCH** **BAD REGULAR EXPRESSION**, "7", " This is a message"
+"2008-02-08T23:47:31", "hostname", "tag", **NO MATCH** **BAD REGULAR EXPRESSION**, "7", " This is a message"
diff --git a/tests/testsuites/samples.snare_ccoff_udp2 b/tests/testsuites/samples.snare_ccoff_udp2
index 7837b82..337cd97 100644
--- a/tests/testsuites/samples.snare_ccoff_udp2
+++ b/tests/testsuites/samples.snare_ccoff_udp2
@@ -9,14 +9,18 @@
 # to be adapted. We do NOT try to preserve misbehaviour on such seriously malformed
 # messages.
 #
+# this is a very simple test, though not snare-based
+test
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('', 1, 'test',5, '20100321185328', '20100321185328', 1, '')
+# and yet another one we have seen in practice
+UX=Abcd-efg-hij-klmno; XXXXX=1111111111, Z123=192.12.231.245:11111, S1234=123456789, XXXXXX=111111111
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' XXXXX=1111111111, Z123=192.12.231.245:11111, S1234=123456789, XXXXXX=111111111', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'UX=Abcd-efg-hij-klmno;')
 # Sample 1 - note the absence of PRI!
 windowsserver	MSWinEventLog	1	Security	1167	Fri Mar 19 15:33:30 2010	540	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306    	733\n
-insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 540 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306     733 ', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1167 Fri')
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 540 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306     733', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1167 Fri')
 # Sample 2
-# the samples below need to be disabled for the "workaround patch" for the message
-# parser to work. They need to be re-enabled once a final solution has been crafted
-#windowsserver	MSWinEventLog	1	Security	1166	Fri Mar 19 15:33:30 2010	576	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege  	732\n
-#insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 576 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege   732', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1166 Fri')
+windowsserver	MSWinEventLog	1	Security	1166	Fri Mar 19 15:33:30 2010	576	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege  	732\n
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 576 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege   732', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1166 Fri')
 # Sample 3
-#windowsserver	MSWinEventLog	1	Security	1165	Fri Mar 19 15:33:30 2010	538	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3    	731\n
-#insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 538 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3     731', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1165 Fri')
+windowsserver	MSWinEventLog	1	Security	1165	Fri Mar 19 15:33:30 2010	538	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3    	731\n
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 538 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3     731', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1165 Fri')