Imported Upstream version 4.6.3upstream/4.6.3

30 files changed, 465 insertions, 105 deletions

diff --git a/ChangeLog b/ChangeLog
index 66eb1e1..7ee6fdb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,26 @@
 ---------------------------------------------------------------------------
+Version 4.6.3  [v4-stable] (rgerhards), 2010-07-07
+- improvded testbench
+  - added test with truly random data received via syslog to test
+    robustness
+  - added new configure option that permits to disable and enable an
+    extended testbench
+- bugfix: segfault on HUP when "HUPIsRestart" was set to "on"
+  thanks varmojfekoj for the patch
+- bugfix: default for $OMFileFlushOnTXEnd was wrong ("off").
+  This, in default mode, caused buffered writing to be used, what
+  means that it looked like no output were written or partial
+  lines. Thanks to Michael Biebl for pointing out this bug.
+- bugfix: testbench failed when not executed in UTC+1 timezone
+  accidently, the time zone information was kept inside some
+  to-be-checked-for responses
+- temporary bugfix replaced by permanent one for
+  message-induced off-by-one error (potential segfault) (see 4.6.2)
+  The analysis has been completed and a better fix been crafted and 
+  integrated.
+- some doc fixes; incorrect config samples could cause confusion
+  thanks to Anthony Edwards for pointing the problems out
+---------------------------------------------------------------------------
 Version 4.6.2  [v4-stable] (rgerhards), 2010-03-26
 - new feature: "." action type added to support writing files to relative
   pathes (this is primarily meant as a debug aid)
diff --git a/Makefile.am b/Makefile.am
index a050e95..f5f9a6e 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -134,5 +134,5 @@ SUBDIRS += tests
 # temporarily be removed below. The intent behind forcing everthing to compile
 # in a make distcheck is so that we detect code that accidently was not updated
 # when some global update happened.
-DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog  --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog --enable-imdiag --enable-shave
+DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog  --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog --enable-imdiag --enable-shave --enable-extended-tests
 ACLOCAL_AMFLAGS = -I m4
diff --git a/Makefile.in b/Makefile.in
index 6de0d21..249a446 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -416,7 +416,7 @@ SUBDIRS = doc runtime . plugins/immark plugins/imuxsock plugins/imtcp \
 # temporarily be removed below. The intent behind forcing everthing to compile
 # in a make distcheck is so that we detect code that accidently was not updated
 # when some global update happened.
-DISTCHECK_CONFIGURE_FLAGS = --enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog  --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog --enable-imdiag --enable-shave
+DISTCHECK_CONFIGURE_FLAGS = --enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog  --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog --enable-imdiag --enable-shave --enable-extended-tests
 ACLOCAL_AMFLAGS = -I m4
 all: config.h
 	$(MAKE) $(AM_MAKEFLAGS) all-recursive
diff --git a/configure b/configure
index 2f175ca..cfe87fe 100755
--- a/configure
+++ b/configure
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.63 for rsyslog 4.6.2.
+# Generated by GNU Autoconf 2.63 for rsyslog 4.6.3.
 #
 # Report bugs to <rsyslog@lists.adiscon.com>.
 #
@@ -745,8 +745,8 @@ SHELL=${CONFIG_SHELL-/bin/sh}
 # Identity of this package.
 PACKAGE_NAME='rsyslog'
 PACKAGE_TARNAME='rsyslog'
-PACKAGE_VERSION='4.6.2'
-PACKAGE_STRING='rsyslog 4.6.2'
+PACKAGE_VERSION='4.6.3'
+PACKAGE_STRING='rsyslog 4.6.3'
 PACKAGE_BUGREPORT='rsyslog@lists.adiscon.com'
 
 ac_unique_file="ChangeLog"
@@ -824,6 +824,8 @@ ENABLE_IMDIAG_FALSE
 ENABLE_IMDIAG_TRUE
 ENABLE_MAIL_FALSE
 ENABLE_MAIL_TRUE
+ENABLE_EXTENDED_TESTS_FALSE
+ENABLE_EXTENDED_TESTS_TRUE
 ENABLE_RSYSLOGD_FALSE
 ENABLE_RSYSLOGD_TRUE
 RSRT_LIBS
@@ -1022,6 +1024,7 @@ enable_snmp
 enable_gnutls
 enable_rsyslogrt
 enable_rsyslogd
+enable_extended_tests
 enable_mail
 enable_imdiag
 enable_relp
@@ -1603,7 +1606,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures rsyslog 4.6.2 to adapt to many kinds of systems.
+\`configure' configures rsyslog 4.6.3 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1673,7 +1676,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of rsyslog 4.6.2:";;
+     short | recursive ) echo "Configuration of rsyslog 4.6.3:";;
    esac
   cat <<\_ACEOF
 
@@ -1709,6 +1712,7 @@ Optional Features:
   --enable-gnutls         Enable GNU TLS support [default=no]
   --enable-rsyslogrt      Build rsyslogrt [default=yes]
   --enable-rsyslogd       Build rsyslogd [default=yes]
+  --enable-extended-tests extended testbench [default=no]
   --enable-mail           Enable mail support [default=no]
   --enable-imdiag         Enable imdiag [default=yes]
   --enable-relp           Enable RELP support [default=no]
@@ -1817,7 +1821,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-rsyslog configure 4.6.2
+rsyslog configure 4.6.3
 generated by GNU Autoconf 2.63
 
 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
@@ -1831,7 +1835,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by rsyslog $as_me 4.6.2, which was
+It was created by rsyslog $as_me 4.6.3, which was
 generated by GNU Autoconf 2.63.  Invocation command line was
 
   $ $0 $@
@@ -2680,7 +2684,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='rsyslog'
- VERSION='4.6.2'
+ VERSION='4.6.3'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -6293,13 +6297,13 @@ if test "${lt_cv_nm_interface+set}" = set; then
 else
   lt_cv_nm_interface="BSD nm"
   echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:6296: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:6300: $ac_compile\"" >&5)
   (eval "$ac_compile" 2>conftest.err)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:6299: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:6303: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
   (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:6302: output\"" >&5)
+  (eval echo "\"\$as_me:6306: output\"" >&5)
   cat conftest.out >&5
   if $GREP 'External.*some_variable' conftest.out > /dev/null; then
     lt_cv_nm_interface="MS dumpbin"
@@ -7504,7 +7508,7 @@ ia64-*-hpux*)
   ;;
 *-*-irix6*)
   # Find out which ABI we are using.
-  echo '#line 7507 "configure"' > conftest.$ac_ext
+  echo '#line 7511 "configure"' > conftest.$ac_ext
   if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
   (eval $ac_compile) 2>&5
   ac_status=$?
@@ -8843,11 +8847,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:8846: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:8850: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:8850: \$? = $ac_status" >&5
+   echo "$as_me:8854: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -9182,11 +9186,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:9185: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:9189: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:9189: \$? = $ac_status" >&5
+   echo "$as_me:9193: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -9287,11 +9291,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:9290: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:9294: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:9294: \$? = $ac_status" >&5
+   echo "$as_me:9298: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -9342,11 +9346,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:9345: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:9349: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:9349: \$? = $ac_status" >&5
+   echo "$as_me:9353: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -12145,7 +12149,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 12148 "configure"
+#line 12152 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -12241,7 +12245,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 12244 "configure"
+#line 12248 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -19725,6 +19729,35 @@ fi
 
 
 
+# capability to enable an extended testbench. By default, this is off. The reason
+# for this switch is that some test simply take too long to execute them on a regular
+# basis. So we enable to skip them, while the majority of tests can still be used. The
+# idea is that at least "make distcheck" executes the extended testbench, and also
+# developers should explicitely enable it after important changes. -- rgerhards, 2010-04-12
+# Check whether --enable-extended_tests was given.
+if test "${enable_extended_tests+set}" = set; then
+  enableval=$enable_extended_tests; case "${enableval}" in
+         yes) enable_rsyslogd="yes" ;;
+          no) enable_rsyslogd="no" ;;
+           *) { { $as_echo "$as_me:$LINENO: error: bad value ${enableval} for --enable-extended-tests" >&5
+$as_echo "$as_me: error: bad value ${enableval} for --enable-extended-tests" >&2;}
+   { (exit 1); exit 1; }; } ;;
+         esac
+else
+  enable_extended_tests=no
+
+fi
+
+ if test x$enable_extended_tests = xyes; then
+  ENABLE_EXTENDED_TESTS_TRUE=
+  ENABLE_EXTENDED_TESTS_FALSE='#'
+else
+  ENABLE_EXTENDED_TESTS_TRUE='#'
+  ENABLE_EXTENDED_TESTS_FALSE=
+fi
+
+
+
 # Mail support (so far we do not need a library, but we need to turn this on and off)
 # Check whether --enable-mail was given.
 if test "${enable_mail+set}" = set; then
@@ -20588,6 +20621,13 @@ $as_echo "$as_me: error: conditional \"ENABLE_RSYSLOGD\" was never defined.
 Usually this means the macro was only invoked conditionally." >&2;}
    { (exit 1); exit 1; }; }
 fi
+if test -z "${ENABLE_EXTENDED_TESTS_TRUE}" && test -z "${ENABLE_EXTENDED_TESTS_FALSE}"; then
+  { { $as_echo "$as_me:$LINENO: error: conditional \"ENABLE_EXTENDED_TESTS\" was never defined.
+Usually this means the macro was only invoked conditionally." >&5
+$as_echo "$as_me: error: conditional \"ENABLE_EXTENDED_TESTS\" was never defined.
+Usually this means the macro was only invoked conditionally." >&2;}
+   { (exit 1); exit 1; }; }
+fi
 if test -z "${ENABLE_MAIL_TRUE}" && test -z "${ENABLE_MAIL_FALSE}"; then
   { { $as_echo "$as_me:$LINENO: error: conditional \"ENABLE_MAIL\" was never defined.
 Usually this means the macro was only invoked conditionally." >&5
@@ -20987,7 +21027,7 @@ exec 6>&1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by rsyslog $as_me 4.6.2, which was
+This file was extended by rsyslog $as_me 4.6.3, which was
 generated by GNU Autoconf 2.63.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -21050,7 +21090,7 @@ Report bugs to <bug-autoconf@gnu.org>."
 _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_version="\\
-rsyslog config.status 4.6.2
+rsyslog config.status 4.6.3
 configured by $0, generated by GNU Autoconf 2.63,
   with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\"
 
@@ -22966,6 +23006,7 @@ echo "    SNMP support enabled:                     $enable_snmp"
 echo
 echo "---{ debugging support }---"
 echo "    Testbench enabled:                        $enable_testbench"
+echo "    Extended Testbench enabled:               $enable_extended_tests"
 echo "    Debug mode enabled:                       $enable_debug"
 echo "    Runtime Instrumentation enabled:          $enable_rtinst"
 echo "    Diagnostic tools enabled:                 $enable_diagtools"
diff --git a/configure.ac b/configure.ac
index b01396d..e41b76f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.61)
-AC_INIT([rsyslog],[4.6.2],[rsyslog@lists.adiscon.com])
+AC_INIT([rsyslog],[4.6.3],[rsyslog@lists.adiscon.com])
 AM_INIT_AUTOMAKE
 AC_CONFIG_SRCDIR([ChangeLog])
 AC_CONFIG_MACRO_DIR([m4])
@@ -612,6 +612,23 @@ AC_ARG_ENABLE(rsyslogd,
 AM_CONDITIONAL(ENABLE_RSYSLOGD, test x$enable_rsyslogd = xyes)
 
 
+# capability to enable an extended testbench. By default, this is off. The reason
+# for this switch is that some test simply take too long to execute them on a regular
+# basis. So we enable to skip them, while the majority of tests can still be used. The
+# idea is that at least "make distcheck" executes the extended testbench, and also
+# developers should explicitely enable it after important changes. -- rgerhards, 2010-04-12
+AC_ARG_ENABLE(extended_tests,
+        [AS_HELP_STRING([--enable-extended-tests],[extended testbench @<:@default=no@:>@])],
+        [case "${enableval}" in
+         yes) enable_rsyslogd="yes" ;;
+          no) enable_rsyslogd="no" ;;
+           *) AC_MSG_ERROR(bad value ${enableval} for --enable-extended-tests) ;;
+         esac],
+        [enable_extended_tests=no]
+)
+AM_CONDITIONAL(ENABLE_EXTENDED_TESTS, test x$enable_extended_tests = xyes)
+
+
 # Mail support (so far we do not need a library, but we need to turn this on and off)
 AC_ARG_ENABLE(mail,
         [AS_HELP_STRING([--enable-mail],[Enable mail support @<:@default=no@:>@])],
@@ -864,6 +881,7 @@ echo "    SNMP support enabled:                     $enable_snmp"
 echo
 echo "---{ debugging support }---"
 echo "    Testbench enabled:                        $enable_testbench"
+echo "    Extended Testbench enabled:               $enable_extended_tests"
 echo "    Debug mode enabled:                       $enable_debug"
 echo "    Runtime Instrumentation enabled:          $enable_rtinst"
 echo "    Diagnostic tools enabled:                 $enable_diagtools"
diff --git a/doc/manual.html b/doc/manual.html
index 455a711..4a29283 100644
--- a/doc/manual.html
+++ b/doc/manual.html
@@ -19,7 +19,7 @@ rsyslog support</a> available directly from the source!</p>
 <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a>
 to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the
 project goals.</p>
-<p><b>This documentation is for version 4.6.2 (v4-stable branch) of rsyslog.</b>
+<p><b>This documentation is for version 4.6.3 (v4-stable branch) of rsyslog.</b>
 Visit the <i><a href="http://www.rsyslog.com/doc-status.html">rsyslog status page</a></i></b>
 to obtain current version information and project status.
 </p><p><b>If you like rsyslog, you might
diff --git a/doc/rsyslog_conf_filter.html b/doc/rsyslog_conf_filter.html
index 841ec9c..63c2981 100644
--- a/doc/rsyslog_conf_filter.html
+++ b/doc/rsyslog_conf_filter.html
@@ -231,7 +231,7 @@ A few quick samples:<br>
 <br>
 <code>
 *.* /var/log/file1 # the traditional way<br>
-if $msg contains 'error' /var/log/errlog # the expression-based way<br>
+if $msg contains 'error' then /var/log/errlog # the expression-based way<br>
 </code>
 <br>
 Right now, you need to specify numerical values if you would like to
diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html
index d5a2754..ce46bac 100644
--- a/doc/rsyslog_conf_global.html
+++ b/doc/rsyslog_conf_global.html
@@ -219,7 +219,7 @@ in that case by sending rsyslogd a HUP signal.
 of the output file. The higher the number, the better the compression, but also the
 more CPU is required for zipping.</li>
 <li><b>$OMFileIOBufferSize</b> &lt;size_nbr&gt;, default 4k, size of the buffer used to writing output data. The larger the buffer, the potentially better performance is. The default of 4k is quite conservative, it is useful to go up to 64k, and 128K if you used gzip compression (then, even higher sizes may make sense)</li>
-<li><b>$OMFileFlushOnTXEnd</b> &lt;[on/<b>off</b>]&gt;, default ff. Omfile has the
+<li><b>$OMFileFlushOnTXEnd</b> &lt;[<b>on</b>/off]&gt;, default on. Omfile has the
 capability to
 write output using a buffered writer. Disk writes are only done when the buffer is
 full. So if an error happens during that write, data is potentially lost. In cases where
@@ -227,7 +227,7 @@ this is unacceptable, set $OMFileFlushOnTXEnd to on. Then, data is written at th
 of each transaction (for pre-v5 this means after <b>each</b> log message) and the usual
 error recovery thus can handle write errors without data loss. Note that this option
 severely reduces the effect of zip compression and should be switched to off
-for that use case. Note that the default -off- is primarily an aid to preserve
+for that use case. Note that the default -on- is primarily an aid to preserve
 the traditional syslogd behaviour.</li>
 <li><b>$RepeatedMsgContainsOriginalMsg</b> [on/<b>off</b>] - "last message repeated n times" messages, if generated,
 have a different format that contains the message that is being repeated.
diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html
index 6c68b80..baa4ce2 100644
--- a/doc/rsyslog_conf_templates.html
+++ b/doc/rsyslog_conf_templates.html
@@ -87,8 +87,7 @@ option. Otherwise you will become vulnerable to SQL injection. <br>
 To escape:<br>
 % = \%<br>
 \ = \\ --&gt; '\' is used to escape (as in C)<br>
-$template TraditionalFormat,%timegenerated% %HOSTNAME%
-%syslogtag%%msg%\n"<br>
+$template TraditionalFormat,"%timegenerated% %HOSTNAME% %syslogtag%%msg%\n"<br>
 <br>
 Properties can be accessed by the <a href="property_replacer.html">property
 replacer</a> (see there for details).</p>
diff --git a/runtime/msg.c b/runtime/msg.c
index 2ce7843..91057f9 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -2010,6 +2010,8 @@ finalize_it:
 
 
 /* set raw message in message object. Size of message is provided.
+ * The function makes sure that the stored rawmsg is properly
+ * terminated by '\0'.
  * rgerhards, 2009-06-16
  */
 void MsgSetRawMsg(msg_t *pThis, char* pszRawMsg, size_t lenMsg)
@@ -2319,13 +2321,6 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 			*pPropLen = sizeof("**INVALID PROPERTY NAME**") - 1;
 			return UCHAR_CONSTANT("**INVALID PROPERTY NAME**");
 	}
-	/* the following line fixes the symptom, but not the root cause -- at least MSG sometimes
-	* returns a size of one too less. To prevent all troubles, we recalculate the sizes based
-	* on what we actually got. TODO: remove once root cause is found.
-	* rgerhards, 2010-03-23
-	*/
-	bufLen = ustrlen(pRes);
-
 
 	/* If we did not receive a template pointer, we are already done... */
 	if(pTpe == NULL) {
diff --git a/runtime/parser.c b/runtime/parser.c
index 466066e..36e88eb 100644
--- a/runtime/parser.c
+++ b/runtime/parser.c
@@ -176,7 +176,10 @@ sanitizeMessage(msg_t *pMsg)
 	pszMsg = pMsg->pszRawMsg;
 	lenMsg = pMsg->iLenRawMsg;
 
-	/* remove NUL character at end of message (see comment in function header) */
+	/* remove NUL character at end of message (see comment in function header)
+	 * Note that we do not need to add a NUL character in this case, because it
+	 * is already present ;)
+	 */
 	if(pszMsg[lenMsg-1] == '\0') {
 		DBGPRINTF("dropped NUL at very end of message\n");
 		bUpdatedLen = TRUE;
@@ -190,8 +193,9 @@ sanitizeMessage(msg_t *pMsg)
 	 */
 	if(bDropTrailingLF && pszMsg[lenMsg-1] == '\n') {
 		DBGPRINTF("dropped LF at very end of message (DropTrailingLF is set)\n");
-		bUpdatedLen = TRUE;
 		lenMsg--;
+		pszMsg[lenMsg] = '\0';
+		bUpdatedLen = TRUE;
 	}
 
 	/* it is much quicker to sweep over the message and see if it actually
@@ -245,6 +249,7 @@ sanitizeMessage(msg_t *pMsg)
 		}
 		++iSrc;
 	}
+	pDst[iDst] = '\0';
 
 	MsgSetRawMsg(pMsg, (char*)pDst, iDst); /* save sanitized string */
 
diff --git a/runtime/ruleset.c b/runtime/ruleset.c
index d98b421..af61f24 100644
--- a/runtime/ruleset.c
+++ b/runtime/ruleset.c
@@ -348,6 +348,7 @@ destructAllActions(void)
 
 	CHKiRet(llDestroy(&llRulesets));
 	CHKiRet(llInit(&llRulesets, rulesetDestructForLinkedList, keyDestruct, strcasecmp));
+	pDfltRuleset = NULL;
 
 finalize_it:
 	RETiRet;
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 42ef32e..0045f00 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,6 +1,6 @@
 if ENABLE_TESTBENCH
 TESTRUNS = rt_init rscript
-check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq
+check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq randomgen
 TESTS = $(TESTRUNS) cfg.sh \
 	validation-run.sh \
 	imtcp-multiport.sh \
@@ -40,6 +40,10 @@ TESTS += omod-if-array.sh \
 	 fieldtest.sh
 endif
 
+if ENABLE_EXTENDED_TESTS
+TESTS += random.sh 
+endif
+
 check_JAVA = DiagTalker.java
 
 endif # if ENABLE_TESTBENCH
@@ -109,6 +113,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/parse3.conf \
 	   testsuites/reallife.parse3 \
 	   testsuites/parse-nodate.conf \
+	   testsuites/samples.parse-nodate \
 	   testsuites/parse_invld_regex.conf \
 	   testsuites/samples.parse_invld_regex \
 	   testsuites/parse-3164-buggyday.conf \
@@ -180,6 +185,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/gzipwr_large_dynfile.conf \
 	   complex1.sh \
 	   testsuites/complex1.conf \
+	   random.sh \
+	   testsuites/random.conf \
 	   dynfile_invld_async.sh \
 	   dynfile_invld_sync.sh \
 	   dynfile_cachemiss.sh \
@@ -203,6 +210,9 @@ chkseq_SOURCES = chkseq.c
 tcpflood_SOURCES = tcpflood.c
 tcpflood_LDADD = $(SOL_LIBS)
 
+randomgen_SOURCES = randomgen.c
+randomgen_LDADD = $(SOL_LIBS)
+
 nettester_SOURCES = nettester.c getline.c
 nettester_LDADD = $(SOL_LIBS)
 
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 2f8b12d..a792be3 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -35,7 +35,8 @@ build_triplet = @build@
 host_triplet = @host@
 @ENABLE_TESTBENCH_TRUE@check_PROGRAMS = $(am__EXEEXT_1) \
 @ENABLE_TESTBENCH_TRUE@	ourtail$(EXEEXT) nettester$(EXEEXT) \
-@ENABLE_TESTBENCH_TRUE@	tcpflood$(EXEEXT) chkseq$(EXEEXT)
+@ENABLE_TESTBENCH_TRUE@	tcpflood$(EXEEXT) chkseq$(EXEEXT) \
+@ENABLE_TESTBENCH_TRUE@	randomgen$(EXEEXT)
 @ENABLE_TESTBENCH_TRUE@TESTS = $(am__EXEEXT_1) cfg.sh \
 @ENABLE_TESTBENCH_TRUE@	validation-run.sh imtcp-multiport.sh \
 @ENABLE_TESTBENCH_TRUE@	diskqueue.sh diskqueue-fsync.sh \
@@ -50,7 +51,7 @@ host_triplet = @host@
 @ENABLE_TESTBENCH_TRUE@	dynfile_invld_sync.sh \
 @ENABLE_TESTBENCH_TRUE@	dynfile_invalid2.sh complex1.sh \
 @ENABLE_TESTBENCH_TRUE@	queue-persist.sh pipeaction.sh \
-@ENABLE_TESTBENCH_TRUE@	$(am__append_1)
+@ENABLE_TESTBENCH_TRUE@	$(am__append_1) $(am__append_2)
 @ENABLE_OMSTDOUT_TRUE@@ENABLE_TESTBENCH_TRUE@am__append_1 = omod-if-array.sh \
 @ENABLE_OMSTDOUT_TRUE@@ENABLE_TESTBENCH_TRUE@	 proprepltest.sh \
 @ENABLE_OMSTDOUT_TRUE@@ENABLE_TESTBENCH_TRUE@	 parsertest.sh \
@@ -60,6 +61,7 @@ host_triplet = @host@
 @ENABLE_OMSTDOUT_TRUE@@ENABLE_TESTBENCH_TRUE@	 threadingmqaq.sh \
 @ENABLE_OMSTDOUT_TRUE@@ENABLE_TESTBENCH_TRUE@	 fieldtest.sh
 
+@ENABLE_EXTENDED_TESTS_TRUE@@ENABLE_TESTBENCH_TRUE@am__append_2 = random.sh 
 subdir = tests
 DIST_COMMON = README $(srcdir)/Makefile.am $(srcdir)/Makefile.in \
 	INSTALL
@@ -87,6 +89,9 @@ nettester_DEPENDENCIES = $(am__DEPENDENCIES_1)
 am_ourtail_OBJECTS = ourtail.$(OBJEXT)
 ourtail_OBJECTS = $(am_ourtail_OBJECTS)
 ourtail_LDADD = $(LDADD)
+am_randomgen_OBJECTS = randomgen.$(OBJEXT)
+randomgen_OBJECTS = $(am_randomgen_OBJECTS)
+randomgen_DEPENDENCIES = $(am__DEPENDENCIES_1)
 am__objects_1 = rscript-runtime-dummy.$(OBJEXT)
 am_rscript_OBJECTS = rscript-rscript.$(OBJEXT) \
 	rscript-getline.$(OBJEXT) $(am__objects_1)
@@ -121,10 +126,11 @@ LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
 	--mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
 	$(LDFLAGS) -o $@
 SOURCES = $(chkseq_SOURCES) $(nettester_SOURCES) $(ourtail_SOURCES) \
-	$(rscript_SOURCES) $(rt_init_SOURCES) $(tcpflood_SOURCES)
-DIST_SOURCES = $(chkseq_SOURCES) $(nettester_SOURCES) \
-	$(ourtail_SOURCES) $(rscript_SOURCES) $(rt_init_SOURCES) \
+	$(randomgen_SOURCES) $(rscript_SOURCES) $(rt_init_SOURCES) \
 	$(tcpflood_SOURCES)
+DIST_SOURCES = $(chkseq_SOURCES) $(nettester_SOURCES) \
+	$(ourtail_SOURCES) $(randomgen_SOURCES) $(rscript_SOURCES) \
+	$(rt_init_SOURCES) $(tcpflood_SOURCES)
 JAVAC = javac
 CLASSPATH_ENV = CLASSPATH=$(JAVAROOT):$(srcdir)/$(JAVAROOT):$$CLASSPATH
 JAVAROOT = $(top_builddir)
@@ -349,6 +355,7 @@ EXTRA_DIST = 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/parse3.conf \
 	   testsuites/reallife.parse3 \
 	   testsuites/parse-nodate.conf \
+	   testsuites/samples.parse-nodate \
 	   testsuites/parse_invld_regex.conf \
 	   testsuites/samples.parse_invld_regex \
 	   testsuites/parse-3164-buggyday.conf \
@@ -420,6 +427,8 @@ EXTRA_DIST = 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/gzipwr_large_dynfile.conf \
 	   complex1.sh \
 	   testsuites/complex1.conf \
+	   random.sh \
+	   testsuites/random.conf \
 	   dynfile_invld_async.sh \
 	   dynfile_invld_sync.sh \
 	   dynfile_cachemiss.sh \
@@ -441,6 +450,8 @@ ourtail_SOURCES = ourtail.c
 chkseq_SOURCES = chkseq.c
 tcpflood_SOURCES = tcpflood.c
 tcpflood_LDADD = $(SOL_LIBS)
+randomgen_SOURCES = randomgen.c
+randomgen_LDADD = $(SOL_LIBS)
 nettester_SOURCES = nettester.c getline.c
 nettester_LDADD = $(SOL_LIBS)
 rt_init_SOURCES = rt-init.c $(test_files)
@@ -503,6 +514,9 @@ nettester$(EXEEXT): $(nettester_OBJECTS) $(nettester_DEPENDENCIES)
 ourtail$(EXEEXT): $(ourtail_OBJECTS) $(ourtail_DEPENDENCIES) 
 	@rm -f ourtail$(EXEEXT)
 	$(LINK) $(ourtail_OBJECTS) $(ourtail_LDADD) $(LIBS)
+randomgen$(EXEEXT): $(randomgen_OBJECTS) $(randomgen_DEPENDENCIES) 
+	@rm -f randomgen$(EXEEXT)
+	$(LINK) $(randomgen_OBJECTS) $(randomgen_LDADD) $(LIBS)
 rscript$(EXEEXT): $(rscript_OBJECTS) $(rscript_DEPENDENCIES) 
 	@rm -f rscript$(EXEEXT)
 	$(rscript_LINK) $(rscript_OBJECTS) $(rscript_LDADD) $(LIBS)
@@ -523,6 +537,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/getline.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/nettester.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ourtail.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/randomgen.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rscript-getline.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rscript-rscript.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rscript-runtime-dummy.Po@am__quote@
diff --git a/tests/complex1.sh b/tests/complex1.sh
index b5dc2c9..7f3cd99 100755
--- a/tests/complex1.sh
+++ b/tests/complex1.sh
@@ -7,8 +7,8 @@ echo ===========================================================================
 echo TEST: \[complex1.sh\]: complex test with gzip and multiple action queues
 source $srcdir/diag.sh init
 # uncomment for debugging support:
-export RSYSLOG_DEBUG="debug nostdout"
-export RSYSLOG_DEBUGLOG="log"
+#export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUGLOG="log"
 source $srcdir/diag.sh startup complex1.conf
 # send 30,000 messages of 400 bytes plus header max, via three dest ports
 source $srcdir/diag.sh tcpflood -m40000 -rd400 -P129 -f5 -n3 -c15 -i1
diff --git a/tests/diag.sh b/tests/diag.sh
index 8f268a5..51ad5f6 100755
--- a/tests/diag.sh
+++ b/tests/diag.sh
@@ -17,10 +17,10 @@ case $1 in
 		cp $srcdir/testsuites/diag-common.conf diag-common.conf
 		cp $srcdir/testsuites/diag-common2.conf diag-common2.conf
 		rm -f rsyslog.action.*.include
-		rm -f rsyslogd.started work-*.conf
+		rm -f rsyslogd.started work-*.conf rsyslog.random.data
 		rm -f rsyslogd2.started work-*.conf
 		rm -f work rsyslog.out.log rsyslog.out.log.save # common work files
-		rm -f rsyslog.out.*.log
+		rm -f rsyslog.out.*.log work-presort
 		rm -rf test-spool
 		rm -f core.* vgcore.*
 		mkdir test-spool
@@ -28,7 +28,7 @@ case $1 in
    'exit')	rm -f rsyslogd.started work-*.conf diag-common.conf
    		rm -f rsyslogd2.started diag-common2.conf rsyslog.action.*.include
 		rm -f work rsyslog.out.log rsyslog.out.log.save # common work files
-		rm -f rsyslog.out.*.log
+		rm -f rsyslog.out.*.log rsyslog.random.data work-presort
 		rm -rf test-spool
 		;;
    'startup')   # start rsyslogd with default params. $2 is the config file name to use
diff --git a/tests/dynfile_invalid2.sh b/tests/dynfile_invalid2.sh
index b6fb394..cb3ef51 100755
--- a/tests/dynfile_invalid2.sh
+++ b/tests/dynfile_invalid2.sh
@@ -9,8 +9,8 @@ echo ===========================================================================
 echo TEST: \[dynfile_invalid2.sh\]: test open fail for dynafiles
 source $srcdir/diag.sh init
 # uncomment for debugging support:
-export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
-export RSYSLOG_DEBUGLOG="log"
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
 source $srcdir/diag.sh startup dynfile_invalid2.conf
 # we send handcrafted message. We have a dynafile cache of 4, and now send one message
 # each to fill up the cache.
diff --git a/tests/gzipwr_large_dynfile.sh b/tests/gzipwr_large_dynfile.sh
index ebd4c55..73d4479 100755
--- a/tests/gzipwr_large_dynfile.sh
+++ b/tests/gzipwr_large_dynfile.sh
@@ -18,8 +18,8 @@ echo ===========================================================================
 echo TEST: \[gzipwr_large_dynfile.sh\]: test for gzip file writing for large message sets
 source $srcdir/diag.sh init
 # uncomment for debugging support:
-export RSYSLOG_DEBUG="debug nostdout"
-export RSYSLOG_DEBUGLOG="log"
+#export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUGLOG="log"
 source $srcdir/diag.sh startup gzipwr_large_dynfile.conf
 # send 4000 messages of 10.000bytes plus header max, randomized
 source $srcdir/diag.sh tcpflood -m4000 -r -d10000 -P129 -f5
diff --git a/tests/random.sh b/tests/random.sh
new file mode 100755
index 0000000..d1f392d
--- /dev/null
+++ b/tests/random.sh
@@ -0,0 +1,20 @@
+# Test if rsyslog survives sending truely random data to it...
+#
+# added 2010-04-01 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[random.sh\]: testing random data
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup random.conf
+# generate random data
+./randomgen -f rsyslog.random.data -s 100000
+ls -l rsyslog.random.data
+source $srcdir/diag.sh tcpflood -B -I rsyslog.random.data -c5 -C10
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+# we do not check anything yet, the point is if rsyslog survived ;)
+# TODO: check for exit message, but we'll notice an abort anyhow, so not that important
+source $srcdir/diag.sh exit
diff --git a/tests/randomgen.c b/tests/randomgen.c
new file mode 100644
index 0000000..9ba5695
--- /dev/null
+++ b/tests/randomgen.c
@@ -0,0 +1,130 @@
+/* generates random data for later use in test cases. Of course,
+ * we could generate random data during the testcase itself, but
+ * the core idea is that we record the random data so that we have
+ * a chance to reproduce a problem should it occur. IMHO this 
+ * provides the best compromise, by a) having randomness but
+ * b) knowing what was used during the test.
+ *
+ * Params
+ * -f	output file name (stdout if not given)
+ * -s	size of test data, plain number is size in k, 1MB default
+ * -u   uses /dev/urandom instead of libc random number generator
+ *      (when available). Note that this is usually much slower.
+ *
+ * Part of the testbench for rsyslog.
+ *
+ * Copyright 2010 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ */
+#include "config.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <signal.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <netinet/in.h>
+
+#define EXIT_FAILURE 1
+
+static char *fileName = NULL;		/* name of output file */
+static int tryUseURandom = 0;		/* try to use /dev/urandom? */
+static long long fileSize = 1024*1024;	/* file size in K, 1MB default */
+
+
+/* generate the random file. This code really can be improved (e.g. read /dev/urandom
+ * when available)
+ */
+static inline void
+genFile()
+{
+	long i;
+	FILE *fp;
+	FILE *rfp = NULL;
+
+	if(fileName == NULL) {
+		fp = stdout;
+	} else {
+		if((fp = fopen(fileName, "w")) == NULL) {
+			perror(fileName);
+		}
+	}
+
+	/* try to use /dev/urandom, if available */
+	if(tryUseURandom)
+		rfp = fopen("/dev/urandom", "r");
+
+	if(rfp == NULL) {
+		/* fallback, use libc random number generator */
+		for(i = 0 ; i < fileSize ; ++i) {
+			if(fputc((char) rand(), fp) == EOF) {
+				perror(fileName);
+				exit(1);
+			}
+		}
+	} else {
+		/* use /dev/urandom */
+		printf("using /dev/urandom");
+		for(i = 0 ; i < fileSize ; ++i) {
+			if(fputc(fgetc(rfp), fp) == EOF) {
+				perror(fileName);
+				exit(1);
+			}
+		}
+	}
+
+	if(fileName != NULL)
+		fclose(fp);
+}
+
+
+/* Run the test.
+ * rgerhards, 2009-04-03
+ */
+int main(int argc, char *argv[])
+{
+	int ret = 0;
+	int opt;
+
+	srand(time(NULL));	/* seed is good enough for our needs */
+
+	while((opt = getopt(argc, argv, "f:s:u")) != -1) {
+		switch (opt) {
+		case 'f':	fileName = optarg;
+				break;
+		case 's':	fileSize = atol(optarg) * 1024;
+				break;
+		case 'u':	tryUseURandom = 1;
+				break;
+		default:	printf("invalid option '%c' or value missing - terminating...\n", opt);
+				exit (1);
+				break;
+		}
+	}
+
+	printf("generating random data file '%s' of %ldkb - may take a short while...\n",
+		fileName, (long) (fileSize / 1024));
+	genFile();
+
+	exit(ret);
+}
diff --git a/tests/tcpflood.c b/tests/tcpflood.c
index 32bf959..d8c3f03 100644
--- a/tests/tcpflood.c
+++ b/tests/tcpflood.c
@@ -20,6 +20,14 @@
  *      one field to the right. Zero (default) disables this functionality.
  * -M   the message to be sent. Disables all message format options, as
  *      only that exact same message is sent.
+ * -I   read specified input file, do NOT generate own test data. The test
+ *      completes when eof is reached.
+ * -B   The specified file (-I) is binary. No data processing is done by
+ *      tcpflood. If multiple connections are specified, data is read in
+ *      chunks and spread across the connections without taking any record
+ *      delemiters into account.
+ * -C	when input from a file is read, this file is transmitted -C times
+ *      (C like cycle, running out of meaningful option switches ;))
  *
  * Part of the testbench for rsyslog.
  *
@@ -75,6 +83,10 @@ static int *sockArray;  /* array of sockets to use */
 static int msgNum = 0;	/* initial message number to start with */
 static int bShowProgress = 1; /* show progress messages */
 static char *MsgToSend = NULL; /* if non-null, this is the actual message to send */
+static int bBinaryFile = 0;	/* is -I file binary */
+static char *dataFile = NULL;	/* name of data file, if NULL, generate own data */
+static int numFileIterations = 1;/* how often is file data to be sent? */
+FILE *dataFP = NULL;		/* file pointer for data file, if used */
 
 
 /* open a single tcp connection
@@ -166,8 +178,8 @@ int openConnections(void)
 void closeConnections(void)
 {
 	int i;
-	char msgBuf[128];
 	size_t lenMsg;
+	char msgBuf[128];
 
 	if(bShowProgress)
 		write(1, "      close connections", sizeof("      close connections")-1);
@@ -186,6 +198,62 @@ void closeConnections(void)
 }
 
 
+/* generate the message to be sent according to program command line parameters.
+ * this has been moved to its own function as we now have various different ways
+ * of constructing test messages. -- rgerhards, 2010-03-31
+ */
+static inline void
+genMsg(char *buf, size_t maxBuf, int *pLenBuf)
+{
+	int edLen; /* actual extra data length to use */
+	char extraData[MAX_EXTRADATA_LEN + 1];
+	char dynFileIDBuf[128] = "";
+	static int numMsgsGen = 0;
+	int done;
+
+	if(dataFP != NULL) {
+		/* get message from file */
+		do {
+			done = 1;
+			*pLenBuf = fread(buf, 1, 1024, dataFP);
+			if(feof(dataFP)) {
+				if(--numFileIterations > 0)  {
+					rewind(dataFP);
+					done = 0; /* need new iteration */
+				} else {
+					*pLenBuf = 0;
+					goto finalize_it;
+				}
+			}
+		} while(!done); /* Attention: do..while()! */
+	} else if(MsgToSend == NULL) {
+		if(dynFileIDs > 0) {
+			snprintf(dynFileIDBuf, maxBuf, "%d:", rand() % dynFileIDs);
+		}
+		if(extraDataLen == 0) {
+			*pLenBuf = snprintf(buf, maxBuf, "<%s>Mar  1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:\n",
+					       msgPRI, dynFileIDBuf, msgNum);
+		} else {
+			if(bRandomizeExtraData)
+				edLen = ((long) rand() + extraDataLen) % extraDataLen + 1;
+			else
+				edLen = extraDataLen;
+			memset(extraData, 'X', edLen);
+			extraData[edLen] = '\0';
+			*pLenBuf = snprintf(buf, maxBuf, "<%s>Mar  1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:%d:%s\n",
+					       msgPRI, dynFileIDBuf, msgNum, edLen, extraData);
+		}
+	} else {
+		/* use fixed message format from command line */
+		*pLenBuf = snprintf(buf, maxBuf, "%s\n", MsgToSend);
+	}
+
+	if(numMsgsGen++ >= numMsgsToSend)
+		*pLenBuf = 0; /* indicate end of run */
+
+finalize_it: ;
+}
+
 /* send messages to the tcp connections we keep open. We use
  * a very basic format that helps identify the message
  * (via msgnum:<number>: e.g. msgnum:00000001:). This format is suitable
@@ -196,52 +264,42 @@ void closeConnections(void)
  */
 int sendMessages(void)
 {
-	int i;
+	int i = 0;
 	int socknum;
 	int lenBuf;
 	int lenSend;
-	int edLen; /* actual extra data length to use */
-	char dynFileIDBuf[128] = "";
+	char *statusText;
 	char buf[MAX_EXTRADATA_LEN + 1024];
-	char extraData[MAX_EXTRADATA_LEN + 1];
 
-	printf("Sending %d messages.\n", numMsgsToSend);
+	if(dataFile == NULL) {
+		printf("Sending %d messages.\n", numMsgsToSend);
+		statusText = "messages";
+	} else {
+		printf("Sending file '%s' %d times.\n", dataFile, numFileIterations);
+		statusText = "kb";
+	}
 	if(bShowProgress)
-		printf("\r%8.8d messages sent", 0);
-	for(i = 0 ; i < numMsgsToSend ; ++i) {
+		printf("\r%8.8d %s sent", 0, statusText);
+	while(1) { /* broken inside loop! */
 		if(i < numConnections)
 			socknum = i;
 		else if(i >= numMsgsToSend - numConnections)
 			socknum = i - (numMsgsToSend - numConnections);
-		else
-			socknum = rand() % numConnections;
-		if(MsgToSend == NULL) {
-			if(dynFileIDs > 0) {
-				sprintf(dynFileIDBuf, "%d:", rand() % dynFileIDs);
-			}
-			if(extraDataLen == 0) {
-				lenBuf = sprintf(buf, "<%s>Mar  1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:\n",
-						       msgPRI, dynFileIDBuf, msgNum);
-			} else {
-				if(bRandomizeExtraData)
-					edLen = ((long) rand() + extraDataLen) % extraDataLen + 1;
-				else
-					edLen = extraDataLen;
-				memset(extraData, 'X', edLen);
-				extraData[edLen] = '\0';
-				lenBuf = sprintf(buf, "<%s>Mar  1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:%d:%s\n",
-						       msgPRI, dynFileIDBuf, msgNum, edLen, extraData);
-			}
-		} else {
-			/* use fixed message format from command line */
-			lenBuf = sprintf(buf, "%s\n", MsgToSend);
+		else {
+			int rnd = rand();
+			//socknum = rand() % numConnections;
+			socknum = rnd % numConnections;
 		}
+		genMsg(buf, sizeof(buf), &lenBuf); /* generate the message to send according to params */
+		if(lenBuf == 0)
+			break; /* end of processing! */
 		lenSend = send(sockArray[socknum], buf, lenBuf, 0);
 		if(lenSend != lenBuf) {
 			printf("\r%5.5d\n", i);
 			fflush(stdout);
 			perror("send test data");
-			printf("send() failed at socket %d, index %d, msgNum %d\n", socknum, i, msgNum);
+			printf("send() failed at socket %d, index %d, msgNum %d\n",
+				sockArray[socknum], i, msgNum);
 			fflush(stderr);
 			return(1);
 		}
@@ -250,8 +308,9 @@ int sendMessages(void)
 				printf("\r%8.8d", i);
 		}
 		++msgNum;
+		++i;
 	}
-	printf("\r%8.8d messages sent\n", i);
+	printf("\r%8.8d %s sent\n", i, statusText);
 
 	return 0;
 }
@@ -335,7 +394,7 @@ int main(int argc, char *argv[])
 	if(!isatty(1))
 		bShowProgress = 0;
 
-	while((opt = getopt(argc, argv, "f:t:p:c:m:i:P:d:n:M:r")) != -1) {
+	while((opt = getopt(argc, argv, "f:t:p:c:C:m:i:I:P:d:n:M:rB")) != -1) {
 		switch (opt) {
 		case 't':	targetIP = optarg;
 				break;
@@ -345,6 +404,8 @@ int main(int argc, char *argv[])
 				break;
 		case 'c':	numConnections = atoi(optarg);
 				break;
+		case 'C':	numFileIterations = atoi(optarg);
+				break;
 		case 'm':	numMsgsToSend = atoi(optarg);
 				break;
 		case 'i':	msgNum = atoi(optarg);
@@ -364,12 +425,27 @@ int main(int argc, char *argv[])
 				break;
 		case 'M':	MsgToSend = optarg;
 				break;
+		case 'I':	dataFile = optarg;
+				/* in this mode, we do not know the num messages to send, so
+				 * we set a (high) number to keep the code happy.
+				 */
+				numMsgsToSend = 1000000;
+				break;
+		case 'B':	bBinaryFile = 1;
+				break;
 		default:	printf("invalid option '%c' or value missing - terminating...\n", opt);
 				exit (1);
 				break;
 		}
 	}
 
+	if(dataFile != NULL) {
+		if((dataFP = fopen(dataFile, "r")) == NULL) {
+			perror(dataFile);
+			exit(1);
+		}
+	}
+
 	if(openConnections() != 0) {
 		printf("error opening connections\n");
 		exit(1);
diff --git a/tests/testsuites/parse3.conf b/tests/testsuites/parse3.conf
index d5cf77d..8a3cb31 100644
--- a/tests/testsuites/parse3.conf
+++ b/tests/testsuites/parse3.conf
@@ -1,8 +1,10 @@
+# note: we need to strip off the TZ designator in the rfc3339 timestamp
+# as this test otherwise fails in different timezones!
 $ModLoad ../plugins/omstdout/.libs/omstdout
 $IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
 
 $ErrorMessagesToStderr off
 
 # use a special format that we can easily parse in expect
-$Template output,"%timereported:1:$:date-rfc3339,csv%, %hostname:::csv%, %programname:::csv%, %syslogtag:R,ERE,0,BLANK:[0-9]+--end:csv%, %syslogseverity:::csv%, %msg:::drop-last-lf,csv%\n" 
+$Template output,"%timereported:1:19:date-rfc3339,csv%, %hostname:::csv%, %programname:::csv%, %syslogtag:R,ERE,0,BLANK:[0-9]+--end:csv%, %syslogseverity:::csv%, %msg:::drop-last-lf,csv%\n" 
 *.* :omstdout:;output
diff --git a/tests/testsuites/parse_invld_regex.conf b/tests/testsuites/parse_invld_regex.conf
index 736aae5..d18a2b3 100644
--- a/tests/testsuites/parse_invld_regex.conf
+++ b/tests/testsuites/parse_invld_regex.conf
@@ -1,8 +1,10 @@
+# note: we need to strip off the TZ designator in the rfc3339 timestamp
+# as this test otherwise fails in different timezones!
 $ModLoad ../plugins/omstdout/.libs/omstdout
 $IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
 
 $ErrorMessagesToStderr off
 
 # use a special format that we can easily parse in expect
-$Template output,"%timereported:1:$:date-rfc3339,csv%, %hostname:::csv%, %programname:::csv%, %syslogtag:R,ERE,0,BLANK:[0-9+--end:csv%, %syslogseverity:::csv%, %msg:::drop-last-lf,csv%\n" 
+$Template output,"%timereported:1:19:date-rfc3339,csv%, %hostname:::csv%, %programname:::csv%, %syslogtag:R,ERE,0,BLANK:[0-9+--end:csv%, %syslogseverity:::csv%, %msg:::drop-last-lf,csv%\n" 
 *.* :omstdout:;output
diff --git a/tests/testsuites/random.conf b/tests/testsuites/random.conf
new file mode 100644
index 0000000..a7079df
--- /dev/null
+++ b/tests/testsuites/random.conf
@@ -0,0 +1,13 @@
+# we write to /dev/null, as we have no chance to verify the output
+# in any case. What we really check is that rsyslogd does not
+# segfault or otherwise abort.
+# rgerhards, 2010-04-01
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%rawmsg%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+*.* /dev/null
diff --git a/tests/testsuites/reallife.parse3 b/tests/testsuites/reallife.parse3
index 465635b..dad3f56 100644
--- a/tests/testsuites/reallife.parse3
+++ b/tests/testsuites/reallife.parse3
@@ -9,7 +9,7 @@
 # documentation by IANA.
 # rgerhards, 2009-10-19
 <175>Oct 16 2009 23:47:31 hostname tag This is a message
-"2009-10-16T23:47:31+01:00", "hostname", "tag", "", "7", " This is a message"
+"2009-10-16T23:47:31", "hostname", "tag", "", "7", " This is a message"
 #
 <175>Oct 16 2009 23:47:31 hostname tag[1234] This is a message
-"2009-10-16T23:47:31+01:00", "hostname", "tag", "1234", "7", " This is a message"
+"2009-10-16T23:47:31", "hostname", "tag", "1234", "7", " This is a message"
diff --git a/tests/testsuites/samples.parse-nodate b/tests/testsuites/samples.parse-nodate
new file mode 100644
index 0000000..7f16181
--- /dev/null
+++ b/tests/testsuites/samples.parse-nodate
@@ -0,0 +1,6 @@
+<27>xapi: [error|xen3|15|Guest liveness monitor D:bca30ab3f1c1|master_connection] Connection to master died. I will continue to retry indefinitely (supressing future logging of this message)
+27,daemon,err,localhost,xapi,xapi:, [error|xen3|15|Guest liveness monitor D:bca30ab3f1c1|master_connection] Connection to master died. I will continue to retry indefinitely (supressing future logging of this message)
+# a message with just text (as permitted by rfc 3164)
+# it is questionable if the current sample result is really correct as of 3164!
+This is a message!
+13,user,notice,This,is,is, a message!
diff --git a/tests/testsuites/samples.parse_invld_regex b/tests/testsuites/samples.parse_invld_regex
index 9ac2c3a..0d0e4ce 100644
--- a/tests/testsuites/samples.parse_invld_regex
+++ b/tests/testsuites/samples.parse_invld_regex
@@ -13,4 +13,4 @@
 # configured format.
 # rgerhards, 2010-02-08
 <175>Feb 08 2008 23:47:31 hostname tag This is a message
-"2008-02-08T23:47:31+01:00", "hostname", "tag", **NO MATCH** **BAD REGULAR EXPRESSION**, "7", " This is a message"
+"2008-02-08T23:47:31", "hostname", "tag", **NO MATCH** **BAD REGULAR EXPRESSION**, "7", " This is a message"
diff --git a/tests/testsuites/samples.snare_ccoff_udp2 b/tests/testsuites/samples.snare_ccoff_udp2
index 7837b82..337cd97 100644
--- a/tests/testsuites/samples.snare_ccoff_udp2
+++ b/tests/testsuites/samples.snare_ccoff_udp2
@@ -9,14 +9,18 @@
 # to be adapted. We do NOT try to preserve misbehaviour on such seriously malformed
 # messages.
 #
+# this is a very simple test, though not snare-based
+test
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('', 1, 'test',5, '20100321185328', '20100321185328', 1, '')
+# and yet another one we have seen in practice
+UX=Abcd-efg-hij-klmno; XXXXX=1111111111, Z123=192.12.231.245:11111, S1234=123456789, XXXXXX=111111111
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' XXXXX=1111111111, Z123=192.12.231.245:11111, S1234=123456789, XXXXXX=111111111', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'UX=Abcd-efg-hij-klmno;')
 # Sample 1 - note the absence of PRI!
 windowsserver	MSWinEventLog	1	Security	1167	Fri Mar 19 15:33:30 2010	540	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306    	733\n
-insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 540 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306     733 ', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1167 Fri')
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 540 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306     733', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1167 Fri')
 # Sample 2
-# the samples below need to be disabled for the "workaround patch" for the message
-# parser to work. They need to be re-enabled once a final solution has been crafted
-#windowsserver	MSWinEventLog	1	Security	1166	Fri Mar 19 15:33:30 2010	576	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege  	732\n
-#insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 576 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege   732', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1166 Fri')
+windowsserver	MSWinEventLog	1	Security	1166	Fri Mar 19 15:33:30 2010	576	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege  	732\n
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 576 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege   732', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1166 Fri')
 # Sample 3
-#windowsserver	MSWinEventLog	1	Security	1165	Fri Mar 19 15:33:30 2010	538	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3    	731\n
-#insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 538 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3     731', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1165 Fri')
+windowsserver	MSWinEventLog	1	Security	1165	Fri Mar 19 15:33:30 2010	538	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3    	731\n
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 538 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3     731', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1165 Fri')
diff --git a/tools/omfile.c b/tools/omfile.c
index 0f47600..24de052 100644
--- a/tools/omfile.c
+++ b/tools/omfile.c
@@ -87,6 +87,7 @@ typedef struct s_dynaFileCacheEntry dynaFileCacheEntry;
 #define IOBUF_DFLT_SIZE 1024	/* default size for io buffers */
 #define FLUSH_INTRVL_DFLT 1 	/* default buffer flush interval (in seconds) */
 #define USE_ASYNCWRITER_DFLT 0 	/* default buffer use async writer */
+#define FLUSHONTX_DFLT 1 	/* default for flush on TX end */
 
 /* globals for default values */
 static int iDynaFileCacheSize = 10; /* max cache for dynamic files */
@@ -100,7 +101,7 @@ static uid_t	dirGID;		/* GID to be used for newly created directories */
 static int	bCreateDirs = 1;/* auto-create directories for dynaFiles: 0 - no, 1 - yes */
 static int	bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */
 static int	iZipLevel = 0;	/* zip compression mode (0..9 as usual) */
-static bool	bFlushOnTXEnd = 0;/* flush write buffers when transaction has ended? */
+static bool	bFlushOnTXEnd = FLUSHONTX_DFLT;/* flush write buffers when transaction has ended? */
 static int64	iIOBufSize = IOBUF_DFLT_SIZE;	/* size of an io buffer */
 static int	iFlushInterval = FLUSH_INTRVL_DFLT; 	/* how often flush the output buffer on inactivity? */
 static int	bUseAsyncWriter = USE_ASYNCWRITER_DFLT;	/* should we enable asynchronous writing? */
@@ -764,7 +765,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
 	bCreateDirs = 1;
 	bEnableSync = 0;
 	iZipLevel = 0;
-	bFlushOnTXEnd = 0;
+	bFlushOnTXEnd = FLUSHONTX_DFLT;
 	iIOBufSize = IOBUF_DFLT_SIZE;
 	iFlushInterval = FLUSH_INTRVL_DFLT;
 	bUseAsyncWriter = USE_ASYNCWRITER_DFLT;
diff --git a/tools/syslogd.c b/tools/syslogd.c
index 64b2356..a03dcf0 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -792,7 +792,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla
 				 * (I couldn't do any more smart things anyway...).
 				 * rgerhards, 2007-9-20
 				 */
-				DBGPRINTF("internal error: iMsg > max msg size in printchopped()\n");
+				DBGPRINTF("internal error: iMsg > max msg size in parseAndSubmitMessage()\n");
 			}
 			FINALIZE; /* in this case, we are done... nothing left we can do */
 		}