Imported Upstream version 7.3.10upstream/7.3.10

28 files changed, 711 insertions, 208 deletions

diff --git a/ChangeLog b/ChangeLog
index c64fe0f..84c5db7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,26 @@
 ---------------------------------------------------------------------------
+Version 7.3.10  [devel] 2013-04-10
+- added RainerScript re_extract() function
+- omrelp: added support for RainerScript-based configuration
+- omrelp: added ability to specify session timeout
+- templates now permit substring extraction relative to end-of-string
+- bugfix: failover/action suspend did not work correctly
+  This was experienced if the retry action took more than one second
+  to complete. For suspending, a cached timestamp was used, and if the
+  retry took longer, that timestamp was already in the past. As a 
+  result, the action never was kept in suspended state, and as such
+  no failover happened. The suspend functionalit now does no longer use
+  the cached timestamp (should not have any performance implication, as
+  action suspend occurs very infrequently).
+- bugfix: gnutls RFC5425 driver had some undersized buffers
+  Thanks to Tomas Heinrich for the patch.
+- bugfix: nested if/prifilt conditions did not work properly
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=415
+- bugfix: imuxsock aborted under some conditions
+  regression from ratelimiting enhancements
+- bugfix: build problems on Solaris
+  Thanks to Martin Carpenter for the patches.
+---------------------------------------------------------------------------
 Version 7.3.9  [devel] 2013-03-27
 - support for signing logs added
 - imudp: now supports user-selectable inputname
@@ -215,6 +237,18 @@ Version 7.2.7  [v7-stable] 2013-03-??
   The rsyslog debug log file is now continued to be written across the
   fork.
 - updated systemd files to match current systemd source
+- bugfix: failover/action suspend did not work correctly
+  This was experienced if the retry action took more than one second
+  to complete. For suspending, a cached timestamp was used, and if the
+  retry took longer, that timestamp was already in the past. As a 
+  result, the action never was kept in suspended state, and as such
+  no failover happened. The suspend functionalit now does no longer use
+  the cached timestamp (should not have any performance implication, as
+  action suspend occurs very infrequently).
+- bugfix: nested if/prifilt conditions did not work properly
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=415
+- bugfix: script == comparison did not work properly on JSON objects
+  [backport from 7.3 branch]
 - bugfix: imudp scheduling parameters did affect main thread, not imudp
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=409
 - bugfix: imuxsock rate-limiting could not be configured via legacy conf
diff --git a/action.c b/action.c
index 8928238..6b5a718 100644
--- a/action.c
+++ b/action.c
@@ -605,13 +605,17 @@ static void actionDisable(action_t *pThis)
  * CPU time. TODO: maybe a config option for that?
  * rgerhards, 2007-08-02
  */
-static inline void actionSuspend(action_t *pThis, time_t ttNow)
+static inline void actionSuspend(action_t *pThis)
 {
-	if(ttNow == NO_TIME_PROVIDED)
-		datetime.GetTime(&ttNow);
+	time_t ttNow;
+
+	/* note: we can NOT use a cached timestamp, as time may have evolved
+	 * since caching, and this would break logic (and it actually did so!)
+	 */
+	datetime.GetTime(&ttNow);
 	pThis->ttResumeRtry = ttNow + pThis->iResumeInterval * (pThis->iNbrResRtry / 10 + 1);
 	actionSetState(pThis, ACT_STATE_SUSP);
-	DBGPRINTF("earliest retry=%d\n", (int) pThis->ttResumeRtry);
+	DBGPRINTF("action suspended, earliest retry=%d\n", (int) pThis->ttResumeRtry);
 }
 
 
@@ -631,7 +635,7 @@ static inline void actionSuspend(action_t *pThis, time_t ttNow)
  * of its inability to recover. -- rgerhards, 2010-04-26.
  */
 static inline rsRetVal
-actionDoRetry(action_t *pThis, time_t ttNow, int *pbShutdownImmediate)
+actionDoRetry(action_t *pThis, int *pbShutdownImmediate)
 {
 	int iRetries;
 	int iSleepPeriod;
@@ -642,7 +646,9 @@ actionDoRetry(action_t *pThis, time_t ttNow, int *pbShutdownImmediate)
 
 	iRetries = 0;
 	while((*pbShutdownImmediate == 0) && pThis->eState == ACT_STATE_RTRY) {
+		DBGPRINTF("actionDoRetry: enter loop, iRetries=%d\n", iRetries);
 		iRet = pThis->pMod->tryResume(pThis->pModData);
+		DBGPRINTF("actionDoRetry: action->tryResume returned %d\n", iRet);
 		if((pThis->iResumeOKinRow > 9) && (pThis->iResumeOKinRow % 10 == 0)) {
 			bTreatOKasSusp = 1;
 			pThis->iResumeOKinRow = 0;
@@ -650,16 +656,18 @@ actionDoRetry(action_t *pThis, time_t ttNow, int *pbShutdownImmediate)
 			bTreatOKasSusp = 0;
 		}
 		if((iRet == RS_RET_OK) && (!bTreatOKasSusp)) {
+			DBGPRINTF("actionDoRetry: had success RDY again (iRet=%d)\n", iRet);
 			actionSetState(pThis, ACT_STATE_RDY);
 		} else if(iRet == RS_RET_SUSPENDED || bTreatOKasSusp) {
 			/* max retries reached? */
+			DBGPRINTF("actionDoRetry: check for max retries, iResumeRetryCount %d, iRetries %d\n",
+				  pThis->iResumeRetryCount, iRetries);
 			if((pThis->iResumeRetryCount != -1 && iRetries >= pThis->iResumeRetryCount)) {
-				actionSuspend(pThis, ttNow);
+				actionSuspend(pThis);
 			} else {
 				++pThis->iNbrResRtry;
 				++iRetries;
 				iSleepPeriod = pThis->iResumeInterval;
-				ttNow += iSleepPeriod; /* not truly exact, but sufficiently... */
 				srSleep(iSleepPeriod, 0);
 				if(*pbShutdownImmediate) {
 					ABORT_FINALIZE(RS_RET_FORCE_TERM);
@@ -706,7 +714,7 @@ static rsRetVal actionTryResume(action_t *pThis, int *pbShutdownImmediate)
 	if(pThis->eState == ACT_STATE_RTRY) {
 		if(ttNow == NO_TIME_PROVIDED) /* use cached result if we have it */
 			datetime.GetTime(&ttNow);
-		CHKiRet(actionDoRetry(pThis, ttNow, pbShutdownImmediate));
+		CHKiRet(actionDoRetry(pThis, pbShutdownImmediate));
 	}
 
 	if(Debug && (pThis->eState == ACT_STATE_RTRY ||pThis->eState == ACT_STATE_SUSP)) {
@@ -953,6 +961,8 @@ actionProcessMessage(action_t *pThis, msg_t *pMsg, void *actParams, int *pbShutd
 	ISOBJ_TYPE_assert(pMsg, msg);
 
 	CHKiRet(actionPrepare(pThis, pbShutdownImmediate));
+	if(pThis->pMod->mod.om.SetShutdownImmdtPtr != NULL)
+		pThis->pMod->mod.om.SetShutdownImmdtPtr(pThis->pModData, pbShutdownImmediate);
 	if(pThis->eState == ACT_STATE_ITX)
 		CHKiRet(actionCallDoAction(pThis, pMsg, actParams));
 
@@ -1111,6 +1121,7 @@ submitBatch(action_t *pAction, batch_t *pBatch, int nElem)
 
 	assert(pBatch != NULL);
 
+	DBGPRINTF("submitBatch: enter, nElem %d\n", nElem);
 	wasDoneTo = pBatch->iDoneUpTo;
 	bDone = 0;
 	do {
@@ -1132,7 +1143,8 @@ submitBatch(action_t *pAction, batch_t *pBatch, int nElem)
 		   || localRet == RS_RET_DEFER_COMMIT) {
 			bDone = 1;
 		} else if(localRet == RS_RET_SUSPENDED) {
-			; /* do nothing, this will retry the full batch */
+			DBGPRINTF("action ret RS_RET_SUSPENDED - retry full batch\n");
+			/* do nothing, this will retry the full batch */
 		} else if(localRet == RS_RET_ACTION_FAILED) {
 			/* in this case, everything not yet committed is BAD */
 			for(i = pBatch->iDoneUpTo ; i < wasDoneTo + nElem ; ++i) {
@@ -1262,8 +1274,11 @@ processBatchMain(action_t *pAction, batch_t *pBatch, int *pbShutdownImmediate)
 
 	assert(pBatch != NULL);
 
-	pbShutdownImmdtSave = pBatch->pbShutdownImmediate;
-	pBatch->pbShutdownImmediate = pbShutdownImmediate;
+	if(pbShutdownImmediate != NULL) {
+		pbShutdownImmdtSave = pBatch->pbShutdownImmediate;
+		pBatch->pbShutdownImmediate = pbShutdownImmediate;
+dbgprintf("DDDD: processBatchMain ShutdownImmediate is %p, was %p\n", pBatch->pbShutdownImmediate, pbShutdownImmdtSave);
+	}
 	CHKiRet(prepareBatch(pAction, pBatch, &activeSave, &bMustRestoreActivePtr));
 
 	/* We now must guard the output module against execution by multiple threads. The
@@ -1294,7 +1309,8 @@ processBatchMain(action_t *pAction, batch_t *pBatch, int *pbShutdownImmediate)
 	}
 
 finalize_it:
-	pBatch->pbShutdownImmediate = pbShutdownImmdtSave;
+	if(pbShutdownImmediate != NULL)
+		pBatch->pbShutdownImmediate = pbShutdownImmdtSave;
 	RETiRet;
 }
 #pragma GCC diagnostic warning "-Wempty-body"
@@ -1864,7 +1880,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData,
 	pAction->eState = ACT_STATE_RDY; /* action is enabled */
 
 	if(bSuspended)
-		actionSuspend(pAction, datetime.GetTime(NULL)); /* "good" time call, only during init and unavoidable */
+		actionSuspend(pAction);
 
 	CHKiRet(actionConstructFinalize(pAction, queueParams));
 	
diff --git a/configure b/configure
index 89bb258..e990e34 100755
--- a/configure
+++ b/configure
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for rsyslog 7.3.9.
+# Generated by GNU Autoconf 2.69 for rsyslog 7.3.10.
 #
 # Report bugs to <rsyslog@lists.adiscon.com>.
 #
@@ -590,8 +590,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='rsyslog'
 PACKAGE_TARNAME='rsyslog'
-PACKAGE_VERSION='7.3.9'
-PACKAGE_STRING='rsyslog 7.3.9'
+PACKAGE_VERSION='7.3.10'
+PACKAGE_STRING='rsyslog 7.3.10'
 PACKAGE_BUGREPORT='rsyslog@lists.adiscon.com'
 PACKAGE_URL=''
 
@@ -1595,7 +1595,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures rsyslog 7.3.9 to adapt to many kinds of systems.
+\`configure' configures rsyslog 7.3.10 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1665,7 +1665,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of rsyslog 7.3.9:";;
+     short | recursive ) echo "Configuration of rsyslog 7.3.10:";;
    esac
   cat <<\_ACEOF
 
@@ -1900,7 +1900,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-rsyslog configure 7.3.9
+rsyslog configure 7.3.10
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2480,7 +2480,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by rsyslog $as_me 7.3.9, which was
+It was created by rsyslog $as_me 7.3.10, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -3304,7 +3304,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='rsyslog'
- VERSION='7.3.9'
+ VERSION='7.3.10'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -13537,6 +13537,10 @@ $as_echo "#define OS_SOLARIS 1" >>confdefs.h
 $as_echo "#define _POSIX_PTHREAD_SEMANTICS 1" >>confdefs.h
 
     SOL_LIBS="-lsocket -lnsl"
+    # Solaris libuuid does not ship with a pkgconfig file so override the appropriate
+    # variables (but only if they have not been set by the user).
+    LIBUUID_CFLAGS=${LIBUUID_CFLAGS:= }
+    LIBUUID_LIBS=${LIBUUID_LIBS:=-luuid}
 
   ;;
 esac
@@ -18155,12 +18159,12 @@ if test -n "$RELP_CFLAGS"; then
     pkg_cv_RELP_CFLAGS="$RELP_CFLAGS"
  elif test -n "$PKG_CONFIG"; then
     if test -n "$PKG_CONFIG" && \
-    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"relp >= 1.0.2\""; } >&5
-  ($PKG_CONFIG --exists --print-errors "relp >= 1.0.2") 2>&5
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"relp >= 1.0.3\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "relp >= 1.0.3") 2>&5
   ac_status=$?
   $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
   test $ac_status = 0; }; then
-  pkg_cv_RELP_CFLAGS=`$PKG_CONFIG --cflags "relp >= 1.0.2" 2>/dev/null`
+  pkg_cv_RELP_CFLAGS=`$PKG_CONFIG --cflags "relp >= 1.0.3" 2>/dev/null`
 		      test "x$?" != "x0" && pkg_failed=yes
 else
   pkg_failed=yes
@@ -18172,12 +18176,12 @@ if test -n "$RELP_LIBS"; then
     pkg_cv_RELP_LIBS="$RELP_LIBS"
  elif test -n "$PKG_CONFIG"; then
     if test -n "$PKG_CONFIG" && \
-    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"relp >= 1.0.2\""; } >&5
-  ($PKG_CONFIG --exists --print-errors "relp >= 1.0.2") 2>&5
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"relp >= 1.0.3\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "relp >= 1.0.3") 2>&5
   ac_status=$?
   $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
   test $ac_status = 0; }; then
-  pkg_cv_RELP_LIBS=`$PKG_CONFIG --libs "relp >= 1.0.2" 2>/dev/null`
+  pkg_cv_RELP_LIBS=`$PKG_CONFIG --libs "relp >= 1.0.3" 2>/dev/null`
 		      test "x$?" != "x0" && pkg_failed=yes
 else
   pkg_failed=yes
@@ -18198,14 +18202,14 @@ else
         _pkg_short_errors_supported=no
 fi
         if test $_pkg_short_errors_supported = yes; then
-	        RELP_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "relp >= 1.0.2" 2>&1`
+	        RELP_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "relp >= 1.0.3" 2>&1`
         else
-	        RELP_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "relp >= 1.0.2" 2>&1`
+	        RELP_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "relp >= 1.0.3" 2>&1`
         fi
 	# Put the nasty error message in config.log where it belongs
 	echo "$RELP_PKG_ERRORS" >&5
 
-	as_fn_error $? "Package requirements (relp >= 1.0.2) were not met:
+	as_fn_error $? "Package requirements (relp >= 1.0.3) were not met:
 
 $RELP_PKG_ERRORS
 
@@ -20381,7 +20385,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by rsyslog $as_me 7.3.9, which was
+This file was extended by rsyslog $as_me 7.3.10, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -20447,7 +20451,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-rsyslog config.status 7.3.9
+rsyslog config.status 7.3.10
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 
diff --git a/configure.ac b/configure.ac
index 0c877ee..c0712a5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.61)
-AC_INIT([rsyslog],[7.3.9],[rsyslog@lists.adiscon.com])
+AC_INIT([rsyslog],[7.3.10],[rsyslog@lists.adiscon.com])
 AM_INIT_AUTOMAKE
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
@@ -56,6 +56,10 @@ case "${host}" in
     AC_DEFINE([OS_SOLARIS], [1], [Indicator for a Solaris OS])
     AC_DEFINE([_POSIX_PTHREAD_SEMANTICS], [1], [Use POSIX pthread semantics])
     SOL_LIBS="-lsocket -lnsl"
+    # Solaris libuuid does not ship with a pkgconfig file so override the appropriate
+    # variables (but only if they have not been set by the user).
+    LIBUUID_CFLAGS=${LIBUUID_CFLAGS:= }
+    LIBUUID_LIBS=${LIBUUID_LIBS:=-luuid}
     AC_SUBST(SOL_LIBS)
   ;;
 esac
@@ -928,7 +932,7 @@ AC_ARG_ENABLE(relp,
         [enable_relp=no]
 )
 if test "x$enable_relp" = "xyes"; then
-	PKG_CHECK_MODULES(RELP, relp >= 1.0.2)
+	PKG_CHECK_MODULES(RELP, relp >= 1.0.3)
 fi
 AM_CONDITIONAL(ENABLE_RELP, test x$enable_relp = xyes)
 
diff --git a/doc/manual.html b/doc/manual.html
index 6fba9a0..a1e39c9 100644
--- a/doc/manual.html
+++ b/doc/manual.html
@@ -19,7 +19,7 @@ professional services</a> available directly from the source!</p>
 <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a>
 to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the
 project goals.</p>
-<p><b>This documentation is for version 7.3.9 (devel branch) of rsyslog.</b>
+<p><b>This documentation is for version 7.3.10 (devel branch) of rsyslog.</b>
 Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b>
 to obtain current version information and project status.
 </p><p><b>If you like rsyslog, you might
diff --git a/doc/omrelp.html b/doc/omrelp.html
index 22e6845..8858f88 100644
--- a/doc/omrelp.html
+++ b/doc/omrelp.html
@@ -16,10 +16,34 @@ RELP protocol. For RELP's advantages over plain tcp syslog, please see
 the documentation for <a href="imrelp.html">imrelp</a>
 (the server counterpart).&nbsp;</p>
 <span style="font-weight: bold;">Setup</span>
-<p>Please note the <a href="http://www.librelp.com">librelp</a>
+<p>Please note that <a href="http://www.librelp.com">librelp</a>
 is required for imrelp (it provides the core relp protocol
 implementation).</p>
-<p><b>Configuration Directives</b>:</p>
+<p><b>Action Configuration Parameters</b>:</p>
+<p>This module supports RainerScript configuration starting with
+rsyslog 7.3.10. For older versions, legacy configuration directives
+must be used.
+<ul>
+	<li><b>target </b>(mandatory)<br>
+	The target server to connect to.
+	</li>
+	<li><b>template </b>(not mandatory, default "RSYSLOG_ForwardFormat")<br>
+	Defines the template to be used for the output.
+	</li>
+	<li><b>timeout </b>(not mandatory, default 90)<br>
+	Timeout for relp sessions. If set too low, valid sessions
+	may be considered dead and tried to recover.
+	</li>
+</ul>
+<p><b>Sample:</b></p>
+<p>The following sample sends all messages to the central server
+"centralserv" at port 2514 (note that that server must run imrelp on
+port 2514).
+</p>
+<textarea rows="3" cols="60">module(load="omrelp")
+action(type="omrelp" target="centralserv" port="2514")
+</textarea>
+<p><b>Legacy Configuration Directives</b>:</p>
 <p>This module uses old-style action configuration to keep
 consistent with the forwarding rule. So far, no additional
 configuration directives can be specified. To send a message via RELP,
@@ -33,18 +57,15 @@ use</p>
 <b>Caveats/Known Bugs:</b>
 <p>See <a href="imrelp.html">imrelp</a>,
 which documents them.&nbsp;</p>
-<p><b>Sample:</b></p>
+<p><b>Legacy Sample:</b></p>
 <p>The following sample sends all messages to the central server
 "centralserv" at port 2514 (note that that server must run imrelp on
-port 2514). Rsyslog's high-precision timestamp format is used, thus the
-special "RSYSLOG_ForwardFormat" (case sensitive!) template is used.<br>
+port 2514).
 </p>
-<textarea rows="15" cols="60">$ModLoad omrelp
-# forward messages to the remote server "myserv" on
-# port 2514
-*.* :omrelp:centralserv:2514;RSYSLOG_ForwardFormat
+<textarea rows="3" cols="60">$ModLoad omrelp
+*.* :omrelp:centralserv:2514
 </textarea>
-Note: to use IPv6 addresses, encode them in [::1] format.
+<p>Note: to use IPv6 addresses, encode them in [::1] format.
 <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>]
 [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
 <p><font size="2">This documentation is part of the
diff --git a/doc/property_replacer.html b/doc/property_replacer.html
index c7624b2..13ff41c 100644
--- a/doc/property_replacer.html
+++ b/doc/property_replacer.html
@@ -413,6 +413,12 @@ option when forwarding to remote hosts - they may treat the date as invalid
 <td>just the subseconds of a timestamp (always 0 for a low precision timestamp)</td>
 </tr>
 <tr>
+<td>pos-end-relative</td>
+  <td>the from and to position is relative to the end of the string
+      instead of the usual start of string. (available since rsyslog v7.3.10)
+  </td>
+</tr>
+<tr>
 <td><b>ControlCharacters</b></td>
 <td>Option values for how to process control characters</td>
 </tr>
diff --git a/doc/rainerscript.html b/doc/rainerscript.html
index d76316e..7cbbfa9 100644
--- a/doc/rainerscript.html
+++ b/doc/rainerscript.html
@@ -66,6 +66,15 @@ variable, if it exists. Returns an empty string if it does not exist.
 <li>cstr(expr) - converts expr to a string value
 <li>cnum(expr) - converts expr to a number (integer)
 <li>re_match(expr, re) - returns 1, if expr matches re, 0 otherwise
+<li>re_extract(expr, re, match, submatch, no-found) - extracts
+data from a string (property) via a regular expression match.
+POSIX ERE regular expressions are used. The variable "match" contains
+the number of the match to use. This permits to pick up more than the
+first expression match. Submatch is the submatch to match (max 50 supported).
+The "no-found" parameter specifies which string is to be returned in case when
+the regular expression is not found. Note that match and submatch start with
+zero. It currently is not possible to extract more than one submatch with
+a single call.
 <li>field(str, delim, matchnbr) - returns a field-based substring. str is the string
 to search, delim is the delimiter and matchnbr is the match to search
 for (the first match starts at 1). This works similar as the field based
diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html
index 0c18910..9a6e161 100644
--- a/doc/rsyslog_conf_templates.html
+++ b/doc/rsyslog_conf_templates.html
@@ -134,6 +134,8 @@ csv-data is generated, "json", which formats proper json content (but without a
 header) and "jsonf", which formats as a complete json field.
 <li>position.from - obtain substring starting from this position (1 is the first position)
 <li>position.to - obtain substring up to this position
+<li>position.relativeToEnd - the from and to position is relative to the end of the string
+   instead of the usual start of string. (available since rsyslog v7.3.10)
 <li>field.number - obtain this field match
 <li>field.delimiter - decimal value of delimiter character for field extraction
 <li>regex.expression - expression to use
diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c
index 8ef7429..89cf946 100644
--- a/grammar/rainerscript.c
+++ b/grammar/rainerscript.c
@@ -2,7 +2,7 @@
  *
  * Module begun 2011-07-01 by Rainer Gerhards
  *
- * Copyright 2011-2012 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2011-2013 Rainer Gerhards and Adiscon GmbH.
  *
  * This file is part of the rsyslog runtime library.
  *
@@ -1228,6 +1228,96 @@ finalize_it:
 	RETiRet;
 }
 
+static inline void
+doFunc_re_extract(struct cnffunc *func, struct var *ret, void* usrptr)
+{
+	size_t submatchnbr;
+	short matchnbr;
+	regmatch_t pmatch[50];
+	int bMustFree;
+	es_str_t *estr;
+	char *str;
+	struct var r[CNFFUNC_MAX_ARGS];
+	int iLenBuf;
+	unsigned iOffs;
+	short iTry = 0;
+	uchar bFound = 0;
+	iOffs = 0;
+	sbool bHadNoMatch = 0;
+
+	cnfexprEval(func->expr[0], &r[0], usrptr);
+	/* search string is already part of the compiled regex, so we don't
+	 * need it here!
+	 */
+	cnfexprEval(func->expr[2], &r[2], usrptr);
+	cnfexprEval(func->expr[3], &r[3], usrptr);
+	str = (char*) var2CString(&r[0], &bMustFree);
+	matchnbr = (short) var2Number(&r[2], NULL);
+	submatchnbr = (size_t) var2Number(&r[3], NULL);
+	if(submatchnbr > sizeof(pmatch)/sizeof(regmatch_t)) {
+		DBGPRINTF("re_extract() submatch %d is too large\n", submatchnbr);
+		bHadNoMatch = 1;
+		goto finalize_it;
+	}
+
+	/* first see if we find a match, iterating through the series of
+	 * potential matches over the string.
+	 */
+	while(!bFound) {
+		int iREstat;
+		iREstat = regexp.regexec(func->funcdata, (char*)(str + iOffs),
+					 submatchnbr+1, pmatch, 0);
+		dbgprintf("re_extract: regexec return is %d\n", iREstat);
+		if(iREstat == 0) {
+			if(pmatch[0].rm_so == -1) {
+				dbgprintf("oops ... start offset of successful regexec is -1\n");
+				break;
+			}
+			if(iTry == matchnbr) {
+				bFound = 1;
+			} else {
+				dbgprintf("re_extract: regex found at offset %d, new offset %d, tries %d\n",
+					  iOffs, (int) (iOffs + pmatch[0].rm_eo), iTry);
+				iOffs += pmatch[0].rm_eo;
+				++iTry;
+			}
+		} else {
+			break;
+		}
+	}
+	dbgprintf("re_extract: regex: end search, found %d\n", bFound);
+	if(!bFound) {
+		bHadNoMatch = 1;
+		goto finalize_it;
+	} else {
+		/* Match- but did it match the one we wanted? */
+		/* we got no match! */
+		if(pmatch[submatchnbr].rm_so == -1) {
+			bHadNoMatch = 1;
+			goto finalize_it;
+		}
+		/* OK, we have a usable match - we now need to malloc pB */
+		iLenBuf = pmatch[submatchnbr].rm_eo - pmatch[submatchnbr].rm_so;
+		estr = es_newStrFromBuf(str + iOffs + pmatch[submatchnbr].rm_so,
+					iLenBuf);
+	}
+
+	if(bMustFree) free(str);
+	if(r[0].datatype == 'S') es_deleteStr(r[0].d.estr);
+	if(r[2].datatype == 'S') es_deleteStr(r[2].d.estr);
+	if(r[3].datatype == 'S') es_deleteStr(r[3].d.estr);
+finalize_it:
+	if(bHadNoMatch) {
+		cnfexprEval(func->expr[4], &r[4], usrptr);
+		estr = var2String(&r[4], &bMustFree);
+		if(r[4].datatype == 'S') es_deleteStr(r[4].d.estr);
+	}
+	ret->datatype = 'S';
+	ret->d.estr = estr;
+	return;
+}
+
+
 /* Perform a function call. This has been moved out of cnfExprEval in order
  * to keep the code small and easier to maintain.
  */
@@ -1331,6 +1421,9 @@ doFuncCall(struct cnffunc *func, struct var *ret, void* usrptr)
 		if(bMustFree) free(str);
 		if(r[0].datatype == 'S') es_deleteStr(r[0].d.estr);
 		break;
+	case CNFFUNC_RE_EXTRACT:
+		doFunc_re_extract(func, ret, usrptr);
+		break;
 	case CNFFUNC_FIELD:
 		cnfexprEval(func->expr[0], &r[0], usrptr);
 		cnfexprEval(func->expr[1], &r[1], usrptr);
@@ -1908,6 +2001,7 @@ cnffuncDestruct(struct cnffunc *func)
 	/* some functions require special destruction */
 	switch(func->fID) {
 		case CNFFUNC_RE_MATCH:
+		case CNFFUNC_RE_EXTRACT:
 			if(func->funcdata != NULL)
 				regexp.regfree(func->funcdata);
 			break;
@@ -2158,31 +2252,33 @@ cnfexprPrint(struct cnfexpr *expr, int indent)
 		break;
 	}
 }
+/* print only the given stmt 
+ * if "subtree" equals 1, the full statement subtree is printed, else
+ * really only the statement.
+ */
 void
-cnfstmtPrint(struct cnfstmt *root, int indent)
+cnfstmtPrintOnly(struct cnfstmt *stmt, int indent, sbool subtree)
 {
-	struct cnfstmt *stmt;
 	char *cstr;
-	//dbgprintf("stmt %p, indent %d, type '%c'\n", expr, indent, expr->nodetype);
-	for(stmt = root ; stmt != NULL ; stmt = stmt->next) {
-		switch(stmt->nodetype) {
-		case S_NOP:
-			doIndent(indent); dbgprintf("NOP\n");
-			break;
-		case S_STOP:
-			doIndent(indent); dbgprintf("STOP\n");
-			break;
-		case S_CALL:
-			cstr = es_str2cstr(stmt->d.s_call.name, NULL);
-			doIndent(indent); dbgprintf("CALL [%s]\n", cstr);
-			free(cstr);
-			break;
-		case S_ACT:
-			doIndent(indent); dbgprintf("ACTION %p [%s]\n", stmt->d.act, stmt->printable);
-			break;
-		case S_IF:
-			doIndent(indent); dbgprintf("IF\n");
-			cnfexprPrint(stmt->d.s_if.expr, indent+1);
+	switch(stmt->nodetype) {
+	case S_NOP:
+		doIndent(indent); dbgprintf("NOP\n");
+		break;
+	case S_STOP:
+		doIndent(indent); dbgprintf("STOP\n");
+		break;
+	case S_CALL:
+		cstr = es_str2cstr(stmt->d.s_call.name, NULL);
+		doIndent(indent); dbgprintf("CALL [%s]\n", cstr);
+		free(cstr);
+		break;
+	case S_ACT:
+		doIndent(indent); dbgprintf("ACTION %p [%s]\n", stmt->d.act, stmt->printable);
+		break;
+	case S_IF:
+		doIndent(indent); dbgprintf("IF\n");
+		cnfexprPrint(stmt->d.s_if.expr, indent+1);
+		if(subtree) {
 			doIndent(indent); dbgprintf("THEN\n");
 			cnfstmtPrint(stmt->d.s_if.t_then, indent+1);
 			if(stmt->d.s_if.t_else != NULL) {
@@ -2190,54 +2286,67 @@ cnfstmtPrint(struct cnfstmt *root, int indent)
 				cnfstmtPrint(stmt->d.s_if.t_else, indent+1);
 			}
 			doIndent(indent); dbgprintf("END IF\n");
-			break;
-		case S_SET:
-			doIndent(indent); dbgprintf("SET %s =\n",
-				          stmt->d.s_set.varname);
-			cnfexprPrint(stmt->d.s_set.expr, indent+1);
-			doIndent(indent); dbgprintf("END SET\n");
-			break;
-		case S_UNSET:
-			doIndent(indent); dbgprintf("UNSET %s\n",
-				          stmt->d.s_unset.varname);
-			break;
-		case S_PRIFILT:
-			doIndent(indent); dbgprintf("PRIFILT '%s'\n", stmt->printable);
-			pmaskPrint(stmt->d.s_prifilt.pmask, indent);
+		}
+		break;
+	case S_SET:
+		doIndent(indent); dbgprintf("SET %s =\n",
+				  stmt->d.s_set.varname);
+		cnfexprPrint(stmt->d.s_set.expr, indent+1);
+		doIndent(indent); dbgprintf("END SET\n");
+		break;
+	case S_UNSET:
+		doIndent(indent); dbgprintf("UNSET %s\n",
+				  stmt->d.s_unset.varname);
+		break;
+	case S_PRIFILT:
+		doIndent(indent); dbgprintf("PRIFILT '%s'\n", stmt->printable);
+		pmaskPrint(stmt->d.s_prifilt.pmask, indent);
+		if(subtree) {
 			cnfstmtPrint(stmt->d.s_prifilt.t_then, indent+1);
 			if(stmt->d.s_prifilt.t_else != NULL) {
 				doIndent(indent); dbgprintf("ELSE\n");
 				cnfstmtPrint(stmt->d.s_prifilt.t_else, indent+1);
 			}
 			doIndent(indent); dbgprintf("END PRIFILT\n");
-			break;
-		case S_PROPFILT:
-			doIndent(indent); dbgprintf("PROPFILT\n");
-			doIndent(indent); dbgprintf("\tProperty.: '%s'\n",
-				propIDToName(stmt->d.s_propfilt.propID));
-			if(stmt->d.s_propfilt.propName != NULL) {
-				cstr = es_str2cstr(stmt->d.s_propfilt.propName, NULL);
-				doIndent(indent);
-				dbgprintf("\tCEE-Prop.: '%s'\n", cstr);
-				free(cstr);
-			}
-			doIndent(indent); dbgprintf("\tOperation: ");
-			if(stmt->d.s_propfilt.isNegated)
-				dbgprintf("NOT ");
-			dbgprintf("'%s'\n", getFIOPName(stmt->d.s_propfilt.operation));
-			if(stmt->d.s_propfilt.pCSCompValue != NULL) {
-				doIndent(indent); dbgprintf("\tValue....: '%s'\n",
-				       rsCStrGetSzStrNoNULL(stmt->d.s_propfilt.pCSCompValue));
-			}
+		}
+		break;
+	case S_PROPFILT:
+		doIndent(indent); dbgprintf("PROPFILT\n");
+		doIndent(indent); dbgprintf("\tProperty.: '%s'\n",
+			propIDToName(stmt->d.s_propfilt.propID));
+		if(stmt->d.s_propfilt.propName != NULL) {
+			cstr = es_str2cstr(stmt->d.s_propfilt.propName, NULL);
+			doIndent(indent);
+			dbgprintf("\tCEE-Prop.: '%s'\n", cstr);
+			free(cstr);
+		}
+		doIndent(indent); dbgprintf("\tOperation: ");
+		if(stmt->d.s_propfilt.isNegated)
+			dbgprintf("NOT ");
+		dbgprintf("'%s'\n", getFIOPName(stmt->d.s_propfilt.operation));
+		if(stmt->d.s_propfilt.pCSCompValue != NULL) {
+			doIndent(indent); dbgprintf("\tValue....: '%s'\n",
+			       rsCStrGetSzStrNoNULL(stmt->d.s_propfilt.pCSCompValue));
+		}
+		if(subtree) {
 			doIndent(indent); dbgprintf("THEN\n");
 			cnfstmtPrint(stmt->d.s_propfilt.t_then, indent+1);
 			doIndent(indent); dbgprintf("END PROPFILT\n");
-			break;
-		default:
-			dbgprintf("error: unknown stmt type %u\n",
-				(unsigned) stmt->nodetype);
-			break;
 		}
+		break;
+	default:
+		dbgprintf("error: unknown stmt type %u\n",
+			(unsigned) stmt->nodetype);
+		break;
+	}
+}
+void
+cnfstmtPrint(struct cnfstmt *root, int indent)
+{
+	struct cnfstmt *stmt;
+	//dbgprintf("stmt %p, indent %d, type '%c'\n", expr, indent, expr->nodetype);
+	for(stmt = root ; stmt != NULL ; stmt = stmt->next) {
+		cnfstmtPrintOnly(stmt, indent, 1);
 	}
 }
 
@@ -2988,7 +3097,6 @@ cnfstmtOptimize(struct cnfstmt *root)
 	struct cnfstmt *stmt;
 	if(root == NULL) goto done;
 	for(stmt = root ; stmt != NULL ; stmt = stmt->next) {
-dbgprintf("RRRR: stmtOptimize: stmt %p, nodetype %u\n", stmt, stmt->nodetype);
 		switch(stmt->nodetype) {
 		case S_IF:
 			cnfstmtOptimizeIf(stmt);
@@ -3088,6 +3196,13 @@ funcName2ID(es_str_t *fname, unsigned short nParams)
 			return CNFFUNC_INVALID;
 		}
 		return CNFFUNC_RE_MATCH;
+	} else if(!es_strbufcmp(fname, (unsigned char*)"re_extract", sizeof("re_extract") - 1)) {
+		if(nParams != 5) {
+			parser_errmsg("number of parameters for re_extract() must be five "
+				      "but is %d.", nParams);
+			return CNFFUNC_INVALID;
+		}
+		return CNFFUNC_RE_EXTRACT;
 	} else if(!es_strbufcmp(fname, (unsigned char*)"field", sizeof("field") - 1)) {
 		if(nParams != 3) {
 			parser_errmsg("number of parameters for field() must be three "
@@ -3118,7 +3233,7 @@ initFunc_re_match(struct cnffunc *func)
 
 	func->funcdata = NULL;
 	if(func->expr[1]->nodetype != 'S') {
-		parser_errmsg("param 2 of re_match() must be a constant string");
+		parser_errmsg("param 2 of re_match/extract() must be a constant string");
 		FINALIZE;
 	}
 
@@ -3196,6 +3311,7 @@ cnffuncNew(es_str_t *fname, struct cnffparamlst* paramlst)
 		/* some functions require special initialization */
 		switch(func->fID) {
 			case CNFFUNC_RE_MATCH:
+			case CNFFUNC_RE_EXTRACT:
 				/* need to compile the regexp in param 2, so this MUST be a constant */
 				initFunc_re_match(func);
 				break;
@@ -3520,3 +3636,52 @@ unescapeStr(uchar *s, int len)
 		s[iDst] = '\0';
 	}
 }
+
+char *
+tokenval2str(int tok)
+{
+	if(tok < 256) return "";
+	switch(tok) {
+	case NAME: return "NAME";
+	case FUNC: return "FUNC";
+	case BEGINOBJ: return "BEGINOBJ";
+	case ENDOBJ: return "ENDOBJ";
+	case BEGIN_ACTION: return "BEGIN_ACTION";
+	case BEGIN_PROPERTY: return "BEGIN_PROPERTY";
+	case BEGIN_CONSTANT: return "BEGIN_CONSTANT";
+	case BEGIN_TPL: return "BEGIN_TPL";
+	case BEGIN_RULESET: return "BEGIN_RULESET";
+	case STOP: return "STOP";
+	case SET: return "SET";
+	case UNSET: return "UNSET";
+	case CONTINUE: return "CONTINUE";
+	case CALL: return "CALL";
+	case LEGACY_ACTION: return "LEGACY_ACTION";
+	case LEGACY_RULESET: return "LEGACY_RULESET";
+	case PRIFILT: return "PRIFILT";
+	case PROPFILT: return "PROPFILT";
+	case BSD_TAG_SELECTOR: return "BSD_TAG_SELECTOR";
+	case BSD_HOST_SELECTOR: return "BSD_HOST_SELECTOR";
+	case IF: return "IF";
+	case THEN: return "THEN";
+	case ELSE: return "ELSE";
+	case OR: return "OR";
+	case AND: return "AND";
+	case NOT: return "NOT";
+	case VAR: return "VAR";
+	case STRING: return "STRING";
+	case NUMBER: return "NUMBER";
+	case CMP_EQ: return "CMP_EQ";
+	case CMP_NE: return "CMP_NE";
+	case CMP_LE: return "CMP_LE";
+	case CMP_GE: return "CMP_GE";
+	case CMP_LT: return "CMP_LT";
+	case CMP_GT: return "CMP_GT";
+	case CMP_CONTAINS: return "CMP_CONTAINS";
+	case CMP_CONTAINSI: return "CMP_CONTAINSI";
+	case CMP_STARTSWITH: return "CMP_STARTSWITH";
+	case CMP_STARTSWITHI: return "CMP_STARTSWITHI";
+	case UMINUS: return "UMINUS";
+	default: return "UNKNOWN TOKEN";
+	}
+}
diff --git a/grammar/rainerscript.h b/grammar/rainerscript.h
index 59ce53f..31b2eb9 100644
--- a/grammar/rainerscript.h
+++ b/grammar/rainerscript.h
@@ -226,6 +226,7 @@ enum cnffuncid {
 	CNFFUNC_CSTR,
 	CNFFUNC_CNUM,
 	CNFFUNC_RE_MATCH,
+	CNFFUNC_RE_EXTRACT,
 	CNFFUNC_FIELD,
 	CNFFUNC_PRIFILT
 };
@@ -317,6 +318,7 @@ int cnfparamvalsIsSet(struct cnfparamblk *params, struct cnfparamvals *vals);
 void varDelete(struct var *v);
 void cnfparamvalsDestruct(struct cnfparamvals *paramvals, struct cnfparamblk *blk);
 struct cnfstmt * cnfstmtNew(unsigned s_type);
+void cnfstmtPrintOnly(struct cnfstmt *stmt, int indent, sbool subtree);
 void cnfstmtPrint(struct cnfstmt *stmt, int indent);
 struct cnfstmt* scriptAddStmt(struct cnfstmt *root, struct cnfstmt *s);
 struct objlst* objlstAdd(struct objlst *root, struct cnfobj *o);
@@ -338,6 +340,7 @@ void cnfarrayContentDestruct(struct cnfarray *ar);
 char* getFIOPName(unsigned iFIOP);
 rsRetVal initRainerscript(void);
 void unescapeStr(uchar *s, int len);
+char * tokenval2str(int tok);
 
 /* debug helper */
 void cstrPrint(char *text, es_str_t *estr);
diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c
index f15773f..0f4ded1 100644
--- a/plugins/imuxsock/imuxsock.c
+++ b/plugins/imuxsock/imuxsock.c
@@ -570,6 +570,10 @@ findRatelimiter(lstn_t *pLstn, struct ucred *cred, ratelimit_t **prl)
 		FINALIZE;
 	}
 #endif
+	if(pLstn->ht == NULL) {
+		*prl = NULL;
+		FINALIZE;
+	}
 
 	rl = hashtable_search(pLstn->ht, &cred->pid);
 	if(rl == NULL) {
diff --git a/plugins/omrelp/omrelp.c b/plugins/omrelp/omrelp.c
index e55836c..c9e3244 100644
--- a/plugins/omrelp/omrelp.c
+++ b/plugins/omrelp/omrelp.c
@@ -7,7 +7,7 @@
  *
  * File begun on 2008-03-13 by RGerhards
  *
- * Copyright 2008-2012 Adiscon GmbH.
+ * Copyright 2008-2013 Adiscon GmbH.
  *
  * This file is part of rsyslog.
  *
@@ -43,6 +43,7 @@
 #include "glbl.h"
 #include "errmsg.h"
 #include "debug.h"
+#include "unicode-helper.h"
 
 MODULE_TYPE_OUTPUT
 MODULE_TYPE_NOKEEP
@@ -57,12 +58,14 @@ DEFobjCurrIf(glbl)
 static relpEngine_t *pRelpEngine;	/* our relp engine */
 
 typedef struct _instanceData {
-	char *f_hname;
+	uchar *target;
 	int compressionLevel; /* 0 - no compression, else level for zlib */
-	char *port;
+	uchar *port;
 	int bInitialConnect; /* is this the initial connection request of our module? (0-no, 1-yes) */
 	int bIsConnected; /* currently connected to server? 0 - no, 1 - yes */
-	relpClt_t *pRelpClt;		/* relp client for this instance */
+	unsigned timeout;
+	relpClt_t *pRelpClt; /* relp client for this instance */
+	uchar *tplName;
 } instanceData;
 
 typedef struct configSettings_s {
@@ -70,30 +73,114 @@ typedef struct configSettings_s {
 } configSettings_t;
 static configSettings_t __attribute__((unused)) cs;
 
+
+/* tables for interfacing with the v6 config system */
+/* action (instance) parameters */
+static struct cnfparamdescr actpdescr[] = {
+	{ "target", eCmdHdlrGetWord, 1 },
+	{ "port", eCmdHdlrGetWord, 0 },
+	{ "timeout", eCmdHdlrInt, 0 },
+	{ "template", eCmdHdlrGetWord, 1 }
+};
+static struct cnfparamblk actpblk =
+	{ CNFPARAMBLK_VERSION,
+	  sizeof(actpdescr)/sizeof(struct cnfparamdescr),
+	  actpdescr
+	};
+
 BEGINinitConfVars		/* (re)set config variables to default values */
 CODESTARTinitConfVars 
 ENDinitConfVars
 
-/* get the syslog forward port from selector_t. The passed in
- * struct must be one that is setup for forwarding.
- * rgerhards, 2007-06-28
- * We may change the implementation to try to lookup the port
- * if it is unspecified. So far, we use the IANA default auf 514.
+/* We may change the implementation to try to lookup the port
+ * if it is unspecified. So far, we use 514 as default (what probably
+ * is not a really bright idea, but kept for backward compatibility).
  */
-static char *getRelpPt(instanceData *pData)
+static uchar *getRelpPt(instanceData *pData)
 {
 	assert(pData != NULL);
 	if(pData->port == NULL)
-		return("514");
+		return((uchar*)"514");
 	else
 		return(pData->port);
 }
 
+static inline rsRetVal
+doCreateRelpClient(instanceData *pData)
+{
+	DEFiRet;
+	if(relpEngineCltConstruct(pRelpEngine, &pData->pRelpClt) != RELP_RET_OK)
+		ABORT_FINALIZE(RS_RET_RELP_ERR);
+	if(relpCltSetTimeout(pData->pRelpClt, pData->timeout) != RELP_RET_OK)
+		ABORT_FINALIZE(RS_RET_RELP_ERR);
+finalize_it:
+	RETiRet;
+}
+
+
 BEGINcreateInstance
 CODESTARTcreateInstance
 	pData->bInitialConnect = 1;
 ENDcreateInstance
 
+BEGINfreeInstance
+CODESTARTfreeInstance
+	if(pData->pRelpClt != NULL)
+		relpEngineCltDestruct(pRelpEngine, &pData->pRelpClt);
+	free(pData->target);
+	free(pData->port);
+	free(pData->tplName);
+ENDfreeInstance
+
+static inline void
+setInstParamDefaults(instanceData *pData)
+{
+	pData->target = NULL;
+	pData->port = NULL;
+	pData->tplName = NULL;
+	pData->timeout = 90;
+}
+
+
+BEGINnewActInst
+	struct cnfparamvals *pvals;
+	int i;
+CODESTARTnewActInst
+	if((pvals = nvlstGetParams(lst, &actpblk, NULL)) == NULL) {
+		ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS);
+	}
+
+	CHKiRet(createInstance(&pData));
+	setInstParamDefaults(pData);
+
+	for(i = 0 ; i < actpblk.nParams ; ++i) {
+		if(!pvals[i].bUsed)
+			continue;
+		if(!strcmp(actpblk.descr[i].name, "target")) {
+			pData->target = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(actpblk.descr[i].name, "port")) {
+			pData->port = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(actpblk.descr[i].name, "template")) {
+			pData->tplName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(actpblk.descr[i].name, "timeout")) {
+			pData->timeout = (unsigned) pvals[i].val.d.n;
+		} else {
+			dbgprintf("omrelp: program error, non-handled "
+			  "param '%s'\n", actpblk.descr[i].name);
+		}
+	}
+	
+	CODE_STD_STRING_REQUESTnewActInst(1)
+
+	CHKiRet(OMSRsetEntry(*ppOMSR, 0, (uchar*)strdup((pData->tplName == NULL) ?
+			    "RSYSLOG_ForwardFormat" : (char*)pData->tplName),
+	   		    OMSR_NO_RQD_TPL_OPTS));
+
+	CHKiRet(doCreateRelpClient(pData));
+
+CODE_STD_FINALIZERnewActInst
+	cnfparamvalsDestruct(pvals, &actpblk);
+ENDnewActInst
 
 BEGINisCompatibleWithFeature
 CODESTARTisCompatibleWithFeature
@@ -101,25 +188,16 @@ CODESTARTisCompatibleWithFeature
 		iRet = RS_RET_OK;
 ENDisCompatibleWithFeature
 
-
-BEGINfreeInstance
-CODESTARTfreeInstance
-	if(pData->port != NULL)
-		free(pData->port);
-
-	/* final cleanup */
-	if(pData->pRelpClt != NULL)
-		relpEngineCltDestruct(pRelpEngine, &pData->pRelpClt);
-
-	if(pData->f_hname != NULL)
-		free(pData->f_hname);
-
-ENDfreeInstance
+BEGINSetShutdownImmdtPtr
+CODESTARTSetShutdownImmdtPtr
+	relpEngineSetShutdownImmdtPtr(pRelpEngine, pPtr);
+	DBGPRINTF("omrelp: shutdownImmediate ptr now is %p\n", pPtr);
+ENDSetShutdownImmdtPtr
 
 
 BEGINdbgPrintInstInfo
 CODESTARTdbgPrintInstInfo
-	printf("RELP/%s", pData->f_hname);
+	dbgprintf("RELP/%s", pData->target);
 ENDdbgPrintInstInfo
 
 
@@ -131,7 +209,7 @@ static rsRetVal doConnect(instanceData *pData)
 	DEFiRet;
 
 	if(pData->bInitialConnect) {
-		iRet = relpCltConnect(pData->pRelpClt, glbl.GetDefPFFamily(), (uchar*) pData->port, (uchar*) pData->f_hname);
+		iRet = relpCltConnect(pData->pRelpClt, glbl.GetDefPFFamily(), pData->port, pData->target);
 		if(iRet == RELP_RET_OK)
 			pData->bInitialConnect = 0;
 	} else {
@@ -160,7 +238,7 @@ BEGINdoAction
 	size_t lenMsg;
 	relpRetVal ret;
 CODESTARTdoAction
-	dbgprintf(" %s:%s/RELP\n", pData->f_hname, getRelpPt(pData));
+	dbgprintf(" %s:%s/RELP\n", pData->target, getRelpPt(pData));
 
 	if(!pData->bIsConnected) {
 		CHKiRet(doConnect(pData));
@@ -309,21 +387,17 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1)
 	/* TODO: make this if go away! */
 	if(*p == ';') {
 		*p = '\0'; /* trick to obtain hostname (later)! */
-		CHKmalloc(pData->f_hname = strdup((char*) q));
+		CHKmalloc(pData->target = ustrdup(q));
 		*p = ';';
 	} else {
-		CHKmalloc(pData->f_hname = strdup((char*) q));
+		CHKmalloc(pData->target = ustrdup(q));
 	}
 
 	/* process template */
 	CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (uchar*) "RSYSLOG_ForwardFormat"));
 
-	/* create our relp client  */
-	CHKiRet(relpEngineCltConstruct(pRelpEngine, &pData->pRelpClt)); /* we use CHKiRet as librelp has a similar return value range */
+	CHKiRet(doCreateRelpClient(pData));
 
-	/* TODO: do we need to call freeInstance if we failed - this is a general question for
-	 * all output modules. I'll address it later as the interface evolves. rgerhards, 2007-07-25
-	 */
 CODE_STD_FINALIZERparseSelectorAct
 ENDparseSelectorAct
 
@@ -342,6 +416,8 @@ BEGINqueryEtryPt
 CODESTARTqueryEtryPt
 CODEqueryEtryPt_STD_OMOD_QUERIES
 CODEqueryEtryPt_STD_CONF2_CNFNAME_QUERIES 
+CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES
+CODEqueryEtryPt_SetShutdownImmdtPtr
 ENDqueryEtryPt
 
 
diff --git a/plugins/omudpspoof/omudpspoof.c b/plugins/omudpspoof/omudpspoof.c
index 4f37fd8..9c4c80b 100644
--- a/plugins/omudpspoof/omudpspoof.c
+++ b/plugins/omudpspoof/omudpspoof.c
@@ -481,7 +481,7 @@ UDPSend(instanceData *pData, uchar *pszSourcename, char *msg, size_t len)
 				0,				/* checksum */
 				source_ip.sin_addr.s_addr,
 				tempaddr->sin_addr.s_addr,
-				(u_int8_t*)(msg+msgOffs),	/* payload */
+				(uint8_t*)(msg+msgOffs),	/* payload */
 				pktLen, 			/* payload size */
 				pData->libnet_handle,		/* libnet handle */
 				ip);				/* libnet id */
diff --git a/runtime/module-template.h b/runtime/module-template.h
index fe74bac..8a958f9 100644
--- a/runtime/module-template.h
+++ b/runtime/module-template.h
@@ -938,6 +938,28 @@ static rsRetVal doHUP(instanceData __attribute__((unused)) *pData)\
 }
 
 
+/* SetShutdownImmdtPtr()
+ * This function is optional. If defined by an output plugin, it is called 
+ * each time the action is invoked to set the "ShutdownImmediate" pointer,
+ * which is used during termination to indicate the action should shutdown
+ * as quickly as possible.
+ */
+#define CODEqueryEtryPt_SetShutdownImmdtPtr \
+	else if(!strcmp((char*) name, "SetShutdownImmdtPtr")) {\
+		*pEtryPoint = SetShutdownImmdtPtr;\
+	}
+#define BEGINSetShutdownImmdtPtr \
+static rsRetVal SetShutdownImmdtPtr(instanceData __attribute__((unused)) *pData, int *pPtr)\
+{\
+	DEFiRet;
+
+#define CODESTARTSetShutdownImmdtPtr 
+
+#define ENDSetShutdownImmdtPtr \
+	RETiRet;\
+}
+
+
 /* parse() - main entry point of parser modules
  */
 #define BEGINparse \
diff --git a/runtime/modules.c b/runtime/modules.c
index 9f7ff31..e9d8d95 100644
--- a/runtime/modules.c
+++ b/runtime/modules.c
@@ -657,6 +657,10 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_
 			if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND)
 				ABORT_FINALIZE(localRet);
 
+			localRet = (*pNew->modQueryEtryPt)((uchar*)"SetShutdownImmdtPtr", &pNew->mod.om.SetShutdownImmdtPtr);
+			if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND)
+				ABORT_FINALIZE(localRet);
+
 			localRet = (*pNew->modQueryEtryPt)((uchar*)"beginTransaction", &pNew->mod.om.beginTransaction);
 			if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND)
 				pNew->mod.om.beginTransaction = dummyBeginTransaction;
diff --git a/runtime/modules.h b/runtime/modules.h
index e42d19e..64644be 100644
--- a/runtime/modules.h
+++ b/runtime/modules.h
@@ -142,6 +142,7 @@ struct modInfo_s {
 			rsRetVal (*endTransaction)(void*);
 			rsRetVal (*parseSelectorAct)(uchar**, void**,omodStringRequest_t**);
 			rsRetVal (*newActInst)(uchar *modName, struct nvlst *lst, void **, omodStringRequest_t **);
+			rsRetVal (*SetShutdownImmdtPtr)(void *pData, void *pPtr);
 		} om;
 		struct { /* data for library modules */
 		    	char dummy;
diff --git a/runtime/msg.c b/runtime/msg.c
index c302a05..a5c5281 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -3230,13 +3230,18 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 		uchar *pSb;
 		iFrom = pTpe->data.field.iFromPos;
 		iTo = pTpe->data.field.iToPos;
-		/* need to zero-base to and from (they are 1-based!) */
-		if(iFrom > 0)
-			--iFrom;
-		if(iTo > 0)
-			--iTo;
 		if(bufLen == -1)
 			bufLen = ustrlen(pRes);
+		if(pTpe->data.field.options.bFromPosEndRelative) {
+			iFrom = (bufLen < iFrom) ? 0 : bufLen - iFrom;
+			iTo = (bufLen < iTo)? 0 : bufLen - iTo;
+		} else {
+			/* need to zero-base to and from (they are 1-based!) */
+			if(iFrom > 0)
+				--iFrom;
+			if(iTo > 0)
+				--iTo;
+		}
 		if(iFrom == 0 && iTo >=  bufLen) { 
 			/* in this case, the requested string is a superset of what we already have,
 			 * so there is no need to do any processing. This is a frequent case for size-limited
diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c
index 0f66c76..6ef4feb 100644
--- a/runtime/nsd_gtls.c
+++ b/runtime/nsd_gtls.c
@@ -259,9 +259,9 @@ gtlsClientCertCallback(gnutls_session session,
 static rsRetVal
 gtlsGetCertInfo(nsd_gtls_t *pThis, cstr_t **ppStr)
 {
-	char dn[128];
-	uchar lnBuf[256];
-	size_t size;
+	uchar szBufA[1024];
+	uchar *szBuf = szBufA;
+	size_t szBufLen = sizeof(szBufA), tmp;
 	unsigned int algo, bits;
 	time_t expiration_time, activation_time;
 	const gnutls_datum *cert_list;
@@ -271,8 +271,6 @@ gtlsGetCertInfo(nsd_gtls_t *pThis, cstr_t **ppStr)
 	int gnuRet;
 	DEFiRet;
 	unsigned iAltName;
-	size_t szAltNameLen;
-	char szAltName[1024]; /* this is sufficient for the DNSNAME... */
 
 	assert(ppStr != NULL);
 	ISOBJ_TYPE_assert(pThis, nsd_gtls);
@@ -281,61 +279,62 @@ gtlsGetCertInfo(nsd_gtls_t *pThis, cstr_t **ppStr)
 		return RS_RET_TLS_CERT_ERR;
 
 	cert_list = gnutls_certificate_get_peers(pThis->sess, &cert_list_size);
-
-	CHKiRet(rsCStrConstruct(&pStr));
-
-	snprintf((char*)lnBuf, sizeof(lnBuf), "peer provided %d certificate(s). ", cert_list_size);
-	CHKiRet(rsCStrAppendStr(pStr, lnBuf));
+	CHKiRet(rsCStrConstructFromszStrf(&pStr, "peer provided %d certificate(s). ", cert_list_size));
 
 	if(cert_list_size > 0) {
 		/* we only print information about the first certificate */
 		CHKgnutls(gnutls_x509_crt_init(&cert));
 		CHKgnutls(gnutls_x509_crt_import(cert, &cert_list[0], GNUTLS_X509_FMT_DER));
 
-		CHKiRet(rsCStrAppendStr(pStr, (uchar*)"Certificate 1 info: "));
-
 		expiration_time = gnutls_x509_crt_get_expiration_time(cert);
 		activation_time = gnutls_x509_crt_get_activation_time(cert);
-		ctime_r(&activation_time, dn);
-		dn[strlen(dn) - 1] = '\0'; /* strip linefeed */
-		snprintf((char*)lnBuf, sizeof(lnBuf), "certificate valid from %s ", dn);
-		CHKiRet(rsCStrAppendStr(pStr, lnBuf));
-
-		ctime_r(&expiration_time, dn);
-		dn[strlen(dn) - 1] = '\0'; /* strip linefeed */
-		snprintf((char*)lnBuf, sizeof(lnBuf), "to %s; ", dn);
-		CHKiRet(rsCStrAppendStr(pStr, lnBuf));
+		ctime_r(&activation_time, szBuf);
+		szBuf[strlen(szBuf) - 1] = '\0'; /* strip linefeed */
+		CHKiRet(rsCStrAppendStrf(pStr, (uchar*)"Certificate 1 info: "
+			"certificate valid from %s ", szBuf));
+		ctime_r(&expiration_time, szBuf);
+		szBuf[strlen(szBuf) - 1] = '\0'; /* strip linefeed */
+		CHKiRet(rsCStrAppendStrf(pStr, "to %s; ", szBuf));
 
 		/* Extract some of the public key algorithm's parameters */
 		algo = gnutls_x509_crt_get_pk_algorithm(cert, &bits);
-
-		snprintf((char*)lnBuf, sizeof(lnBuf), "Certificate public key: %s; ",
-			 gnutls_pk_algorithm_get_name(algo));
-		CHKiRet(rsCStrAppendStr(pStr, lnBuf));
+		CHKiRet(rsCStrAppendStrf(pStr, "Certificate public key: %s; ",
+			gnutls_pk_algorithm_get_name(algo)));
 
 		/* names */
-		size = sizeof(dn);
-		gnutls_x509_crt_get_dn(cert, dn, &size);
-		snprintf((char*)lnBuf, sizeof(lnBuf), "DN: %s; ", dn);
-		CHKiRet(rsCStrAppendStr(pStr, lnBuf));
-
-		size = sizeof(dn);
-		gnutls_x509_crt_get_issuer_dn(cert, dn, &size);
-		snprintf((char*)lnBuf, sizeof(lnBuf), "Issuer DN: %s; ", dn);
-		CHKiRet(rsCStrAppendStr(pStr, lnBuf));
+		tmp = szBufLen;
+		if(gnutls_x509_crt_get_dn(cert, szBuf, &tmp)
+		    == GNUTLS_E_SHORT_MEMORY_BUFFER) {
+			szBufLen = tmp;
+			szBuf = malloc(tmp);
+			gnutls_x509_crt_get_dn(cert, szBuf, &tmp);
+		}
+		CHKiRet(rsCStrAppendStrf(pStr, "DN: %s; ", szBuf));
+
+		tmp = szBufLen;
+		if(gnutls_x509_crt_get_issuer_dn(cert, szBuf, &tmp)
+		    == GNUTLS_E_SHORT_MEMORY_BUFFER) {
+			szBufLen = tmp;
+			szBuf = realloc((szBuf == szBufA) ? NULL : szBuf, tmp);
+			gnutls_x509_crt_get_issuer_dn(cert, szBuf, &tmp);
+		}
+		CHKiRet(rsCStrAppendStrf(pStr, "Issuer DN: %s; ", szBuf));
 
 		/* dNSName alt name */
 		iAltName = 0;
 		while(1) { /* loop broken below */
-			szAltNameLen = sizeof(szAltName);
+			tmp = szBufLen;
 			gnuRet = gnutls_x509_crt_get_subject_alt_name(cert, iAltName,
-					szAltName, &szAltNameLen, NULL);
-			if(gnuRet < 0)
+					szBuf, &tmp, NULL);
+			if(gnuRet == GNUTLS_E_SHORT_MEMORY_BUFFER) {
+				szBufLen = tmp;
+				szBuf = realloc((szBuf == szBufA) ? NULL : szBuf, tmp);
+				continue;
+			} else if(gnuRet < 0)
 				break;
 			else if(gnuRet == GNUTLS_SAN_DNSNAME) {
 				/* we found it! */
-				snprintf((char*)lnBuf, sizeof(lnBuf), "SAN:DNSname: %s; ", szAltName);
-				CHKiRet(rsCStrAppendStr(pStr, lnBuf));
+				CHKiRet(rsCStrAppendStrf(pStr, "SAN:DNSname: %s; ", szBuf));
 				/* do NOT break, because there may be multiple dNSName's! */
 			}
 			++iAltName;
@@ -352,6 +351,8 @@ finalize_it:
 		if(pStr != NULL)
 			rsCStrDestruct(&pStr);
 	}
+	if(szBuf != szBufA)
+		free(szBuf);
 
 	RETiRet;
 }
diff --git a/runtime/queue.c b/runtime/queue.c
index 4c8d3ac..74090a4 100644
--- a/runtime/queue.c
+++ b/runtime/queue.c
@@ -991,7 +991,7 @@ rsRetVal qqueueEnqObjDirectBatch(qqueue_t *pThis, batch_t *pBatch)
 	 * We use our knowledge about the batch_t structure below, but without that, we
 	 * pay a too-large performance toll... -- rgerhards, 2009-04-22
 	 */
-	iRet = pThis->pConsumer(pThis->pAction, pBatch, &pThis->bShutdownImmediate);
+	iRet = pThis->pConsumer(pThis->pAction, pBatch, NULL);
 
 	RETiRet;
 }
@@ -1150,6 +1150,7 @@ tryShutdownWorkersWithinActionTimeout(qqueue_t *pThis)
 	DBGOPRINT((obj_t*) pThis, "trying to shutdown workers within Action Timeout");
 	DBGOPRINT((obj_t*) pThis, "setting EnqOnly mode\n");
 	pThis->bEnqOnly = 1;
+dbgprintf("DDDD: setting shutdownImmediate mode, ptr %p!\n", &pThis->bShutdownImmediate);
 	pThis->bShutdownImmediate = 1;
 	/* now DA queue */
 	if(pThis->bIsDA) {
@@ -1291,8 +1292,6 @@ finalize_it:
 	RETiRet;
 }
 
-
-
 /* Constructor for the queue object
  * This constructs the data structure, but does not yet start the queue. That
  * is done by queueStart(). The reason is that we want to give the caller a chance
@@ -1856,6 +1855,8 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti)
 	/* at this spot, we may be cancelled */
 	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave);
 
+
+dbgprintf("DDDD: calling consumer with shutdownImmeditate ptr %p\n", &pThis->bShutdownImmediate);
 	CHKiRet(pThis->pConsumer(pThis->pAction, &pWti->batch, &pThis->bShutdownImmediate));
 
 	/* we now need to check if we should deliberately delay processing a bit
diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index 5ba6ede..b1f7d63 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -387,7 +387,9 @@ enum rsRetVal_				/** return value. All methods return this if not specified oth
 	RS_RET_MOD_NO_INPUT_STMT = -2224, /**< (input) module does not support input() statement */
 	RS_RET_NO_CEE_MSG = -2225, /**< the message being processed is NOT CEE-enhanced */
 
-	/**** up to 2300 is reserved for v6 use ****/
+	/**** up to 2290 is reserved for v6 use ****/
+	RS_RET_RELP_ERR = -2291,	/**<< error in RELP processing */
+	/**** up to 3000 is reserved for c7 use ****/
 	RS_RET_JNAME_NO_ROOT = -2301, /**< root element is missing in JSON path */
 	RS_RET_JNAME_INVALID = -2302, /**< JSON path is invalid */
 	RS_RET_JSON_PARSE_ERR = -2303, /**< we had a problem parsing JSON (or extra data) */
diff --git a/runtime/ruleset.c b/runtime/ruleset.c
index 66b38fc..e334893 100644
--- a/runtime/ruleset.c
+++ b/runtime/ruleset.c
@@ -11,7 +11,7 @@
  *
  * Module begun 2009-06-10 by Rainer Gerhards
  *
- * Copyright 2009-2012 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2009-2013 Rainer Gerhards and Adiscon GmbH.
  *
  * This file is part of the rsyslog runtime library.
  *
@@ -297,6 +297,7 @@ execIf(struct cnfstmt *stmt, batch_t *pBatch, sbool *active)
 	sbool *newAct;
 	int i;
 	sbool bRet;
+	sbool allInactive = 1;
 	DEFiRet;
 	newAct = newActive(pBatch);
 	for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) {
@@ -306,12 +307,19 @@ execIf(struct cnfstmt *stmt, batch_t *pBatch, sbool *active)
 			continue; /* will be ignored in any case */
 		if(active == NULL || active[i]) {
 			bRet = cnfexprEvalBool(stmt->d.s_if.expr, pBatch->pElem[i].pMsg);
+			allInactive = 0;
 		} else 
 			bRet = 0;
 		newAct[i] = bRet;
 		DBGPRINTF("batch: item %d: expr eval: %d\n", i, bRet);
 	}
 
+	if(allInactive) {
+		DBGPRINTF("execIf: all batch elements are inactive, holding execution\n");
+		freeActive(newAct);
+		FINALIZE;
+	}
+
 	if(stmt->d.s_if.t_then != NULL) {
 		scriptExec(stmt->d.s_if.t_then, pBatch, newAct);
 	}
@@ -319,7 +327,8 @@ execIf(struct cnfstmt *stmt, batch_t *pBatch, sbool *active)
 		for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) {
 			if(*(pBatch->pbShutdownImmediate))
 				FINALIZE;
-			if(pBatch->eltState[i] != BATCH_STATE_DISC)
+			if(pBatch->eltState[i] != BATCH_STATE_DISC
+			   && (active == NULL || active[i]))
 				newAct[i] = !newAct[i];
 			}
 		scriptExec(stmt->d.s_if.t_else, pBatch, newAct);
@@ -364,7 +373,8 @@ execPRIFILT(struct cnfstmt *stmt, batch_t *pBatch, sbool *active)
 		for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) {
 			if(*(pBatch->pbShutdownImmediate))
 				return;
-			if(pBatch->eltState[i] != BATCH_STATE_DISC)
+			if(pBatch->eltState[i] != BATCH_STATE_DISC
+			   && (active == NULL || active[i]))
 				newAct[i] = !newAct[i];
 			}
 		scriptExec(stmt->d.s_prifilt.t_else, pBatch, newAct);
@@ -504,7 +514,11 @@ scriptExec(struct cnfstmt *root, batch_t *pBatch, sbool *active)
 	struct cnfstmt *stmt;
 
 	for(stmt = root ; stmt != NULL ; stmt = stmt->next) {
-dbgprintf("RRRR: scriptExec: batch of %d elements, active %p, stmt %p, nodetype %u\n", batchNumMsgs(pBatch), active, stmt, stmt->nodetype);
+		if(Debug) {
+			dbgprintf("scriptExec: batch of %d elements, active %p, active[0]:%d\n",
+				  batchNumMsgs(pBatch), active, (active == NULL ? 1 : active[0]));
+			cnfstmtPrintOnly(stmt, 2, 0);
+		}
 		switch(stmt->nodetype) {
 		case S_NOP:
 			break;
@@ -521,7 +535,6 @@ dbgprintf("RRRR: scriptExec: batch of %d elements, active %p, stmt %p, nodetype
 			execUnset(stmt, pBatch, active);
 			break;
 		case S_CALL:
-			DBGPRINTF("calling ruleset\n"); // TODO: add Name
 			scriptExec(stmt->d.s_call.stmt, pBatch, active);
 			break;
 		case S_IF:
diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c
index d9f8023..75d2eac 100644
--- a/runtime/stringbuf.c
+++ b/runtime/stringbuf.c
@@ -32,6 +32,7 @@
 #include <assert.h>
 #include <string.h>
 #include <ctype.h>
+#include <stdarg.h>
 #include <sys/types.h>
 #include <libestr.h>
 #include "rsyslog.h"
@@ -104,6 +105,56 @@ finalize_it:
 }
 
 
+/* a helper function for rsCStr*Strf()
+ */
+static rsRetVal rsCStrConstructFromszStrv(cstr_t **ppThis, uchar *fmt, va_list ap)
+{
+	DEFiRet;
+	cstr_t *pThis;
+	va_list ap2;
+	uchar *sz;
+	int len;
+
+	assert(ppThis != NULL);
+
+	va_copy(ap2, ap);
+	len = vsnprintf(NULL, 0, fmt, ap2);
+	va_end(ap2);
+
+	if(len < 0)
+		ABORT_FINALIZE(RS_RET_ERR);
+
+	CHKiRet(rsCStrConstruct(&pThis));
+
+	pThis->iBufSize = pThis->iStrLen = len;
+	len++; /* account for the \0 written by vsnprintf */
+	if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * len)) == NULL) {
+		RSFREEOBJ(pThis);
+		ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
+	}
+
+	vsnprintf(pThis->pBuf, len, fmt, ap);
+	*ppThis = pThis;
+finalize_it:
+	RETiRet;
+}
+
+
+/* construct from a printf-style formated string
+ */
+rsRetVal rsCStrConstructFromszStrf(cstr_t **ppThis, uchar *fmt, ...)
+{
+	DEFiRet;
+	va_list ap;
+
+	va_start(ap, fmt);
+	iRet = rsCStrConstructFromszStrv(ppThis, fmt, ap);
+	va_end(ap);
+
+	RETiRet;
+}
+
+
 /* construct from es_str_t string
  * rgerhards 2010-12-03
  */
@@ -256,6 +307,27 @@ rsRetVal cstrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend)
 }
 
 
+/* append a printf-style formated string
+ */
+rsRetVal rsCStrAppendStrf(cstr_t *pThis, uchar *fmt, ...)
+{
+	DEFiRet;
+	va_list ap;
+	cstr_t *pStr;
+
+	va_start(ap, fmt);
+	iRet = rsCStrConstructFromszStrv(&pStr, fmt, ap);
+	va_end(ap);
+
+	CHKiRet(iRet);
+
+	iRet = cstrAppendCStr(pThis, pStr);
+	rsCStrDestruct(pStr);
+finalize_it:
+	RETiRet;
+}
+
+
 rsRetVal rsCStrAppendInt(cstr_t *pThis, long i)
 {
 	DEFiRet;
diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h
index bba004a..b301f4b 100644
--- a/runtime/stringbuf.h
+++ b/runtime/stringbuf.h
@@ -58,6 +58,7 @@ rsRetVal cstrConstruct(cstr_t **ppThis);
 rsRetVal cstrConstructFromESStr(cstr_t **ppThis, es_str_t *str);
 rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz);
 rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom);
+rsRetVal rsCStrConstructFromszStrf(cstr_t **ppThis, uchar *fmt, ...);
 
 /**
  * Destruct the string buffer object.
@@ -173,6 +174,12 @@ rsRetVal rsCStrAppendStr(cstr_t *pThis, uchar* psz);
  */
 rsRetVal rsCStrAppendStrWithLen(cstr_t *pThis, uchar* psz, size_t iStrLen);
 
+/**
+ * Append a printf-style formated string to the buffer.
+ *
+ * \param fmt pointer to the format string (see man 3 printf for details). Must not be NULL.
+ */
+rsRetVal rsCStrAppendStrf(cstr_t *pThis, uchar *fmt, ...);
 
 /**
  * Append an integer to the string. No special formatting is
diff --git a/template.c b/template.c
index 1ccb375..c48bf4b 100644
--- a/template.c
+++ b/template.c
@@ -79,6 +79,7 @@ static struct cnfparamdescr cnfparamdescrProperty[] = {
 	{ "format", eCmdHdlrString, 0 },
 	{ "position.from", eCmdHdlrInt, 0 },
 	{ "position.to", eCmdHdlrInt, 0 },
+	{ "position.relativetoend", eCmdHdlrBinary, 0 },
 	{ "field.number", eCmdHdlrInt, 0 },
 	{ "field.delimiter", eCmdHdlrInt, 0 },
 	{ "regex.expression", eCmdHdlrString, 0 },
@@ -712,6 +713,8 @@ static void doOptions(unsigned char **pp, struct templateEntry *pTpe)
 			pTpe->data.field.options.bSecPathDrop = 1;
 		 } else if(!strcmp((char*)Buf, "secpath-replace")) {
 			pTpe->data.field.options.bSecPathReplace = 1;
+		 } else if(!strcmp((char*)Buf, "pos-end-relative")) {
+			pTpe->data.field.options.bFromPosEndRelative = 1;
 		 } else if(!strcmp((char*)Buf, "csv")) {
 		 	if(pTpe->data.field.options.bJSON || pTpe->data.field.options.bJSONf) {
 				errmsg.LogError(0, NO_ERRCODE, "error: can only specify "
@@ -1057,18 +1060,27 @@ do_Parameter(uchar **pp, struct template *pTpl)
 #endif /* #ifdef FEATURE_REGEXP */
 	}
 
-	if(pTpe->data.field.iToPos < pTpe->data.field.iFromPos) {
-		iNum = pTpe->data.field.iToPos;
-		pTpe->data.field.iToPos = pTpe->data.field.iFromPos;
-		pTpe->data.field.iFromPos = iNum;
-	}
-
 	/* check options */
 	if(*p == ':') {
 		++p; /* eat ':' */
 		doOptions(&p, pTpe);
 	}
 
+	if(pTpe->data.field.options.bFromPosEndRelative) {
+		if(pTpe->data.field.iToPos > pTpe->data.field.iFromPos) {
+			iNum = pTpe->data.field.iToPos;
+			pTpe->data.field.iToPos = pTpe->data.field.iFromPos;
+			pTpe->data.field.iFromPos = iNum;
+		}
+	} else {
+		if(pTpe->data.field.iToPos < pTpe->data.field.iFromPos) {
+			iNum = pTpe->data.field.iToPos;
+			pTpe->data.field.iToPos = pTpe->data.field.iFromPos;
+			pTpe->data.field.iFromPos = iNum;
+		}
+	}
+
+
 	/* check field name */
 	if(*p == ':') {
 		++p; /* eat ':' */
@@ -1356,6 +1368,7 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 	int re_matchToUse = 0;
 	int re_submatchToUse = 0;
 	int bComplexProcessing = 0;
+	int bPosRelativeToEnd = 0;
 	char *re_expr = NULL;
 	struct cnfparamvals *pvals = NULL;
 	enum {F_NONE, F_CSV, F_JSON, F_JSONF} formatType = F_NONE;
@@ -1395,6 +1408,8 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 		} else if(!strcmp(pblkProperty.descr[i].name, "position.to")) {
 			topos = pvals[i].val.d.n;
 			bComplexProcessing = 1;
+		} else if(!strcmp(pblkProperty.descr[i].name, "position.relativetoend")) {
+			bPosRelativeToEnd = pvals[i].val.d.n;
 		} else if(!strcmp(pblkProperty.descr[i].name, "field.number")) {
 			fieldnum = pvals[i].val.d.n;
 			bComplexProcessing = 1;
@@ -1537,10 +1552,18 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 		topos = 2000000000; /* large enough ;) */
 	if(frompos == -1 && topos != -1)
 		frompos = 0;
-	if(topos < frompos) {
-		errmsg.LogError(0, RS_RET_ERR, "position.to=%d is lower than postion.from=%d\n",
-			topos, frompos);
-		ABORT_FINALIZE(RS_RET_ERR);
+	if(bPosRelativeToEnd) {
+		if(topos > frompos) {
+			errmsg.LogError(0, RS_RET_ERR, "position.to=%d is higher than postion.from=%d in 'relativeToEnd' mode\n",
+				topos, frompos);
+			ABORT_FINALIZE(RS_RET_ERR);
+		}
+	} else {
+		if(topos < frompos) {
+			errmsg.LogError(0, RS_RET_ERR, "position.to=%d is lower than postion.from=%d\n",
+				topos, frompos);
+			ABORT_FINALIZE(RS_RET_ERR);
+		}
 	}
 	if(fieldnum != -1 && re_expr != NULL) {
 		errmsg.LogError(0, RS_RET_ERR, "both field extraction and regex extraction "
@@ -1613,6 +1636,7 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 	if(frompos != -1) {
 		pTpe->data.field.iFromPos = frompos;
 		pTpe->data.field.iToPos = topos;
+		pTpe->data.field.options.bFromPosEndRelative = bPosRelativeToEnd;
 	}
 	if(re_expr != NULL) {
 		rsRetVal iRetLocal;
diff --git a/template.h b/template.h
index 018e2f5..318db6f 100644
--- a/template.h
+++ b/template.h
@@ -119,6 +119,7 @@ struct templateEntry {
 				unsigned bJSON: 1;		/* format field JSON escaped */
 				unsigned bJSONf: 1;		/* format field JSON *field* (n/v pair) */
 				unsigned bMandatory: 1;		/* mandatory field - emit even if empty */
+				unsigned bFromPosEndRelative: 1;/* is From/To-Pos relative to end of string? */
 			} options;		/* options as bit fields */
 		} field;
 	} data;
diff --git a/tests/queue-persist-drvr.sh b/tests/queue-persist-drvr.sh
index de59730..823fed6 100755
--- a/tests/queue-persist-drvr.sh
+++ b/tests/queue-persist-drvr.sh
@@ -26,5 +26,10 @@ source $srcdir/diag.sh startup queue-persist.conf
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 ./msleep 500
 $srcdir/diag.sh wait-shutdown
-source $srcdir/diag.sh seq-check 0 4999
+# note: we need to permit duplicate messages, as due to the forced
+# shutdown some messages may be flagged as "unprocessed" while they
+# actually were processed. This is inline with rsyslog's philosophy
+# to better duplicate than loose messages. Duplicate messages are
+# permitted by the -d seq-check option.
+source $srcdir/diag.sh seq-check 0 4999 -d
 source $srcdir/diag.sh exit
diff --git a/tools/syslogd.c b/tools/syslogd.c
index e291ba4..03fb2bd 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -567,6 +567,7 @@ msgConsumer(void __attribute__((unused)) *notNeeded, batch_t *pBatch, int *pbShu
 	assert(pBatch != NULL);
 	pBatch->pbShutdownImmediate = pbShutdownImmediate; /* TODO: move this to batch creation! */
 	preprocessBatch(pBatch);
+dbgprintf("DDDD: batches ShutdownImmediate is %p\n", pBatch->pbShutdownImmediate);
 	ruleset.ProcessBatch(pBatch);
 //TODO: the BATCH_STATE_COMM must be set somewhere down the road, but we 
 //do not have this yet and so we emulate -- 2010-06-10