mail/rspamd: Update to 1.8.0.

- [Feature] Add arguments schemas to processors and extractors
- [Feature] Add functional selectors library
- [Feature] Add generic selector to reputation module
- [Feature] Add more ratelimits: by digest, by attachments data, by
  filenames
- [Feature] Add preliminary stop words detection support
- [Feature] Add pure Lua debugm function
- [Feature] Add schema validation for Redis settings
- [Feature] Add selectors combine function
- [Feature] Add some recursion protection to lua logger
- [Feature] Add support for Lua API tracing
- [Feature] Allow to apply schema to arguments
- [Feature] Allow to get dkim signing data directly from HTTP headers
- [Feature] Allow to reuse existing authentication results
- [Feature] Cache selectors results in re runtime
- [Feature] Implement new text tokenizer based on libicu
- [Feature] Integrate selectors framework to multimap
- [Feature] Relax FORGED_RECIPIENTS
- [Feature] Support (almost) all html entities
- [Feature] Support adding and deletion of recipients in the milter
  block
- [Feature] Support gathering HTTP body from fragments in lua_http
- [Feature] Support multi flag in regexp and glob maps
- [Feature] Support selectors in ratelimit module
- [Feature] Support selectors in settings
- [Feature] Use khash in HTML parser
- [Feature] Use pure Lua debugm function
- [Fix] Add fail-safety for destroying sessions
- [Fix] Allow to add result-less fake DNS records
- [Fix] Another try to fix race conditions on config unload
- [Fix] Call Lua callback on DNS timeouts
- [Fix] Deprecate task:inc_dns_req as it is redundant
- [Fix] Do not allow events deletions on cleanup
- [Fix] Do not try to process skipped messages
- [Fix] Fix HTTP requests with no body
- [Fix] Fix another cleanup race condition
- [Fix] Fix bug in processing of pcre regexps
- [Fix] Fix byte array allocation in the pool
- [Fix] Fix crashes on task cleanup
- [Fix] Fix dynamic buckets in ratelimits
- [Fix] Fix endless loop when waiting for Rspamd to stop
- [Fix] Fix lua_util.str_split in case of delimiters set
- [Fix] Fix more issues with watching of async events
- [Fix] Fix stop words detection and loading logic
- [Fix] Fix various corner cases for language detection
- [Fix] Fix watchers in lua_tcp
- [Fix] Fix words decay algorithm
- [Fix] Implement watchers replacement to handle nested calls
- [Fix] Save faked code into fake dns record
- [Fix] Show the proper frame when using lua_util.debugm
- [Fix] Use fake dns records in tests
- [Fix] Use unicode replacements for HTML entities
- [Fix] fixed "cannot find dependency on symbol 1" issue when using
  replaced symbols in spamassassin rules
- [Fix] partition_id is not available in old versions of CH
- [Project] Add implicit conversion logic to selectors
- [Project] Add initial support for selectors in regexps
- [Project] Add method concept
- [Project] Further changes in unicode operations
- [Project] Implement Clickhouse migrations
- [Project] Implement implicit conversions to userdata
- [Project] Implement insert method
- [Project] Implement selectors registration for regular expressions
- [Project] Implement selectors support in re_cache
- [Project] Improve language detector: cleanup unused files,
  categorize
- [Project] Migrate CH data to a fat table
- [Project] Rework selectors logic
- [Project] Start Clickhouse utilities library
- [Project] Start unicode rework
- [Project] coroutine threaded model for API calls: thread pool
- [Rework] Move phishtank to a DNS based service
- [Rework] Rework Clickhouse plugin to use the new API
- [Rework] Rework language detector
- [Rework] Rework utf content processing in text parts
- [WebUI] Add progress bar for AJAX requests
- [WebUI] Avoid errors table reinitialization
- [WebUI] Avoid history table reinitialization
- [WebUI] Avoid throughput summary table reinitialization
- [WebUI] Destroy summary table on disconnect
- [WebUI] Fix "auth" request URL
- [WebUI] Fix disabling and hiding controls on page reload
- [WebUI] Fix maps loading from neighbours
- [WebUI] Fix symbols sorting by score
- [WebUI] Fix tables destroying
- [WebUI] Fix throughput data consolidation
- [WebUI] Fix upload buttons disabling

6 files changed, 63 insertions, 42 deletions

diff --git a/mail/rspamd/Makefile b/mail/rspamd/Makefile
index 42a881005bc..8ac2d18979c 100644
--- a/mail/rspamd/Makefile
+++ b/mail/rspamd/Makefile
@@ -1,6 +1,6 @@
-# $NetBSD: Makefile,v 1.47 2018/09/07 11:54:16 fhajny Exp $
+# $NetBSD: Makefile,v 1.48 2018/10/07 20:10:57 fhajny Exp $
 
-DISTNAME=	rspamd-1.7.9
+DISTNAME=	rspamd-1.8.0
 CATEGORIES=	mail
 MASTER_SITES=	${MASTER_SITE_GITHUB:=vstakhov/}
 
@@ -82,7 +82,7 @@ CONF_FILES+=	${EGDIR}/${file} ${PKG_SYSCONFDIR}/${file}
 .PHONY: cfgfiles
 cfgfiles: do-extract
 	@${ECHO} "# \$$NetBSD\$$" > ${.CURDIR}/cfgfiles.mk
-	@${ECHO} "# Do not edit, regenerate using 'make cfgfiles'" >> ${.CURDIR}/cfgfiles.mk
+	@${ECHO} "# Do not edit, regenerate post-extract using 'make cfgfiles'" >> ${.CURDIR}/cfgfiles.mk
 	@${ECHO} "CFGFILES= \\" >> ${.CURDIR}/cfgfiles.mk
 	cd ${WRKSRC}/conf && ${FIND} * -type f | ${SORT} | ${TR} '\n' ' ' >> ${.CURDIR}/cfgfiles.mk
 	@${ECHO} >> ${.CURDIR}/cfgfiles.mk
diff --git a/mail/rspamd/PLIST b/mail/rspamd/PLIST
index cf281f2a948..2ca4d78262f 100644
--- a/mail/rspamd/PLIST
+++ b/mail/rspamd/PLIST
@@ -1,10 +1,10 @@
-@comment $NetBSD: PLIST,v 1.14 2018/09/07 11:19:07 fhajny Exp $
+@comment $NetBSD: PLIST,v 1.15 2018/10/07 20:10:57 fhajny Exp $
+bin/${PKGNAME}
 bin/rspamadm
 bin/rspamadm-${PKGVERSION}
 bin/rspamc
 bin/rspamc-${PKGVERSION}
 bin/rspamd
-bin/${PKGNAME}
 bin/rspamd_stats
 lib/rspamd/librspamd-actrie.so
 man/man1/rspamadm.1
@@ -82,6 +82,7 @@ share/examples/rspamd/scores.d/rbl_group.conf
 share/examples/rspamd/scores.d/statistics_group.conf
 share/examples/rspamd/scores.d/subject_group.conf
 share/examples/rspamd/scores.d/surbl_group.conf
+share/examples/rspamd/settings.conf
 share/examples/rspamd/spf_dkim_whitelist.inc
 share/examples/rspamd/statistic.conf
 share/examples/rspamd/surbl-whitelist.inc
@@ -102,7 +103,6 @@ share/rspamd/languages/cs.json
 share/rspamd/languages/cy.json
 share/rspamd/languages/da.json
 share/rspamd/languages/de.json
-share/rspamd/languages/el.json
 share/rspamd/languages/en.json
 share/rspamd/languages/es.json
 share/rspamd/languages/et.json
@@ -111,22 +111,14 @@ share/rspamd/languages/fa.json
 share/rspamd/languages/fi.json
 share/rspamd/languages/fr.json
 share/rspamd/languages/ga.json
-share/rspamd/languages/gl.json
-share/rspamd/languages/gu.json
-share/rspamd/languages/he.json
 share/rspamd/languages/hi.json
 share/rspamd/languages/hr.json
 share/rspamd/languages/hu.json
 share/rspamd/languages/id.json
 share/rspamd/languages/is.json
 share/rspamd/languages/it.json
-share/rspamd/languages/ja.json
-share/rspamd/languages/km.json
-share/rspamd/languages/kn.json
-share/rspamd/languages/ko.json
 share/rspamd/languages/lt.json
 share/rspamd/languages/lv.json
-share/rspamd/languages/ml.json
 share/rspamd/languages/mr.json
 share/rspamd/languages/ms.json
 share/rspamd/languages/ne.json
@@ -141,30 +133,29 @@ share/rspamd/languages/sl.json
 share/rspamd/languages/so.json
 share/rspamd/languages/sq.json
 share/rspamd/languages/sr.json
+share/rspamd/languages/stop_words
 share/rspamd/languages/sv.json
 share/rspamd/languages/sw.json
-share/rspamd/languages/ta.json
-share/rspamd/languages/te.json
-share/rspamd/languages/th.json
 share/rspamd/languages/tr.json
 share/rspamd/languages/uk.json
 share/rspamd/languages/ur.json
 share/rspamd/languages/vi.json
-share/rspamd/languages/yi.json
-share/rspamd/languages/zh-CN.json
 share/rspamd/lib/ansicolors.lua
 share/rspamd/lib/argparse.lua
 share/rspamd/lib/fun.lua
 share/rspamd/lib/global_functions.lua
 share/rspamd/lib/lua_auth_results.lua
 share/rspamd/lib/lua_cfg_transform.lua
+share/rspamd/lib/lua_clickhouse.lua
 share/rspamd/lib/lua_dkim_tools.lua
 share/rspamd/lib/lua_maps.lua
 share/rspamd/lib/lua_meta.lua
 share/rspamd/lib/lua_nn.lua
 share/rspamd/lib/lua_redis.lua
+share/rspamd/lib/lua_selectors.lua
 share/rspamd/lib/lua_squeeze_rules.lua
 share/rspamd/lib/lua_stat.lua
+share/rspamd/lib/lua_tcp_sync.lua
 share/rspamd/lib/lua_util.lua
 share/rspamd/lib/plugins_stats.lua
 share/rspamd/lib/rescore_utility.lua
@@ -178,6 +169,7 @@ share/rspamd/lib/rspamadm/keypair.lua
 share/rspamd/lib/rspamadm/mime.lua
 share/rspamd/lib/rspamadm/rescore.lua
 share/rspamd/lib/rspamadm/stat_convert.lua
+share/rspamd/lib/tableshape.lua
 share/rspamd/lua/antivirus.lua
 share/rspamd/lua/arc.lua
 share/rspamd/lua/asn.lua
@@ -238,6 +230,7 @@ share/rspamd/www/README.md
 share/rspamd/www/css/bootstrap.min.css
 share/rspamd/www/css/d3evolution.css
 share/rspamd/www/css/footable.bootstrap.min.css
+share/rspamd/www/css/nprogress.css
 share/rspamd/www/css/rspamd.css
 share/rspamd/www/favicon.ico
 share/rspamd/www/fonts/glyphicons-halflings-regular.ttf
@@ -262,6 +255,7 @@ share/rspamd/www/js/lib/domReady.js
 share/rspamd/www/js/lib/footable.min.js
 share/rspamd/www/js/lib/humanize.min.js
 share/rspamd/www/js/lib/jquery-3.3.1.min.js
+share/rspamd/www/js/lib/nprogress.min.js
 share/rspamd/www/js/lib/require.min.js
 share/rspamd/www/js/lib/visibility.min.js
 share/rspamd/www/js/main.js
diff --git a/mail/rspamd/cfgfiles.mk b/mail/rspamd/cfgfiles.mk
index 01940f228f4..a8f9a9c2489 100644
--- a/mail/rspamd/cfgfiles.mk
+++ b/mail/rspamd/cfgfiles.mk
@@ -1,4 +1,4 @@
-# $NetBSD: cfgfiles.mk,v 1.1 2018/09/07 11:19:07 fhajny Exp $
+# $NetBSD: cfgfiles.mk,v 1.2 2018/10/07 20:10:57 fhajny Exp $
 # Do not edit, regenerate post-extract using 'make cfgfiles'
 CFGFILES= \
-2tld.inc actions.conf cgp.inc common.conf composites.conf dmarc_whitelist.inc groups.conf logging.inc maillist.inc metrics.conf mid.inc mime_types.inc modules.conf modules.d/antivirus.conf modules.d/arc.conf modules.d/asn.conf modules.d/chartable.conf modules.d/clickhouse.conf modules.d/dcc.conf modules.d/dkim.conf modules.d/dkim_signing.conf modules.d/dmarc.conf modules.d/elastic.conf modules.d/emails.conf modules.d/force_actions.conf modules.d/forged_recipients.conf modules.d/fuzzy_check.conf modules.d/greylist.conf modules.d/hfilter.conf modules.d/history_redis.conf modules.d/ip_score.conf modules.d/maillist.conf modules.d/metadata_exporter.conf modules.d/metric_exporter.conf modules.d/mid.conf modules.d/milter_headers.conf modules.d/mime_types.conf modules.d/multimap.conf modules.d/mx_check.conf modules.d/neural.conf modules.d/once_received.conf modules.d/phishing.conf modules.d/ratelimit.conf modules.d/rbl.conf modules.d/redis.conf modules.d/regexp.conf modules.d/replies.conf modules.d/reputation.conf modules.d/rspamd_update.conf modules.d/spamassassin.conf modules.d/spamtrap.conf modules.d/spf.conf modules.d/surbl.conf modules.d/trie.conf modules.d/url_redirector.conf modules.d/url_reputation.conf modules.d/url_tags.conf modules.d/whitelist.conf options.inc redirectors.inc rspamd.conf scores.d/fuzzy_group.conf scores.d/headers_group.conf scores.d/hfilter_group.conf scores.d/mime_types_group.conf scores.d/mua_group.conf scores.d/phishing_group.conf scores.d/policies_group.conf scores.d/rbl_group.conf scores.d/statistics_group.conf scores.d/subject_group.conf scores.d/surbl_group.conf spf_dkim_whitelist.inc statistic.conf surbl-whitelist.inc worker-controller.inc worker-fuzzy.inc worker-normal.inc worker-proxy.inc 
+2tld.inc actions.conf cgp.inc common.conf composites.conf dmarc_whitelist.inc groups.conf logging.inc maillist.inc metrics.conf mid.inc mime_types.inc modules.conf modules.d/antivirus.conf modules.d/arc.conf modules.d/asn.conf modules.d/clickhouse.conf modules.d/dcc.conf modules.d/dkim_signing.conf modules.d/dkim.conf modules.d/dmarc.conf modules.d/elastic.conf modules.d/emails.conf modules.d/force_actions.conf modules.d/forged_recipients.conf modules.d/fuzzy_check.conf modules.d/greylist.conf modules.d/hfilter.conf modules.d/history_redis.conf modules.d/chartable.conf modules.d/ip_score.conf modules.d/maillist.conf modules.d/metadata_exporter.conf modules.d/metric_exporter.conf modules.d/mid.conf modules.d/milter_headers.conf modules.d/mime_types.conf modules.d/multimap.conf modules.d/mx_check.conf modules.d/neural.conf modules.d/once_received.conf modules.d/phishing.conf modules.d/ratelimit.conf modules.d/rbl.conf modules.d/redis.conf modules.d/regexp.conf modules.d/replies.conf modules.d/reputation.conf modules.d/rspamd_update.conf modules.d/spamassassin.conf modules.d/spamtrap.conf modules.d/spf.conf modules.d/surbl.conf modules.d/trie.conf modules.d/url_redirector.conf modules.d/url_reputation.conf modules.d/url_tags.conf modules.d/whitelist.conf options.inc redirectors.inc rspamd.conf scores.d/fuzzy_group.conf scores.d/headers_group.conf scores.d/hfilter_group.conf scores.d/mime_types_group.conf scores.d/mua_group.conf scores.d/phishing_group.conf scores.d/policies_group.conf scores.d/rbl_group.conf scores.d/statistics_group.conf scores.d/subject_group.conf scores.d/surbl_group.conf settings.conf spf_dkim_whitelist.inc statistic.conf surbl-whitelist.inc worker-controller.inc worker-fuzzy.inc worker-normal.inc worker-proxy.inc 
diff --git a/mail/rspamd/distinfo b/mail/rspamd/distinfo
index a2d1308613c..dc972411e50 100644
--- a/mail/rspamd/distinfo
+++ b/mail/rspamd/distinfo
@@ -1,10 +1,10 @@
-$NetBSD: distinfo,v 1.26 2018/09/07 11:19:07 fhajny Exp $
+$NetBSD: distinfo,v 1.27 2018/10/07 20:10:57 fhajny Exp $
 
-SHA1 (rspamd-1.7.9.tar.gz) = 38b3134ac171643b59dd54c68e94e6ce492b2589
-RMD160 (rspamd-1.7.9.tar.gz) = dc519118a4d90552ea29051ba347b9ee350acd91
-SHA512 (rspamd-1.7.9.tar.gz) = d11d8c03668b85feae665ed491cd156c7430fbad30f42fe44103ea3c3a8bc988e5f8981e784e8f31aa35ba1ff9f7abd23605638e4ee3086a056b22fb3109563e
-Size (rspamd-1.7.9.tar.gz) = 4557492 bytes
+SHA1 (rspamd-1.8.0.tar.gz) = 918331a060246d921c89ffc42e99feed0335a89a
+RMD160 (rspamd-1.8.0.tar.gz) = 2a263670bf68b22792f52e39b2fb44a6c2e42da5
+SHA512 (rspamd-1.8.0.tar.gz) = 6e254abc70da5db1562273a5772ed90bc43a9cfdcb04f45b3e99706e69a98ff16cef8081284c8a8d73eae2ba5ae6870402280b5391804c14e3ca3206f7f7b4d6
+Size (rspamd-1.8.0.tar.gz) = 4261319 bytes
 SHA1 (patch-CMakeLists.txt) = 1d3e66b52ef38638674baf4c10b68c97b2ca38ad
 SHA1 (patch-src_CMakeLists.txt) = c80015bb275ba8b9ad31491311275bf353f8f30a
-SHA1 (patch-src_lua_lua__task.c) = 19e0c8433ad08fda075f50d93bff7cedf3d4cda1
 SHA1 (patch-test_CMakeLists.txt) = 2e4aafd860f0f120928cce49b0995c739558bf0d
+SHA1 (patch-test_rspamd__lua__pcall__vs__resume__test.c) = 65e901af64ebcd983310707da9814fa09aa11410
diff --git a/mail/rspamd/patches/patch-src_lua_lua__task.c b/mail/rspamd/patches/patch-src_lua_lua__task.c
deleted file mode 100644
index 3618a453fab..00000000000
--- a/mail/rspamd/patches/patch-src_lua_lua__task.c
+++ /dev/null
@@ -1,15 +0,0 @@
-$NetBSD: patch-src_lua_lua__task.c,v 1.3 2018/09/07 11:19:07 fhajny Exp $
-
-Fix deprecated Lua call.
-
---- src/lua/lua_task.c.orig	2018-08-01 14:21:24.000000000 +0000
-+++ src/lua/lua_task.c
-@@ -1307,7 +1307,7 @@ lua_task_load_from_string (lua_State * L
- {
- 	struct rspamd_task *task = NULL, **ptask;
- 	const gchar *str_message = luaL_checkstring (L, 1), *err = NULL;
--	gsize message_len = lua_strlen (L, 1);
-+	gsize message_len = lua_rawlen (L, 1);
- 	struct rspamd_config *cfg = NULL;
- 	gboolean res = FALSE;
- 
diff --git a/mail/rspamd/patches/patch-test_rspamd__lua__pcall__vs__resume__test.c b/mail/rspamd/patches/patch-test_rspamd__lua__pcall__vs__resume__test.c
new file mode 100644
index 00000000000..c1050be3712
--- /dev/null
+++ b/mail/rspamd/patches/patch-test_rspamd__lua__pcall__vs__resume__test.c
@@ -0,0 +1,42 @@
+$NetBSD: patch-test_rspamd__lua__pcall__vs__resume__test.c,v 1.1 2018/10/07 20:10:57 fhajny Exp $
+
+Add Lua 5.3 support.
+
+--- test/rspamd_lua_pcall_vs_resume_test.c.orig	2018-09-24 13:53:53.000000000 +0000
++++ test/rspamd_lua_pcall_vs_resume_test.c
+@@ -53,7 +53,11 @@ test_resume(lua_State *L, gint function_
+ 
+ 	for (i = 0; i < N; i ++) {
+ 		lua_rawgeti (L, LUA_REGISTRYINDEX, function_call);
++#if LUA_VERSION_NUM < 503
+ 		lua_resume (L, 0);
++#else
++		lua_resume (L, NULL, 0);
++#endif
+ 		lua_pop (L, 1);
+ 	}
+ 
+@@ -75,7 +79,11 @@ test_resume_get_thread(gint function_cal
+ 		ent = lua_thread_pool_get_for_config (rspamd_main->cfg);
+ 
+ 		lua_rawgeti (ent->lua_state, LUA_REGISTRYINDEX, function_call);
++#if LUA_VERSION_NUM < 503
+ 		lua_resume (ent->lua_state, 0);
++#else
++		lua_resume (ent->lua_state, NULL, 0);
++#endif
+ 		lua_pop (ent->lua_state, 1);
+ 
+ 		lua_thread_pool_return (rspamd_main->cfg->lua_thread_pool, ent);
+@@ -99,7 +107,11 @@ test_resume_get_new_thread(gint function
+ 		ent = lua_thread_pool_get_for_task (rspamd_main->cfg->lua_thread_pool);
+ 
+ 		lua_rawgeti (ent->lua_state, LUA_REGISTRYINDEX, function_call);
++#if LUA_VERSION_NUM < 503
+ 		lua_resume (ent->lua_state, 0);
++#else
++		lua_resume (ent->lua_state, NULL, 0);
++#endif
+ 		lua_pop (ent->lua_state, 1);
+ 
+ 		/* lua_thread_pool_return (rspamd_main->cfg->lua_thread_pool, ent); */