py-nltk: updated to 3.5

Version 3.5
* add support for Python 3.8
* drop support for Python 2
* create NLTK's own Tokenizer class distinct from the Treebank reference tokeniser
* update Vader sentiment analyser
* fix JSON serialization of some PoS taggers
* minor improvements in grammar.CFG, Vader, pl196x corpus reader, StringTokenizer
* change implementation <= and >= for FreqDist so they are partial orders
* make FreqDist iterable
* correctly handle Penn Treebank trees with a unlabeled branching top node.

3 files changed, 55 insertions, 26 deletions

diff --git a/textproc/py-nltk/Makefile b/textproc/py-nltk/Makefile
index 4e47dce5179..9db320a0bbb 100644
--- a/textproc/py-nltk/Makefile
+++ b/textproc/py-nltk/Makefile
@@ -1,24 +1,34 @@
-# $NetBSD: Makefile,v 1.2 2020/01/26 17:32:17 rillig Exp $
+# $NetBSD: Makefile,v 1.3 2020/08/10 14:43:10 adam Exp $
 
-DISTNAME=	nltk-3.4.1
+DISTNAME=	nltk-3.5
 PKGNAME=	${PYPKGPREFIX}-${DISTNAME}
-CATEGORIES=	textproc
-MASTER_SITES=	${MASTER_SITE_GITHUB:=nltk/}
-GITHUB_PROJECT=	nltk
+CATEGORIES=	textproc python
+MASTER_SITES=	${MASTER_SITE_PYPI:=n/nltk/}
+EXTRACT_SUFX=	.zip
 
 MAINTAINER=	pkgsrc-users@NetBSD.org
 HOMEPAGE=	https://www.nltk.org/
 COMMENT=	Natural Language Toolkit (NLTK)
 LICENSE=	apache-2.0
 
-DEPENDS+=	${PYPKGPREFIX}-sqlite3-[0-9]*:../../databases/py-sqlite3
+DEPENDS+=	${PYPKGPREFIX}-click-[0-9]*:../../devel/py-click
+DEPENDS+=	${PYPKGPREFIX}-expat-[0-9]*:../../textproc/py-expat
+DEPENDS+=	${PYPKGPREFIX}-joblib-[0-9]*:../../devel/py-joblib
 DEPENDS+=	${PYPKGPREFIX}-nose>=1.1.2:../../devel/py-nose
+DEPENDS+=	${PYPKGPREFIX}-regex-[0-9]*:../../textproc/py-regex
+DEPENDS+=	${PYPKGPREFIX}-sqlite3-[0-9]*:../../databases/py-sqlite3
+DEPENDS+=	${PYPKGPREFIX}-tqdm-[0-9]*:../../misc/py-tqdm
 DEPENDS+=	${PYPKGPREFIX}-yaml-[0-9]*:../../textproc/py-yaml
-DEPENDS+=	${PYPKGPREFIX}-expat-[0-9]*:../../textproc/py-expat
 
-REPLACE_PYTHON=			nltk/corpus/reader/knbc.py
-REPLACE_PYTHON+=		nltk/stem/rslp.py
-REPLACE_PYTHON+=		nltk/test/runtests.py
+REPLACE_PYTHON=		nltk/corpus/reader/knbc.py
+REPLACE_PYTHON+=	nltk/stem/rslp.py
+REPLACE_PYTHON+=	nltk/test/runtests.py
+
+PYTHON_VERSIONS_INCOMPATIBLE=	27
+
+post-install:
+	cd ${DESTDIR}${PREFIX}/bin && \
+	${MV} nltk nltk-${PYVERSSUFFIX} || ${TRUE}
 
 .include "../../lang/python/application.mk"
 .include "../../lang/python/egg.mk"
diff --git a/textproc/py-nltk/PLIST b/textproc/py-nltk/PLIST
index 0a773496f1e..2f676c8c830 100644
--- a/textproc/py-nltk/PLIST
+++ b/textproc/py-nltk/PLIST
@@ -1,7 +1,9 @@
-@comment $NetBSD: PLIST,v 1.1 2019/05/28 14:10:04 wiz Exp $
+@comment $NetBSD: PLIST,v 1.2 2020/08/10 14:43:10 adam Exp $
+bin/nltk-${PYVERSSUFFIX}
 ${PYSITELIB}/${EGG_INFODIR}/PKG-INFO
 ${PYSITELIB}/${EGG_INFODIR}/SOURCES.txt
 ${PYSITELIB}/${EGG_INFODIR}/dependency_links.txt
+${PYSITELIB}/${EGG_INFODIR}/entry_points.txt
 ${PYSITELIB}/${EGG_INFODIR}/not-zip-safe
 ${PYSITELIB}/${EGG_INFODIR}/requires.txt
 ${PYSITELIB}/${EGG_INFODIR}/top_level.txt
@@ -141,6 +143,9 @@ ${PYSITELIB}/nltk/classify/util.pyo
 ${PYSITELIB}/nltk/classify/weka.py
 ${PYSITELIB}/nltk/classify/weka.pyc
 ${PYSITELIB}/nltk/classify/weka.pyo
+${PYSITELIB}/nltk/cli.py
+${PYSITELIB}/nltk/cli.pyc
+${PYSITELIB}/nltk/cli.pyo
 ${PYSITELIB}/nltk/cluster/__init__.py
 ${PYSITELIB}/nltk/cluster/__init__.pyc
 ${PYSITELIB}/nltk/cluster/__init__.pyo
@@ -252,6 +257,9 @@ ${PYSITELIB}/nltk/corpus/reader/opinion_lexicon.pyo
 ${PYSITELIB}/nltk/corpus/reader/panlex_lite.py
 ${PYSITELIB}/nltk/corpus/reader/panlex_lite.pyc
 ${PYSITELIB}/nltk/corpus/reader/panlex_lite.pyo
+${PYSITELIB}/nltk/corpus/reader/panlex_swadesh.py
+${PYSITELIB}/nltk/corpus/reader/panlex_swadesh.pyc
+${PYSITELIB}/nltk/corpus/reader/panlex_swadesh.pyo
 ${PYSITELIB}/nltk/corpus/reader/pl196x.py
 ${PYSITELIB}/nltk/corpus/reader/pl196x.pyc
 ${PYSITELIB}/nltk/corpus/reader/pl196x.pyo
@@ -709,10 +717,6 @@ ${PYSITELIB}/nltk/test/classify_fixt.pyc
 ${PYSITELIB}/nltk/test/classify_fixt.pyo
 ${PYSITELIB}/nltk/test/collections.doctest
 ${PYSITELIB}/nltk/test/collocations.doctest
-${PYSITELIB}/nltk/test/compat.doctest
-${PYSITELIB}/nltk/test/compat_fixt.py
-${PYSITELIB}/nltk/test/compat_fixt.pyc
-${PYSITELIB}/nltk/test/compat_fixt.pyo
 ${PYSITELIB}/nltk/test/concordance.doctest
 ${PYSITELIB}/nltk/test/corpus.doctest
 ${PYSITELIB}/nltk/test/corpus_fixt.py
@@ -725,9 +729,6 @@ ${PYSITELIB}/nltk/test/discourse.doctest
 ${PYSITELIB}/nltk/test/discourse_fixt.py
 ${PYSITELIB}/nltk/test/discourse_fixt.pyc
 ${PYSITELIB}/nltk/test/discourse_fixt.pyo
-${PYSITELIB}/nltk/test/doctest_nose_plugin.py
-${PYSITELIB}/nltk/test/doctest_nose_plugin.pyc
-${PYSITELIB}/nltk/test/doctest_nose_plugin.pyo
 ${PYSITELIB}/nltk/test/drt.doctest
 ${PYSITELIB}/nltk/test/featgram.doctest
 ${PYSITELIB}/nltk/test/featstruct.doctest
@@ -815,15 +816,15 @@ ${PYSITELIB}/nltk/test/unit/lm/test_preprocessing.pyo
 ${PYSITELIB}/nltk/test/unit/lm/test_vocabulary.py
 ${PYSITELIB}/nltk/test/unit/lm/test_vocabulary.pyc
 ${PYSITELIB}/nltk/test/unit/lm/test_vocabulary.pyo
-${PYSITELIB}/nltk/test/unit/test_2x_compat.py
-${PYSITELIB}/nltk/test/unit/test_2x_compat.pyc
-${PYSITELIB}/nltk/test/unit/test_2x_compat.pyo
 ${PYSITELIB}/nltk/test/unit/test_aline.py
 ${PYSITELIB}/nltk/test/unit/test_aline.pyc
 ${PYSITELIB}/nltk/test/unit/test_aline.pyo
 ${PYSITELIB}/nltk/test/unit/test_brill.py
 ${PYSITELIB}/nltk/test/unit/test_brill.pyc
 ${PYSITELIB}/nltk/test/unit/test_brill.pyo
+${PYSITELIB}/nltk/test/unit/test_cfd_mutation.py
+${PYSITELIB}/nltk/test/unit/test_cfd_mutation.pyc
+${PYSITELIB}/nltk/test/unit/test_cfd_mutation.pyo
 ${PYSITELIB}/nltk/test/unit/test_cfg2chomsky.py
 ${PYSITELIB}/nltk/test/unit/test_cfg2chomsky.pyc
 ${PYSITELIB}/nltk/test/unit/test_cfg2chomsky.pyo
@@ -854,15 +855,27 @@ ${PYSITELIB}/nltk/test/unit/test_data.pyo
 ${PYSITELIB}/nltk/test/unit/test_disagreement.py
 ${PYSITELIB}/nltk/test/unit/test_disagreement.pyc
 ${PYSITELIB}/nltk/test/unit/test_disagreement.pyo
+${PYSITELIB}/nltk/test/unit/test_freqdist.py
+${PYSITELIB}/nltk/test/unit/test_freqdist.pyc
+${PYSITELIB}/nltk/test/unit/test_freqdist.pyo
 ${PYSITELIB}/nltk/test/unit/test_hmm.py
 ${PYSITELIB}/nltk/test/unit/test_hmm.pyc
 ${PYSITELIB}/nltk/test/unit/test_hmm.pyo
 ${PYSITELIB}/nltk/test/unit/test_json2csv_corpus.py
 ${PYSITELIB}/nltk/test/unit/test_json2csv_corpus.pyc
 ${PYSITELIB}/nltk/test/unit/test_json2csv_corpus.pyo
+${PYSITELIB}/nltk/test/unit/test_json_serialization.py
+${PYSITELIB}/nltk/test/unit/test_json_serialization.pyc
+${PYSITELIB}/nltk/test/unit/test_json_serialization.pyo
 ${PYSITELIB}/nltk/test/unit/test_naivebayes.py
 ${PYSITELIB}/nltk/test/unit/test_naivebayes.pyc
 ${PYSITELIB}/nltk/test/unit/test_naivebayes.pyo
+${PYSITELIB}/nltk/test/unit/test_nombank.py
+${PYSITELIB}/nltk/test/unit/test_nombank.pyc
+${PYSITELIB}/nltk/test/unit/test_nombank.pyo
+${PYSITELIB}/nltk/test/unit/test_pl196x.py
+${PYSITELIB}/nltk/test/unit/test_pl196x.pyc
+${PYSITELIB}/nltk/test/unit/test_pl196x.pyo
 ${PYSITELIB}/nltk/test/unit/test_pos_tag.py
 ${PYSITELIB}/nltk/test/unit/test_pos_tag.pyc
 ${PYSITELIB}/nltk/test/unit/test_pos_tag.pyo
@@ -951,6 +964,9 @@ ${PYSITELIB}/nltk/tokenize/api.pyo
 ${PYSITELIB}/nltk/tokenize/casual.py
 ${PYSITELIB}/nltk/tokenize/casual.pyc
 ${PYSITELIB}/nltk/tokenize/casual.pyo
+${PYSITELIB}/nltk/tokenize/destructive.py
+${PYSITELIB}/nltk/tokenize/destructive.pyc
+${PYSITELIB}/nltk/tokenize/destructive.pyo
 ${PYSITELIB}/nltk/tokenize/mwe.py
 ${PYSITELIB}/nltk/tokenize/mwe.pyc
 ${PYSITELIB}/nltk/tokenize/mwe.pyo
@@ -972,6 +988,9 @@ ${PYSITELIB}/nltk/tokenize/sexpr.pyo
 ${PYSITELIB}/nltk/tokenize/simple.py
 ${PYSITELIB}/nltk/tokenize/simple.pyc
 ${PYSITELIB}/nltk/tokenize/simple.pyo
+${PYSITELIB}/nltk/tokenize/sonority_sequencing.py
+${PYSITELIB}/nltk/tokenize/sonority_sequencing.pyc
+${PYSITELIB}/nltk/tokenize/sonority_sequencing.pyo
 ${PYSITELIB}/nltk/tokenize/stanford.py
 ${PYSITELIB}/nltk/tokenize/stanford.pyc
 ${PYSITELIB}/nltk/tokenize/stanford.pyo
diff --git a/textproc/py-nltk/distinfo b/textproc/py-nltk/distinfo
index 4985de1591b..5917280aa78 100644
--- a/textproc/py-nltk/distinfo
+++ b/textproc/py-nltk/distinfo
@@ -1,6 +1,6 @@
-$NetBSD: distinfo,v 1.1 2019/05/28 14:10:04 wiz Exp $
+$NetBSD: distinfo,v 1.2 2020/08/10 14:43:10 adam Exp $
 
-SHA1 (nltk-3.4.1.tar.gz) = fbb5cabe8cf9a17111307a2e66d69534f0dc6de9
-RMD160 (nltk-3.4.1.tar.gz) = 18e7920a9f66395c71228eb7da73f236ea83cb34
-SHA512 (nltk-3.4.1.tar.gz) = 738718cb21bae385528cc171d5949ffe85f3c35ae64949838064e889db640a46ce03ca5c2689f74236c44869fd81dcbf23d908f869c0d38968beb34e49949eb4
-Size (nltk-3.4.1.tar.gz) = 2814434 bytes
+SHA1 (nltk-3.5.zip) = 8cacfc7a0f98c1a391e256483999651da8736213
+RMD160 (nltk-3.5.zip) = e9c69bef8b27bb6e60f18a1184bc1a5f68db05c6
+SHA512 (nltk-3.5.zip) = 93cb2c031bbb6d2f20a3bc739f6a2760b9073483834506b5b828f4562fd41435f30339b998d158ac90792b7908d60ebff31e9ba49b4ed4e8c38bdb0b753fc70a
+Size (nltk-3.5.zip) = 1433531 bytes