pkgtools/url2pkg: update to 19.3.4

Changes since 19.3.3:

The code that handles license names is the same for Python and Perl
modules.

Python packages that use distutils.extension can be handled. An example
package is devel/py-pysha3.

The DESCR file is filled from the package's README file.

The PLIST is generated with some comments providing further assistance.

7 files changed, 87 insertions, 25 deletions

diff --git a/pkgtools/url2pkg/Makefile b/pkgtools/url2pkg/Makefile
index 346a1a4ce4b..51083af7473 100644
--- a/pkgtools/url2pkg/Makefile
+++ b/pkgtools/url2pkg/Makefile
@@ -1,6 +1,6 @@
-# $NetBSD: Makefile,v 1.106 2019/10/06 12:55:30 rillig Exp $
+# $NetBSD: Makefile,v 1.107 2019/10/27 19:19:54 rillig Exp $
 
-PKGNAME=	url2pkg-19.3.3
+PKGNAME=	url2pkg-19.3.4
 CATEGORIES=	pkgtools
 
 MAINTAINER=	rillig@NetBSD.org
@@ -41,6 +41,7 @@ do-install:
 	${INSTALL_DATA} ${WRKSRC}/setuptools.py ${DESTDIR}${PREFIX}/lib/url2pkg/
 	${INSTALL_DATA} /dev/null ${DESTDIR}${PREFIX}/lib/url2pkg/distutils/__init__.py
 	${INSTALL_DATA} ${WRKSRC}/setuptools.py ${DESTDIR}${PREFIX}/lib/url2pkg/distutils/core.py
+	${INSTALL_DATA} ${WRKSRC}/setuptools.py ${DESTDIR}${PREFIX}/lib/url2pkg/distutils/extension.py
 
 .include "../../lang/python/application.mk"
 .include "../../mk/bsd.pkg.mk"
diff --git a/pkgtools/url2pkg/PLIST b/pkgtools/url2pkg/PLIST
index 8fa88d6613c..66cb3d20c07 100644
--- a/pkgtools/url2pkg/PLIST
+++ b/pkgtools/url2pkg/PLIST
@@ -1,8 +1,9 @@
-@comment $NetBSD: PLIST,v 1.8 2019/10/06 12:55:30 rillig Exp $
+@comment $NetBSD: PLIST,v 1.9 2019/10/27 19:19:54 rillig Exp $
 bin/url2pkg
 lib/url2pkg/ExtUtils/MakeMaker.pm
 lib/url2pkg/Module/Build.pm
 lib/url2pkg/distutils/__init__.py
 lib/url2pkg/distutils/core.py
+lib/url2pkg/distutils/extension.py
 lib/url2pkg/setuptools.py
 man/man8/url2pkg.8
diff --git a/pkgtools/url2pkg/files/Build.pm b/pkgtools/url2pkg/files/Build.pm
index 50c9838ece8..400c910d502 100644
--- a/pkgtools/url2pkg/files/Build.pm
+++ b/pkgtools/url2pkg/files/Build.pm
@@ -56,6 +56,11 @@ sub url2pkg_write_var($$$) {
 	printf("var\t%s\t%s\n", $varname, $value);
 }
 
+sub url2pkg_write_cmd($$) {
+	my ($self, $cmd, $arg) = @_;
+	printf("cmd\t%s\t%s\n", $cmd, $arg);
+}
+
 sub VERSION($$) {
 	my ($class, $min_version) = @_;
 	return $min_version <= $VERSION;
@@ -79,10 +84,9 @@ sub create_build_script($) {
 
 	$self->url2pkg_write_var("COMMENT", $self->{"dist_abstract"});
 	my $license = $self->{"license"} || "";
-	if ($license eq "perl") {
-		$self->url2pkg_write_var("LICENSE", "\${PERL5_LICENSE}");
-	} elsif ($license ne "") {
-		$self->url2pkg_write_var("#LICENSE", "# TODO: $license (from Build.PL)")
+	if ($license ne "") {
+		$self->url2pkg_write_cmd("license", $license);
+		$self->url2pkg_write_var("license_default", "# TODO: $license (from Build.PL)")
 	}
 }
 
diff --git a/pkgtools/url2pkg/files/MakeMaker.pm b/pkgtools/url2pkg/files/MakeMaker.pm
index 37b0b50e437..849b48e43e1 100644
--- a/pkgtools/url2pkg/files/MakeMaker.pm
+++ b/pkgtools/url2pkg/files/MakeMaker.pm
@@ -99,10 +99,15 @@ sub url2pkg_write_var($$) {
 	printf("var\t%s\t%s\n", $varname, $value);
 }
 
+sub url2pkg_write_cmd($$) {
+	my ($cmd, $arg) = @_;
+	printf("cmd\t%s\t%s\n", $cmd, $arg);
+}
+
 sub url2pkg_write_depends($$) {
 	my ($type, $deps) = @_;
 
-	return unless $deps;
+	return unless defined $deps;
 	foreach my $dep (sort(keys(%$deps))) {
 		url2pkg_write_dependency($type, $dep, $deps->{$dep});
 	}
@@ -115,10 +120,9 @@ sub WriteMakefile(%) {
 	url2pkg_write_depends("TEST_DEPENDS", $options{"TEST_DEPENDS"});
 
 	my $license = $options{"LICENSE"} || "";
-	if ($license eq "perl") {
-		url2pkg_write_var("LICENSE", "\${PERL5_LICENSE}");
-	} elsif ($license ne "") {
-		url2pkg_write_var("#LICENSE", "# TODO: $license (from Makefile.PL)")
+	if ($license ne "") {
+		url2pkg_write_cmd("license", $license);
+		url2pkg_write_cmd("license_default", "# TODO: $license (from Makefile.PL)");
 	}
 }
 
diff --git a/pkgtools/url2pkg/files/setuptools.py b/pkgtools/url2pkg/files/setuptools.py
index 12c5a18f30a..a24685ce7c1 100644
--- a/pkgtools/url2pkg/files/setuptools.py
+++ b/pkgtools/url2pkg/files/setuptools.py
@@ -1,4 +1,4 @@
-# $NetBSD: setuptools.py,v 1.6 2019/10/27 13:15:04 rillig Exp $
+# $NetBSD: setuptools.py,v 1.7 2019/10/27 19:19:55 rillig Exp $
 
 # This is a drop-in replacement for the setuptools Python module. Instead
 # of actually searching for the dependencies, it extracts the dependency
@@ -44,3 +44,14 @@ def setup(**kwargs):
 
 def find_packages(where='.', exclude=(), include=('*',)):
     return []
+
+
+# for distutils.core; see devel/py-pysha3-1.0.2 for an example
+class Command:
+    pass
+
+
+# for distutils.extension; see devel/py-pysha3-1.0.2 for an example
+class Extension:
+    def __init__(self, *args, **kwargs):
+        pass
diff --git a/pkgtools/url2pkg/files/url2pkg.py b/pkgtools/url2pkg/files/url2pkg.py
index 5706e994764..041a2a6f3c1 100644
--- a/pkgtools/url2pkg/files/url2pkg.py
+++ b/pkgtools/url2pkg/files/url2pkg.py
@@ -1,5 +1,5 @@
 #! @PYTHONBIN@
-# $NetBSD: url2pkg.py,v 1.23 2019/10/27 13:15:04 rillig Exp $
+# $NetBSD: url2pkg.py,v 1.24 2019/10/27 19:19:55 rillig Exp $
 
 # Copyright (c) 2019 The NetBSD Foundation, Inc.
 # All rights reserved.
@@ -122,6 +122,7 @@ class Globals:
              'BSD'),  # too unspecific, may be 2-clause, 3-clause, 4-clause
             ('${PERL5_LICENSE}', 'perl'),
             ('apache-2.0', 'Apache 2', 'Apache 2.0'),
+            ('artistic-2.0', 'artistic_2'),
             ('gnu-gpl-v3', 'GNU Lesser General Public License (LGPL), Version 3'),
             ('gnu-lgpl-v2', 'LGPL'),
             ('mit', 'MIT', 'MIT License'),
@@ -137,10 +138,11 @@ class Globals:
             return license_name
         return ''
 
+
 class Lines:
     """
-    A list of Makefile lines with high-level methods for manipulating
-    variable assignments.
+    A list of lines (typically from a Makefile, but other file types work as
+    well) with high-level methods for manipulating variable assignments.
     """
     lines: List[str]
 
@@ -483,7 +485,6 @@ class Generator:
     def generate_package(self, g: Globals) -> Lines:
         pkgdir = g.pkgdir
         makefile = pkgdir / 'Makefile'
-        descr = pkgdir / 'DESCR'
         plist = pkgdir / 'PLIST'
 
         initial_lines = self.generate_Makefile()
@@ -493,8 +494,14 @@ class Generator:
         except OSError:
             pass
         initial_lines.write_to(makefile)
-        plist.is_file() or Lines('@comment $''NetBSD$').write_to(plist)
-        descr.is_file() or Lines().write_to(descr)
+
+        plist_lines = [
+            f'@comment $''NetBSD$',
+            f'@comment TODO: to fill this file with the file listing:',
+            f'@comment TODO: 1. run "{g.make} package"',
+            f'@comment TODO: 2. run "{g.make} print-PLIST"'
+        ]
+        plist.is_file() or Lines(*plist_lines).write_to(plist)
 
         subprocess.check_call([g.editor, makefile])
 
@@ -570,6 +577,8 @@ class Adjuster:
 
     regenerate_distinfo: bool
 
+    descr_lines: List[str]
+
     def __init__(self, g: Globals, url: str, initial_lines: Lines):
         self.g = g
         self.url = url
@@ -591,6 +600,7 @@ class Adjuster:
         self.pkgname_transform = ''
         self.makefile_lines = Lines()
         self.regenerate_distinfo = False
+        self.descr_lines = []
 
     def add_dependency(self, kind: str, pkgbase: str, constraint: str, dep_dir: str) -> None:
         """ add_dependency('DEPENDS', 'package', '>=1', '../../category/package') """
@@ -698,6 +708,13 @@ class Adjuster:
     def wrksrc_open(self, relative_pathname: str):
         return (self.abs_wrksrc / relative_pathname).open()
 
+    def wrksrc_head(self, relative_pathname: str, n: int):
+        try:
+            with (self.abs_wrksrc / relative_pathname).open(encoding="UTF-8") as f:
+                return f.read().splitlines()[:n]
+        except IOError:
+            return []
+
     def wrksrc_find(self, what: Union[str, Callable[[str], bool]]) -> List[str]:
         def search(f):
             return re.search(what, f) if type(what) == str else what(f)
@@ -721,6 +738,22 @@ class Adjuster:
     def wrksrc_isfile(self, relative_pathname: str) -> bool:
         return (self.abs_wrksrc / relative_pathname).is_file()
 
+    def adjust_descr(self):
+        for filename in ('README', 'README.txt', 'README.md'):
+            lines = self.wrksrc_head(filename, 21)
+            if len(lines) == 21:
+                lines[-1] = '...'
+            if lines:
+                self.descr_lines = [
+                    f'TODO: Adjust the following lines from {filename}',
+                    '',
+                    *lines]
+                return
+
+        self.descr_lines = [
+            'TODO: Fill in a short description of the package.',
+            'TODO: It should be between 3 and 20 lines.']
+
     def adjust_configure(self):
         if not self.wrksrc_isfile('configure'):
             return
@@ -765,7 +798,7 @@ class Adjuster:
 
     def adjust_perl_module_Makefile_PL(self):
         # Example packages:
-        # devel/p5-Algorithm-Diff (no dependencies)
+        # devel/p5-Algorithm-Diff (no dependencies, no license)
         # devel/p5-Carp-Assert-More (dependencies without version numbers)
         # www/p5-HTML-Quoted (dependency with version number)
 
@@ -990,6 +1023,7 @@ class Adjuster:
         self.wrksrc_dirs = scan(self.abs_wrksrc, Path.is_dir)
         self.wrksrc_files = scan(self.abs_wrksrc, Path.is_file)
 
+        self.adjust_descr()
         self.adjust_configure()
         self.adjust_cmake()
         self.adjust_meson()
@@ -1003,6 +1037,8 @@ class Adjuster:
         self.adjust_use_languages()
 
         self.generate_lines().write_to(self.g.pkgdir / 'Makefile')
+        descr = (self.g.pkgdir / 'DESCR')
+        descr.is_file() or Lines(*self.descr_lines).write_to(descr)
 
         if self.regenerate_distinfo:
             self.g.bmake('distinfo')
diff --git a/pkgtools/url2pkg/files/url2pkg_test.py b/pkgtools/url2pkg/files/url2pkg_test.py
index 0478bc05072..115ca02ec1e 100644
--- a/pkgtools/url2pkg/files/url2pkg_test.py
+++ b/pkgtools/url2pkg/files/url2pkg_test.py
@@ -1,4 +1,4 @@
-# $NetBSD: url2pkg_test.py,v 1.22 2019/10/27 13:15:04 rillig Exp $
+# $NetBSD: url2pkg_test.py,v 1.23 2019/10/27 19:19:55 rillig Exp $
 
 import pytest
 from url2pkg import *
@@ -99,7 +99,7 @@ def test_Global_pkgsrc_license():
     assert g.pkgsrc_license('Apache 2') == 'apache-2.0'
 
     # Not explicitly in the list, looked up from PKGSRCDIR.
-    assert g.pkgsrc_license('artistic-2.0') == 'artistic-2.0'
+    assert g.pkgsrc_license('skype21-license') == 'skype21-license'
 
     # Neither in the list nor in PKGSRCDIR/licenses.
     assert g.pkgsrc_license('unknown') == ''
@@ -647,12 +647,17 @@ def test_Generator_generate_package(tmp_path: Path):
 
     Generator(url).generate_package(g)
 
-    assert (tmp_path / 'DESCR').read_text() == ''
+    assert not (tmp_path / 'DESCR').is_file()  # is created later
     assert len((tmp_path / 'Makefile').read_text().splitlines()) == 13
-    assert (tmp_path / 'PLIST').read_text() == '@comment $''NetBSD$\n'
+    assert (tmp_path / 'PLIST').read_text().splitlines() == [
+        '@comment $''NetBSD$',
+        '@comment TODO: to fill this file with the file listing:',
+        '@comment TODO: 1. run "true package"',
+        '@comment TODO: 2. run "true print-PLIST"',
+    ]
 
     # Since bmake is only fake in this test, the distinfo file is not created.
-    expected_files = ['DESCR', 'Makefile', 'PLIST']
+    expected_files = ['Makefile', 'PLIST']
     assert sorted([f.name for f in tmp_path.glob("*")]) == expected_files