pkgtools/url2pkg: update to 19.3.0

Changes since 2.37:

* Test dependencies in Perl MakeMaker modules are copied to the
  package Makefile.

* Unknown dependencies in Perl MakeMaker modules no longer abort url2pkg
  but instead are added as "# TODO" dependencies to the package Makefile.

* The license from Perl MakeMaker and Python modules is copied to the
  package Makefile.

* The build dependencies, COMMENT and HOMEPAGE from Python setuptools
  modules are copied to the package Makefile.

* No more "undefined $url" when running url2pkg in a directory that
  already has a package Makefile and is extracted.

5 files changed, 162 insertions, 35 deletions

diff --git a/pkgtools/url2pkg/Makefile b/pkgtools/url2pkg/Makefile
index d99c8b6e36e..a31bcd6cca8 100644
--- a/pkgtools/url2pkg/Makefile
+++ b/pkgtools/url2pkg/Makefile
@@ -1,6 +1,6 @@
-# $NetBSD: Makefile,v 1.101 2019/10/01 18:43:46 rillig Exp $
+# $NetBSD: Makefile,v 1.102 2019/10/01 19:41:23 rillig Exp $
 
-PKGNAME=	url2pkg-2.37
+PKGNAME=	url2pkg-19.3.0
 CATEGORIES=	pkgtools
 
 MAINTAINER=	rillig@NetBSD.org
diff --git a/pkgtools/url2pkg/files/MakeMaker.pm b/pkgtools/url2pkg/files/MakeMaker.pm
index 16d8ea64855..46c89010368 100644
--- a/pkgtools/url2pkg/files/MakeMaker.pm
+++ b/pkgtools/url2pkg/files/MakeMaker.pm
@@ -75,14 +75,14 @@ sub url2pkg_find_category($) {
 	return "";
 }
 
-sub url2pkg_write_dependency($$) {
-	my ($dep, $ver) = @_;
+sub url2pkg_write_dependency($$$) {
+	my ($type, $dep, $ver) = @_;
 
 	my $pkgbase = "p5-$dep" =~ s/::/-/gr;
 	my $category = url2pkg_find_category($pkgbase);
 
 	if ($category ne "") {
-		printf("DEPENDS\t%s>=%s:../../%s/%s\n", $pkgbase, $ver, $category, $pkgbase);
+		printf("%s\t%s>=%s:../../%s/%s\n", $type, $pkgbase, $ver, $category, $pkgbase);
 		return;
 	}
 
@@ -90,15 +90,35 @@ sub url2pkg_write_dependency($$) {
 	# that it is a built-in module and no dependency declaration is needed.
 	return if eval("use $dep $ver; 1;");
 
-	die("$0: ERROR: No pkgsrc package found for dependency $dep>=$ver.\n$@\n");
+	printf("%s\t%s>=%s\n", $type, $pkgbase, $ver);
+}
+
+sub url2pkg_write_var($$) {
+	my ($varname, $value) = @_;
+	return unless defined($value) && $value ne "";
+	printf("var\t%s\t%s\n", $varname, $value);
+}
+
+sub url2pkg_write_depends($$) {
+	my ($type, $deps) = @_;
+
+	return unless $deps;
+	foreach my $dep (sort(keys(%$deps))) {
+		url2pkg_write_dependency($type, $dep, $deps->{$dep});
+	}
 }
 
 sub WriteMakefile(%) {
 	my (%options) = @_;
 
-	my $deps = $options{"PREREQ_PM"} || {};
-	foreach my $dep (sort(keys(%$deps))) {
-		url2pkg_write_dependency($dep, $deps->{$dep});
+	url2pkg_write_depends("DEPENDS", $options{"PREREQ_PM"});
+	url2pkg_write_depends("TEST_DEPENDS", $options{"TEST_DEPENDS"});
+
+	my $license = $options{"LICENSE"} || "";
+	if ($license eq "perl") {
+		url2pkg_write_var("LICENSE", "\${PERL5_LICENSE}");
+	} elsif ($license ne "") {
+		url2pkg_write_var("#LICENSE", "# TODO: $license (from Build.PL)")
 	}
 }
 
diff --git a/pkgtools/url2pkg/files/setuptools.py b/pkgtools/url2pkg/files/setuptools.py
index e7a7cc71055..429f6a07dfb 100644
--- a/pkgtools/url2pkg/files/setuptools.py
+++ b/pkgtools/url2pkg/files/setuptools.py
@@ -1,15 +1,52 @@
-# $NetBSD: setuptools.py,v 1.2 2019/09/11 05:25:55 rillig Exp $
+# $NetBSD: setuptools.py,v 1.3 2019/10/01 19:41:23 rillig Exp $
+
+# This is a drop-in replacement for the setuptools Python module. Instead
+# of actually searching for the dependencies, it extracts the dependency
+# information and includes it in the generated pkgsrc package Makefile.
+
+url2pkg_license_mapping = {
+    'Apache 2': 'apache-2.0',
+    'Apache 2.0': 'apache-2.0',
+    'Apache Software License': '', # too unspecific; needs a version number
+    'BSD': '', # too unspecific
+    'GNU Lesser General Public License (LGPL), Version 3': 'gnu-lgpl-v3',
+    'LGPL': 'gnu-lgpl-v2',
+    'MIT': 'mit',
+    'MIT License': 'mit',
+    'PSF': 'python-software-foundation',
+    'PSF license': 'python-software-foundation',
+    'Python Software Foundation License': 'python-software-foundation',
+    'ZPL 2.1': 'zpl-2.1',
+}
 
 def url2pkg_print_depends(keyword, depends):
     for dep in depends:
         print('%s\t%s%s' % (keyword, dep.replace(' ', ''), '' if '>' in dep else '>=0'))
 
+def url2pkg_print_var(varname, value):
+    if value != '':
+        print('var\t%s\t%s' % (varname, value))
+
+def url2pkg_print_license(license):
+    if license == '':
+        return
+    pkgsrc_license = url2pkg_license_mapping.get(license, '')
+    if pkgsrc_license == '':
+        url2pkg_print_var('#LICENSE', '%s # TODO: from setup.py; needs to be adjusted' % license)
+    else:
+        url2pkg_print_var('LICENSE', pkgsrc_license)
+
 def setup(**kwargs):
-    if 'install_requires' in kwargs:
-        url2pkg_print_depends('DEPENDS', kwargs['install_requires'])
+    url2pkg_print_depends('DEPENDS', kwargs.get('install_requires', []))
+    url2pkg_print_depends('TEST_DEPENDS', kwargs.get('tests_require', []))
+    url2pkg_print_depends('BUILD_DEPENDS', kwargs.get('extras_require', {}).get('dev', []))
+    url2pkg_print_depends('BUILD_DEPENDS', kwargs.get('setup_requires', []))
+    url2pkg_print_var('COMMENT', kwargs.get('description', '').rstrip('.'))
+    url2pkg_print_var('HOMEPAGE', kwargs.get('url', ''))
+    url2pkg_print_license(kwargs.get('license', ''))
 
-    if 'tests_require' in kwargs:
-        url2pkg_print_depends('TEST_DEPENDS', kwargs['tests_require'])
+    # TODO: python_requires (see devel/py-futures)
+    #       example: '>=2.6, <3, >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*')
 
 def find_packages(where='.', exclude=(), include=('*',)):
     return []
diff --git a/pkgtools/url2pkg/files/url2pkg.pl b/pkgtools/url2pkg/files/url2pkg.pl
index f7d98d570e5..665931baa2a 100644
--- a/pkgtools/url2pkg/files/url2pkg.pl
+++ b/pkgtools/url2pkg/files/url2pkg.pl
@@ -1,5 +1,5 @@
 #! @PERL5@
-# $NetBSD: url2pkg.pl,v 1.74 2019/10/01 18:43:46 rillig Exp $
+# $NetBSD: url2pkg.pl,v 1.75 2019/10/01 19:41:23 rillig Exp $
 #
 
 # Copyright (c) 2010 The NetBSD Foundation, Inc.
@@ -457,7 +457,7 @@ sub adjust_perl_module_Build_PL() {
 sub adjust_perl_module_Makefile_PL() {
 
 	# To avoid fix_up_makefile error for p5-HTML-Quoted, generate Makefile first.
-	system("cd '$abs_wrksrc' && perl -I. Makefile.PL < /dev/null") == 0 or do {};
+	system("cd '$abs_wrksrc' && perl -I. Makefile.PL < /dev/null 1>&0 2>&0") == 0 or do {};
 
 	read_dependencies("cd '$abs_wrksrc' && perl -I$libdir -I. Makefile.PL", {}, "");
 }
@@ -498,6 +498,8 @@ sub adjust_perl_module($) {
 # Example packages:
 #
 # devel/py-ZopeComponent (dependencies, test dependencies)
+# devel/py-gflags (uses distutils.core instead of setuptools; BSD license)
+# devel/py-gcovr (uses setuptools; BSD license)
 sub adjust_python_module() {
 
 	return unless -f "$abs_wrksrc/setup.py";
@@ -804,6 +806,16 @@ sub adjust_package_from_extracted_distfiles($) {
 	adjust_po();
 	adjust_use_languages();
 
+	generate_adjusted_Makefile_lines($url)->write_to("Makefile");
+
+	if ($regenerate_distinfo) {
+		make("distinfo");
+	}
+}
+
+sub generate_adjusted_Makefile_lines($) {
+	my ($url) = @_;
+
 	my $marker_index = $makefile_lines->index(qr"^# url2pkg-marker");
 	if ($marker_index == -1) {
 		die("$0: ERROR: didn't find the url2pkg marker in the Makefile.\n");
@@ -848,32 +860,28 @@ sub adjust_package_from_extracted_distfiles($) {
 		$lines->set($varname, $update_vars{$varname});
 	}
 
-	$lines->write_to("Makefile");
-
-	if ($regenerate_distinfo) {
-		make("distinfo");
-	}
+	return $lines;
 }
 
 sub main() {
-	my $url;
-
 	if (!-f "../../mk/bsd.pkg.mk") {
 		die("ERROR: $0 must be run from a package directory (.../pkgsrc/category/package).\n");
 	}
 
-	my @extract_cookie = <w*/.extract_done>;
-	if (scalar(@extract_cookie) == 0) {
-		if (scalar(@ARGV) == 0) {
-			print("URL: ");
-			if (!defined($url = <STDIN>)) {
-				print("\n");
-				exit(0);
-			}
-			chomp($url);
-		} else {
-			$url = shift(@ARGV);
+	my $url;
+	if (scalar(@ARGV) == 0) {
+		print("URL: ");
+		if (!defined($url = <STDIN>)) {
+			print("\n");
+			exit(0);
 		}
+		chomp($url);
+	} else {
+		$url = shift(@ARGV);
+	}
+
+	my @extract_cookie = <w*/.extract_done>;
+	if (scalar(@extract_cookie) == 0 || !-f "Makefile") {
 
 		generate_initial_package($url);
 	} else {
diff --git a/pkgtools/url2pkg/files/url2pkg.t b/pkgtools/url2pkg/files/url2pkg.t
index 88b1c8dba29..ed3fa68b733 100644
--- a/pkgtools/url2pkg/files/url2pkg.t
+++ b/pkgtools/url2pkg/files/url2pkg.t
@@ -1,5 +1,5 @@
 # -*- perl -*-
-# $NetBSD: url2pkg.t,v 1.12 2019/09/13 13:31:39 rillig Exp $
+# $NetBSD: url2pkg.t,v 1.13 2019/10/01 19:41:23 rillig Exp $
 
 require "url2pkg.pl";
 
@@ -102,6 +102,14 @@ sub test_Lines_set__previously_with_comment() {
 	is_deeply($lines, ["LICENSE=\t\${PERL5_LICENSE}"]);
 }
 
+sub test_Lines_set__overwrite_comment_with_comment() {
+	my $lines = Lines->new("#LICENSE=\t# TODO: see mk/license.mk");
+
+	$lines->set("#LICENSE", "\${PERL5_LICENSE}");
+
+	is_deeply($lines, ["#LICENSE=\t\${PERL5_LICENSE}"]);
+}
+
 sub test_Lines_set__not_found() {
 	my $lines = Lines->new("OLD_VAR=\told value # old comment");
 
@@ -264,6 +272,60 @@ sub test_read_dependencies() {
 	]);
 }
 
+sub test_generate_adjusted_Makefile_lines() {
+	$main::makefile_lines = Lines->new(
+		"# before 1",
+		"# before 2",
+		"# url2pkg-marker",
+		"# after 1",
+		"# after 2"
+	);
+
+	my $lines = generate_adjusted_Makefile_lines("https://example.org/pkgname-1.0.tar.gz");
+
+	is_deeply($lines, [
+		"# before 1",
+		"# before 2",
+		"# after 1",
+		"# after 2"
+	]);
+}
+
+sub test_generate_adjusted_Makefile_lines__dependencies() {
+	$main::makefile_lines = Lines->new(
+		"# \$NetBSD\$",
+		"",
+		"# url2pkg-marker",
+		".include \"../../mk/bsd.pkg.mk\""
+	);
+	# some dependencies whose directory will not be found
+	add_dependency("DEPENDS", "depends", ">=5.0", "../../devel/depends");
+	add_dependency("TOOL_DEPENDS", "tool-depends", ">=6.0", "../../devel/tool-depends");
+	add_dependency("BUILD_DEPENDS", "build-depends", ">=7.0", "../../devel/build-depends");
+	add_dependency("TEST_DEPENDS", "test-depends", ">=8.0", "../../devel/test-depends");
+	# some dependencies whose directory is explicitly given
+	push(@main::depends, "depends>=11.0:../../devel/depends");
+	push(@main::build_depends, "build-depends>=12.0:../../devel/build-depends");
+	push(@main::test_depends, "test-depends>=13.0:../../devel/test-depends");
+
+	my $lines = generate_adjusted_Makefile_lines("https://example.org/pkgname-1.0.tar.gz");
+
+	is_deeply($lines, [
+		"# \$NetBSD\$",
+		"",
+		"# TODO: dependency TOOL_DEPENDS # TODO: tool-depends>=6.0",
+		"",
+		"BUILD_DEPENDS+=\t# TODO: build-depends>=7.0",
+		"BUILD_DEPENDS+=\tbuild-depends>=12.0:../../devel/build-depends",
+		"DEPENDS+=\t# TODO: depends>=5.0",
+		"DEPENDS+=\tdepends>=11.0:../../devel/depends",
+		"TEST_DEPENDS+=\t# TODO: test-depends>=8.0",
+		"TEST_DEPENDS+=\ttest-depends>=13.0:../../devel/test-depends",
+		"",
+		".include \"../../mk/bsd.pkg.mk\""
+	]);
+}
+
 sub set_up_test() {
 	no warnings 'once';