pkgtools/url2pkg: update to 2.34

Changes since 2.33:

* code cleanup in MakeMaker.pm
* don't create a PLIST for Perl packages

* allow Python dependencies from setup.py to use spaces around >=
* properly indented Python code according to PEP 8
* added mock for setuptools.find_package
* only try to migrate a package from GitHub to PyPI if it is really
  a Python package on GitHub

4 files changed, 56 insertions, 46 deletions

diff --git a/pkgtools/url2pkg/Makefile b/pkgtools/url2pkg/Makefile
index a232cc803c8..4c74304b366 100644
--- a/pkgtools/url2pkg/Makefile
+++ b/pkgtools/url2pkg/Makefile
@@ -1,6 +1,6 @@
-# $NetBSD: Makefile,v 1.96 2019/09/09 08:08:02 maya Exp $
+# $NetBSD: Makefile,v 1.97 2019/09/11 05:25:55 rillig Exp $
 
-PKGNAME=	url2pkg-2.33
+PKGNAME=	url2pkg-2.34
 CATEGORIES=	pkgtools
 
 MAINTAINER=	pkgsrc-users@NetBSD.org
@@ -29,11 +29,7 @@ SUBST_MESSAGE.up=	Replacing variable placeholders
 SUBST_FILES.up=		url2pkg.pl MakeMaker.pm
 SUBST_VARS.up=		MAKE PERL5 PYTHONBIN
 SUBST_SED.up+=		-e 's,@LIBDIR@,${PREFIX}/lib/url2pkg,g'
-.if defined(BATCH)
-SUBST_SED.up+=		-e 's,@PKGSRCDIR@,/usr/pkgsrc,g'
-.else
-SUBST_VARS.up+=		PKGSRCDIR
-.endif
+SUBST_SED.up+=		-e 's,@PKGSRCDIR@,${BATCH:D/usr/pkgsrc:U${PKGSRCDIR}},g'
 
 do-install:
 	${INSTALL_SCRIPT} ${WRKSRC}/url2pkg.pl ${DESTDIR}${PREFIX}/bin/url2pkg
diff --git a/pkgtools/url2pkg/files/MakeMaker.pm b/pkgtools/url2pkg/files/MakeMaker.pm
index 3f8c8d03976..7b9610ae997 100644
--- a/pkgtools/url2pkg/files/MakeMaker.pm
+++ b/pkgtools/url2pkg/files/MakeMaker.pm
@@ -39,7 +39,7 @@ require 5.013002;
 use strict;
 use warnings;
 
-use constant conf_pkgsrcdir	=> '@PKGSRCDIR@';
+my $url2pkg_pkgsrcdir = '@PKGSRCDIR@';
 
 BEGIN {
 	use Exporter;
@@ -55,51 +55,50 @@ our @EXPORT	= qw(&WriteMakefile &prompt $Verbose $version);
 our @EXPORT_OK	= qw(&neatvalue &_sprintf562);
 
 # Finds and returns the category a given package lies in.
-# If the package does not exist, C<undef> is returned.
+# If the package does not exist, an empty string is returned.
 # If the package exists more than once, it is unspecified which
 # of the categories is returned.
-sub find_category($) {
+sub url2pkg_find_category($) {
 	my ($pkg) = @_;
-	my ($retval, $pkgsrcdir);
 
-	opendir(D, conf_pkgsrcdir) or die;
-	foreach my $cat (readdir(D)) {
-		next if ($cat =~ qr"^\.");
+	opendir(D, $url2pkg_pkgsrcdir) or die;
+	my @categories = readdir(D);
+	closedir(D) or die;
+
+	foreach my $cat (@categories) {
+		next if $cat =~ qr"^\.";
 
-		if (-f (conf_pkgsrcdir."/${cat}/${pkg}/Makefile")) {
-			$retval = $cat;
+		if (-f "$url2pkg_pkgsrcdir/$cat/$pkg/Makefile") {
+			return $cat;
 		}
 	}
-	closedir(D) or die;
-	return $retval;
+	return "";
 }
 
-sub writeDependency($$) {
+sub url2pkg_write_dependency($$) {
 	my ($dep, $ver) = @_;
 
-	my $pkgbase = "p5-" . ($dep =~ s/::/-/gr);
-	my $category = find_category($pkgbase);
+	my $pkgbase = "p5-$dep" =~ s/::/-/gr;
+	my $category = url2pkg_find_category($pkgbase);
 
-	if (defined($category)) {
+	if ($category ne "") {
 		printf("DEPENDS\t%s>=%s:../../%s/%s\n", $pkgbase, $ver, $category, $pkgbase);
 		return;
 	}
 
-	# If the package does not exist but the Perl module can be
-	# loaded, assume that no extra dependency is needed.
+	# If the package does not exist but the Perl module can be loaded, assume
+	# that it is a built-in module and no dependency declaration is needed.
 	return if eval("use $dep $ver; 1;");
 
-	die("$0: ERROR: No pkgsrc package found for dependency ${dep}>=${ver}.\n$@\n");
+	die("$0: ERROR: No pkgsrc package found for dependency $dep>=$ver.\n$@\n");
 }
 
 sub WriteMakefile(%) {
 	my (%options) = @_;
 
-	if (exists($options{"PREREQ_PM"})) {
-		my $deps = $options{"PREREQ_PM"};
-		foreach my $dep (sort(keys(%{$deps}))) {
-			writeDependency($dep, $deps->{$dep});
-		}
+	my $deps = $options{"PREREQ_PM"} || {};
+	foreach my $dep (sort(keys(%$deps))) {
+		url2pkg_write_dependency($dep, $deps->{$dep});
 	}
 }
 
diff --git a/pkgtools/url2pkg/files/setuptools.py b/pkgtools/url2pkg/files/setuptools.py
index 489adc036a9..e7a7cc71055 100644
--- a/pkgtools/url2pkg/files/setuptools.py
+++ b/pkgtools/url2pkg/files/setuptools.py
@@ -1,15 +1,15 @@
-# $NetBSD: setuptools.py,v 1.1 2019/08/18 11:26:33 rillig Exp $
+# $NetBSD: setuptools.py,v 1.2 2019/09/11 05:25:55 rillig Exp $
 
 def url2pkg_print_depends(keyword, depends):
-	for dep in depends:
-		if '>' not in dep:
-			dep = dep + '>=0'
-		print('%s\t%s' % (keyword, dep))
+    for dep in depends:
+        print('%s\t%s%s' % (keyword, dep.replace(' ', ''), '' if '>' in dep else '>=0'))
 
 def setup(**kwargs):
+    if 'install_requires' in kwargs:
+        url2pkg_print_depends('DEPENDS', kwargs['install_requires'])
 
-	if 'install_requires' in kwargs:
-		url2pkg_print_depends('DEPENDS', kwargs['install_requires'])
+    if 'tests_require' in kwargs:
+        url2pkg_print_depends('TEST_DEPENDS', kwargs['tests_require'])
 
-	if 'tests_require' in kwargs:
-		url2pkg_print_depends('TEST_DEPENDS', kwargs['tests_require'])
+def find_packages(where='.', exclude=(), include=('*',)):
+    return []
diff --git a/pkgtools/url2pkg/files/url2pkg.pl b/pkgtools/url2pkg/files/url2pkg.pl
index b09d1c0180f..bffb2751a78 100644
--- a/pkgtools/url2pkg/files/url2pkg.pl
+++ b/pkgtools/url2pkg/files/url2pkg.pl
@@ -1,5 +1,5 @@
 #! @PERL5@
-# $NetBSD: url2pkg.pl,v 1.64 2019/09/09 08:08:02 maya Exp $
+# $NetBSD: url2pkg.pl,v 1.65 2019/09/11 05:25:55 rillig Exp $
 #
 
 # Copyright (c) 2010 The NetBSD Foundation, Inc.
@@ -353,14 +353,23 @@ sub adjust_libtool() {
 sub adjust_perl_module() {
 
 	if (-f "$abs_wrksrc/Build.PL") {
+		# Example packages:
+		# devel/p5-Algorithm-CheckDigits
 
 		# It's a Module::Build module. Dependencies cannot yet be
 		# extracted automatically.
+		#
+		# TODO: Implement this similarly to the Makefile.PL mock below.
+
 		push(@todos, "Look for the dependencies in Build.PL.");
 
 		push(@build_vars, var("PERL5_MODULE_TYPE", "=", "Module::Build"));
 
 	} elsif (-f "$abs_wrksrc/Makefile.PL") {
+		# Example packages:
+		# devel/p5-Algorithm-Diff (no dependencies)
+		# devel/p5-Carp-Assert-More (dependencies without version numbers)
+		# www/p5-HTML-Quoted (dependency with version number)
 
 		# To avoid fix_up_makefile error for p5-HTML-Quoted, generate Makefile first.
 		system("cd '$abs_wrksrc' && perl -I. Makefile.PL < /dev/null") or do {};
@@ -384,8 +393,12 @@ sub adjust_perl_module() {
 	push(@includes, "../../lang/perl5/module.mk");
 	$pkgname = "p5-\${DISTNAME}";
 	push(@categories, "perl5");
+	unlink("PLIST") or do {};
 }
 
+# Example packages:
+#
+# devel/py-ZopeComponent (dependencies, test dependencies)
 sub adjust_python_module() {
 
 	return unless -f "$abs_wrksrc/setup.py";
@@ -626,12 +639,6 @@ sub adjust_lines_python_module($$) {
 	my @initial_lines = generate_initial_package_Makefile_lines($url);
 	my @current_lines = read_lines("Makefile");
 
-	# don't risk to overwrite any changes made by the package developer.
-	if (join('\n', @current_lines) ne join('\n', @initial_lines)) {
-		splice(@$lines, -2, 0, "# TODO: Migrate MASTER_SITES to PYPI");
-		return;
-	}
-
 	my %old;
 	foreach my $line (@initial_lines) {
 		if ($line =~ qr"^(\w+)(\+?=)([ \t]+)([^#\\]*?)(\s*)(#.*|)$") {
@@ -643,10 +650,18 @@ sub adjust_lines_python_module($$) {
 		}
 	}
 
+	return unless $old{"CATEGORIES"} =~ qr"python";
 	my $pkgbase = $old{"GITHUB_PROJECT"};
+	return unless defined($pkgbase);
 	my $pkgbase1 = substr($pkgbase, 0, 1);
 	my $pkgversion_norev = $old{"DISTNAME"} =~ s/^v//r;
 
+	# don't risk to overwrite any changes made by the package developer.
+	if (join('\n', @current_lines) ne join('\n', @initial_lines)) {
+		splice(@$lines, -2, 0, "# TODO: Migrate MASTER_SITES to PYPI");
+		return;
+	}
+
 	my @lines = @$lines;
 	if (lines_remove(\@lines, "GITHUB_PROJECT")
 		&& lines_set(\@lines, "DISTNAME", "$pkgbase-$pkgversion_norev")