url2pkg: split support files for Python distutils and setuptools

When url2pkg determines the metadata of a Python module, it had merged
the two implementations for Python's distutils and setuptools, which
made the code hard to understand.  Split them again.

Extend the support for setuptools, so that url2pkg correctly handles
wip/pytorch; it still fails because pytorch does some Git stuff from
inside setup.py, but that may be fixed later.

Bump version to 22.2.0.

16 files changed, 185 insertions, 76 deletions

diff --git a/pkgtools/url2pkg/Makefile b/pkgtools/url2pkg/Makefile
index 8bbae29d5cf..f5521cf5a78 100644
--- a/pkgtools/url2pkg/Makefile
+++ b/pkgtools/url2pkg/Makefile
@@ -1,7 +1,6 @@
-# $NetBSD: Makefile,v 1.126 2022/06/30 11:18:45 nia Exp $
+# $NetBSD: Makefile,v 1.127 2022/07/16 08:59:38 rillig Exp $
 
-PKGNAME=	url2pkg-21.4.2
-PKGREVISION=	2
+PKGNAME=	url2pkg-22.2.0
 CATEGORIES=	pkgtools
 
 MAINTAINER=	rillig@NetBSD.org
@@ -13,20 +12,18 @@ WRKSRC=		${WRKDIR}
 NO_CHECKSUM=	yes
 NO_BUILD=	yes
 USE_LANGUAGES=	# none
-USE_TOOLS+=	perl:run
+USE_TOOLS+=	perl:run pax
 AUTO_MKDIRS=	yes
 
 PYTHON_VERSIONED_DEPENDENCIES=	test:test
 
 do-extract:
-	${RUN} cd ${FILESDIR} && cp *.* ${WRKSRC}/
+	${RUN} cd ${FILESDIR} && pax -rw *.* python/*.py python/*/*.py python/*/*/*.py ${WRKSRC}/
 
 do-test:
 	${RUN} cd ${WRKSRC} && env PKGSRCDIR=${PKGSRCDIR} MAKE=${MAKE:Q} \
 		${PREFIX}/bin/pytest-${PYVERSSUFFIX} ${PY_TEST_FLAGS:U}
 
-.include "../../mk/bsd.prefs.mk"
-
 SUBST_CLASSES+=		up
 SUBST_STAGE.up=		do-configure
 SUBST_MESSAGE.up=	Replacing variable placeholders
@@ -39,10 +36,9 @@ do-install:
 	${INSTALL_MAN} ${FILESDIR}/url2pkg.8 ${DESTDIR}${PREFIX}/${PKGMANDIR}/man8
 	${INSTALL_DATA} ${WRKSRC}/Build.pm ${DESTDIR}${PREFIX}/lib/url2pkg/Module/
 	${INSTALL_DATA} ${WRKSRC}/MakeMaker.pm ${DESTDIR}${PREFIX}/lib/url2pkg/ExtUtils/
-	${INSTALL_DATA} ${WRKSRC}/setuptools.py ${DESTDIR}${PREFIX}/lib/url2pkg/
-	${INSTALL_DATA} /dev/null ${DESTDIR}${PREFIX}/lib/url2pkg/distutils/__init__.py
-	${INSTALL_DATA} ${WRKSRC}/setuptools.py ${DESTDIR}${PREFIX}/lib/url2pkg/distutils/core.py
-	${INSTALL_DATA} ${WRKSRC}/setuptools.py ${DESTDIR}${PREFIX}/lib/url2pkg/distutils/extension.py
+	${RUN} cd ${WRKSRC}; for f in $$(find python -name '*.py' -print); do \
+		${INSTALL_DATA} "$$f" ${DESTDIR}${PREFIX}/lib/url2pkg/"$$f"; \
+	done
 
 .include "../../lang/python/application.mk"
 .include "../../lang/python/versioned_dependencies.mk"
diff --git a/pkgtools/url2pkg/PLIST b/pkgtools/url2pkg/PLIST
index 66cb3d20c07..56752139cdf 100644
--- a/pkgtools/url2pkg/PLIST
+++ b/pkgtools/url2pkg/PLIST
@@ -1,9 +1,16 @@
-@comment $NetBSD: PLIST,v 1.9 2019/10/27 19:19:54 rillig Exp $
+@comment $NetBSD: PLIST,v 1.10 2022/07/16 08:59:38 rillig Exp $
 bin/url2pkg
 lib/url2pkg/ExtUtils/MakeMaker.pm
 lib/url2pkg/Module/Build.pm
-lib/url2pkg/distutils/__init__.py
-lib/url2pkg/distutils/core.py
-lib/url2pkg/distutils/extension.py
-lib/url2pkg/setuptools.py
+lib/url2pkg/python/distutils/__init__.py
+lib/url2pkg/python/distutils/core.py
+lib/url2pkg/python/distutils/extension.py
+lib/url2pkg/python/distutils/version.py
+lib/url2pkg/python/setuptools/__init__.py
+lib/url2pkg/python/setuptools/command/__init__.py
+lib/url2pkg/python/setuptools/command/build_ext.py
+lib/url2pkg/python/setuptools/command/install.py
+lib/url2pkg/python/setuptools/command/sdist.py
+lib/url2pkg/python/setuptools/dist.py
+lib/url2pkg/python/url2pkg.py
 man/man8/url2pkg.8
diff --git a/pkgtools/url2pkg/files/python/distutils/__init__.py b/pkgtools/url2pkg/files/python/distutils/__init__.py
new file mode 100644
index 00000000000..0a3f1933fdf
--- /dev/null
+++ b/pkgtools/url2pkg/files/python/distutils/__init__.py
@@ -0,0 +1,9 @@
+# $NetBSD: __init__.py,v 1.1 2022/07/16 08:59:39 rillig Exp $

+#

+# This is a drop-in replacement for the distutils Python module. Instead

+# of actually searching for the dependencies, it extracts the dependency

+# information and includes it in the generated pkgsrc package Makefile.

+#

+# https://docs.python.org/3/distutils/

+# As of Python 3.10, distutils is deprecated.

+# The successor is setuptools.

diff --git a/pkgtools/url2pkg/files/python/distutils/core.py b/pkgtools/url2pkg/files/python/distutils/core.py
new file mode 100644
index 00000000000..e14b1138b28
--- /dev/null
+++ b/pkgtools/url2pkg/files/python/distutils/core.py
@@ -0,0 +1,18 @@
+# $NetBSD: core.py,v 1.1 2022/07/16 08:59:39 rillig Exp $

+#

+# Fake implementation of distutils.core.

+#

+# https://docs.python.org/3/distutils/apiref.html#module-distutils.core

+

+import url2pkg

+

+

+# used by pkgsrc package devel/py-pysha3 1.0.2

+def setup(**kwargs):

+    url2pkg.setup(**kwargs)

+

+

+# Originally from distutils.cmd.

+# used by pkgsrc package devel/py-pysha3 1.0.2

+class Command:

+    pass

diff --git a/pkgtools/url2pkg/files/python/distutils/extension.py b/pkgtools/url2pkg/files/python/distutils/extension.py
new file mode 100644
index 00000000000..14418e2bfca
--- /dev/null
+++ b/pkgtools/url2pkg/files/python/distutils/extension.py
@@ -0,0 +1,10 @@
+# $NetBSD: extension.py,v 1.1 2022/07/16 08:59:39 rillig Exp $

+#

+# Fake implementation of distutils.extension

+#

+# https://docs.python.org/3/distutils/apiref.html#module-distutils.extension

+

+# used by pkgsrc package devel/py-pysha3 1.0.2

+class Extension:

+    def __init__(self, *args, **kwargs):

+        pass

diff --git a/pkgtools/url2pkg/files/python/distutils/version.py b/pkgtools/url2pkg/files/python/distutils/version.py
new file mode 100644
index 00000000000..0ffe35a2fd6
--- /dev/null
+++ b/pkgtools/url2pkg/files/python/distutils/version.py
@@ -0,0 +1,9 @@
+# $NetBSD: version.py,v 1.1 2022/07/16 08:59:39 rillig Exp $

+#

+# Fake implementation of distutils.version.

+#

+# https://docs.python.org/3/distutils/apiref.html#module-distutils.version

+

+# used by pkgsrc package wip/py-torch 1.12.0

+class LooseVersion:

+    pass

diff --git a/pkgtools/url2pkg/files/python/setuptools/__init__.py b/pkgtools/url2pkg/files/python/setuptools/__init__.py
new file mode 100644
index 00000000000..f4b713bffc8
--- /dev/null
+++ b/pkgtools/url2pkg/files/python/setuptools/__init__.py
@@ -0,0 +1,33 @@
+# $NetBSD: __init__.py,v 1.1 2022/07/16 08:59:39 rillig Exp $
+#
+# This is a drop-in replacement for the setuptools Python module. Instead
+# of actually searching for the dependencies, it extracts the dependency
+# information and includes it in the generated pkgsrc package Makefile.
+#
+# https://setuptools.pypa.io/en/latest/index.html
+# https://setuptools.pypa.io/en/latest/setuptools.html
+# https://setuptools.pypa.io/en/latest/userguide/index.html
+# https://github.com/pypa/setuptools/tree/main/setuptools
+
+import url2pkg
+# used by pkgsrc package wip/py-torch 1.12.0
+import distutils  # only re-export
+from setuptools.dist import Distribution
+
+
+# used by pkgsrc package devel/py-pysha3 1.0.2
+# used by pkgsrc package wip/py-torch 1.12.0
+def setup(**kwargs):
+    url2pkg.setup(**kwargs)
+
+
+# used by pkgsrc package wip/py-torch 1.12.0
+def find_packages(where='.', exclude=(), include=('*',)):
+    return []
+
+
+# used by pkgsrc package devel/py-pysha3 1.0.2
+# used by pkgsrc package wip/py-torch 1.12.0
+class Extension:
+    def __init__(self, name, sources, *args, **kw) -> None:
+        pass
diff --git a/pkgtools/url2pkg/files/python/setuptools/command/__init__.py b/pkgtools/url2pkg/files/python/setuptools/command/__init__.py
new file mode 100644
index 00000000000..1696368c3f9
--- /dev/null
+++ b/pkgtools/url2pkg/files/python/setuptools/command/__init__.py
@@ -0,0 +1,7 @@
+# $NetBSD: __init__.py,v 1.1 2022/07/16 08:59:39 rillig Exp $
+#
+# Fake implementation of setuptools.command.
+#
+# https://github.com/pypa/setuptools/blob/main/setuptools/command/__init__.py
+
+# This module is empty, it's just a container for submodules.
diff --git a/pkgtools/url2pkg/files/python/setuptools/command/build_ext.py b/pkgtools/url2pkg/files/python/setuptools/command/build_ext.py
new file mode 100644
index 00000000000..d48c6f859f3
--- /dev/null
+++ b/pkgtools/url2pkg/files/python/setuptools/command/build_ext.py
@@ -0,0 +1,7 @@
+# $NetBSD: build_ext.py,v 1.1 2022/07/16 08:59:39 rillig Exp $

+#

+# Fake implementation of setuptools.command.build_ext.

+#

+# https://github.com/pypa/setuptools/blob/main/setuptools/command/build_ext.py

+

+# used by pkgsrc package wip/pytorch 1.12.0

diff --git a/pkgtools/url2pkg/files/python/setuptools/command/install.py b/pkgtools/url2pkg/files/python/setuptools/command/install.py
new file mode 100644
index 00000000000..a4a7f18dd12
--- /dev/null
+++ b/pkgtools/url2pkg/files/python/setuptools/command/install.py
@@ -0,0 +1,7 @@
+# $NetBSD: install.py,v 1.1 2022/07/16 08:59:39 rillig Exp $

+#

+# Fake implementation of setuptools.command.install.

+#

+# https://github.com/pypa/setuptools/blob/main/setuptools/command/install.py

+

+# used by pkgsrc package wip/pytorch 1.12.0

diff --git a/pkgtools/url2pkg/files/python/setuptools/command/sdist.py b/pkgtools/url2pkg/files/python/setuptools/command/sdist.py
new file mode 100644
index 00000000000..eeda10630e1
--- /dev/null
+++ b/pkgtools/url2pkg/files/python/setuptools/command/sdist.py
@@ -0,0 +1,7 @@
+# $NetBSD: sdist.py,v 1.1 2022/07/16 08:59:39 rillig Exp $

+#

+# Fake implementation of setuptools.command.sdist.

+#

+# https://github.com/pypa/setuptools/blob/main/setuptools/command/sdist.py

+

+# used by pkgsrc package wip/pytorch 1.12.0

diff --git a/pkgtools/url2pkg/files/python/setuptools/dist.py b/pkgtools/url2pkg/files/python/setuptools/dist.py
new file mode 100644
index 00000000000..ffcfed9fb13
--- /dev/null
+++ b/pkgtools/url2pkg/files/python/setuptools/dist.py
@@ -0,0 +1,7 @@
+# $NetBSD: dist.py,v 1.1 2022/07/16 08:59:39 rillig Exp $

+#

+# Fake implementation for setuptools.dist.

+

+# used by pkgsrc package wip/py-torch 1.12.0

+class Distribution:

+    pass

diff --git a/pkgtools/url2pkg/files/python/url2pkg.py b/pkgtools/url2pkg/files/python/url2pkg.py
new file mode 100644
index 00000000000..3f3ede6e9a2
--- /dev/null
+++ b/pkgtools/url2pkg/files/python/url2pkg.py
@@ -0,0 +1,40 @@
+# $NetBSD: url2pkg.py,v 1.1 2022/07/16 08:59:38 rillig Exp $

+#

+# Forwards the module metadata from distutils and setuptools to url2pkg.

+

+def print_depends(varname, depends):

+    for dep in depends:

+        print('%s\t%s%s' % (varname, dep.replace(' ', ''), '' if '>' in dep else '>=0'))

+

+

+def print_var(varname, value):

+    if value != '':

+        print('var\t%s\t%s' % (varname, value))

+

+

+def print_cmd(cmd, arg):

+    print('\t'.join(('cmd', cmd, arg)))

+

+

+def print_license(license_name):

+    if license_name == '':

+        return

+    print_cmd('license', license_name)

+    print_cmd(

+        'license_default',

+        '%s # TODO: from setup.py; needs to be adjusted' % license_name)

+

+

+def setup(**kwargs):

+    print_depends('DEPENDS', kwargs.get('requires', []))  # only for distutils.core

+    print_depends('DEPENDS', kwargs.get('install_requires', []))

+    print_depends('TEST_DEPENDS', kwargs.get('tests_require', []))

+    print_depends('BUILD_DEPENDS', kwargs.get('extras_require', {}).get('dev', []))

+    print_depends('BUILD_DEPENDS', kwargs.get('setup_requires', []))

+    print_var('COMMENT', kwargs.get('description', '').rstrip('.'))

+    print_var('HOMEPAGE', kwargs.get('url', ''))

+    print_license(kwargs.get('license', ''))

+

+    # TODO: implement 'python_requires'

+    #  example pkgsrc package: devel/py-futures

+    #  example values: >=2.6, <3, >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*

diff --git a/pkgtools/url2pkg/files/setuptools.py b/pkgtools/url2pkg/files/setuptools.py
deleted file mode 100644
index a24685ce7c1..00000000000
--- a/pkgtools/url2pkg/files/setuptools.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# $NetBSD: setuptools.py,v 1.7 2019/10/27 19:19:55 rillig Exp $
-
-# This is a drop-in replacement for the setuptools Python module. Instead
-# of actually searching for the dependencies, it extracts the dependency
-# information and includes it in the generated pkgsrc package Makefile.
-
-
-def url2pkg_print_depends(varname, depends):
-    for dep in depends:
-        print('%s\t%s%s' % (varname, dep.replace(' ', ''), '' if '>' in dep else '>=0'))
-
-
-def url2pkg_print_var(varname, value):
-    if value != '':
-        print('var\t%s\t%s' % (varname, value))
-
-
-def url2pkg_print_cmd(cmd, arg):
-    print('\t'.join(('cmd', cmd, arg)))
-
-
-def url2pkg_print_license(license_name):
-    if license_name == '':
-        return
-    url2pkg_print_cmd('license', license_name)
-    url2pkg_print_cmd(
-        'license_default',
-        '%s # TODO: from setup.py; needs to be adjusted' % license_name)
-
-
-def setup(**kwargs):
-    url2pkg_print_depends('DEPENDS', kwargs.get('requires', []))  # for distutils.core
-    url2pkg_print_depends('DEPENDS', kwargs.get('install_requires', []))
-    url2pkg_print_depends('TEST_DEPENDS', kwargs.get('tests_require', []))
-    url2pkg_print_depends('BUILD_DEPENDS', kwargs.get('extras_require', {}).get('dev', []))
-    url2pkg_print_depends('BUILD_DEPENDS', kwargs.get('setup_requires', []))
-    url2pkg_print_var('COMMENT', kwargs.get('description', '').rstrip('.'))
-    url2pkg_print_var('HOMEPAGE', kwargs.get('url', ''))
-    url2pkg_print_license(kwargs.get('license', ''))
-
-    # TODO: python_requires (see devel/py-futures)
-    #       example: '>=2.6, <3, >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*')
-
-
-def find_packages(where='.', exclude=(), include=('*',)):
-    return []
-
-
-# for distutils.core; see devel/py-pysha3-1.0.2 for an example
-class Command:
-    pass
-
-
-# for distutils.extension; see devel/py-pysha3-1.0.2 for an example
-class Extension:
-    def __init__(self, *args, **kwargs):
-        pass
diff --git a/pkgtools/url2pkg/files/url2pkg.py b/pkgtools/url2pkg/files/url2pkg.py
index d432fabd91b..53580cf1a5c 100644
--- a/pkgtools/url2pkg/files/url2pkg.py
+++ b/pkgtools/url2pkg/files/url2pkg.py
@@ -1,5 +1,5 @@
 #! @PYTHONBIN@
-# $NetBSD: url2pkg.py,v 1.42 2022/02/08 20:48:09 rillig Exp $
+# $NetBSD: url2pkg.py,v 1.43 2022/07/16 08:59:38 rillig Exp $
 
 # Copyright (c) 2019 The NetBSD Foundation, Inc.
 # All rights reserved.
@@ -1021,7 +1021,7 @@ class Adjuster:
         cmd = f'{self.g.pythonbin} setup.py build'
         env = {
             'PYTHONDONTWRITEBYTECODE': 'x',
-            'PYTHONPATH': self.g.libdir
+            'PYTHONPATH': f'{self.g.libdir}/python'
         }
         self.read_dependencies(cmd, env, self.abs_wrksrc, 'py-', '${PYPKGPREFIX}-')
 
diff --git a/pkgtools/url2pkg/files/url2pkg_test.py b/pkgtools/url2pkg/files/url2pkg_test.py
index 8d0f885482d..4d4224a59ca 100644
--- a/pkgtools/url2pkg/files/url2pkg_test.py
+++ b/pkgtools/url2pkg/files/url2pkg_test.py
@@ -1,4 +1,13 @@
-# $NetBSD: url2pkg_test.py,v 1.40 2022/02/06 21:07:44 rillig Exp $
+# $NetBSD: url2pkg_test.py,v 1.41 2022/07/16 08:59:38 rillig Exp $
+
+# URLs for manual testing:
+#
+# https://files.pythonhosted.org/packages/source/p/pysha3/pysha3-1.0.2.tar.gz
+#   Prefers distutils over setuptools, has no external dependencies.
+#
+# https://github.com/pytorch/pytorch/archive/refs/tags/v1.12.0.tar.gz
+#   Uses setuptools with an extension.
+#   Runs Git from 'setup.py', which is rather unusual.
 
 import pytest
 from url2pkg import *