pkgtools/url2pkg: fixed migration of Python packages from GitHub to PyPI

2 files changed, 350 insertions, 84 deletions

diff --git a/pkgtools/url2pkg/files/url2pkg.py b/pkgtools/url2pkg/files/url2pkg.py
index a882e39380a..eb4c7dcc69c 100644
--- a/pkgtools/url2pkg/files/url2pkg.py
+++ b/pkgtools/url2pkg/files/url2pkg.py
@@ -1,5 +1,5 @@
 #! @PYTHONBIN@
-# $NetBSD: url2pkg.py,v 1.10 2019/10/05 12:22:51 rillig Exp $
+# $NetBSD: url2pkg.py,v 1.11 2019/10/05 18:00:09 rillig Exp $
 
 # Copyright (c) 2019 The NetBSD Foundation, Inc.
 # All rights reserved.
@@ -44,6 +44,7 @@
 import getopt
 import glob
 import os
+import pathlib
 import re
 import subprocess
 import sys
@@ -81,10 +82,11 @@ class Url2Pkg:
         self.perl5 = '@PERL5@'
         self.pkgsrcdir = '@PKGSRCDIR@'
         self.pythonbin = '@PYTHONBIN@'
+        self.editor = os.getenv('PKGEDITOR') or os.getenv('EDITOR') or 'vi'
 
         self.verbose = False
 
-        self.pkgdir = '.'  # only overridable for tests
+        self.pkgdir = pathlib.Path('.')  # only overridable for tests
         self.out = sys.stdout  # only overridable for tests
         self.err = sys.stderr  # only overridable for tests
 
@@ -121,30 +123,32 @@ class Lines:
             self.add(line)
 
     @classmethod
-    def read_from(cls, filename: str) -> 'Lines':
-        pass
+    def read_from(cls, filename: Union[str, pathlib.Path]) -> 'Lines':
+        return Lines(*pathlib.Path(filename).read_text().splitlines())
 
-        lines = Lines()
-        with open(filename) as f:
-            for line in f:
-                lines.add(line.rstrip('\n'))
-        return lines
-
-    def write_to(self, filename: str):
-        with open(f'{filename}.tmp', 'w') as f:
+    def write_to(self, filename: Union[str, pathlib.Path]):
+        target = pathlib.Path(filename)
+        tmp = target.with_name(f'{target.name}.tmp')
+        with tmp.open('w') as f:
             f.writelines(line + '\n' for line in self.lines)
-        try:
-            os.remove(filename)
-        except OSError:
-            pass
-        os.rename(f'{filename}.tmp', filename)
+        tmp.replace(target)
 
     def all_varassigns(self, varname: str) -> Sequence[Varassign]:
         varassigns = []
         for (i, line) in enumerate(self.lines):
-            m = re.search(r'^(#?[\w+\-]+?)([!+:?]?=)([ \t]*)([^#\\]*?)(\s*)(#.*|)$', line)
+            pattern = r'''(?x)
+                ^
+                ([#]?[\w+\-]+?)  # varname
+                ([!+:?]?=)       # op
+                ([ \t]*)         # indent
+                ([^#\\]*?)       # value
+                (\s*)            # space_after_value
+                ([#].*|)         # comment
+                $
+                '''
+            m = re.search(pattern, line)
             if m and m[1].lstrip('#') == varname:
-                varassigns.append(Varassign(i, m[1], m[2], m[3], m[4], m[5], m[6]))
+                varassigns.append(Varassign(i, *m.groups()))
         return varassigns
 
     def unique_varassign(self, varname: str) -> Optional[Varassign]:
@@ -239,6 +243,7 @@ class Generator:
         self.extract_sufx = ''
         self.categories = ''
         self.github_project = ''
+        self.github_tag = ''
         self.github_release = ''
         self.dist_subdir = ''
         self.pkgname_prefix = ''
@@ -290,12 +295,15 @@ class Generator:
             self.homepage = self.url[:-len(self.distfile)] + ' # TODO: check'
 
     def adjust_site_SourceForge(self):
-        pattern = r'^https?://downloads\.sourceforge\.net/' \
-                  r'(?:project|sourceforge)/' \
-                  r'([^/?]+)/' \
-                  r'((?:[^/?]+/)*)' \
-                  r'([^/?]+)' \
-                  r'(?:\?.*)?$'
+        pattern = r'''(?x)
+            ^
+            https?://downloads\.sourceforge\.net/(?:project|sourceforge)/
+            ([^/?]+)/       # project name
+            ((?:[^/?]+/)*)  # subdirectories
+            ([^/?]+)        # filename
+            (?:\?.*)?       # query parameters
+            $
+            '''
         m = re.search(pattern, self.url)
         if not m:
             return
@@ -306,11 +314,15 @@ class Generator:
         self.distfile = filename
 
     def adjust_site_GitHub_archive(self):
-        pattern = r'^https://github\.com/' \
-                  r'(.+)/' \
-                  r'(.+)/archive/' \
-                  r'(.+)' \
-                  r'(\.tar\.gz|\.zip)$'
+        pattern = r'''(?x)
+            ^
+            https://github\.com/
+            (.+)/               # org
+            (.+)/archive/       # proj
+            (.+)                # tag
+            (\.tar\.gz|\.zip)   # ext
+            $
+            '''
         m = re.search(pattern, self.url)
         if not m:
             return
@@ -318,6 +330,7 @@ class Generator:
         org, proj, tag, ext = m.groups()
 
         self.github_project = proj
+        self.github_tag = tag
         self.master_sites = f'${{MASTER_SITE_GITHUB:={org}/}}'
         self.homepage = f'https://github.com/{org}/{proj}/'
         if proj not in tag:
@@ -326,12 +339,15 @@ class Generator:
         self.distfile = tag + ext
 
     def adjust_site_GitHub_release(self):
-        pattern = r'^https://github\.com/' \
-                  r'(.+)/' \
-                  r'(.+)/releases/download/' \
-                  r'(.+)/' \
-                  r'(.+)' \
-                  r'(\.tar\.gz|\.zip)$'
+        pattern = r'''(?x)
+            ^https://github\.com/
+            (.+)/               # org
+            (.+)/               # proj   
+            releases/download/
+            (.+)/               # tag
+            (.+)                # base
+            (\.tar\.gz|\.zip)$  # ext
+            '''
         m = re.search(pattern, self.url)
         if not m:
             return
@@ -395,6 +411,7 @@ class Generator:
 
         lines.add_vars(
             Var('GITHUB_PROJECT', '=', self.github_project),
+            Var('GITHUB_TAG', '=', self.github_tag),
             Var('DISTNAME', '=', self.distname),
             Var('PKGNAME', '=', self.pkgname),
             Var('CATEGORIES', '=', self.categories),
@@ -416,7 +433,7 @@ class Generator:
 
         return lines
 
-    def generate_Makefile(self):
+    def generate_Makefile(self) -> Lines:
         self.adjust_site_SourceForge()
         self.adjust_site_GitHub_archive()
         self.adjust_site_GitHub_release()
@@ -427,14 +444,14 @@ class Generator:
 
     def generate_package(self, up: Url2Pkg) -> Lines:
         pkgdir = up.pkgdir
-        makefile = f'{pkgdir}/Makefile'
-        descr = f'{pkgdir}/DESCR'
-        plist = f'{pkgdir}/PLIST'
+        makefile = pkgdir / 'Makefile'
+        descr = pkgdir / 'DESCR'
+        plist = pkgdir / 'PLIST'
 
         initial_lines = self.generate_Makefile()
 
         try:
-            os.rename(makefile, f'{makefile}.url2pkg~')
+            makefile.replace(f'{makefile}.url2pkg~')
         except OSError:
             pass
         initial_lines.write_to(makefile)
@@ -443,11 +460,9 @@ class Generator:
         if not os.path.isfile(descr):
             Lines().write_to(descr)
 
-        editor = os.getenv('PKGEDITOR') or os.getenv('EDITOR') or 'vi'
-        subprocess.check_call([editor, makefile])
+        subprocess.check_call([up.editor, makefile])
 
-        up.bmake('distinfo')
-        up.bmake('extract')
+        up.bmake('distinfo', 'extract')
 
         return initial_lines
 
@@ -568,10 +583,10 @@ class Adjuster:
         effective_env.update(env)
 
         self.up.debug('reading dependencies: cd {0} && env {1} {2}', cwd, env, cmd)
-        output = subprocess.check_output(args=cmd, shell=True, env=effective_env, cwd=cwd)
+        output: bytes = subprocess.check_output(args=cmd, shell=True, env=effective_env, cwd=cwd)
 
         dep_lines: List[Tuple[str, str, str, str]] = []
-        for line in output.decode('utf-8').split('\n'):
+        for line in output.decode('utf-8').splitlines():
             # example: DEPENDS   pkgbase>=1.2.3:../../category/pkgbase
             m = re.search(r'^(\w+)\t([^\s:>]+)(>[^\s:]+|)(?::(\.\./\.\./\S+))?$', line)
             if m:
@@ -604,7 +619,6 @@ class Adjuster:
 
     def wrksrc_find(self, what: Union[str, Callable[[str], bool]]) -> Iterator[str]:
         def search(f):
-            print('search', f)
             return re.search(what, f) if type(what) == str else what(f)
 
         return list(sorted(filter(search, self.wrksrc_files)))
@@ -706,7 +720,7 @@ class Adjuster:
         self.adjust_perl_module_homepage()
 
         try:
-            os.unlink('PLIST')
+            (self.up.pkgdir / 'PLIST').unlink()
         except OSError:
             pass
 
@@ -801,36 +815,51 @@ class Adjuster:
 
     def adjust_lines_python_module(self, lines: Lines):
 
-        initial_lines = self.initial_lines
-        current_lines = self.makefile_lines
+        initial_lines = self.initial_lines  # as generated by url2pkg
+        edited_lines = self.makefile_lines  # as edited by the package developer
 
-        if 'python' not in initial_lines.get('CATEGORIES'):
+        if 'python' not in lines.get('CATEGORIES'):
             return
-        pkgbase = initial_lines.get('GITHUB_PROJECT')
-        if pkgbase == '':
+        if lines.get('GITHUB_PROJECT') == '':
             return
-        pkgbase1 = pkgbase[:1]
-        pkgversion_norev = re.sub(r'^v', '', initial_lines.get('DISTNAME'))
 
         # don't risk to overwrite any changes made by the package developer.
-        if '\n'.join(current_lines.lines) != '\n'.join(initial_lines.lines):
-            lines.lines.insert(-2, '# TODO: Migrate MASTER_SITES to PYPI')
+        if edited_lines.lines != initial_lines.lines:
+            lines.lines.insert(-2, '# TODO: Migrate MASTER_SITES to MASTER_SITE_PYPI')
             return
 
+        pkgbase = initial_lines.get('GITHUB_PROJECT')
+        pkgbase1 = pkgbase[:1] if pkgbase != '' else ''
+        pkgversion_norev = re.sub(r'^v', '', initial_lines.get('DISTNAME'))
+
         tx_lines = Lines(*self.makefile_lines.lines)
-        if (tx_lines.remove('GITHUB_PROJECT')
+        if not (tx_lines.remove('GITHUB_PROJECT')
                 and tx_lines.set('DISTNAME', f'{pkgbase}-{pkgversion_norev}')
                 and tx_lines.set('PKGNAME', '${PYPKGPREFIX}-${DISTNAME}')
                 and tx_lines.set('MASTER_SITES', f'${{MASTER_SITE_PYPI:={pkgbase1}/{pkgbase}/}}')
                 and tx_lines.remove('DIST_SUBDIR')):
-            tx_lines.remove_if('EXTRACT_SUFX', '.zip')
-            self.makefile_lines = tx_lines
-            self.regenerate_distinfo = True
+            return
+
+        tx_lines.remove_if('GITHUB_TAG', initial_lines.get('DISTNAME'))
+        tx_lines.remove_if('EXTRACT_SUFX', '.zip')
+
+        up = self.up
+        try_mk = up.pkgdir / 'try-pypi.mk'
+        tx_lines.write_to(try_mk)
+        args = [up.make, '-f', str(try_mk), 'distinfo']
+        up.debug('running {0} to try PyPI', args)
+        fetch_ok = subprocess.call(args, cwd=up.pkgdir) == 0
+        try_mk.unlink()
+        if not fetch_ok:
+            return
+
+        lines.lines = tx_lines.lines
+        self.regenerate_distinfo = True
 
     def generate_lines(self) -> Lines:
         marker_index = self.makefile_lines.index(r'^# url2pkg-marker')
         if marker_index == -1:
-            raise Exception('ERROR: didn\'t find the url2pkg marker in the Makefile.')
+            sys.exit('error: didn\'t find the url2pkg marker in the Makefile.')
 
         lines = Lines(*self.makefile_lines.lines[: marker_index])
 
@@ -877,7 +906,7 @@ class Adjuster:
             return list(f[len(basedir) + 1:] for f in full_paths)
 
         self.up.debug('Adjusting the Makefile')
-        self.makefile_lines = Lines.read_from(self.up.pkgdir + '/Makefile')
+        self.makefile_lines = Lines.read_from(self.up.pkgdir / 'Makefile')
 
         self.abs_wrkdir = self.up.show_var('WRKDIR')
         self.determine_wrksrc()
@@ -896,7 +925,7 @@ class Adjuster:
         self.adjust_po()
         self.adjust_use_languages()
 
-        self.generate_lines().write_to(self.up.pkgdir + '/Makefile')
+        self.generate_lines().write_to(self.up.pkgdir / 'Makefile')
 
         if self.regenerate_distinfo:
             self.up.bmake('distinfo')
@@ -917,7 +946,7 @@ def main():
 
     url = args[0] if args else input('URL: ')
 
-    if not glob.glob('w*/.extract_done') or not os.path.isfile('Makefile'):
+    if not up.pkgdir.glob('w*/.extract_done') or not (up.pkgdir / 'Makefile').is_file():
         initial_lines = Generator(url).generate_package(up)
     else:
         initial_lines = Generator(url).generate_lines()
diff --git a/pkgtools/url2pkg/files/url2pkg_test.py b/pkgtools/url2pkg/files/url2pkg_test.py
index 89c7451fcff..810c6017563 100644
--- a/pkgtools/url2pkg/files/url2pkg_test.py
+++ b/pkgtools/url2pkg/files/url2pkg_test.py
@@ -1,4 +1,4 @@
-# $NetBSD: url2pkg_test.py,v 1.9 2019/10/05 12:22:51 rillig Exp $
+# $NetBSD: url2pkg_test.py,v 1.10 2019/10/05 18:00:09 rillig Exp $
 
 import pytest
 from url2pkg import *
@@ -91,7 +91,7 @@ def test_Url2Pkg_bmake():
     assert up.err.output() == 'url2pkg: running bmake (\'hello\', \'world\')\n'
 
 
-def test_Lines__write_and_read(tmp_path):
+def test_Lines__write_and_read(tmp_path: pathlib.Path):
     example = tmp_path / 'example'
 
     lines = Lines('1', '2', '3')
@@ -392,6 +392,7 @@ def test_Generator_adjust_site_GitHub_archive():
         mkcvsid,
         '',
         'GITHUB_PROJECT= proj',
+        'GITHUB_TAG=     v1.0.0',
         'DISTNAME=       v1.0.0',
         'PKGNAME=        ${GITHUB_PROJECT}-${DISTNAME:S,^v,,}',
         'CATEGORIES=     pkgtools',
@@ -530,6 +531,12 @@ def test_Generator_adjust_site_from_sites_mk__GNU():
     ]
 
 
+def test_Generator_adjust_site_other__malformed_URL():
+    error = 'error: URL "localhost" must have at least one slash'
+    with pytest.raises(SystemExit, match=error):
+        Generator('localhost').generate_Makefile()
+
+
 def test_Generator_adjust_everything_else__distname_version_with_v():
     # Some version numbers have a leading 'v', derived from the Git tag name.
 
@@ -555,6 +562,29 @@ def test_Generator_adjust_everything_else__distname_version_with_v():
     ]
 
 
+def test_Generator_adjust_everything_else__distfile_without_extension():
+    url = 'https://example.org/app-2019-10-05'
+
+    lines = Generator(url).generate_Makefile()
+
+    assert detab(lines) == [
+        mkcvsid,
+        '',
+        'DISTNAME=       app-2019-10-05',
+        'CATEGORIES=     pkgtools',
+        'MASTER_SITES=   https://example.org/',
+        'EXTRACT_SUFX=   # none',
+        '',
+        'MAINTAINER=     INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
+        'HOMEPAGE=       https://example.org/',
+        'COMMENT=        TODO: Short description of the package',
+        '#LICENSE=       # TODO: (see mk/license.mk)',
+        '',
+        '# url2pkg-marker (please do not remove this line.)',
+        '.include "../../mk/bsd.pkg.mk"'
+    ]
+
+
 def test_Generator_determine_distname__v8():
     generator = Generator('https://example.org/v8-1.0.zip')
 
@@ -579,6 +609,23 @@ def test_Generator_determine_distname__v8():
     ]
 
 
+def test_Generator_generate_package(tmp_path: pathlib.Path):
+    url = 'https://ftp.gnu.org/pub/gnu/cflow/cflow-1.6.tar.gz'
+    up.editor = 'true'  # the shell command
+    up.make = 'true'  # the shell command
+    up.pkgdir = tmp_path
+
+    Generator(url).generate_package(up)
+
+    assert (tmp_path / 'DESCR').read_text() == ''
+    assert len((tmp_path / 'Makefile').read_text().splitlines()) == 13
+    assert (tmp_path / 'PLIST').read_text() == '@comment $''NetBSD$\n'
+
+    # Since bmake is only fake in this test, the distinfo file is not created.
+    expected_files = ['DESCR', 'Makefile', 'PLIST']
+    assert sorted([f.name for f in tmp_path.glob("*")]) == expected_files
+
+
 def test_Adjuster_read_dependencies():
     child_process_output = [
         'DEPENDS\tpackage>=112.0:../../pkgtools/pkglint',
@@ -611,6 +658,22 @@ def test_Adjuster_read_dependencies():
     assert adjuster.update_vars == {'HOMEPAGE': 'https://homepage.example.org/'}
 
 
+def test_Adjuster_read_dependencies__lookup_with_prefix():
+    child_process_output = [
+        'DEPENDS\tpyobjc-framework-Quartz>=0',
+        ''
+    ]
+    env = {'URL2PKG_DEPENDENCIES': '\n'.join(child_process_output)}
+    cmd = "printf '%s\n' \"$URL2PKG_DEPENDENCIES\""
+
+    adjuster = Adjuster(up, '', Lines())
+    adjuster.read_dependencies(cmd, env, '.', 'py-')
+
+    assert adjuster.depends == [
+        'py-pyobjc-framework-Quartz>=0:../../devel/py-pyobjc-framework-Quartz',
+    ]
+
+
 def test_Adjuster_generate_adjusted_Makefile_lines():
     adjuster = Adjuster(up, 'https://example.org/pkgname-1.0.tar.gz', Lines())
     adjuster.makefile_lines = Lines(
@@ -748,7 +811,17 @@ def test_Adjuster_add_dependency__buildlink():
     ]
 
 
-def test_Adjuster_adjust_configure__none(tmp_path):
+def test_Adjuster_adjust_cmake(tmp_path: pathlib.Path):
+    adjuster = Adjuster(up, '', Lines())
+    adjuster.abs_wrksrc = str(tmp_path)
+    (tmp_path / 'CMakeLists.txt').touch()
+
+    adjuster.adjust_cmake()
+
+    assert str_vars(adjuster.build_vars) == ['USE_CMAKE=yes']
+
+
+def test_Adjuster_adjust_configure__none(tmp_path: pathlib.Path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrksrc = str(tmp_path)
 
@@ -757,7 +830,7 @@ def test_Adjuster_adjust_configure__none(tmp_path):
     assert adjuster.build_vars == []
 
 
-def test_Adjuster_adjust_configure__GNU(tmp_path):
+def test_Adjuster_adjust_configure__GNU(tmp_path: pathlib.Path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrksrc = str(tmp_path)
     adjuster.wrksrc_files.append('configure')
@@ -770,7 +843,7 @@ def test_Adjuster_adjust_configure__GNU(tmp_path):
     ]
 
 
-def test_Adjuster_adjust_configure__other(tmp_path):
+def test_Adjuster_adjust_configure__other(tmp_path: pathlib.Path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrksrc = str(tmp_path)
     adjuster.wrksrc_files.append('configure')
@@ -783,7 +856,7 @@ def test_Adjuster_adjust_configure__other(tmp_path):
     ]
 
 
-def test_Adjuster_adjust_cargo__not_found(tmp_path):
+def test_Adjuster_adjust_cargo__not_found(tmp_path: pathlib.Path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrksrc = str(tmp_path)
 
@@ -792,7 +865,7 @@ def test_Adjuster_adjust_cargo__not_found(tmp_path):
     assert str_vars(adjuster.build_vars) == []
 
 
-def test_Adjuster_adjust_cargo__found(tmp_path):
+def test_Adjuster_adjust_cargo__found(tmp_path: pathlib.Path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrksrc = str(tmp_path)
     (tmp_path / 'Cargo.lock').write_text('"checksum cargo-pkg 1.2.3 1234"')
@@ -824,7 +897,7 @@ def test_Adjuster_adjust_gconf2():
     ]
 
 
-def test_Adjuster_adjust_libtool__ltconfig(tmp_path):
+def test_Adjuster_adjust_libtool__ltconfig(tmp_path: pathlib.Path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrksrc = str(tmp_path)
     (tmp_path / 'ltconfig').write_text('')
@@ -834,7 +907,7 @@ def test_Adjuster_adjust_libtool__ltconfig(tmp_path):
     assert str_vars(adjuster.build_vars) == ['USE_LIBTOOL=yes']
 
 
-def test_Adjuster_adjust_libtool__libltdl(tmp_path):
+def test_Adjuster_adjust_libtool__libltdl(tmp_path: pathlib.Path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrksrc = str(tmp_path)
     (tmp_path / 'libltdl').mkdir()
@@ -846,6 +919,140 @@ def test_Adjuster_adjust_libtool__libltdl(tmp_path):
     ]
 
 
+def test_Adjuster_adjust_meson(tmp_path: pathlib.Path):
+    adjuster = Adjuster(up, '', Lines())
+    adjuster.abs_wrksrc = str(tmp_path)
+    (tmp_path / 'meson.build').touch()
+
+    adjuster.adjust_meson()
+
+    assert adjuster.includes == ['../../devel/py-meson/build.mk']
+
+
+def test_Adjuster_adjust_perl_module_Build_PL(tmp_path: pathlib.Path):
+    up.perl5 = 'echo perl5'
+    up.libdir = '/libdir'
+    up.verbose = True
+    adjuster = Adjuster(up, '', Lines())
+    adjuster.abs_wrksrc = str(tmp_path)
+
+    adjuster.adjust_perl_module_Build_PL()
+
+    assert str_vars(adjuster.build_vars) == ['PERL5_MODULE_TYPE=Module::Build']
+    assert up.err.output().splitlines() == [
+        f'url2pkg: reading dependencies: cd \'{tmp_path}\' && env {{}} \'echo perl5 -I/libdir -I. Build.PL\'',
+        'url2pkg: unknown dependency line: \'perl5 -I/libdir -I. Build.PL\''
+    ]
+
+
+def test_Adjuster_adjust_perl_module_Makefile_PL(tmp_path: pathlib.Path):
+    up.perl5 = 'echo perl5'
+    up.libdir = '/libdir'
+    up.verbose = True
+    adjuster = Adjuster(up, '', Lines())
+    adjuster.abs_wrksrc = str(tmp_path)
+
+    adjuster.adjust_perl_module_Makefile_PL()
+
+    assert str_vars(adjuster.build_vars) == []
+    assert up.err.output().splitlines() == [
+        f'url2pkg: reading dependencies: cd \'{tmp_path}\' && env {{}} \'echo perl5 -I/libdir -I. Makefile.PL\'',
+        'url2pkg: unknown dependency line: \'perl5 -I/libdir -I. Makefile.PL\''
+    ]
+
+
+def test_Adjuster_adjust_perl_module_homepage():
+    adjuster = Adjuster(up, 'https://example.org/Perl-Module-1.0.tar.gz', Lines())
+    adjuster.makefile_lines.add_vars(
+        Var('DISTNAME', '=', 'Perl-Module-1.0.tar.gz'),
+        Var('MASTER_SITES', '=', '${MASTER_SITE_PERL_CPAN:=subdir/}'),
+        Var('HOMEPAGE', '=', 'https://example.org/'),
+    )
+
+    adjuster.adjust_perl_module_homepage()
+
+    assert adjuster.makefile_lines.get('HOMEPAGE') == 'https://metacpan.org/pod/Perl::Module'
+
+
+def test_Adjuster_adjust_perl_module__Build_PL(tmp_path: pathlib.Path):
+    up.perl5 = 'echo perl5'
+    up.pkgdir = tmp_path  # for removing the PLIST
+    adjuster = Adjuster(up, 'https://example.org/Perl-Module-1.0.tar.gz', Lines())
+    adjuster.abs_wrksrc = str(tmp_path)
+    adjuster.makefile_lines.add_vars(
+        Var('DISTNAME', '=', 'Perl-Module-1.0.tar.gz'),
+        Var('MASTER_SITES', '=', '${MASTER_SITE_PERL_CPAN:=subdir/}'),
+        Var('HOMEPAGE', '=', 'https://example.org/'),
+    )
+    adjuster.makefile_lines.add('# url2pkg-marker')
+    (tmp_path / 'Build.PL').touch()
+    (tmp_path / 'PLIST').touch()
+
+    adjuster.adjust_perl_module()
+
+    assert detab(adjuster.generate_lines()) == [
+        'DISTNAME=       Perl-Module-1.0.tar.gz',
+        'PKGNAME=        p5-${DISTNAME}',
+        'MASTER_SITES=   ${MASTER_SITE_PERL_CPAN:=subdir/}',
+        'HOMEPAGE=       https://metacpan.org/pod/Perl::Module',
+        '',
+        'PERL5_MODULE_TYPE=      Module::Build',
+        'PERL5_PACKLIST=         auto/Perl/Module/.packlist',
+        '',
+        '.include "../../lang/perl5/module.mk"',
+    ]
+    assert not (tmp_path / 'PLIST').exists()
+
+
+def test_Adjuster_adjust_perl_module__Makefile_PL_without_PLIST(tmp_path: pathlib.Path):
+    # For code coverage, when PLIST cannot be unlinked.
+
+    up.perl5 = 'echo perl5'
+    up.pkgdir = tmp_path
+    adjuster = Adjuster(up, 'https://example.org/Mod-1.0.tar.gz', Lines())
+    adjuster.abs_wrksrc = str(tmp_path)
+    adjuster.makefile_lines.add_vars(
+        Var('DISTNAME', '=', 'Mod-1.0.tar.gz'),
+        Var('MASTER_SITES', '=', '${MASTER_SITE_PERL_CPAN:=subdir/}'),
+        Var('HOMEPAGE', '=', 'https://example.org/'),
+    )
+    adjuster.makefile_lines.add('# url2pkg-marker')
+    (tmp_path / 'Makefile.PL').touch()
+
+    adjuster.adjust_perl_module()
+
+    assert not (tmp_path / 'PLIST').exists()
+
+
+def test_Adjuster_adjust_python_module(tmp_path: pathlib.Path):
+    url = 'https://example.org/Mod-1.0.tar.gz'
+    up.pythonbin = 'echo python'
+    up.pkgdir = tmp_path
+    adjuster = Adjuster(up, url, Lines())
+    adjuster.abs_wrksrc = str(tmp_path)
+    adjuster.makefile_lines = Generator(url).generate_Makefile()
+    (tmp_path / 'setup.py').touch()
+
+    adjuster.adjust_python_module()
+
+    assert detab(adjuster.generate_lines()) == [
+        mkcvsid,
+        '',
+        'DISTNAME=       Mod-1.0',
+        'PKGNAME=        ${PYPKGPREFIX}-${DISTNAME}',
+        'CATEGORIES=     pkgtools python',
+        'MASTER_SITES=   https://example.org/',
+        '',
+        'MAINTAINER=     INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
+        'HOMEPAGE=       https://example.org/',
+        'COMMENT=        TODO: Short description of the package',
+        '#LICENSE=       # TODO: (see mk/license.mk)',
+        '',
+        '.include "../../lang/python/egg.mk"',
+        '.include "../../mk/bsd.pkg.mk"',
+    ]
+
+
 def test_Adjuster_adjust_po__not_found():
     adjuster = Adjuster(up, '', Lines())
 
@@ -988,7 +1195,7 @@ def test_Adjuster__adjust_homepage():
     ]
 
 
-def test_Adjuster_determine_wrksrc__no_files(tmp_path):
+def test_Adjuster_determine_wrksrc__no_files(tmp_path: pathlib.Path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrkdir = str(tmp_path)
 
@@ -997,7 +1204,7 @@ def test_Adjuster_determine_wrksrc__no_files(tmp_path):
     assert adjuster.abs_wrksrc == adjuster.abs_wrkdir
 
 
-def test_Adjuster_determine_wrksrc__single_dir(tmp_path):
+def test_Adjuster_determine_wrksrc__single_dir(tmp_path: pathlib.Path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrkdir = str(tmp_path)
     (tmp_path / 'subdir').mkdir()
@@ -1007,7 +1214,7 @@ def test_Adjuster_determine_wrksrc__single_dir(tmp_path):
     assert adjuster.abs_wrksrc == adjuster.abs_wrkdir + '/subdir'
 
 
-def test_Adjuster_determine_wrksrc__several_dirs(tmp_path):
+def test_Adjuster_determine_wrksrc__several_dirs(tmp_path: pathlib.Path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrkdir = str(tmp_path)
     (tmp_path / 'subdir1').mkdir()
@@ -1021,7 +1228,7 @@ def test_Adjuster_determine_wrksrc__several_dirs(tmp_path):
     ]
 
 
-def test_Adjuster_adjust_package_from_extracted_distfiles__empty_wrkdir(tmp_path):
+def test_Adjuster_adjust_package_from_extracted_distfiles__empty_wrkdir(tmp_path: pathlib.Path):
     pkgdir = tmp_path
     wrkdir = tmp_path / 'wrkdir'
     fake = '''\
@@ -1031,13 +1238,13 @@ case $* in
 (*) "unknown: $*" ;;
 esac
 ''' % str(wrkdir)
-    up.pkgdir = str(tmp_path)
+    up.pkgdir = tmp_path
     wrkdir.mkdir()
     url = 'https://example.org/distfile-1.0.zip'
     adjuster = Adjuster(up, url, Lines())
     adjuster.abs_wrkdir = str(wrkdir)
     (pkgdir / 'Makefile').write_text('# url2pkg-marker\n')
-    fake_path = (tmp_path / 'fake')
+    fake_path = tmp_path / 'fake'
     fake_path.write_text(fake)
     fake_path.chmod(0o755)
 
@@ -1055,17 +1262,22 @@ esac
     ]
 
 
-def test_Adjuster_adjust_lines_python_module():
+def test_Adjuster_adjust_lines_python_module(tmp_path: pathlib.Path):
     url = 'https://github.com/espressif/esptool/archive/v2.7.tar.gz'
+    up.pkgdir = tmp_path
+    up.make = 'true'  # the shell command
+    up.verbose = True
     initial_lines = Generator(url).generate_Makefile()
     initial_lines.append('CATEGORIES', 'python')
     adjuster = Adjuster(up, url, initial_lines)
     adjuster.makefile_lines = Lines(*initial_lines.lines)
+    (up.pkgdir / 'Makefile').touch()
 
     assert detab(adjuster.makefile_lines) == [
         mkcvsid,
         '',
         'GITHUB_PROJECT= esptool',
+        'GITHUB_TAG=     v2.7',
         'DISTNAME=       v2.7',
         'PKGNAME=        ${GITHUB_PROJECT}-${DISTNAME:S,^v,,}',
         'CATEGORIES=     pkgtools python',
@@ -1081,13 +1293,13 @@ def test_Adjuster_adjust_lines_python_module():
         '.include "../../mk/bsd.pkg.mk"',
     ]
 
-    adjuster.adjust_lines_python_module(initial_lines)
+    lines = adjuster.generate_lines()
 
     # FIXME: Currently url2pkg assumes that all Python modules that are on
     #  GitHub are also available from PyPI. That is wrong. Probably url2pkg
     #  should try to fetch the file from PyPI, and only switch to PyPI if
     #  they are the same.
-    assert detab(adjuster.makefile_lines) == [
+    assert detab(lines) == [
         mkcvsid,
         '',
         'DISTNAME=       esptool-2.7',
@@ -1103,3 +1315,28 @@ def test_Adjuster_adjust_lines_python_module():
         '# url2pkg-marker (please do not remove this line.)',
         '.include "../../mk/bsd.pkg.mk"',
     ]
+    assert up.err.output() == (f"url2pkg: running ['true', "
+                               f"'-f', '{tmp_path / 'try-pypi.mk'}', "
+                               f"'distinfo'] to try PyPI\n")
+
+
+def test_Adjuster_adjust_lines_python_module__edited():
+    # When the package developer has edited the Makefile, it's unclear
+    # what has changed. To not damage anything, let the package
+    # developer migrate manually.
+
+    url = 'https://github.com/espressif/esptool/archive/v2.7.tar.gz'
+    initial_lines = Generator(url).generate_Makefile()
+    initial_lines.append('CATEGORIES', 'python')
+    adjuster = Adjuster(up, url, initial_lines)
+    adjuster.makefile_lines = Lines(*initial_lines.lines)
+    initial_lines.add('')  # to make the lines different
+
+    lines = adjuster.generate_lines()
+
+    assert lines.get('GITHUB_PROJECT') == 'esptool'
+
+    adjuster.adjust_lines_python_module(lines)
+
+    assert lines.get('GITHUB_PROJECT') == 'esptool'
+    assert lines.index('TODO: Migrate MASTER_SITES to MASTER_SITE_PYPI') == 14