pkgtools/url2pkg: improve URL handling for SourceForge

* Improved handling of SourceForge URLs
* Fixed wrong HOMEPAGE for URLs composed from MASTER_SITE_*
* Improved the existing tests and added some more

2 files changed, 538 insertions, 410 deletions

diff --git a/pkgtools/url2pkg/files/url2pkg.py b/pkgtools/url2pkg/files/url2pkg.py
index 3577250b4d7..3f04934cbcf 100644
--- a/pkgtools/url2pkg/files/url2pkg.py
+++ b/pkgtools/url2pkg/files/url2pkg.py
@@ -1,5 +1,5 @@
 #! @PYTHONBIN@
-# $NetBSD: url2pkg.py,v 1.8 2019/10/04 22:26:34 rillig Exp $
+# $NetBSD: url2pkg.py,v 1.9 2019/10/05 11:02:30 rillig Exp $
 
 # Copyright (c) 2019 The NetBSD Foundation, Inc.
 # All rights reserved.
@@ -47,7 +47,7 @@ import os
 import re
 import subprocess
 import sys
-from typing import Callable, Dict, Iterator, List, Optional, Sequence, Union, Tuple
+from typing import Callable, Dict, Iterator, List, Optional, Sequence, Tuple, Union
 
 
 class Var:
@@ -81,10 +81,12 @@ class Url2Pkg:
         self.perl5 = '@PERL5@'
         self.pkgsrcdir = '@PKGSRCDIR@'
         self.pythonbin = '@PYTHONBIN@'
-        self.pkgdir = '.'  # only overridable for tests
+
         self.verbose = False
-        self.out = sys.stdout
-        self.err = sys.stderr
+
+        self.pkgdir = '.'  # only overridable for tests
+        self.out = sys.stdout  # only overridable for tests
+        self.err = sys.stderr  # only overridable for tests
 
     def debug(self, fmt: str, *args):
         if self.verbose:
@@ -137,6 +139,33 @@ class Lines:
             pass
         os.rename(f'{filename}.tmp', filename)
 
+    def all_varassigns(self, varname: str) -> Sequence[Varassign]:
+        varassigns = []
+        for (i, line) in enumerate(self.lines):
+            m = re.search(r'^(#?[\w+\-]+?)([!+:?]?=)([ \t]*)([^#\\]*?)(\s*)(#.*|)$', line)
+            if m and m[1].lstrip('#') == varname:
+                varassigns.append(Varassign(i, m[1], m[2], m[3], m[4], m[5], m[6]))
+        return varassigns
+
+    def unique_varassign(self, varname: str) -> Optional[Varassign]:
+        varassigns = self.all_varassigns(varname)
+        return varassigns[0] if len(varassigns) == 1 else None
+
+    def get(self, varname: str) -> str:
+        """
+        Returns the value from the only variable assignment, or an empty
+        string.
+        """
+        varassign = self.unique_varassign(varname)
+        return varassign.value if varassign is not None and varassign.varname == varname else ''
+
+    def index(self, pattern: str) -> int:
+        """ Returns the first index where the pattern is found, or -1. """
+        for (i, line) in enumerate(self.lines):
+            if re.search(pattern, line):
+                return i
+        return -1
+
     def add(self, *lines: Sequence[str]):
         for line in lines:
             assert type(line) == str, type(line)
@@ -162,18 +191,6 @@ class Lines:
             self.add(var.name + var.op + '\t' * tabs + var.value)
         self.add('')
 
-    def unique_varassign(self, varname: str) -> Optional[Varassign]:
-        varassigns = self.all_varassigns(varname)
-        return varassigns[0] if len(varassigns) == 1 else None
-
-    def all_varassigns(self, varname: str) -> Sequence[Varassign]:
-        varassigns = []
-        for (i, line) in enumerate(self.lines):
-            m = re.search(r'^(#?[\w+\-]+?)([!+:?]?=)([ \t]*)([^#\\]*?)(\s*)(#.*|)$', line)
-            if m and m[1].lstrip('#') == varname:
-                varassigns.append(Varassign(i, m[1], m[2], m[3], m[4], m[5], m[6]))
-        return varassigns
-
     def set(self, varname: str, new_value: str) -> bool:
         """ Updates the value of an existing variable in the lines. """
 
@@ -202,14 +219,6 @@ class Lines:
             self.lines.pop(varassign.index)
         return varassign is not None
 
-    def get(self, varname: str) -> str:
-        """
-        Returns the value from the only variable assignment, or an empty
-        string.
-        """
-        varassign = self.unique_varassign(varname)
-        return varassign.value if varassign is not None and varassign.varname == varname else ''
-
     def remove_if(self, varname: str, expected_value: str) -> bool:
         """ Removes a variable assignment if its value is the expected one. """
         for varassign in self.all_varassigns(varname):
@@ -218,13 +227,6 @@ class Lines:
                 return True
         return False
 
-    def index(self, pattern: str) -> int:
-        """ Returns the first index where the pattern is found, or -1. """
-        for (i, line) in enumerate(self.lines):
-            if re.search(pattern, line):
-                return i
-        return -1
-
 
 class Generator:
     """ Generates the initial package Makefile. """
@@ -245,8 +247,10 @@ class Generator:
         self.distname = ''
         self.pkgname = ''
 
-    @staticmethod
-    def foreach_site(action: Callable[[str, str], None]):
+    def foreach_site_from_sites_mk(self, action: Callable[[str, str], None]):
+        if self.master_sites != '':
+            return
+
         varname = ''
         with open('../../mk/fetch/sites.mk') as sites_mk:
             for line in sites_mk:
@@ -262,38 +266,51 @@ class Generator:
                 site_url = m[1]
                 action(varname, site_url)
 
-    def adjust_site(self, varname: str, site_url: str):
+    def adjust_site_from_sites_mk(self, varname: str, site_url: str):
         if not self.url.startswith(site_url):
             return
 
         rest = self.url[len(site_url):]
-        m = re.search(r'^(.+)/([^/]+)$', rest)
-        if not m:
+        if '/' not in rest:
             self.master_sites = f'${{{varname}}}'
+            self.distfile = rest
+            self.homepage = '# TODO'
             return
 
-        subdir, self.distfile = m.groups()
+        subdir, self.distfile = re.search(r'^(.*/)(.*)$', rest).groups()
 
-        self.master_sites = f'${{{varname}:={subdir}/}}'
-        if varname == 'MASTER_SITE_SOURCEFORGE':
-            self.homepage = f'https://{subdir}.sourceforge.net/'
-        elif varname == 'MASTER_SITE_GNU':
-            self.homepage = f'https://www.gnu.org/software/{subdir}/'
+        self.master_sites = f'${{{varname}:={subdir}}}'
+        if varname == 'MASTER_SITE_GNU':
+            self.homepage = f'https://www.gnu.org/software/{subdir}'
         else:
-            self.homepage = site_url[:-len(self.distfile)]
-
-    def adjust_site_sourceforge(self):
-        m = re.search(r'^https://downloads\.sourceforge\.net/project/([^/?]+)/[^?]+/([^/?]+)(?:[?].*)?$', self.url)
+            print('site_url', site_url)
+            print('distfile', self.distfile)
+            self.homepage = self.url[:-len(self.distfile)] + ' # TODO: check'
+            print('homepage', self.homepage)
+
+    def adjust_site_SourceForge(self):
+        pattern = r'^https?://downloads\.sourceforge\.net/' \
+                  r'(?:project|sourceforge)/' \
+                  r'([^/?]+)/' \
+                  r'((?:[^/?]+/)*)' \
+                  r'([^/?]+)' \
+                  r'(?:\?.*)?$'
+        m = re.search(pattern, self.url)
         if not m:
             return
 
-        project, filename = m.groups()
-        self.master_sites = f'${{MASTER_SITE_SOURCEFORGE:={project}/}}'
+        project, subdir, filename = m.groups()
+        self.master_sites = f'${{MASTER_SITE_SOURCEFORGE:={project}/{subdir}}}'
         self.homepage = f'https://{project}.sourceforge.net/'
         self.distfile = filename
 
     def adjust_site_GitHub_archive(self):
-        m = re.search(r'^https://github\.com/(.+)/(.+)/archive/(.+)(\.tar\.gz|\.zip)$', self.url)
+        pattern = r'^https://github\.com/' \
+                  r'(.+)/' \
+                  r'(.+)/archive/' \
+                  r'(.+)' \
+                  r'(\.tar\.gz|\.zip)$'
+        m = re.search(pattern, self.url)
         if not m:
             return
 
@@ -308,7 +325,13 @@ class Generator:
         self.distfile = tag + ext
 
     def adjust_site_GitHub_release(self):
-        m = re.search(r'^https://github\.com/(.+)/(.+)/releases/download/(.+)/(.+)(\.tar\.gz|\.zip)$', self.url)
+        pattern = r'^https://github\.com/' \
+                  r'(.+)/' \
+                  r'(.+)/releases/download/' \
+                  r'(.+)/' \
+                  r'(.+)' \
+                  r'(\.tar\.gz|\.zip)$'
+        m = re.search(pattern, self.url)
         if not m:
             return
 
@@ -327,15 +350,14 @@ class Generator:
         if self.master_sites != '':
             return
 
-        m = re.search(r'^(.*/)(.*)$', self.url)
-        if not m:
+        if '/' not in self.url:
             sys.exit(f'error: URL "{self.url}" must have at least one slash')
+        base_url, self.distfile = re.search(r'^(.*/)(.*)$', self.url).groups()
 
-        self.master_sites = m[1]
-        self.distfile = m[2]
-        self.homepage = self.master_sites
+        self.master_sites = base_url
+        self.homepage = base_url
 
-    def determine_distname(self):
+    def adjust_everything_else(self):
         m = re.search(r'^(.*?)((?:\.tar)?\.\w+)$', self.distfile)
         if m:
             distname, extract_sufx = m.groups()
@@ -394,12 +416,12 @@ class Generator:
         return lines
 
     def generate_Makefile(self):
-        self.foreach_site(self.adjust_site)
-        self.adjust_site_sourceforge()
+        self.adjust_site_SourceForge()
         self.adjust_site_GitHub_archive()
         self.adjust_site_GitHub_release()
+        self.foreach_site_from_sites_mk(self.adjust_site_from_sites_mk)
         self.adjust_site_other()
-        self.determine_distname()
+        self.adjust_everything_else()
         return self.generate_lines()
 
     def generate_package(self, up: Url2Pkg) -> Lines:
@@ -804,7 +826,7 @@ class Adjuster:
             self.makefile_lines = tx_lines
             self.regenerate_distinfo = True
 
-    def generate_adjusted_Makefile_lines(self) -> Lines:
+    def generate_lines(self) -> Lines:
         marker_index = self.makefile_lines.index(r'^# url2pkg-marker')
         if marker_index == -1:
             raise Exception('ERROR: didn\'t find the url2pkg marker in the Makefile.')
@@ -847,7 +869,7 @@ class Adjuster:
 
         return lines
 
-    def adjust_package_from_extracted_distfiles(self):
+    def adjust(self):
 
         def scan(basedir: str, pattern: str) -> List[str]:
             full_paths = glob.glob(f'{basedir}/{pattern}', recursive=True)
@@ -873,7 +895,7 @@ class Adjuster:
         self.adjust_po()
         self.adjust_use_languages()
 
-        self.generate_adjusted_Makefile_lines().write_to(self.up.pkgdir + '/Makefile')
+        self.generate_lines().write_to(self.up.pkgdir + '/Makefile')
 
         if self.regenerate_distinfo:
             self.up.bmake('distinfo')
@@ -899,7 +921,7 @@ def main():
     else:
         initial_lines = Generator(url).generate_lines()
 
-    Adjuster(up, url, initial_lines).adjust_package_from_extracted_distfiles()
+    Adjuster(up, url, initial_lines).adjust()
 
     print('')
     print('Remember to run pkglint when you\'re done.')
diff --git a/pkgtools/url2pkg/files/url2pkg_test.py b/pkgtools/url2pkg/files/url2pkg_test.py
index 7f47c7e40cf..899e7609ad8 100644
--- a/pkgtools/url2pkg/files/url2pkg_test.py
+++ b/pkgtools/url2pkg/files/url2pkg_test.py
@@ -1,7 +1,9 @@
-# $NetBSD: url2pkg_test.py,v 1.7 2019/10/04 22:26:34 rillig Exp $
+# $NetBSD: url2pkg_test.py,v 1.8 2019/10/05 11:02:30 rillig Exp $
 
+import pytest
 from url2pkg import *
 
+mkcvsid = '# $''NetBSD$'
 up: Url2Pkg
 
 
@@ -15,16 +17,57 @@ def setup_function(_):
 
 
 def str_vars(vars: List[Var]) -> List[str]:
-    return list(map(lambda var: var.name + var.op + var.value, vars))
+    def to_string(var):
+        return var.name + var.op + var.value
+
+    return list(map(to_string, vars))
+
+
+def str_varassigns(varassigns: Sequence[Varassign]) -> List[str]:
+    def to_string(v: Varassign) -> str:
+        return f'{v.varname}{v.op}{v.indent}' \
+               f'{v.value}{v.space_after_value}{v.comment}'
+
+    return list(map(to_string, varassigns))
+
+
+def detab(lines: Lines) -> List[str]:
+    """ Replaces tabs with the appropriate amount of spaces. """
+    def detab_line(line: str) -> str:
+        detabbed = []
+        for ch in line:
+            if ch == '\t':
+                detabbed.append('        '[:8 - len(detabbed) % 8])
+            else:
+                detabbed.append(ch)
+        return ''.join(detabbed)
+
+    return list(map(detab_line, lines.lines))
 
 
 def test_debug():
-    """ Just ensure that the debug calls do not crash. """
+    class Wr:
+        def __init__(self) -> None:
+            self.output = ''
+
+        def write(self, s: str):
+            self.output += s
+
+    up.verbose = True
+    up.err = Wr()
+
     up.debug('plain message')
     up.debug('list {0}', [1, 2, 3])
     up.debug('tuple {0}', (1, 2, 3))
     up.debug('cwd {0} env {1} cmd {2}', 'directory', {'VAR': 'value'}, 'command')
 
+    assert up.err.output.splitlines() == [
+        'url2pkg: plain message',
+        'url2pkg: list [1, 2, 3]',
+        'url2pkg: tuple (1, 2, 3)',
+        'url2pkg: cwd \'directory\' env {\'VAR\': \'value\'} cmd \'command\'',
+    ]
+
 
 def test_Lines__write_and_read(tmp_path):
     example = tmp_path / 'example'
@@ -37,25 +80,92 @@ def test_Lines__write_and_read(tmp_path):
 
     back = Lines.read_from(str(example))
 
-    assert back.lines == [
-        '1',
-        '2',
-        '3'
+    assert back.lines == ['1', '2', '3']
+
+
+def test_Lines_all_varassigns():
+    lines = Lines(
+        'OTHER=\tvalue',  # unrelated variable name
+        'VAR=\tvalue',
+        'VAR=value',  # no space between operator and value
+        'VAR=\t# only comment',
+        '#VAR=\t# commented variable assignment',
+        '#VAR=',
+        '# VAR=',  # This is a regular comment
+    )
+
+    assert str_varassigns(lines.all_varassigns('VAR')) == [
+        'VAR=\tvalue',
+        'VAR=value',
+        'VAR=\t# only comment',
+        '#VAR=\t# commented variable assignment',
+        '#VAR=',
     ]
 
 
+def test_Lines_unique_varassign():
+    lines = Lines(
+        'UNIQUE=\tunique',
+        'REPEATED=\tfirst',
+        'REPEATED+=\tlast',
+    )
+
+    assert lines.unique_varassign('UNIQUE') is not None
+    assert lines.unique_varassign('REPEATED') is None
+
+
+def test_Lines_get():
+    lines = Lines(
+        'VAR=value',
+        'VAR=\tvalue # comment',
+        'UNIQUE=\tunique',
+        '#COMMENTED=\tvalue',
+    )
+
+    assert lines.get('VAR') == ''  # too many values
+    assert lines.get('ENOENT') == ''  # not found
+    assert lines.get('UNIQUE') == 'unique'
+    assert lines.get('COMMENTED') == ''  # commented out
+
+
+def test_Lines_index():
+    lines = Lines('1', '2', '345')
+
+    assert lines.index('1') == 0
+    assert lines.index('2') == 1
+    assert lines.index('345') == 2
+    assert lines.index('4') == 2
+
+    assert lines.index(r'^(\d\d)\d$') == 2
+    assert lines.index(r'^\d\s\d$') == -1
+    assert lines.index(r'(\d)') == 0
+
+
+def test_Lines_add():
+    lines = Lines()
+
+    lines.add('')
+
+    # Adding variables might also be supported one day.
+    with pytest.raises(AssertionError):
+        lines.add(Var('VAR', '=', 'value'))
+
+    with pytest.raises(AssertionError):
+        lines.add(1)
+
+
 def test_Lines_add_vars__simple():
     lines = Lines()
 
     lines.add_vars(
-        Var("1", "=", "one"),
-        Var("6", "=", "six"),
+        Var('1', '=', 'one'),
+        Var('6', '=', 'six'),
     )
 
     assert lines.lines == [
-        "1=\tone",
-        "6=\tsix",
-        "",
+        '1=\tone',
+        '6=\tsix',
+        '',
     ]
 
 
@@ -63,116 +173,111 @@ def test_Lines_add_vars__alignment():
     lines = Lines()
 
     lines.add_vars(
-        Var("short", "=", "value"),
-        Var("long_name", "=", "value # comment"),
+        Var('short', '=', 'value'),
+        Var('long_name', '=', 'value # comment'),
     )
 
     assert lines.lines == [
-        "short=\t\tvalue",
-        "long_name=\tvalue # comment",
-        "",
+        'short=\t\tvalue',
+        'long_name=\tvalue # comment',
+        '',
     ]
 
 
 def test_Lines_add_vars__operators():
     lines = Lines()
 
-    lines.add_vars(
-        Var("123456", "+=", "value"),
-    )
+    lines.add_vars(Var('123456', '=', 'value'))
+    lines.add_vars(Var('1234567', '=', 'value'))
+    lines.add_vars(Var('123456', '+=', 'value'))
 
     assert lines.lines == [
-        "123456+=\tvalue",
-        "",
+        '123456=\tvalue',
+        '',
+        '1234567=\tvalue',
+        '',
+        '123456+=\tvalue',
+        '',
     ]
 
 
 def test_Lines_add_vars__empty():
-    lines = Lines("# initial")
+    lines = Lines('# initial')
 
     lines.add_vars()
 
     # No empty line is added.
-    assert lines.lines == ["# initial"]
-
-
-def test_Lines_append__not_found():
-    lines = Lines()
-
-    lines.append("VARNAME", "value")
-
-    assert lines.lines == []
-
-
-def test_Lines_append__only_comment():
-    lines = Lines("VARNAME=\t\t\t# none")
+    assert lines.lines == ['# initial']
 
-    lines.append("VARNAME", "value")
 
-    assert lines.lines == ["VARNAME=\t\t\tvalue # none"]
+def test_Lines_set__replace_comment():
+    lines = Lines('LICENSE=\t# TODO: see mk/license.mk')
 
+    assert lines.set('LICENSE', '${PERL5_LICENSE}')
 
-def test_Lines_append__value_with_comment():
-    lines = Lines("VARNAME=\tvalue # comment")
+    assert lines.lines == ['LICENSE=\t${PERL5_LICENSE}']
 
-    lines.append("VARNAME", "appended")
 
-    assert lines.lines == ["VARNAME=\tvalue appended # comment"]
+def test_Lines_set__overwrite_commented_comment_with_comment():
+    lines = Lines('#LICENSE=\t# TODO: see mk/license.mk')
 
+    assert lines.set('LICENSE', '${PERL5_LICENSE}')
 
-def test_Lines_append__value_without_comment():
-    lines = Lines("VARNAME+=\tvalue")
+    assert lines.lines == ['LICENSE=\t${PERL5_LICENSE}']
 
-    assert lines.append("VARNAME", "appended")
 
-    assert lines.lines == ["VARNAME+=\tvalue appended"]
+def test_Lines_set__not_found():
+    lines = Lines('OLD_VAR=\told value # old comment')
 
+    assert not lines.set('NEW_VAR', 'new value')
 
-def test_Lines_append__multiple_assignments():
-    lines = Lines("VARNAME+=\tvalue1", "VARNAME+=\tvalue2")
+    assert lines.lines == ['OLD_VAR=\told value # old comment']
 
-    assert not lines.append("VARNAME", "appended")
 
-    assert lines.lines == ["VARNAME+=\tvalue1", "VARNAME+=\tvalue2"]
+def test_Lines_append__not_found():
+    lines = Lines()
 
+    lines.append('VARNAME', 'value')
 
-def test_Lines_set__previously_with_comment():
-    lines = Lines("LICENSE=\t# TODO: see mk/license.mk")
+    assert lines.lines == []
 
-    assert lines.set("LICENSE", "${PERL5_LICENSE}")
 
-    assert lines.lines == ["LICENSE=\t${PERL5_LICENSE}"]
+def test_Lines_append__no_value_only_comment():
+    lines = Lines('VARNAME=\t\t\t# none')
 
+    lines.append('VARNAME', 'value')
 
-def test_Lines_unique_varassign__commented_out_no_value():
-    lines = Lines("#LICENSE=\t# TODO: see mk/license.mk")
+    assert lines.lines == ['VARNAME=\t\t\tvalue # none']
 
-    assert len(lines.all_varassigns('LICENSE')) == 1
 
+def test_Lines_append__value_with_comment():
+    lines = Lines('VARNAME=\tvalue # comment')
 
-def test_Lines_set__overwrite_comment_with_comment():
-    lines = Lines("#LICENSE=\t# TODO: see mk/license.mk")
+    lines.append('VARNAME', 'appended')
 
-    assert len(lines.all_varassigns('LICENSE')) == 1
-    assert lines.set("LICENSE", "${PERL5_LICENSE}")
+    assert lines.lines == ['VARNAME=\tvalue appended # comment']
 
-    assert lines.lines == ["LICENSE=\t${PERL5_LICENSE}"]
 
+def test_Lines_append__value_without_comment():
+    lines = Lines('VARNAME+=\tvalue')
 
-def test_Lines_set__overwrite_commented_with_comment():
-    lines = Lines("#LICENSE=\t# TODO: see mk/license.mk")
+    assert lines.append('VARNAME', 'appended')
 
-    assert lines.set("LICENSE", "${PERL5_LICENSE}")
+    assert lines.lines == ['VARNAME+=\tvalue appended']
 
-    assert lines.lines == ["LICENSE=\t${PERL5_LICENSE}"]
 
+def test_Lines_append__multiple_assignments():
+    # When there is more than one assignment for a variable,
+    # it may not be clear which to append to.
+    # The assignments might be in an .if statement.
+    # Therefore, rather do nothing.
+    # Assuming no .if statements, appending to the last one makes sense.
 
-def test_Lines_set__not_found():
-    lines = Lines("OLD_VAR=\told value # old comment")
+    lines = Lines('VARNAME+=\tvalue1', 'VARNAME+=\tvalue2')
 
-    assert not lines.set("NEW_VAR", "new value")
+    assert not lines.append('VARNAME', 'appended')
 
-    assert lines.lines == ["OLD_VAR=\told value # old comment"]
+    assert lines.lines == ['VARNAME+=\tvalue1', 'VARNAME+=\tvalue2']
 
 
 def test_Lines_remove__not_found():
@@ -235,150 +340,170 @@ def test_Lines_remove_if__multiple():
     assert lines.lines == []
 
 
-def test_Lines_index():
-    lines = Lines("1", "2", "345")
+def test_Generator_adjust_site_SourceForge():
+    url = 'http://downloads.sourceforge.net/sourceforge/rfcascade/cascade-1.4.tar.gz'
 
-    assert lines.index("1") == 0
-    assert lines.index("2") == 1
-    assert lines.index("345") == 2
-    assert lines.index("4") == 2
+    lines = Generator(url).generate_Makefile()
 
-    assert lines.index(r'^(\d\d)\d$') == 2
-    assert lines.index(r"^\d\s\d$") == -1
-    assert lines.index(r"(\d)") == 0
+    assert lines.lines == [
+        mkcvsid,
+        '',
+        'DISTNAME=\tcascade-1.4',
+        'CATEGORIES=\tpkgtools',
+        'MASTER_SITES=\t${MASTER_SITE_SOURCEFORGE:=rfcascade/}',
+        '',
+        'MAINTAINER=\tINSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
+        'HOMEPAGE=\thttps://rfcascade.sourceforge.net/',
+        'COMMENT=\tTODO: Short description of the package',
+        '#LICENSE=\t# TODO: (see mk/license.mk)',
+        '',
+        '# url2pkg-marker (please do not remove this line.)',
+        ".include \"../../mk/bsd.pkg.mk\"",
+    ]
 
 
-def test_Lines_get():
-    lines = Lines(
-        "VAR=value",
-        "VAR=\tvalue # comment",
-        "UNIQUE=\tunique",
-        "#COMMENTED=\tvalue",
-    )
+def test_Generator_adjust_site_GitHub_archive():
+    url = 'https://github.com/org/proj/archive/v1.0.0.tar.gz'
 
-    assert lines.get("VAR") == ""  # too many values
-    assert lines.get("ENOENT") == ""  # no value at all
-    assert lines.get("UNIQUE") == "unique"
-    assert lines.get("COMMENTED") == ""  # commented out
+    lines = Generator(url).generate_Makefile()
+    assert detab(lines) == [
+        mkcvsid,
+        '',
+        'GITHUB_PROJECT= proj',
+        'DISTNAME=       v1.0.0',
+        'PKGNAME=        ${GITHUB_PROJECT}-${DISTNAME:S,^v,,}',
+        'CATEGORIES=     pkgtools',
+        'MASTER_SITES=   ${MASTER_SITE_GITHUB:=org/}',
+        'DIST_SUBDIR=    ${GITHUB_PROJECT}',
+        '',
+        'MAINTAINER=     INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
+        'HOMEPAGE=       https://github.com/org/proj/',
+        'COMMENT=        TODO: Short description of the package',
+        '#LICENSE=       # TODO: (see mk/license.mk)',
+        '',
+        '# url2pkg-marker (please do not remove this line.)',
+        ".include \"../../mk/bsd.pkg.mk\"",
+    ]
 
 
-def test_generate_initial_package_Makefile_lines__GitHub_archive():
-    url = "https://github.com/org/proj/archive/v1.0.0.tar.gz"
+def test_Generator_adjust_site_GitHub_release__containing_project_name():
+    url = 'https://github.com/org/proj/releases/download/1.0.0/proj.zip'
 
     lines = Generator(url).generate_Makefile()
 
-    assert lines.lines == [
-        "# $" + "NetBSD$",
-        "",
-        "GITHUB_PROJECT=\tproj",
-        "DISTNAME=\tv1.0.0",
-        "PKGNAME=\t${GITHUB_PROJECT}-${DISTNAME:S,^v,,}",
-        "CATEGORIES=\tpkgtools",
-        "MASTER_SITES=\t${MASTER_SITE_GITHUB:=org/}",
-        "DIST_SUBDIR=\t${GITHUB_PROJECT}",
-        "",
-        "MAINTAINER=\tINSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org",
-        "HOMEPAGE=\thttps://github.com/org/proj/",
-        "COMMENT=\tTODO: Short description of the package",
-        "#LICENSE=\t# TODO: (see mk/license.mk)",
-        "",
-        "# url2pkg-marker (please do not remove this line.)",
+    assert detab(lines) == [
+        mkcvsid,
+        '',
+        'GITHUB_PROJECT= proj',
+        'DISTNAME=       proj',
+        'CATEGORIES=     pkgtools',
+        'MASTER_SITES=   ${MASTER_SITE_GITHUB:=org/}',
+        'GITHUB_RELEASE= 1.0.0',
+        'EXTRACT_SUFX=   .zip',
+        '',
+        'MAINTAINER=     INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
+        'HOMEPAGE=       https://github.com/org/proj/',
+        'COMMENT=        TODO: Short description of the package',
+        '#LICENSE=       # TODO: (see mk/license.mk)',
+        '',
+        '# url2pkg-marker (please do not remove this line.)',
         ".include \"../../mk/bsd.pkg.mk\""
     ]
 
 
-def test_generate_initial_package_Makefile_lines__GitHub_release_containing_project_name():
-    url = "https://github.com/org/proj/releases/download/1.0.0/proj.zip"
+def test_Generator_adjust_site_GitHub_release__not_containing_project_name():
+    url = 'https://github.com/org/proj/releases/download/1.0.0/data.zip'
 
     lines = Generator(url).generate_Makefile()
 
-    assert lines.lines == [
-        "# $" + "NetBSD$",
-        "",
-        "GITHUB_PROJECT=\tproj",
-        "DISTNAME=\tproj",
-        "CATEGORIES=\tpkgtools",
-        "MASTER_SITES=\t${MASTER_SITE_GITHUB:=org/}",
-        "GITHUB_RELEASE=\t1.0.0",
-        "EXTRACT_SUFX=\t.zip",
-        "",
-        "MAINTAINER=\tINSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org",
-        "HOMEPAGE=\thttps://github.com/org/proj/",
-        "COMMENT=\tTODO: Short description of the package",
-        "#LICENSE=\t# TODO: (see mk/license.mk)",
-        "",
-        "# url2pkg-marker (please do not remove this line.)",
+    assert detab(lines) == [
+        mkcvsid,
+        '',
+        'GITHUB_PROJECT= proj',
+        'DISTNAME=       data',
+        'CATEGORIES=     pkgtools',
+        'MASTER_SITES=   ${MASTER_SITE_GITHUB:=org/}',
+        'GITHUB_RELEASE= 1.0.0',
+        'EXTRACT_SUFX=   .zip',
+        'DIST_SUBDIR=    ${GITHUB_PROJECT}',
+        '',
+        'MAINTAINER=     INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
+        'HOMEPAGE=       https://github.com/org/proj/',
+        'COMMENT=        TODO: Short description of the package',
+        '#LICENSE=       # TODO: (see mk/license.mk)',
+        '',
+        '# url2pkg-marker (please do not remove this line.)',
         ".include \"../../mk/bsd.pkg.mk\""
     ]
 
 
-def test_generate_initial_package_Makefile_lines__GitHub_release_not_containing_project_name():
-    url = "https://github.com/org/proj/releases/download/1.0.0/data.zip"
+def test_Generator_adjust_site_from_sites_mk__with_subdir():
+    url = 'https://files.pythonhosted.org/packages/source/i/irc/irc-11.1.1.zip'
+    generator = Generator(url)
 
-    lines = Generator(url).generate_Makefile()
+    lines = generator.generate_Makefile()
 
-    assert lines.lines == [
-        "# $" + "NetBSD$",
-        "",
-        "GITHUB_PROJECT=\tproj",
-        "DISTNAME=\tdata",
-        "CATEGORIES=\tpkgtools",
-        "MASTER_SITES=\t${MASTER_SITE_GITHUB:=org/}",
-        "GITHUB_RELEASE=\t1.0.0",
-        "EXTRACT_SUFX=\t.zip",
-        "DIST_SUBDIR=\t${GITHUB_PROJECT}",
-        "",
-        "MAINTAINER=\tINSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org",
-        "HOMEPAGE=\thttps://github.com/org/proj/",
-        "COMMENT=\tTODO: Short description of the package",
-        "#LICENSE=\t# TODO: (see mk/license.mk)",
-        "",
-        "# url2pkg-marker (please do not remove this line.)",
-        ".include \"../../mk/bsd.pkg.mk\""
+    assert detab(lines) == [
+        mkcvsid,
+        '',
+        'DISTNAME=       irc-11.1.1',
+        'CATEGORIES=     pkgtools',
+        'MASTER_SITES=   ${MASTER_SITE_PYPI:=i/irc/}',
+        'EXTRACT_SUFX=   .zip',
+        '',
+        'MAINTAINER=     INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
+        'HOMEPAGE=       https://files.pythonhosted.org/packages/source/i/irc/ # TODO: check',
+        'COMMENT=        TODO: Short description of the package',
+        '#LICENSE=       # TODO: (see mk/license.mk)',
+        '',
+        '# url2pkg-marker (please do not remove this line.)',
+        '.include "../../mk/bsd.pkg.mk"',
     ]
 
 
-def test_generate_initial_package_Makefile_lines__distname_version_with_v():
-    url = "https://cpan.example.org/Algorithm-CheckDigits-v1.3.2.tar.gz"
+def test_Generator_adjust_site_from_sites_mk__without_subdir():
+    url = 'https://files.pythonhosted.org/packages/source/irc-11.1.1.zip'
+    generator = Generator(url)
 
-    lines = Generator(url).generate_Makefile()
+    lines = generator.generate_Makefile()
 
-    assert lines.lines == [
-        "# $" + "NetBSD$",
-        "",
-        "DISTNAME=\tAlgorithm-CheckDigits-v1.3.2",
-        "PKGNAME=\t${DISTNAME:S,-v,-,}",
-        "CATEGORIES=\tpkgtools",
-        "MASTER_SITES=\thttps://cpan.example.org/",
-        "",
-        "MAINTAINER=\tINSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org",
-        "HOMEPAGE=\thttps://cpan.example.org/",
-        "COMMENT=\tTODO: Short description of the package",
-        "#LICENSE=\t# TODO: (see mk/license.mk)",
-        "",
-        "# url2pkg-marker (please do not remove this line.)",
-        ".include \"../../mk/bsd.pkg.mk\""
+    assert detab(lines) == [
+        mkcvsid,
+        '',
+        'DISTNAME=       irc-11.1.1',
+        'CATEGORIES=     pkgtools',
+        'MASTER_SITES=   ${MASTER_SITE_PYPI}',
+        'EXTRACT_SUFX=   .zip',
+        '',
+        'MAINTAINER=     INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
+        'HOMEPAGE=       # TODO',
+        'COMMENT=        TODO: Short description of the package',
+        '#LICENSE=       # TODO: (see mk/license.mk)',
+        '',
+        '# url2pkg-marker (please do not remove this line.)',
+        '.include "../../mk/bsd.pkg.mk"',
     ]
 
 
-def test_Generator_adjust_site():
-    url = 'https://files.pythonhosted.org/packages/source/i/irc/irc-11.1.1.zip'
-    generator = Generator(url)
+def test_Generator_adjust_everything_else__distname_version_with_v():
+    # Some version numbers have a leading 'v', derived from the Git tag name.
 
-    lines = generator.generate_Makefile()
+    url = 'https://cpan.example.org/Algorithm-CheckDigits-v1.3.2.tar.gz'
 
-    assert lines.lines == [
-        '# $NetBSD: url2pkg_test.py,v 1.7 2019/10/04 22:26:34 rillig Exp $',
+    lines = Generator(url).generate_Makefile()
+
+    assert detab(lines) == [
+        mkcvsid,
         '',
-        'DISTNAME=\tirc-11.1.1',
-        'CATEGORIES=\tpkgtools',
-        'MASTER_SITES=\t${MASTER_SITE_PYPI:=i/irc/}',
-        'EXTRACT_SUFX=\t.zip',
+        'DISTNAME=       Algorithm-CheckDigits-v1.3.2',
+        'PKGNAME=        ${DISTNAME:S,-v,-,}',
+        'CATEGORIES=     pkgtools',
+        'MASTER_SITES=   https://cpan.example.org/',
         '',
-        'MAINTAINER=\tINSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
-        'HOMEPAGE=\thttps://files.pythonhosted.org/pa',
-        'COMMENT=\tTODO: Short description of the package',
-        '#LICENSE=\t# TODO: (see mk/license.mk)',
+        'MAINTAINER=     INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
+        'HOMEPAGE=       https://cpan.example.org/',
+        'COMMENT=        TODO: Short description of the package',
+        '#LICENSE=       # TODO: (see mk/license.mk)',
         '',
         '# url2pkg-marker (please do not remove this line.)',
         '.include "../../mk/bsd.pkg.mk"'
@@ -390,176 +515,166 @@ def test_Generator_determine_distname__v8():
 
     lines = generator.generate_Makefile()
 
-    assert lines.lines == [
-        '# $NetBSD: url2pkg_test.py,v 1.7 2019/10/04 22:26:34 rillig Exp $',
+    assert detab(lines) == [
+        mkcvsid,
         '',
-        'DISTNAME=\tv8-1.0',
-        'PKGNAME=\t${DISTNAME:S,^v,,}',  # FIXME: v8 is part of the PKGBASE
-        'CATEGORIES=\tpkgtools',
-        'MASTER_SITES=\thttps://example.org/',
-        'EXTRACT_SUFX=\t.zip',
+        'DISTNAME=       v8-1.0',
+        'PKGNAME=        ${DISTNAME:S,^v,,}',  # FIXME: v8 is part of the PKGBASE
+        'CATEGORIES=     pkgtools',
+        'MASTER_SITES=   https://example.org/',
+        'EXTRACT_SUFX=   .zip',
         '',
-        'MAINTAINER=\tINSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
-        'HOMEPAGE=\thttps://example.org/',
-        'COMMENT=\tTODO: Short description of the package',
-        '#LICENSE=\t# TODO: (see mk/license.mk)',
+        'MAINTAINER=     INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
+        'HOMEPAGE=       https://example.org/',
+        'COMMENT=        TODO: Short description of the package',
+        '#LICENSE=       # TODO: (see mk/license.mk)',
         '',
         '# url2pkg-marker (please do not remove this line.)',
-        '.include "../../mk/bsd.pkg.mk"'
+        '.include "../../mk/bsd.pkg.mk"',
     ]
 
 
 def test_Adjuster_read_dependencies():
-    dep_lines = [
-        "DEPENDS\tpackage>=80.0:../../pkgtools/pkglint",
-        "DEPENDS\tpackage>=120.0:../../pkgtools/x11-links",
-        "BUILD_DEPENDS\turl2pkg>=1.0",
-        "TEST_DEPENDS\tpkglint",
-        "A line that is not a dependency at all",
-        "",
-        "var\tHOMEPAGE\thttps://homepage.example.org/"
-        ""
-    ]
-    env = {"URL2PKG_DEPENDENCIES": '\n'.join(dep_lines)}
+    child_process_output = [
+        'DEPENDS\tpackage>=112.0:../../pkgtools/pkglint',
+        'DEPENDS\tpackage>=120.0:../../pkgtools/x11-links',
+        'BUILD_DEPENDS\turl2pkg>=1.0',
+        'TEST_DEPENDS\tpkglint',
+        'A line that is not a dependency at all',
+        '',
+        'var\tHOMEPAGE\thttps://homepage.example.org/',
+        ''
+    ]
+    env = {'URL2PKG_DEPENDENCIES': '\n'.join(child_process_output)}
     cmd = "printf '%s\n' \"$URL2PKG_DEPENDENCIES\""
 
     adjuster = Adjuster(up, '', Lines())
     adjuster.read_dependencies(cmd, env, '.', '')
 
     assert os.getenv('URL2PKG_DEPENDENCIES') is None
-
-    assert adjuster.depends == [
-        "package>=80.0:../../pkgtools/pkglint"
-    ]
+    assert adjuster.depends == ['package>=112.0:../../pkgtools/pkglint']
     assert adjuster.bl3_lines == [
         'BUILDLINK_API_DEPENDS.x11-links+=\tx11-links>=120.0',
-        ".include \"../../pkgtools/x11-links/buildlink3.mk\""
-    ]
-    assert adjuster.build_depends == [
-        "url2pkg>=1.0:../../pkgtools/url2pkg"
-    ]
-    assert adjuster.test_depends == [
-        "pkglint>=0:../../pkgtools/pkglint"
+        ".include \"../../pkgtools/x11-links/buildlink3.mk\"",
     ]
-    assert adjuster.update_vars == {
-        'HOMEPAGE': 'https://homepage.example.org/'
-    }
+    assert adjuster.build_depends == ['url2pkg>=1.0:../../pkgtools/url2pkg']
+    assert adjuster.test_depends == ['pkglint>=0:../../pkgtools/pkglint']
+    assert adjuster.update_vars == {'HOMEPAGE': 'https://homepage.example.org/'}
 
 
 def test_Adjuster_generate_adjusted_Makefile_lines():
-    adjuster = Adjuster(up, "https://example.org/pkgname-1.0.tar.gz", Lines())
+    adjuster = Adjuster(up, 'https://example.org/pkgname-1.0.tar.gz', Lines())
     adjuster.makefile_lines = Lines(
-        "# before 1",
-        "# before 2",
-        "# url2pkg-marker",
-        "# after 1",
-        "# after 2"
+        '# before 1',
+        '# before 2',
+        '# url2pkg-marker',
+        '# after 1',
+        '# after 2'
     )
 
-    lines = adjuster.generate_adjusted_Makefile_lines()
+    lines = adjuster.generate_lines()
 
     assert lines.lines == [
-        "# before 1",
-        "# before 2",
-        "# after 1",
-        "# after 2"
+        '# before 1',
+        '# before 2',
+        '# after 1',
+        '# after 2',
     ]
 
 
 def test_Adjuster_generate_adjusted_Makefile_lines__dependencies():
-    adjuster = Adjuster(up, "https://example.org/pkgname-1.0.tar.gz", Lines())
+    adjuster = Adjuster(up, 'https://example.org/pkgname-1.0.tar.gz', Lines())
     adjuster.makefile_lines.add(
-        "# $" + "NetBSD$",
-        "",
-        "# url2pkg-marker",
+        mkcvsid,
+        '',
+        '# url2pkg-marker',
         ".include \"../../mk/bsd.pkg.mk\""
     )
-
     # some dependencies whose directory will not be found
-    adjuster.add_dependency("DEPENDS", "depends", ">=5.0", "../../devel/depends")
-    adjuster.add_dependency("TOOL_DEPENDS", "tool-depends", ">=6.0", "../../devel/tool-depends")
-    adjuster.add_dependency("BUILD_DEPENDS", "build-depends", ">=7.0", "../../devel/build-depends")
-    adjuster.add_dependency("TEST_DEPENDS", "test-depends", ">=8.0", "../../devel/test-depends")
+    adjuster.add_dependency('DEPENDS', 'depends', '>=5.0', '../../devel/depends')
+    adjuster.add_dependency('TOOL_DEPENDS', 'tool-depends', '>=6.0', '../../devel/tool-depends')
+    adjuster.add_dependency('BUILD_DEPENDS', 'build-depends', '>=7.0', '../../devel/build-depends')
+    adjuster.add_dependency('TEST_DEPENDS', 'test-depends', '>=8.0', '../../devel/test-depends')
     # some dependencies whose directory is explicitly given
-    adjuster.depends.append("depends>=11.0:../../devel/depends")
-    adjuster.build_depends.append("build-depends>=12.0:../../devel/build-depends")
-    adjuster.test_depends.append("test-depends>=13.0:../../devel/test-depends")
+    adjuster.depends.append('depends>=11.0:../../devel/depends')
+    adjuster.build_depends.append('build-depends>=12.0:../../devel/build-depends')
+    adjuster.test_depends.append('test-depends>=13.0:../../devel/test-depends')
 
-    lines = adjuster.generate_adjusted_Makefile_lines()
+    lines = adjuster.generate_lines()
 
-    assert lines.lines == [
-        "# $" + "NetBSD$",
-        "",
-        "# TODO: dependency TOOL_DEPENDS # TODO: tool-depends>=6.0",
-        "",
-        "BUILD_DEPENDS+=\t# TODO: build-depends>=7.0",
-        "BUILD_DEPENDS+=\tbuild-depends>=12.0:../../devel/build-depends",
-        "DEPENDS+=\t# TODO: depends>=5.0",
-        "DEPENDS+=\tdepends>=11.0:../../devel/depends",
-        "TEST_DEPENDS+=\t# TODO: test-depends>=8.0",
-        "TEST_DEPENDS+=\ttest-depends>=13.0:../../devel/test-depends",
-        "",
+    assert detab(lines) == [
+        mkcvsid,
+        '',
+        '# TODO: dependency TOOL_DEPENDS # TODO: tool-depends>=6.0',
+        '',
+        'BUILD_DEPENDS+= # TODO: build-depends>=7.0',
+        'BUILD_DEPENDS+= build-depends>=12.0:../../devel/build-depends',
+        'DEPENDS+=       # TODO: depends>=5.0',
+        'DEPENDS+=       depends>=11.0:../../devel/depends',
+        'TEST_DEPENDS+=  # TODO: test-depends>=8.0',
+        'TEST_DEPENDS+=  test-depends>=13.0:../../devel/test-depends',
+        '',
         ".include \"../../mk/bsd.pkg.mk\""
     ]
 
 
 def test_Adjuster_generate_adjusted_Makefile_lines__dont_overwrite_PKGNAME():
-    adjuster = Adjuster(up, "https://example.org/pkgname-1.0.tar.gz", Lines())
+    adjuster = Adjuster(up, 'https://example.org/pkgname-1.0.tar.gz', Lines())
     adjuster.makefile_lines.add(
-        "# $" + "NetBSD$",
-        "DISTNAME=\tdistname-1.0",
-        "PKGNAME=\tmanually-edited-pkgname-1.0"
-        "",
-        "# url2pkg-marker",
+        mkcvsid,
+        'DISTNAME=\tdistname-1.0',
+        'PKGNAME=\tmanually-edited-pkgname-1.0'
+        '',
+        '# url2pkg-marker',
         ".include \"../../mk/bsd.pkg.mk\""
     )
 
-    lines = adjuster.generate_adjusted_Makefile_lines()
+    lines = adjuster.generate_lines()
 
-    assert lines.lines == [
-        "# $" + "NetBSD$",
-        "DISTNAME=\tdistname-1.0",
-        'PKGNAME=\tmanually-edited-pkgname-1.0',
+    assert detab(lines) == [
+        mkcvsid,
+        'DISTNAME=       distname-1.0',
+        'PKGNAME=        manually-edited-pkgname-1.0',
         ".include \"../../mk/bsd.pkg.mk\""
     ]
 
 
 def test_Adjuster_generate_adjusted_Makefile_lines__add_PKGNAME():
-    adjuster = Adjuster(up, "https://example.org/pkgname-1.0.tar.gz", Lines())
+    adjuster = Adjuster(up, 'https://example.org/pkgname-1.0.tar.gz', Lines())
     adjuster.makefile_lines.add(
-        "# $" + "NetBSD$",
-        "DISTNAME=\tdistname-1.0",
-        "",
-        "# url2pkg-marker",
+        mkcvsid,
+        'DISTNAME=\tdistname-1.0',
+        '',
+        '# url2pkg-marker',
         ".include \"../../mk/bsd.pkg.mk\""
     )
 
-    lines = adjuster.generate_adjusted_Makefile_lines()
+    lines = adjuster.generate_lines()
 
     assert lines.lines == [
-        "# $" + "NetBSD$",
-        "DISTNAME=\tdistname-1.0",
+        mkcvsid,
+        'DISTNAME=\tdistname-1.0',
         '',
         ".include \"../../mk/bsd.pkg.mk\""
     ]
 
 
 def test_Adjuster_generate_adjusted_Makefile_lines__add_PKGNAME_with_prefix():
-    adjuster = Adjuster(up, "https://example.org/pkgname-1.0.tar.gz", Lines())
+    adjuster = Adjuster(up, 'https://example.org/pkgname-1.0.tar.gz', Lines())
     adjuster.makefile_lines.add(
-        "# $" + "NetBSD$",
-        "DISTNAME=\tdistname-1.0",
-        "",
-        "# url2pkg-marker",
+        mkcvsid,
+        'DISTNAME=\tdistname-1.0',
+        '',
+        '# url2pkg-marker',
         ".include \"../../mk/bsd.pkg.mk\""
     )
     adjuster.pkgname_prefix = '${PYPKGPREFIX}-'
 
-    lines = adjuster.generate_adjusted_Makefile_lines()
+    lines = adjuster.generate_lines()
 
     assert lines.lines == [
-        "# $" + "NetBSD$",
-        "DISTNAME=\tdistname-1.0",
+        mkcvsid,
+        'DISTNAME=\tdistname-1.0',
         'PKGNAME=\t${PYPKGPREFIX}-${DISTNAME}',
         '',
         ".include \"../../mk/bsd.pkg.mk\""
@@ -567,12 +682,15 @@ def test_Adjuster_generate_adjusted_Makefile_lines__add_PKGNAME_with_prefix():
 
 
 def test_Adjuster_add_dependency__buildlink():
+    # Note: this test only works because it runs in pkgtools/url2pkg,
+    # and from there the file ../../devel/libusb/buildlink3.mk is visible.
+
     adjuster = Adjuster(up, 'https://example.org/distfile-1.0.zip', Lines())
     adjuster.makefile_lines.add('# url2pkg-marker')
 
     adjuster.add_dependency('BUILD_DEPENDS', 'libusb', '>=2019', '../../devel/libusb')
 
-    lines = adjuster.generate_adjusted_Makefile_lines()
+    lines = adjuster.generate_lines()
 
     assert lines.lines == [
         'BUILDLINK_DEPENDS.libusb+=\tbuild',
@@ -581,7 +699,7 @@ def test_Adjuster_add_dependency__buildlink():
     ]
 
 
-def test_Adjuster_adjust_configure__not_found(tmp_path):
+def test_Adjuster_adjust_configure__none(tmp_path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrksrc = str(tmp_path)
 
@@ -590,7 +708,7 @@ def test_Adjuster_adjust_configure__not_found(tmp_path):
     assert adjuster.build_vars == []
 
 
-def test_Adjuster_adjust_configure__GNU_configure(tmp_path):
+def test_Adjuster_adjust_configure__GNU(tmp_path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrksrc = str(tmp_path)
     adjuster.wrksrc_files.append('configure')
@@ -603,7 +721,7 @@ def test_Adjuster_adjust_configure__GNU_configure(tmp_path):
     ]
 
 
-def test_Adjuster_adjust_configure__other_configure(tmp_path):
+def test_Adjuster_adjust_configure__other(tmp_path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrksrc = str(tmp_path)
     adjuster.wrksrc_files.append('configure')
@@ -628,12 +746,12 @@ def test_Adjuster_adjust_cargo__not_found(tmp_path):
 def test_Adjuster_adjust_cargo__found(tmp_path):
     adjuster = Adjuster(up, '', Lines())
     adjuster.abs_wrksrc = str(tmp_path)
-    (tmp_path / 'Cargo.lock').write_text('"checksum cargo-package-name cargo-package-version 1234"')
+    (tmp_path / 'Cargo.lock').write_text('"checksum cargo-pkg 1.2.3 1234"')
 
     adjuster.adjust_cargo()
 
     assert str_vars(adjuster.build_vars) == [
-        'CARGO_CRATE_DEPENDS+=cargo-package-name-cargo-package-version',
+        'CARGO_CRATE_DEPENDS+=cargo-pkg-1.2.3',
     ]
 
 
@@ -664,9 +782,7 @@ def test_Adjuster_adjust_libtool__ltconfig(tmp_path):
 
     adjuster.adjust_libtool()
 
-    assert str_vars(adjuster.build_vars) == [
-        'USE_LIBTOOL=yes'
-    ]
+    assert str_vars(adjuster.build_vars) == ['USE_LIBTOOL=yes']
 
 
 def test_Adjuster_adjust_libtool__libltdl(tmp_path):
@@ -695,9 +811,7 @@ def test_Adjuster_adjust_po__mo_found():
 
     adjuster.adjust_po()
 
-    assert str_vars(adjuster.build_vars) == [
-        'USE_PKGLOCALEDIR=yes'
-    ]
+    assert str_vars(adjuster.build_vars) == ['USE_PKGLOCALEDIR=yes']
 
 
 def test_Adjuster_adjust_po__po_found():
@@ -706,9 +820,7 @@ def test_Adjuster_adjust_po__po_found():
 
     adjuster.adjust_po()
 
-    assert str_vars(adjuster.build_vars) == [
-        'USE_PKGLOCALEDIR=yes'
-    ]
+    assert str_vars(adjuster.build_vars) == ['USE_PKGLOCALEDIR=yes']
 
 
 def test_Adjuster_adjust_use_languages__none():
@@ -716,9 +828,7 @@ def test_Adjuster_adjust_use_languages__none():
 
     adjuster.adjust_use_languages()
 
-    assert str_vars(adjuster.build_vars) == [
-        'USE_LANGUAGES=# none'
-    ]
+    assert str_vars(adjuster.build_vars) == ['USE_LANGUAGES=# none']
 
 
 def test_Adjuster_adjust_use_languages__c():
@@ -745,9 +855,7 @@ def test_Adjuster_adjust_use_languages__cplusplus_in_subdir():
 
     adjuster.adjust_use_languages()
 
-    assert str_vars(adjuster.build_vars) == [
-        'USE_LANGUAGES=c++'
-    ]
+    assert str_vars(adjuster.build_vars) == ['USE_LANGUAGES=c++']
 
 
 def test_Adjuster_adjust_use_languages__cplusplus_and_fortran():
@@ -756,9 +864,7 @@ def test_Adjuster_adjust_use_languages__cplusplus_and_fortran():
 
     adjuster.adjust_use_languages()
 
-    assert str_vars(adjuster.build_vars) == [
-        'USE_LANGUAGES=c++ fortran'
-    ]
+    assert str_vars(adjuster.build_vars) == ['USE_LANGUAGES=c++ fortran']
 
 
 def test_Adjuster_adjust_pkg_config__none():
@@ -794,7 +900,7 @@ def test_Adjuster_adjust_pkg_config__both():
     adjuster = Adjuster(up, '', Lines())
     adjuster.wrksrc_files = [
         'library.pc.in',
-        'library-uninstalled.pc.in'
+        'library-uninstalled.pc.in',
     ]
 
     adjuster.adjust_pkg_config()
@@ -811,25 +917,25 @@ def test_Adjuster__adjust_homepage():
     adjuster.depends.append('dependency>=0:../../category/dependency')
     adjuster.todos.append('Run pkglint')
 
-    lines = adjuster.generate_adjusted_Makefile_lines()
+    lines = adjuster.generate_lines()
 
-    assert lines.lines == [
-        '# $' + 'NetBSD$',
+    assert detab(lines) == [
+        mkcvsid,
         '',
-        'DISTNAME=\tpackage-1.0',
-        'CATEGORIES=\tpkgtools',
-        'MASTER_SITES=\thttps://dummy.example.org/',
+        'DISTNAME=       package-1.0',
+        'CATEGORIES=     pkgtools',
+        'MASTER_SITES=   https://dummy.example.org/',
         '',
-        'MAINTAINER=\tINSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
-        'HOMEPAGE=\thttps://example.org/',
-        'COMMENT=\tTODO: Short description of the package',
-        '#LICENSE=\t# TODO: (see mk/license.mk)',
+        'MAINTAINER=     INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
+        'HOMEPAGE=       https://example.org/',
+        'COMMENT=        TODO: Short description of the package',
+        '#LICENSE=       # TODO: (see mk/license.mk)',
         '',
         '# TODO: Run pkglint',
         '',
-        'DEPENDS+=\tdependency>=0:../../category/dependency',
+        'DEPENDS+=       dependency>=0:../../category/dependency',
         '',
-        '.include "../../mk/bsd.pkg.mk"'
+        '.include "../../mk/bsd.pkg.mk"',
     ]
 
 
@@ -862,7 +968,7 @@ def test_Adjuster_determine_wrksrc__several_dirs(tmp_path):
 
     assert adjuster.abs_wrksrc == adjuster.abs_wrkdir
     assert str_vars(adjuster.build_vars) == [
-        'WRKSRC=${WRKDIR} # More than one possibility -- please check manually.'
+        'WRKSRC=${WRKDIR} # More than one possibility -- please check manually.',
     ]
 
 
@@ -882,20 +988,20 @@ esac
     adjuster = Adjuster(up, url, Lines())
     adjuster.abs_wrkdir = str(wrkdir)
     (pkgdir / 'Makefile').write_text('# url2pkg-marker\n')
-    fake_path = (tmp_path / "fake")
+    fake_path = (tmp_path / 'fake')
     fake_path.write_text(fake)
     fake_path.chmod(0o755)
 
     prev_make = up.make
     up.make = fake_path
     try:
-        adjuster.adjust_package_from_extracted_distfiles()
+        adjuster.adjust()
     finally:
         up.make = prev_make
 
-    assert adjuster.generate_adjusted_Makefile_lines().lines == [
-        'WRKSRC=\t\t${WRKDIR}',
-        'USE_LANGUAGES=\t# none',
+    assert detab(adjuster.generate_lines()) == [
+        'WRKSRC=         ${WRKDIR}',
+        'USE_LANGUAGES=  # none',
         '',
     ]
 
@@ -907,23 +1013,23 @@ def test_Adjuster_adjust_lines_python_module():
     adjuster = Adjuster(up, url, initial_lines)
     adjuster.makefile_lines = Lines(*initial_lines.lines)
 
-    assert adjuster.makefile_lines.lines == [
-        '# $NetBSD: url2pkg_test.py,v 1.7 2019/10/04 22:26:34 rillig Exp $',
+    assert detab(adjuster.makefile_lines) == [
+        mkcvsid,
         '',
-        'GITHUB_PROJECT=\tesptool',
-        'DISTNAME=\tv2.7',
-        'PKGNAME=\t${GITHUB_PROJECT}-${DISTNAME:S,^v,,}',
-        'CATEGORIES=\tpkgtools python',
-        'MASTER_SITES=\t${MASTER_SITE_GITHUB:=espressif/}',
-        'DIST_SUBDIR=\t${GITHUB_PROJECT}',
+        'GITHUB_PROJECT= esptool',
+        'DISTNAME=       v2.7',
+        'PKGNAME=        ${GITHUB_PROJECT}-${DISTNAME:S,^v,,}',
+        'CATEGORIES=     pkgtools python',
+        'MASTER_SITES=   ${MASTER_SITE_GITHUB:=espressif/}',
+        'DIST_SUBDIR=    ${GITHUB_PROJECT}',
         '',
-        'MAINTAINER=\tINSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
-        'HOMEPAGE=\thttps://github.com/espressif/esptool/',
-        'COMMENT=\tTODO: Short description of the package',
-        '#LICENSE=\t# TODO: (see mk/license.mk)',
+        'MAINTAINER=     INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
+        'HOMEPAGE=       https://github.com/espressif/esptool/',
+        'COMMENT=        TODO: Short description of the package',
+        '#LICENSE=       # TODO: (see mk/license.mk)',
         '',
         '# url2pkg-marker (please do not remove this line.)',
-        '.include "../../mk/bsd.pkg.mk"'
+        '.include "../../mk/bsd.pkg.mk"',
     ]
 
     adjuster.adjust_lines_python_module(initial_lines)
@@ -932,19 +1038,19 @@ def test_Adjuster_adjust_lines_python_module():
     #  GitHub are also available from PyPI. That is wrong. Probably url2pkg
     #  should try to fetch the file from PyPI, and only switch to PyPI if
     #  they are the same.
-    assert adjuster.makefile_lines.lines == [
-        '# $NetBSD: url2pkg_test.py,v 1.7 2019/10/04 22:26:34 rillig Exp $',
+    assert detab(adjuster.makefile_lines) == [
+        mkcvsid,
         '',
-        'DISTNAME=\tesptool-2.7',
-        'PKGNAME=\t${PYPKGPREFIX}-${DISTNAME}',
-        'CATEGORIES=\tpkgtools python',
-        'MASTER_SITES=\t${MASTER_SITE_PYPI:=e/esptool/}',
+        'DISTNAME=       esptool-2.7',
+        'PKGNAME=        ${PYPKGPREFIX}-${DISTNAME}',
+        'CATEGORIES=     pkgtools python',
+        'MASTER_SITES=   ${MASTER_SITE_PYPI:=e/esptool/}',
         '',
-        'MAINTAINER=\tINSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
-        'HOMEPAGE=\thttps://github.com/espressif/esptool/',
-        'COMMENT=\tTODO: Short description of the package',
-        '#LICENSE=\t# TODO: (see mk/license.mk)',
+        'MAINTAINER=     INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
+        'HOMEPAGE=       https://github.com/espressif/esptool/',
+        'COMMENT=        TODO: Short description of the package',
+        '#LICENSE=       # TODO: (see mk/license.mk)',
         '',
         '# url2pkg-marker (please do not remove this line.)',
-        '.include "../../mk/bsd.pkg.mk"'
+        '.include "../../mk/bsd.pkg.mk"',
     ]