diff options
author | rillig <rillig@pkgsrc.org> | 2019-10-03 12:52:54 +0000 |
---|---|---|
committer | rillig <rillig@pkgsrc.org> | 2019-10-03 12:52:54 +0000 |
commit | f2d2f0f4f74302b0fc732a99a9dda32961c7571a (patch) | |
tree | dc3511606fd006cb0c16d5394354c626661a22d0 /pkgtools/url2pkg | |
parent | fc1b4c07c308a23235647158414be08a8264b290 (diff) | |
download | pkgsrc-f2d2f0f4f74302b0fc732a99a9dda32961c7571a.tar.gz |
pkgtools/url2pkg: improve Python implementation
* verbose mode no longer crashes
* licenses and other variables are copied to the package Makefile
* several more automatic tests
Diffstat (limited to 'pkgtools/url2pkg')
-rw-r--r-- | pkgtools/url2pkg/files/url2pkg.py | 42 | ||||
-rw-r--r-- | pkgtools/url2pkg/files/url2pkg_test.py | 314 |
2 files changed, 309 insertions, 47 deletions
diff --git a/pkgtools/url2pkg/files/url2pkg.py b/pkgtools/url2pkg/files/url2pkg.py index 8b7a4be8f8d..612477a153a 100644 --- a/pkgtools/url2pkg/files/url2pkg.py +++ b/pkgtools/url2pkg/files/url2pkg.py @@ -1,5 +1,5 @@ #! @PYTHONBIN@ -# $NetBSD: url2pkg.py,v 1.1 2019/10/03 09:37:41 rillig Exp $ +# $NetBSD: url2pkg.py,v 1.2 2019/10/03 12:52:54 rillig Exp $ # Copyright (c) 2019 The NetBSD Foundation, Inc. # All rights reserved. @@ -50,6 +50,7 @@ class Config: self.perl5 = '@PERL5@' self.pkgsrcdir = '@PKGSRCDIR@' self.pythonbin = '@PYTHONBIN@' + self.pkgdir = '.' # only overridable for tests self.verbose = False @@ -59,7 +60,7 @@ distname = '' def debug(fmt: str, *args): if config.verbose: - msg = fmt % map(repr, args) + msg = fmt.format(*map(repr, args)) if len(args) else fmt sys.stderr.write('url2pkg: %s\n' % msg) @@ -219,10 +220,10 @@ def generate_initial_package(url): os.rename('Makefile', 'Makefile-url2pkg.bak') except OSError: pass - generate_initial_package_Makefile_lines(url).write_to('Makefile') - Lines(cvsid('@comment %s')).write_to('PLIST') - Lines().write_to('DESCR') - run_editor('Makefile', 5) + generate_initial_package_Makefile_lines(url).write_to(config.pkgdir + '/Makefile') + Lines(cvsid('@comment %s')).write_to(config.pkgdir + '/PLIST') + Lines().write_to(config.pkgdir + '/DESCR') + run_editor(config.pkgdir + '/Makefile', 5) bmake('distinfo') bmake('extract') @@ -271,15 +272,15 @@ class Varassign: def find_package(pkgbase: str) -> str: candidates = glob.glob(config.pkgsrcdir + '/*/' + pkgbase) - debug('candidates for package %s are %s', pkgbase, candidates) + debug('candidates for package {0} are {1}', pkgbase, candidates) if len(candidates) != 1: return '' return candidates[0].replace(config.pkgsrcdir, '../..') def bmake(*args: str) -> None: - debug('running bmake %s', args) - subprocess.check_call([config.make] + list(args)) + debug('running bmake {0}', args) + subprocess.check_call([config.make] + list(args), cwd=config.pkgdir) def show_var(varname: str) -> str: @@ -346,7 +347,7 @@ class Lines: varassigns = [] for (i, line) in enumerate(self.lines): m = re.search(r'^(#?[\w+\-]+?)([!+:?]?=)([ \t]*)([^#\\]*?)(\s*)(#.*|)$', line) - if m and m[1] == varname: + if m and m[1].lstrip('#') == varname: varassigns.append(Varassign(i, m[1], m[2], m[3], m[4], m[5], m[6])) return varassigns @@ -355,7 +356,7 @@ class Lines: varassign = self.unique_varassign(varname) if varassign is not None: - self.lines[varassign.index] = varassign.varname + varassign.op + varassign.indent + new_value + self.lines[varassign.index] = varname + varassign.op + varassign.indent + new_value return varassign is not None def append(self, varname: str, value: str) -> None: @@ -502,6 +503,7 @@ class Adjuster: effective_env = dict(os.environ) effective_env.update(env) + debug('reading dependencies: cd {0} && env {1} {2}', cwd, env, cmd) output = subprocess.check_output( args=cmd, shell=True, @@ -519,7 +521,7 @@ class Adjuster: self.update_vars[m[1]] = m[2] continue if line != '': - debug('unknown dependency line: %s', line) + debug('unknown dependency line: {0}', line) for dep_line in dep_lines: type, pkgbase, constraint, dir = dep_line @@ -531,7 +533,7 @@ class Adjuster: if dir == '': dir = find_package(pkgbase) - debug('add_dependency: %s %s %s %s', type, pkgbase, constraint, dir) + debug('add_dependency: {0} {1} {2} {3}', type, pkgbase, constraint, dir) self.add_dependency(type, pkgbase, constraint, dir) def wrksrc_find(self, what: Union[str, Callable]) -> Iterator[str]: @@ -727,7 +729,7 @@ class Adjuster: def adjust_lines_python_module(self, lines: Lines, url: str): initial_lines = generate_initial_package_Makefile_lines(url) - current_lines = Lines.read_from('Makefile') + current_lines = self.makefile_lines if 'python' not in initial_lines.get('CATEGORIES'): return @@ -752,7 +754,7 @@ class Adjuster: self.makefile_lines = tx_lines self.regenerate_distinfo = True - def generate_adjusted_Makefile_lines(self, url): + def generate_adjusted_Makefile_lines(self, url) -> Lines: marker_index = self.makefile_lines.index(r'^# url2pkg-marker') if marker_index == -1: raise Exception('ERROR: didn\'t find the url2pkg marker in the Makefile.') @@ -789,6 +791,7 @@ class Adjuster: self.adjust_lines_python_module(lines, url) for varname in self.update_vars: + debug('update_var {0} {1}', varname, self.update_vars[varname]) lines.set(varname, self.update_vars[varname]) return lines @@ -802,7 +805,7 @@ class Adjuster: self.wrksrc_files = glob.glob(f'{self.abs_wrksrc}/**', recursive=True) self.wrksrc_dirs = glob.glob(f'{self.abs_wrksrc}/**/', recursive=True) - self.makefile_lines = Lines.read_from('Makefile') + self.makefile_lines = Lines.read_from(config.pkgdir + '/Makefile') self.adjust_configure() self.adjust_cmake() @@ -816,7 +819,7 @@ class Adjuster: self.adjust_po() self.adjust_use_languages() - self.generate_adjusted_Makefile_lines(url).write_to('Makefile') + self.generate_adjusted_Makefile_lines(url).write_to(config.pkgdir + '/Makefile') if self.regenerate_distinfo: bmake('distinfo') @@ -830,8 +833,9 @@ def main(): try: opts, args = getopt.getopt(sys.argv[1:], 'v', ['verbose']) - for opt in opts: - if opt in ('v', 'verbose'): + for (opt, optarg) in opts: + print('opt', repr(opt)) + if opt in ('-v', '--verbose'): config.verbose = True except getopt.GetoptError: sys.exit(f'usage: {sys.argv[0]} [-v|--verbose] [URL]') diff --git a/pkgtools/url2pkg/files/url2pkg_test.py b/pkgtools/url2pkg/files/url2pkg_test.py index aef7b538339..a71dcf15dd2 100644 --- a/pkgtools/url2pkg/files/url2pkg_test.py +++ b/pkgtools/url2pkg/files/url2pkg_test.py @@ -1,20 +1,47 @@ -# $NetBSD: url2pkg_test.py,v 1.1 2019/10/03 09:37:41 rillig Exp $ +# $NetBSD: url2pkg_test.py,v 1.2 2019/10/03 12:52:54 rillig Exp $ -import os -from typing import List from url2pkg import * -def setup_function(fn): +def setup_function(_): config.pkgsrcdir = os.getenv('PKGSRCDIR') assert config.pkgsrcdir is not None os.chdir(config.pkgsrcdir + '/pkgtools/url2pkg') -def vars(vars: List[Var]) -> List[str]: +def str_vars(vars: List[Var]) -> List[str]: return list(map(lambda var: var.name + var.op + var.value, vars)) +def test_debug(): + """ Just ensure that the debug calls do not crash. """ + config.verbose = True + try: + debug('plain message') + debug('list {0}', [1, 2, 3]) + debug('tuple {0}', (1, 2, 3)) + debug('cwd {0} env {1} cmd {2}', 'directory', {'VAR': 'value'}, 'command') + finally: + config.verbose = False + + +def test_aligned__empty(): + assert aligned([]) == [] + + +def test_aligned__variables(): + vars = [ + Var('V', '=', 'value'), + Var('LONG_NAME', '=', 'value # comment') + ] + lines = [ + 'V=\t\tvalue', + 'LONG_NAME=\tvalue # comment', + '' + ] + assert aligned(vars) == lines + + def test_Lines_add_vars__simple(): lines = Lines() @@ -94,7 +121,7 @@ def test_Lines_append__value_with_comment(): def test_Lines_append__value_without_comment(): lines = Lines("VARNAME+=\tvalue") - assert lines.append("VARNAME", "appended") == True + assert lines.append("VARNAME", "appended") assert lines.lines == ["VARNAME+=\tvalue appended"] @@ -102,27 +129,102 @@ def test_Lines_append__value_without_comment(): def test_Lines_set__previously_with_comment(): lines = Lines("LICENSE=\t# TODO: see mk/license.mk") - lines.set("LICENSE", "${PERL5_LICENSE}") + assert lines.set("LICENSE", "${PERL5_LICENSE}") assert lines.lines == ["LICENSE=\t${PERL5_LICENSE}"] +def test_Lines_unique_varassign__commented_out_no_value(): + lines = Lines("#LICENSE=\t# TODO: see mk/license.mk") + + assert len(lines.all_varassigns('LICENSE')) == 1 + + def test_Lines_set__overwrite_comment_with_comment(): lines = Lines("#LICENSE=\t# TODO: see mk/license.mk") - lines.set("#LICENSE", "${PERL5_LICENSE}") + assert len(lines.all_varassigns('LICENSE')) == 1 + assert lines.set("LICENSE", "${PERL5_LICENSE}") - assert lines.lines == ["#LICENSE=\t${PERL5_LICENSE}"] + assert lines.lines == ["LICENSE=\t${PERL5_LICENSE}"] + + +def test_Lines_set__overwrite_commented_with_comment(): + lines = Lines("#LICENSE=\t# TODO: see mk/license.mk") + + assert lines.set("LICENSE", "${PERL5_LICENSE}") + + assert lines.lines == ["LICENSE=\t${PERL5_LICENSE}"] def test_Lines_set__not_found(): lines = Lines("OLD_VAR=\told value # old comment") - lines.set("NEW_VAR", "new value") + assert not lines.set("NEW_VAR", "new value") assert lines.lines == ["OLD_VAR=\told value # old comment"] +def test_Lines_remove__not_found(): + lines = Lines('VAR=\tvalue') + + assert not lines.remove('VARIABLE') + + assert lines.lines == ['VAR=\tvalue'] + + +def test_Lines_remove__found(): + lines = Lines('VAR=\tvalue') + + assert lines.remove('VAR') + + assert lines.lines == [] + + +def test_Lines_remove__found_several_times(): + lines = Lines('VAR=\tvalue1', 'VAR=\tvalue2') + + assert not lines.remove('VAR') + + assert lines.lines == ['VAR=\tvalue1', 'VAR=\tvalue2'] + + +def test_Lines_remove_if__different_name(): + lines = Lines('VAR=\tvalue') + + assert not lines.remove_if('VARIABLE', 'value') + + assert lines.lines == ['VAR=\tvalue'] + + +def test_Lines_remove_if__different_value(): + lines = Lines('VAR=\tvalue') + + assert not lines.remove_if('VAR', 'something') + + assert lines.lines == ['VAR=\tvalue'] + + +def test_Lines_remove_if__found(): + lines = Lines('VAR=\tvalue') + + assert lines.remove_if('VAR', 'value') + + assert lines.lines == [] + + +def test_Lines_remove_if__multiple(): + lines = Lines('VAR=\tvalue', 'VAR=\tvalue') + + assert lines.remove_if('VAR', 'value') + + assert lines.lines == ['VAR=\tvalue'] + + assert lines.remove_if('VAR', 'value') + + assert lines.lines == [] + + def test_Lines_index(): lines = Lines("1", "2", "345") @@ -255,6 +357,7 @@ def test_Adjuster_read_dependencies(): "TEST_DEPENDS\tpkglint", "A line that is not a dependency at all", "", + "var\tHOMEPAGE\thttps://homepage.example.org/" "" ] env = {"URL2PKG_DEPENDENCIES": '\n'.join(dep_lines)} @@ -277,6 +380,9 @@ def test_Adjuster_read_dependencies(): assert adjuster.test_depends == [ "pkglint>=0:../../pkgtools/pkglint" ] + assert adjuster.update_vars == { + 'HOMEPAGE': 'https://homepage.example.org/' + } def test_Adjuster_generate_adjusted_Makefile_lines(): @@ -309,10 +415,10 @@ def test_Adjuster_generate_adjusted_Makefile_lines__dependencies(): ) # some dependencies whose directory will not be found - adjuster.add_dependency("DEPENDS", "depends", ">=5.0", "../../devel/depends"); - adjuster.add_dependency("TOOL_DEPENDS", "tool-depends", ">=6.0", "../../devel/tool-depends"); - adjuster.add_dependency("BUILD_DEPENDS", "build-depends", ">=7.0", "../../devel/build-depends"); - adjuster.add_dependency("TEST_DEPENDS", "test-depends", ">=8.0", "../../devel/test-depends"); + adjuster.add_dependency("DEPENDS", "depends", ">=5.0", "../../devel/depends") + adjuster.add_dependency("TOOL_DEPENDS", "tool-depends", ">=6.0", "../../devel/tool-depends") + adjuster.add_dependency("BUILD_DEPENDS", "build-depends", ">=7.0", "../../devel/build-depends") + adjuster.add_dependency("TEST_DEPENDS", "test-depends", ">=8.0", "../../devel/test-depends") # some dependencies whose directory is explicitly given adjuster.depends.append("depends>=11.0:../../devel/depends") adjuster.build_depends.append("build-depends>=12.0:../../devel/build-depends") @@ -336,6 +442,60 @@ def test_Adjuster_generate_adjusted_Makefile_lines__dependencies(): ] +def test_Adjuster_adjust_configure__not_found(tmp_path): + adjuster = Adjuster() + adjuster.abs_wrksrc = str(tmp_path) + + adjuster.adjust_configure() + + assert adjuster.build_vars == [] + + +def test_Adjuster_adjust_configure__GNU_configure(tmp_path): + adjuster = Adjuster() + adjuster.abs_wrksrc = str(tmp_path) + (tmp_path / 'configure').write_text('# Free Software Foundation\n') + + adjuster.adjust_configure() + + assert str_vars(adjuster.build_vars) == [ + 'GNU_CONFIGURE=yes', + ] + + +def test_Adjuster_adjust_configure__other_configure(tmp_path): + adjuster = Adjuster() + adjuster.abs_wrksrc = str(tmp_path) + (tmp_path / 'configure').write_text('# A generic configure script\n') + + adjuster.adjust_configure() + + assert str_vars(adjuster.build_vars) == [ + 'HAS_CONFIGURE=yes', + ] + + +def test_Adjuster_adjust_cargo__not_found(tmp_path): + adjuster = Adjuster() + adjuster.abs_wrksrc = str(tmp_path) + + adjuster.adjust_cargo() + + assert str_vars(adjuster.build_vars) == [] + + +def test_Adjuster_adjust_cargo__found(tmp_path): + adjuster = Adjuster() + adjuster.abs_wrksrc = str(tmp_path) + (tmp_path / 'Cargo.lock').write_text('"checksum cargo-package-name cargo-package-version 1234"') + + adjuster.adjust_cargo() + + assert str_vars(adjuster.build_vars) == [ + 'CARGO_CRATE_DEPENDS+=cargo-package-name-cargo-package-version', + ] + + def test_Adjuster_adjust_po__not_found(): adjuster = Adjuster() @@ -350,7 +510,7 @@ def test_Adjuster_adjust_po__found(): adjuster.adjust_po() - assert vars(adjuster.build_vars) == [ + assert str_vars(adjuster.build_vars) == [ 'USE_PKGLOCALEDIR=yes' ] @@ -360,7 +520,7 @@ def test_Adjuster_adjust_use_languages__none(): adjuster.adjust_use_languages() - assert vars(adjuster.build_vars) == [ + assert str_vars(adjuster.build_vars) == [ 'USE_LANGUAGES=# none' ] @@ -371,7 +531,7 @@ def test_Adjuster_adjust_use_languages__c(): adjuster.adjust_use_languages() - assert vars(adjuster.build_vars) == [] + assert str_vars(adjuster.build_vars) == [] def test_Adjuster_adjust_use_languages__c_in_subdir(): @@ -380,7 +540,7 @@ def test_Adjuster_adjust_use_languages__c_in_subdir(): adjuster.adjust_use_languages() - assert vars(adjuster.build_vars) == [] + assert str_vars(adjuster.build_vars) == [] def test_Adjuster_adjust_use_languages__cplusplus_in_subdir(): @@ -389,7 +549,7 @@ def test_Adjuster_adjust_use_languages__cplusplus_in_subdir(): adjuster.adjust_use_languages() - assert vars(adjuster.build_vars) == [ + assert str_vars(adjuster.build_vars) == [ 'USE_LANGUAGES=c++' ] @@ -400,7 +560,7 @@ def test_Adjuster_adjust_use_languages__cplusplus_and_fortran(): adjuster.adjust_use_languages() - assert vars(adjuster.build_vars) == [ + assert str_vars(adjuster.build_vars) == [ 'USE_LANGUAGES=c++ fortran' ] @@ -410,8 +570,8 @@ def test_Adjuster_adjust_pkg_config__none(): adjuster.adjust_pkg_config() - assert vars(adjuster.build_vars) == [] - assert vars(adjuster.extra_vars) == [] + assert str_vars(adjuster.build_vars) == [] + assert str_vars(adjuster.extra_vars) == [] def test_Adjuster_adjust_pkg_config__pc_in(): @@ -420,8 +580,8 @@ def test_Adjuster_adjust_pkg_config__pc_in(): adjuster.adjust_pkg_config() - assert vars(adjuster.build_vars) == ['USE_TOOLS+=pkg-config'] - assert vars(adjuster.extra_vars) == ['PKGCONFIG_OVERRIDE+=library.pc.in'] + assert str_vars(adjuster.build_vars) == ['USE_TOOLS+=pkg-config'] + assert str_vars(adjuster.extra_vars) == ['PKGCONFIG_OVERRIDE+=library.pc.in'] def test_Adjuster_adjust_pkg_config__uninstalled_pc_in(): @@ -430,8 +590,8 @@ def test_Adjuster_adjust_pkg_config__uninstalled_pc_in(): adjuster.adjust_pkg_config() - assert vars(adjuster.build_vars) == [] - assert vars(adjuster.extra_vars) == [] + assert str_vars(adjuster.build_vars) == [] + assert str_vars(adjuster.extra_vars) == [] def test_Adjuster_adjust_pkg_config__both(): @@ -443,5 +603,103 @@ def test_Adjuster_adjust_pkg_config__both(): adjuster.adjust_pkg_config() - assert vars(adjuster.build_vars) == ['USE_TOOLS+=pkg-config'] - assert vars(adjuster.extra_vars) == ['PKGCONFIG_OVERRIDE+=library.pc.in'] + assert str_vars(adjuster.build_vars) == ['USE_TOOLS+=pkg-config'] + assert str_vars(adjuster.extra_vars) == ['PKGCONFIG_OVERRIDE+=library.pc.in'] + + +def test_Adjuster__adjust_homepage(): + url = 'https://dummy.example.org/package-1.0.tar.gz' + adjuster = Adjuster() + adjuster.makefile_lines = generate_initial_package_Makefile_lines(url) + adjuster.update_vars['HOMEPAGE'] = 'https://example.org/' + adjuster.depends.append('dependency>=0:../../category/dependency') + adjuster.todos.append('Run pkglint') + + lines = adjuster.generate_adjusted_Makefile_lines(url) + + assert lines.lines == [ + '# $' + 'NetBSD$', + '', + 'DISTNAME=\tpackage-1.0', + 'PKGNAME=\t${DISTNAME}', + 'CATEGORIES=\tpkgtools', + 'MASTER_SITES=\thttps://dummy.example.org/', + '', + 'MAINTAINER=\tINSERT_YOUR_MAIL_ADDRESS_HERE', + 'HOMEPAGE=\thttps://example.org/', + 'COMMENT=\tTODO: Short description of the package', + '#LICENSE=\t# TODO: (see mk/license.mk)', + '', + '# TODO: Run pkglint', + '', + 'DEPENDS+=\tdependency>=0:../../category/dependency', + '', + '.include "../../mk/bsd.pkg.mk"' + ] + + +def test_Adjuster_determine_wrksrc__no_files(tmp_path): + adjuster = Adjuster() + adjuster.abs_wrkdir = str(tmp_path) + + adjuster.determine_wrksrc() + + assert adjuster.abs_wrksrc == adjuster.abs_wrkdir + + +def test_Adjuster_determine_wrksrc__single_dir(tmp_path): + adjuster = Adjuster() + adjuster.abs_wrkdir = str(tmp_path) + (tmp_path / 'subdir').mkdir() + + adjuster.determine_wrksrc() + + assert adjuster.abs_wrksrc == adjuster.abs_wrkdir + '/subdir' + + +def test_Adjuster_determine_wrksrc__several_dirs(tmp_path): + adjuster = Adjuster() + adjuster.abs_wrkdir = str(tmp_path) + (tmp_path / 'subdir1').mkdir() + (tmp_path / 'subdir2').mkdir() + + adjuster.determine_wrksrc() + + assert adjuster.abs_wrksrc == adjuster.abs_wrkdir + assert str_vars(adjuster.build_vars) == [ + 'WRKSRC=${WRKDIR} # More than one possibility -- please check manually.' + ] + + +def test_Adjuster_adjust_package_from_extracted_distfiles__empty_wrkdir(tmp_path): + pkgdir = tmp_path + wrkdir = tmp_path / 'wrkdir' + fake = '''\ +#! /bin/sh +case $* in +("show-var VARNAME=WRKDIR") echo '%s' ;; +(*) "unknown: $*" ;; +esac +''' % str(wrkdir) + config.pkgdir = str(tmp_path) + wrkdir.mkdir() + url = 'https://example.org/distfile-1.0.zip' + adjuster = Adjuster() + adjuster.abs_wrkdir = str(wrkdir) + (pkgdir / 'Makefile').write_text('# url2pkg-marker\n') + fake_path = (tmp_path / "fake") + fake_path.write_text(fake) + fake_path.chmod(0o755) + + prev_make = config.make + config.make = fake_path + try: + adjuster.adjust_package_from_extracted_distfiles(url) + finally: + config.make = prev_make + + assert adjuster.generate_adjusted_Makefile_lines(url).lines == [ + 'WRKSRC=\t\t${WRKDIR}', + 'USE_LANGUAGES=\t# none', + '', + ] |