diff options
author | rillig <rillig@pkgsrc.org> | 2019-10-05 21:05:50 +0000 |
---|---|---|
committer | rillig <rillig@pkgsrc.org> | 2019-10-05 21:05:50 +0000 |
commit | f8aac6544049a0bbd516ea58e84a497484897b41 (patch) | |
tree | 339720ad56b56aa743510bf588cf4d105a3d7497 /pkgtools | |
parent | 9178c689ca22e59a8673914a6541e22e584b560d (diff) | |
download | pkgsrc-f8aac6544049a0bbd516ea58e84a497484897b41.tar.gz |
pkgtools/url2pkg: clean up path handling
Diffstat (limited to 'pkgtools')
-rw-r--r-- | pkgtools/url2pkg/files/url2pkg.py | 81 | ||||
-rw-r--r-- | pkgtools/url2pkg/files/url2pkg_test.py | 133 |
2 files changed, 110 insertions, 104 deletions
diff --git a/pkgtools/url2pkg/files/url2pkg.py b/pkgtools/url2pkg/files/url2pkg.py index 44168ede925..2c4b3cecfdb 100644 --- a/pkgtools/url2pkg/files/url2pkg.py +++ b/pkgtools/url2pkg/files/url2pkg.py @@ -1,5 +1,5 @@ #! @PYTHONBIN@ -# $NetBSD: url2pkg.py,v 1.13 2019/10/05 19:59:04 rillig Exp $ +# $NetBSD: url2pkg.py,v 1.14 2019/10/05 21:05:50 rillig Exp $ # Copyright (c) 2019 The NetBSD Foundation, Inc. # All rights reserved. @@ -42,13 +42,12 @@ import getopt -import glob import os -import pathlib import re import subprocess import sys -from typing import Callable, Dict, Iterator, List, Optional, Sequence, Tuple, Union +from pathlib import Path +from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union class Var: @@ -80,12 +79,12 @@ class Url2Pkg: self.make = os.getenv('MAKE') or '@MAKE@' self.libdir = '@LIBDIR@' self.perl5 = '@PERL5@' - self.pkgsrcdir = pathlib.Path(os.getenv('PKGSRCDIR') or '@PKGSRCDIR@') + self.pkgsrcdir = Path(os.getenv('PKGSRCDIR') or '@PKGSRCDIR@') self.pythonbin = '@PYTHONBIN@' self.editor = os.getenv('PKGEDITOR') or os.getenv('EDITOR') or 'vi' # the following are overridden in tests - self.pkgdir = pathlib.Path('.') + self.pkgdir = Path('.') self.out = sys.stdout self.err = sys.stderr @@ -124,17 +123,16 @@ class Lines: self.add(line) @classmethod - def read_from(cls, filename: Union[str, pathlib.Path]) -> 'Lines': - return Lines(*pathlib.Path(filename).read_text().splitlines()) + def read_from(cls, src: Path) -> 'Lines': + return Lines(*src.read_text().splitlines()) - def write_to(self, filename: Union[str, pathlib.Path]): - target = pathlib.Path(filename) - tmp = target.with_name(f'{target.name}.tmp') + def write_to(self, dst: Path): + tmp = dst.with_name(f'{dst.name}.tmp') with tmp.open('w') as f: f.writelines(line + '\n' for line in self.lines) - tmp.replace(target) + tmp.replace(dst) - def all_varassigns(self, varname: str) -> Sequence[Varassign]: + def all_varassigns(self, varname: str) -> List[Varassign]: varassigns = [] for (i, line) in enumerate(self.lines): pattern = r'''(?x) @@ -384,7 +382,7 @@ class Generator: elif re.search(r'-v\d', distname) and not re.search(r'-v.*-v\d', distname): self.pkgname_transform = ':S,-v,-,' - main_category = pathlib.Path.cwd().parts[-2] + main_category = Path.cwd().parts[-2] self.categories = main_category \ if main_category not in ('local', 'wip') \ else '# TODO: add primary category' @@ -476,11 +474,11 @@ class Adjuster: # the absolute pathname to the working directory, containing # the extracted distfiles. - self.abs_wrkdir = '' + self.abs_wrkdir = Path('') # the absolute pathname to a subdirectory of abs_wrkdir, typically # containing package-provided Makefiles or configure scripts. - self.abs_wrksrc = '' + self.abs_wrksrc = Path('') # the regular files and directories relative to abs_wrksrc. self.wrksrc_files: List[str] = [] @@ -576,7 +574,7 @@ class Adjuster: effective_env = dict(os.environ) effective_env.update(env) - self.up.debug('reading dependencies: cd {0} && env {1} {2}', cwd, env, cmd) + self.up.debug('reading dependencies: cd {0} && env {1} {2}', str(cwd), env, cmd) output: bytes = subprocess.check_output(args=cmd, shell=True, env=effective_env, cwd=cwd) dep_lines: List[Tuple[str, str, str, str]] = [] @@ -609,35 +607,39 @@ class Adjuster: self.add_dependency(kind, pkgbase, constraint, dep_dir) def wrksrc_open(self, relative_pathname: str): - return open(self.abs_wrksrc + '/' + relative_pathname) + return (self.abs_wrksrc / relative_pathname).open() - def wrksrc_find(self, what: Union[str, Callable[[str], bool]]) -> Iterator[str]: + def wrksrc_find(self, what: Union[str, Callable[[str], bool]]) -> List[str]: def search(f): return re.search(what, f) if type(what) == str else what(f) return list(sorted(filter(search, self.wrksrc_files))) - def wrksrc_grep(self, filename: str, pattern: str) -> List[str]: + def wrksrc_grep(self, filename: str, pattern: str) -> List[Union[str, List[str]]]: with self.wrksrc_open(filename) as f: - return [line for line in f if re.search(pattern, line)] + matches = [] + for line in f: + line = line.rstrip('\n') + m = re.search(pattern, line) + if m: + groups = list(m.groups()) + matches.append(groups if groups else line) + return matches def wrksrc_isdir(self, relative_pathname: str) -> bool: - return os.path.isdir(self.abs_wrksrc + '/' + relative_pathname) + return (self.abs_wrksrc / relative_pathname).is_dir() def wrksrc_isfile(self, relative_pathname: str) -> bool: - return os.path.isfile(self.abs_wrksrc + '/' + relative_pathname) + return (self.abs_wrksrc / relative_pathname).is_file() def adjust_configure(self): if not self.wrksrc_isfile('configure'): return - gnu = False - some = False - for configure in self.wrksrc_find(r'(^|/)configure$'): - some = True - if self.wrksrc_grep(configure, r'\b(Free Software Foundation|autoconf)\b'): - gnu = True - if some: + configures = self.wrksrc_find(r'(^|/)configure$') + if configures: + gnu = any(self.wrksrc_grep(configure, r'\b(Free Software Foundation|autoconf)\b') + for configure in configures) varname = 'GNU_CONFIGURE' if gnu else 'HAS_CONFIGURE' self.build_vars.append(Var(varname, '=', 'yes')) @@ -742,12 +744,9 @@ class Adjuster: if not self.wrksrc_isfile('Cargo.lock'): return - with self.wrksrc_open('Cargo.lock') as f: - for line in f: - # "checksum cargo-package-name cargo-package-version - m = re.search(r'^"checksum\s(\S+)\s(\S+)', line) - if m: - self.build_vars.append(Var('CARGO_CRATE_DEPENDS', '+=', m[1] + '-' + m[2])) + # "checksum cargo-package-name cargo-package-version + for (name, version) in self.wrksrc_grep('Cargo.lock', r'^"checksum\s(\S+)\s(\S+)'): + self.build_vars.append(Var('CARGO_CRATE_DEPENDS', '+=', f'{name}-{version}')) self.includes.append('../../lang/rust/cargo.mk') @@ -798,7 +797,7 @@ class Adjuster: if len(files) == 1: if files[0] != self.makefile_lines.get('DISTNAME'): self.build_vars.append(Var('WRKSRC', '=', '${WRKDIR}/' + files[0])) - self.abs_wrksrc = self.abs_wrkdir + '/' + files[0] + self.abs_wrksrc = self.abs_wrkdir / files[0] elif len(files) == 0: self.build_vars.append(Var('WRKSRC', '=', '${WRKDIR}')) self.abs_wrksrc = self.abs_wrkdir @@ -895,14 +894,14 @@ class Adjuster: def adjust(self): - def scan(basedir: str, pattern: str) -> List[str]: - full_paths = glob.glob(f'{basedir}/{pattern}', recursive=True) - return list(f[len(basedir) + 1:] for f in full_paths) + def scan(basedir: Path, pattern: str) -> List[str]: + full_paths = basedir.rglob(pattern) + return [str(f.relative_to(basedir)) for f in full_paths] self.up.debug('Adjusting the Makefile') self.makefile_lines = Lines.read_from(self.up.pkgdir / 'Makefile') - self.abs_wrkdir = self.up.show_var('WRKDIR') + self.abs_wrkdir = Path(self.up.show_var('WRKDIR')) self.determine_wrksrc() self.wrksrc_files = scan(self.abs_wrksrc, '**') self.wrksrc_dirs = scan(self.abs_wrksrc, '**/') diff --git a/pkgtools/url2pkg/files/url2pkg_test.py b/pkgtools/url2pkg/files/url2pkg_test.py index 463fb45e25c..f9cb5753b95 100644 --- a/pkgtools/url2pkg/files/url2pkg_test.py +++ b/pkgtools/url2pkg/files/url2pkg_test.py @@ -1,11 +1,11 @@ -# $NetBSD: url2pkg_test.py,v 1.12 2019/10/05 19:59:04 rillig Exp $ +# $NetBSD: url2pkg_test.py,v 1.13 2019/10/05 21:05:50 rillig Exp $ import pytest from url2pkg import * mkcvsid = '# $''NetBSD$' up: Url2Pkg -prev_dir = pathlib.Path.cwd() +prev_dir = Path.cwd() def setup_function(_): @@ -31,7 +31,7 @@ def setup_function(_): def teardown_function(_): - os.chdir(str(prev_dir)) + os.chdir(prev_dir) assert up.out.written() == [] assert up.err.written() == [] @@ -93,16 +93,16 @@ def test_Url2Pkg_bmake(): ] -def test_Lines__write_and_read(tmp_path: pathlib.Path): +def test_Lines__write_and_read(tmp_path: Path): example = tmp_path / 'example' lines = Lines('1', '2', '3') - lines.write_to(str(example)) + lines.write_to(example) assert example.read_text() == '1\n2\n3\n' - back = Lines.read_from(str(example)) + back = Lines.read_from(example) assert back.lines == ['1', '2', '3'] @@ -613,7 +613,7 @@ def test_Generator_determine_distname__v8(): ] -def test_Generator_generate_package(tmp_path: pathlib.Path): +def test_Generator_generate_package(tmp_path: Path): url = 'https://ftp.gnu.org/pub/gnu/cflow/cflow-1.6.tar.gz' up.editor = 'true' # the shell command up.make = 'true' # the shell command @@ -678,6 +678,21 @@ def test_Adjuster_read_dependencies__lookup_with_prefix(): ] +def test_Adjuster_wrksrc_grep(tmp_path: Path): + adjuster = Adjuster(up, '', Lines()) + adjuster.abs_wrksrc = tmp_path + (tmp_path / 'file').write_text('\n'.join( + ('a', 'b', 'c', 'd', 'e', 'abc', 'def', 'ghi') + )) + + assert adjuster.wrksrc_grep('file', r'e') == ['e', 'def'] + assert adjuster.wrksrc_grep('file', r'(.)(.)(.)') == [ + ['a', 'b', 'c'], + ['d', 'e', 'f'], + ['g', 'h', 'i'], + ] + + def test_Adjuster_generate_adjusted_Makefile_lines(): adjuster = Adjuster(up, 'https://example.org/pkgname-1.0.tar.gz', Lines()) adjuster.makefile_lines = Lines( @@ -815,9 +830,9 @@ def test_Adjuster_add_dependency__buildlink(): ] -def test_Adjuster_adjust_cmake(tmp_path: pathlib.Path): +def test_Adjuster_adjust_cmake(tmp_path: Path): adjuster = Adjuster(up, '', Lines()) - adjuster.abs_wrksrc = str(tmp_path) + adjuster.abs_wrksrc = tmp_path (tmp_path / 'CMakeLists.txt').touch() adjuster.adjust_cmake() @@ -825,18 +840,18 @@ def test_Adjuster_adjust_cmake(tmp_path: pathlib.Path): assert str_vars(adjuster.build_vars) == ['USE_CMAKE=yes'] -def test_Adjuster_adjust_configure__none(tmp_path: pathlib.Path): +def test_Adjuster_adjust_configure__none(tmp_path: Path): adjuster = Adjuster(up, '', Lines()) - adjuster.abs_wrksrc = str(tmp_path) + adjuster.abs_wrksrc = tmp_path adjuster.adjust_configure() assert adjuster.build_vars == [] -def test_Adjuster_adjust_configure__GNU(tmp_path: pathlib.Path): +def test_Adjuster_adjust_configure__GNU(tmp_path: Path): adjuster = Adjuster(up, '', Lines()) - adjuster.abs_wrksrc = str(tmp_path) + adjuster.abs_wrksrc = tmp_path adjuster.wrksrc_files.append('configure') (tmp_path / 'configure').write_text('# Free Software Foundation\n') @@ -847,9 +862,9 @@ def test_Adjuster_adjust_configure__GNU(tmp_path: pathlib.Path): ] -def test_Adjuster_adjust_configure__other(tmp_path: pathlib.Path): +def test_Adjuster_adjust_configure__other(tmp_path: Path): adjuster = Adjuster(up, '', Lines()) - adjuster.abs_wrksrc = str(tmp_path) + adjuster.abs_wrksrc = tmp_path adjuster.wrksrc_files.append('configure') (tmp_path / 'configure').write_text('# A generic configure script\n') @@ -860,18 +875,18 @@ def test_Adjuster_adjust_configure__other(tmp_path: pathlib.Path): ] -def test_Adjuster_adjust_cargo__not_found(tmp_path: pathlib.Path): +def test_Adjuster_adjust_cargo__not_found(tmp_path: Path): adjuster = Adjuster(up, '', Lines()) - adjuster.abs_wrksrc = str(tmp_path) + adjuster.abs_wrksrc = tmp_path adjuster.adjust_cargo() assert str_vars(adjuster.build_vars) == [] -def test_Adjuster_adjust_cargo__found(tmp_path: pathlib.Path): +def test_Adjuster_adjust_cargo__found(tmp_path: Path): adjuster = Adjuster(up, '', Lines()) - adjuster.abs_wrksrc = str(tmp_path) + adjuster.abs_wrksrc = tmp_path (tmp_path / 'Cargo.lock').write_text('"checksum cargo-pkg 1.2.3 1234"') adjuster.adjust_cargo() @@ -901,9 +916,9 @@ def test_Adjuster_adjust_gconf2(): ] -def test_Adjuster_adjust_libtool__ltconfig(tmp_path: pathlib.Path): +def test_Adjuster_adjust_libtool__ltconfig(tmp_path: Path): adjuster = Adjuster(up, '', Lines()) - adjuster.abs_wrksrc = str(tmp_path) + adjuster.abs_wrksrc = tmp_path (tmp_path / 'ltconfig').write_text('') adjuster.adjust_libtool() @@ -911,9 +926,9 @@ def test_Adjuster_adjust_libtool__ltconfig(tmp_path: pathlib.Path): assert str_vars(adjuster.build_vars) == ['USE_LIBTOOL=yes'] -def test_Adjuster_adjust_libtool__libltdl(tmp_path: pathlib.Path): +def test_Adjuster_adjust_libtool__libltdl(tmp_path: Path): adjuster = Adjuster(up, '', Lines()) - adjuster.abs_wrksrc = str(tmp_path) + adjuster.abs_wrksrc = tmp_path (tmp_path / 'libltdl').mkdir() adjuster.adjust_libtool() @@ -923,9 +938,9 @@ def test_Adjuster_adjust_libtool__libltdl(tmp_path: pathlib.Path): ] -def test_Adjuster_adjust_meson(tmp_path: pathlib.Path): +def test_Adjuster_adjust_meson(tmp_path: Path): adjuster = Adjuster(up, '', Lines()) - adjuster.abs_wrksrc = str(tmp_path) + adjuster.abs_wrksrc = tmp_path (tmp_path / 'meson.build').touch() adjuster.adjust_meson() @@ -933,12 +948,12 @@ def test_Adjuster_adjust_meson(tmp_path: pathlib.Path): assert adjuster.includes == ['../../devel/py-meson/build.mk'] -def test_Adjuster_adjust_perl_module_Build_PL(tmp_path: pathlib.Path): +def test_Adjuster_adjust_perl_module_Build_PL(tmp_path: Path): up.perl5 = 'echo perl5' up.libdir = '/libdir' up.verbose = True adjuster = Adjuster(up, '', Lines()) - adjuster.abs_wrksrc = str(tmp_path) + adjuster.abs_wrksrc = tmp_path adjuster.adjust_perl_module_Build_PL() @@ -949,12 +964,12 @@ def test_Adjuster_adjust_perl_module_Build_PL(tmp_path: pathlib.Path): ] -def test_Adjuster_adjust_perl_module_Makefile_PL(tmp_path: pathlib.Path): +def test_Adjuster_adjust_perl_module_Makefile_PL(tmp_path: Path): up.perl5 = 'echo perl5' up.libdir = '/libdir' up.verbose = True adjuster = Adjuster(up, '', Lines()) - adjuster.abs_wrksrc = str(tmp_path) + adjuster.abs_wrksrc = tmp_path adjuster.adjust_perl_module_Makefile_PL() @@ -978,11 +993,11 @@ def test_Adjuster_adjust_perl_module_homepage(): assert adjuster.makefile_lines.get('HOMEPAGE') == 'https://metacpan.org/pod/Perl::Module' -def test_Adjuster_adjust_perl_module__Build_PL(tmp_path: pathlib.Path): +def test_Adjuster_adjust_perl_module__Build_PL(tmp_path: Path): up.perl5 = 'echo perl5' up.pkgdir = tmp_path # for removing the PLIST adjuster = Adjuster(up, 'https://example.org/Perl-Module-1.0.tar.gz', Lines()) - adjuster.abs_wrksrc = str(tmp_path) + adjuster.abs_wrksrc = tmp_path adjuster.makefile_lines.add_vars( Var('DISTNAME', '=', 'Perl-Module-1.0.tar.gz'), Var('MASTER_SITES', '=', '${MASTER_SITE_PERL_CPAN:=subdir/}'), @@ -1008,13 +1023,13 @@ def test_Adjuster_adjust_perl_module__Build_PL(tmp_path: pathlib.Path): assert not (tmp_path / 'PLIST').exists() -def test_Adjuster_adjust_perl_module__Makefile_PL_without_PLIST(tmp_path: pathlib.Path): +def test_Adjuster_adjust_perl_module__Makefile_PL_without_PLIST(tmp_path: Path): # For code coverage, when PLIST cannot be unlinked. up.perl5 = 'echo perl5' up.pkgdir = tmp_path adjuster = Adjuster(up, 'https://example.org/Mod-1.0.tar.gz', Lines()) - adjuster.abs_wrksrc = str(tmp_path) + adjuster.abs_wrksrc = tmp_path adjuster.makefile_lines.add_vars( Var('DISTNAME', '=', 'Mod-1.0.tar.gz'), Var('MASTER_SITES', '=', '${MASTER_SITE_PERL_CPAN:=subdir/}'), @@ -1028,12 +1043,12 @@ def test_Adjuster_adjust_perl_module__Makefile_PL_without_PLIST(tmp_path: pathli assert not (tmp_path / 'PLIST').exists() -def test_Adjuster_adjust_python_module(tmp_path: pathlib.Path): +def test_Adjuster_adjust_python_module(tmp_path: Path): url = 'https://example.org/Mod-1.0.tar.gz' up.pythonbin = 'echo python' up.pkgdir = tmp_path adjuster = Adjuster(up, url, Lines()) - adjuster.abs_wrksrc = str(tmp_path) + adjuster.abs_wrksrc = tmp_path adjuster.makefile_lines = Generator(url).generate_Makefile() (tmp_path / 'setup.py').touch() @@ -1199,28 +1214,28 @@ def test_Adjuster__adjust_homepage(): ] -def test_Adjuster_determine_wrksrc__no_files(tmp_path: pathlib.Path): +def test_Adjuster_determine_wrksrc__no_files(tmp_path: Path): adjuster = Adjuster(up, '', Lines()) - adjuster.abs_wrkdir = str(tmp_path) + adjuster.abs_wrkdir = tmp_path adjuster.determine_wrksrc() assert adjuster.abs_wrksrc == adjuster.abs_wrkdir -def test_Adjuster_determine_wrksrc__single_dir(tmp_path: pathlib.Path): +def test_Adjuster_determine_wrksrc__single_dir(tmp_path: Path): adjuster = Adjuster(up, '', Lines()) - adjuster.abs_wrkdir = str(tmp_path) + adjuster.abs_wrkdir = tmp_path (tmp_path / 'subdir').mkdir() adjuster.determine_wrksrc() - assert adjuster.abs_wrksrc == adjuster.abs_wrkdir + '/subdir' + assert adjuster.abs_wrksrc == adjuster.abs_wrkdir / 'subdir' -def test_Adjuster_determine_wrksrc__several_dirs(tmp_path: pathlib.Path): +def test_Adjuster_determine_wrksrc__several_dirs(tmp_path: Path): adjuster = Adjuster(up, '', Lines()) - adjuster.abs_wrkdir = str(tmp_path) + adjuster.abs_wrkdir = tmp_path (tmp_path / 'subdir1').mkdir() (tmp_path / 'subdir2').mkdir() @@ -1232,32 +1247,24 @@ def test_Adjuster_determine_wrksrc__several_dirs(tmp_path: pathlib.Path): ] -def test_Adjuster_adjust_package_from_extracted_distfiles__empty_wrkdir(tmp_path: pathlib.Path): - pkgdir = tmp_path +def test_Adjuster_adjust__empty_wrkdir(tmp_path: Path): wrkdir = tmp_path / 'wrkdir' - fake = '''\ -#! /bin/sh -case $* in -("show-var VARNAME=WRKDIR") echo '%s' ;; -(*) "unknown: $*" ;; -esac -''' % str(wrkdir) up.pkgdir = tmp_path wrkdir.mkdir() - url = 'https://example.org/distfile-1.0.zip' - adjuster = Adjuster(up, url, Lines()) - adjuster.abs_wrkdir = str(wrkdir) - (pkgdir / 'Makefile').write_text('# url2pkg-marker\n') + adjuster = Adjuster(up, 'https://example.org/distfile-1.0.zip', Lines()) + adjuster.abs_wrkdir = wrkdir + (tmp_path / 'Makefile').write_text('# url2pkg-marker\n') fake_path = tmp_path / 'fake' - fake_path.write_text(fake) + fake_path.write_text( + '#! /bin/sh\n' + 'case $* in\n' + f'("show-var VARNAME=WRKDIR") echo "{wrkdir}" ;;\n' + '(*) "unknown: $*" ;;\n' + 'esac\n') fake_path.chmod(0o755) - - prev_make = up.make up.make = fake_path - try: - adjuster.adjust() - finally: - up.make = prev_make + + adjuster.adjust() assert detab(adjuster.generate_lines()) == [ 'WRKSRC= ${WRKDIR}', @@ -1266,7 +1273,7 @@ esac ] -def test_Adjuster_adjust_lines_python_module(tmp_path: pathlib.Path): +def test_Adjuster_adjust_lines_python_module(tmp_path: Path): url = 'https://github.com/espressif/esptool/archive/v2.7.tar.gz' up.pkgdir = tmp_path up.make = 'true' # the shell command |