diff options
Diffstat (limited to 'debian/scripts')
| -rwxr-xr-x | debian/scripts/build-cffi-modules.py | 24 | ||||
| -rwxr-xr-x | debian/scripts/cleanup-lib.sh | 32 | ||||
| -rwxr-xr-x | debian/scripts/gen-backend-versions.py | 74 | ||||
| -rwxr-xr-x | debian/scripts/multiarch-extensions.sh | 17 | ||||
| -rwxr-xr-x | debian/scripts/pypyclean | 200 | ||||
| -rwxr-xr-x | debian/scripts/pypycompile | 129 | ||||
| -rwxr-xr-x | debian/scripts/timeout-inactive.py | 84 | ||||
| -rwxr-xr-x | debian/scripts/translate.sh | 121 |
8 files changed, 681 insertions, 0 deletions
diff --git a/debian/scripts/build-cffi-modules.py b/debian/scripts/build-cffi-modules.py new file mode 100755 index 0000000..f321aa3 --- /dev/null +++ b/debian/scripts/build-cffi-modules.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +import os +import sys +sys.path.insert(0, '.') + +import py.path + +from pypy.tool.build_cffi_imports import create_cffi_import_libraries + + +class FakeOptions(object): + def __getattr__(self, name): + # Build all the modules + if name.startswith('no_'): + return False + + raise AttributeError() + + +os.environ['LD_LIBRARY_PATH'] = 'pypy/goal' +pypy_c = py.path.local('pypy/goal/pypy-c') +options = FakeOptions() +create_cffi_import_libraries(pypy_c, options, '.') diff --git a/debian/scripts/cleanup-lib.sh b/debian/scripts/cleanup-lib.sh new file mode 100755 index 0000000..a1d0e0e --- /dev/null +++ b/debian/scripts/cleanup-lib.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +set -euxf + +pkg=$1 + +find debian/$pkg \ + -name '*.pyc' -delete +find debian/$pkg \ + -name '__pycache__' -delete + +# Don't need these, and lintian will make a noise about them +find debian/$pkg \( \ + -name 'regen' \ + -o -name '*.bat' \ + -o -name 'fetch_*' \ + -o -name '*.pickle' \ + \) -delete + +# Remove empty directories, because pypyclean will +find debian/$pkg/usr/lib/pypy/lib-python -type d -empty -delete + +# Nothing in the stdlib should be executable +chmod -R a-x+X debian/$pkg/usr/lib/pypy/lib-python/ +if [ "$pkg" = "pypy-lib" ]; then + chmod -R a-x+X debian/$pkg/usr/lib/pypy/lib_pypy/ +fi + +# Fix interpreters +find debian/$pkg \ + -name '*.py' -print0 \ + | xargs -0 sed -i -e '1s|^#!.*python.*|#!/usr/bin/pypy|' diff --git a/debian/scripts/gen-backend-versions.py b/debian/scripts/gen-backend-versions.py new file mode 100755 index 0000000..9dd5fa8 --- /dev/null +++ b/debian/scripts/gen-backend-versions.py @@ -0,0 +1,74 @@ +#!/usr/bin/python + +import codecs +import os +import re +import subprocess + + +def cffi_version(): + with codecs.open('lib_pypy/cffi/__init__.py', encoding='UTF-8') as f: + for line in f: + m = re.match(r'^__version__ = "([0-9.]+)"$', line) + if m: + return m.group(1) + raise Exception('Version not found') + + +def target_version(): + with codecs.open('lib_pypy/cffi/recompiler.py', encoding='UTF-8') as f: + for line in f: + m = re.match(r'^VERSION_CHAR16CHAR32 = (0x[0-9A-F]+)$', line) + if m: + return int(m.group(1), 16) + raise Exception('Version not found') + + +def backend_supported_versions(): + versions = {} + with codecs.open('pypy/module/_cffi_backend/cffi1_module.py', + encoding='UTF-8') as f: + for line in f: + m = re.match(r'^VERSION_(MIN|MAX) *= (0x[0-9A-F]+)$', + line) + if m: + versions[m.group(1)] = int(m.group(2), 16) + if len(versions) == 2: + return versions['MIN'], versions['MAX'] + raise Exception('Versions not found') + + +def pypy_abi(): + if 'pypy' not in subprocess.check_output(('dh_listpackages',)).split(): + return '' + soabi = subprocess.check_output(( + 'pypy/goal/pypy-c', '-c', + 'import sysconfig; print sysconfig.get_config_var("SOABI")')) + return soabi.strip().replace('-', '-abi-') + + +cffi = cffi_version() +versions = backend_supported_versions() +target = target_version() + +subst = { + 'cffi': cffi, + 'min': versions[0], + 'max': versions[1], + 'target': target, +} +with codecs.open('debian/pypy-lib.substvars', 'a', encoding='UTF-8') as f: + f.write('cffi:Provides=pypy-cffi (= {cffi}), ' + 'pypy-cffi-backend-api-min (= {min}), ' + 'pypy-cffi-backend-api-max (= {max}), ' + 'pypy-cffi-backend-api-{target}\n'.format(**subst)) + +with codecs.open('debian/pypy.substvars', 'a', encoding='UTF-8') as f: + f.write('pypy-abi={}\n'.format(pypy_abi())) + +path = 'debian/pypy-lib/usr/share/pypy/dist' +os.makedirs(path) +with codecs.open(os.path.join(path, 'pypy-cffi'), 'w', encoding='UTF-8') as f: + f.write('cffi pypy-cffi-backend-api-min (<= {target}), ' + 'pypy-cffi-backend-api-max (>= {target})\n' + .format(**subst)) diff --git a/debian/scripts/multiarch-extensions.sh b/debian/scripts/multiarch-extensions.sh new file mode 100755 index 0000000..d396eb4 --- /dev/null +++ b/debian/scripts/multiarch-extensions.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# Rename libraries to fully-qualified multiarch filenames. + +set -euf + +# Skip if we built without multiarch (backport-friendly) +if ! pypy/goal/pypy-c -c 'import sys; sys.exit(0 if hasattr(sys, "_multiarch") else 1)'; then + exit 0 +fi + +find lib_pypy -name '*.so' | while read extension; do + if echo "$extension" | grep -q '\.pypy-[0-9]*[a-z]*\.so'; then + dest="${extension%.so}-${DEB_HOST_MULTIARCH}.so" + mv "$extension" "$dest" + fi +done diff --git a/debian/scripts/pypyclean b/debian/scripts/pypyclean new file mode 100755 index 0000000..83c8907 --- /dev/null +++ b/debian/scripts/pypyclean @@ -0,0 +1,200 @@ +#!/usr/bin/pypy + +import argparse +import collections +import itertools +import os +import shutil +import subprocess +import sys + + +def abort(message): + print >> sys.stderr, message + sys.exit(1) + + +def package_modules(package): + '''Iterate through all python modules in an installed Debian package''' + p = subprocess.Popen(('dpkg', '-L', package), stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + files, stderr = p.communicate() + if p.returncode != 0: + abort('Unable to list files in %s. Is it installed?' % package) + + for fn in files.splitlines(): + if fn.endswith('.py'): + if fn.startswith('/usr/share/doc/'): + continue + yield fn + + +def installed_namespaces(): + '''Return a dictionary of package: frozenset(namespaces)''' + ns_dir = '/usr/share/pypy/ns' + ns_by_pkg = {} + for pkg in os.listdir(ns_dir): + ns_file = os.path.join(ns_dir, pkg) + with open(ns_file) as f: + contents = f.read().decode('utf-8').strip() + namespaces = [line.strip() for line in contents.splitlines()] + ns_by_pkg[pkg] = frozenset(namespaces) + return ns_by_pkg + + +def cleanup_namespaces(package, verbose): + '''Check if a namespace is still being used and, if not: + Remove the __init__.py. + Remove any pycs related to it. + ''' + ns_by_pkg = installed_namespaces() + pkg_namespaces = ns_by_pkg.pop(package, None) + if not pkg_namespaces: + return + foreign_namespaces = reduce(lambda a, b: a | b, ns_by_pkg.values(), set()) + orphan_namespaces = pkg_namespaces - foreign_namespaces + inits_to_remove = [] + for namespace in orphan_namespaces: + init = os.path.join('/usr/lib/pypy/dist-packages', + namespace.replace('.', '/'), + '__init__.py') + if not os.path.exists(init): + print 'Missing namespace init: %s' % init + continue + if os.path.getsize(init) != 0: + print 'Non-empty init, ignoring: %s' % init + continue + inits_to_remove.append(init) + + clean_modules(inits_to_remove, verbose) + for init in inits_to_remove: + os.unlink(init) + + +def cleanup_package_modules(package, verbose): + '''Iterate through all python modules in an installed Debian package that + were in /usr/share/doc/, and previously byte-compiled by + pypy << 6.0.0+dfsg-2. See #904521 + + Can be removed once every pypy library has been re-uploaded, since this was + added. + ''' + p = subprocess.Popen(('dpkg', '-L', package), stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + files, stderr = p.communicate() + if p.returncode != 0: + abort('Unable to list files in %s. Is it installed?' % package) + + modules = [ + fn for fn in files.splitlines() + if fn.startswith('/usr/share/doc/') and fn.endswith('.py')] + + try: + clean_modules(modules, verbose) + except IOError as e: + if e.errno != errno.ENOENT: + raise + + +def find_modules(root): + '''Iterate through all python modules in directory tree root''' + if os.path.isfile(root): + yield root + return + + for dirpath, dirnames, filenames in os.walk(root): + for fn in filenames: + if fn.endswith('.py'): + yield os.path.join(dirpath, fn) + + +def clean_modules(modules, verbose): + '''Remove all .pyc files for every module specified''' + clean = collections.defaultdict(list) + for module in modules: + dir_, basename = os.path.split(module) + clean[dir_].append(os.path.splitext(basename)[0]) + + for dir_, basenames in clean.iteritems(): + pycache = os.path.join(dir_, '__pycache__') + if not os.path.exists(pycache): + continue + + empty = True + for fn in os.listdir(pycache): + if fn.endswith('.pyc') and fn.rsplit('.', 2)[0] in basenames: + if verbose: + print 'Removing %s' % os.path.join(pycache, fn) + os.unlink(os.path.join(pycache, fn)) + else: + empty = False + + if empty: + if verbose: + print 'Pruning %s' % pycache + os.rmdir(pycache) + + +def clean_directories(directories, verbose): + '''Indiscriminately remove __pycache__ directories''' + for root in directories: + for dirpath, dirnames, filenames in os.walk(root): + for dir_ in dirnames: + if dir_ == '__pycache__': + if verbose: + print 'Removing %s' % os.path.join(dirpath, dir_) + shutil.rmtree(os.path.join(dirpath, dir_)) + + +def main(): + parser = argparse.ArgumentParser( + description='Remove byte-compiled files for a package') + parser.add_argument('-p', '--package', metavar='PACKAGE', + action='append', default=[], + help='Debian package to byte-compile ' + '(may be specified multiple times)') + parser.add_argument('directory', nargs='*', + help='Directory tree (or file) to byte-compile') + parser.add_argument('-v', '--verbose', action='store_true', + help='Be more verbose') + parser.add_argument('-q', '--quiet', action='store_true', + help='Be quiet') + args = parser.parse_args() + + if not (args.package or args.directory): + parser.error('Either a package or a directory must be specified') + if args.quiet and args.verbose: + parser.error('--quiet and --verbose cannot both be specified') + + modules_p = set(itertools.chain(*( + package_modules(package) for package in args.package))) + modules_d = set(itertools.chain(*( + find_modules(dir_) for dir_ in args.directory))) + + if args.package and args.directory: + modules = modules_d & modules_p + elif args.package: + modules = modules_p + else: + # Split files from directories, so that we can completely clean any + # specified directories. + modules = set() + directories = set() + for fn in args.directory: + if os.path.isfile(fn) and fn.endswith('.py'): + modules.add(fn) + else: + directories.add(fn) + clean_directories(directories, args.verbose) + + clean_modules(modules, args.verbose) + + for package in args.package: + cleanup_namespaces(package, args.verbose) + cleanup_package_modules(package, args.verbose) + + +if __name__ == '__main__': + main() + +# vim: ft=python diff --git a/debian/scripts/pypycompile b/debian/scripts/pypycompile new file mode 100755 index 0000000..31abe2d --- /dev/null +++ b/debian/scripts/pypycompile @@ -0,0 +1,129 @@ +#!/usr/bin/pypy + +import argparse +import itertools +import os +import re +import py_compile +import subprocess +import sys + + +def abort(message): + print >> sys.stderr, message + sys.exit(1) + + +def package_modules(package): + '''Iterate through all python modules in an installed Debian package''' + p = subprocess.Popen(('dpkg', '-L', package), stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + files, stderr = p.communicate() + if p.returncode != 0: + abort('Unable to list files in %s. Is it installed?' % package) + + for fn in files.splitlines(): + if fn.endswith('.py'): + if fn.startswith('/usr/share/doc/'): + continue + yield fn + + +def find_modules(root): + '''Iterate through all python modules in directory tree root''' + if os.path.isfile(root): + yield root + return + + for dirpath, dirnames, filenames in os.walk(root): + for fn in filenames: + if fn.endswith('.py'): + yield os.path.join(dirpath, fn) + + +def generate_namespace_init(package, verbose): + '''Iterate through a package's ns file. + Create all necessary__init__.pys, and yield them. + ''' + ns_file = os.path.join('/usr/share/pypy/ns', package) + if not os.path.exists(ns_file): + return + with open(ns_file) as f: + contents = f.read().decode('utf-8').strip() + namespaces = [line.strip() for line in contents.splitlines()] + + for namespace in namespaces: + init = os.path.join('/usr/lib/pypy/dist-packages', + namespace.replace('.', '/'), + '__init__.py') + if verbose: + print 'Ensuring %s exists' % init + with open(init, 'w') as f: + pass + yield init + + +def main(): + parser = argparse.ArgumentParser( + description='Byte-compile Python source files in a package, for PyPy') + parser.add_argument('-p', '--package', metavar='PACKAGE', + action='append', default=[], + help='Debian package to byte-compile ' + '(may be specified multiple times)') + parser.add_argument('directory', nargs='*', + help='Directory tree (or file) to byte-compile') + parser.add_argument('-X', '--exclude', metavar='REGEXPR', + action='append', default=[], type=re.compile, + help='Exclude items that match given REGEXPR ' + '(may be specified multiple times)') + parser.add_argument('-v', '--verbose', action='store_true', + help='Be more verbose') + parser.add_argument('-q', '--quiet', action='store_true', + help='Be quiet') + parser.add_argument('-V', metavar='VRANGE', dest='vrange', + help=argparse.SUPPRESS) + parser.add_argument('-O', action='store_true', dest='pyo', + help=argparse.SUPPRESS) + args = parser.parse_args() + + if not (args.package or args.directory): + parser.error('Either a package or a directory must be specified') + if args.quiet and args.verbose: + parser.error('--quiet and --verbose cannot both be specified') + + if args.vrange: + print >> sys.stderr, '-V is ignored in pypycompile' + if args.pyo: + print >> sys.stderr, '-O is ignored in pypycompile' + + modules_p = set(itertools.chain(*( + package_modules(package) for package in args.package))) + modules_d = set(itertools.chain(*( + find_modules(dir_) for dir_ in args.directory))) + + if args.package and args.directory: + modules = modules_d & modules_p + else: + modules = modules_d | modules_p + + for package in args.package: + modules |= set(generate_namespace_init(package, verbose=args.verbose)) + + modules = filter(lambda module: not any(pattern.match(module) + for pattern in args.exclude), + modules) + + for module in modules: + if args.verbose: + print 'Byte-compiling %s' % module + try: + py_compile.compile(module, doraise=True) + except py_compile.PyCompileError as e: + if not args.quiet: + print >> sys.stderr, ('Failed to byte-compile %s: %s' + % (module, e.msg)) + +if __name__ == '__main__': + main() + +# vim: ft=python diff --git a/debian/scripts/timeout-inactive.py b/debian/scripts/timeout-inactive.py new file mode 100755 index 0000000..d865fc8 --- /dev/null +++ b/debian/scripts/timeout-inactive.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +import argparse +import re +import signal +import subprocess +import sys +import threading + + +def main(): + p = argparse.ArgumentParser( + description='Run a program, and kill it if it is silent for too long.' + 'After the arguments, specify the program to run prefixed ' + 'with -- .') + p.add_argument('--timeout', type=parse_timeout, + help='Timeout (in seconds, if no unit specified') + + our_args = sys.argv[1:] + cmd = [] + if '--' in our_args: + index = our_args.index('--') + cmd = our_args[index + 1:] + our_args = our_args[:index] + + args = p.parse_args(our_args) + if not cmd: + p.error('No command specified') + + run_cmd(cmd, args.timeout) + + +def parse_timeout(arg): + m = re.match(r'(\d+) *(s(ec(ond)?s?)?|m(in(ute)?s?)?|h((ou)?rs?)?)?', arg) + if not m: + raise argparse.ArgumentTypeError('Un-parseable time') + value = int(m.group(1)) + unit = m.group(2) + if unit: + unit = unit[0] + if unit == 'm': + return value * 60 + if unit == 'h': + return value * 60 * 60 + return value + + +def run_cmd(cmd, timeout): + p = subprocess.Popen(cmd, bufsize=1, stdout=subprocess.PIPE) + watchdog = Watchdog(timeout, p) + watchdog.start() + while True: + line = p.stdout.readline() + watchdog.event.set() + sys.stdout.write(line) + sys.stdout.flush() + if p.poll() != None: + sys.exit(p.returncode) + + +class Watchdog(threading.Thread): + def __init__(self, timeout, process): + super(Watchdog, self).__init__() + self.daemon = True + self.timeout = timeout + self.process = process + self.event = threading.Event() + + def run(self): + while True: + ev = self.event.wait(self.timeout) + if ev is True: + self.event.clear() + else: + sys.stderr.write( + '\nTimed out after {} seconds of inactivity...\n' + .format(self.timeout)) + sys.stderr.flush() + self.process.send_signal(signal.SIGINT) + return + + +if __name__ == '__main__': + main() diff --git a/debian/scripts/translate.sh b/debian/scripts/translate.sh new file mode 100755 index 0000000..ab01086 --- /dev/null +++ b/debian/scripts/translate.sh @@ -0,0 +1,121 @@ +#!/bin/sh +# Determine the translation options and translate + +set -e -u + +usage() { + name=$(basename $0) + echo <<EOF +Usage: $name [options...]" +Options: + --python=PYTHON Use PYTHON to tranlate PyPy + +EOF +} + +PYTHON=python + +while [ $# -ge 1 ]; do + case "$1" in + --python) + PYTHON="$2" + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Unkown option: $1" >&2 + usage + exit 1 + ;; + esac + shift +done + +RPYOPTS="--batch --source" +TARGETOPTS="" + +OPT=3 +if echo "$DEB_BUILD_OPTIONS" | egrep -q '\bnoopt\b'; then + OPT=0 +elif dpkg-architecture -iany-i386; then + OPT=jit + #RPYOPTS="$RPYOPTS --jit-backend=x86-without-sse2" +elif dpkg-architecture -iany-amd64; then + # No JIT on x32 + if ! dpkg-architecture -elinux-x32; then + OPT=jit + fi +elif dpkg-architecture -iany-arm; then + # No JIT support for ARMv5 + if ! dpkg-architecture -elinux-armel; then + OPT=jit + fi +elif dpkg-architecture -iany-arm64; then + OPT=jit +elif dpkg-architecture -iany-ppc64el || dpkg-architecture -iany-ppc64; then + OPT=jit +elif dpkg-architecture -iany-s390x; then + OPT=jit +fi +RPYOPTS="$RPYOPTS --opt=$OPT --shared" + +if [ $OPT = 3 ]; then + RPYOPTS="$RPYOPTS --no-profopt" +fi + +CONTINUATION=0 +if dpkg-architecture -iany-i386 \ + || dpkg-architecture -iany-amd64 \ + || dpkg-architecture -iany-arm \ + || dpkg-architecture -iany-arm64 \ + || dpkg-architecture -iany-mips64el \ + || dpkg-architecture -iany-ppc64 \ + || dpkg-architecture -iany-ppc64el \ + || dpkg-architecture -iany-s390x; then + CONTINUATION=1 +fi +if [ $CONTINUATION -eq 0 ]; then + TARGETOPTS="$TARGETOPTS --withoutmod-_continuation" +fi + +if dpkg-architecture --is kfreebsd-any; then + # No support for F_GETPATH + TARGETOPTS="$TARGETOPTS --withoutmod-_vmprof" +fi + +if dpkg-architecture -iany-s390x; then + # Target the oldest s390x supported by upstream + export CFLAGS="$CFLAGS -march=z10" +fi + +if echo "$PYTHON" | grep -Fq pypy; then + if [ $(dpkg-architecture -q DEB_HOST_ARCH_BITS) -eq 32 ]; then + export PYPY_GC_MAX_DELTA=200MB + PYTHON="$PYTHON --jit loop_longevity=300" + fi +fi + +CURDIR=$(pwd) +RPYTHON=$CURDIR/rpython/bin/rpython +TMPDIR=$CURDIR/build-tmp +PYPY_USESSION_BASENAME=debian +C_SRC_DIR=$TMPDIR/usession-$PYPY_USESSION_BASENAME-0/testing_1 +SUB_MAKEFLAGS=$(echo ${DEB_BUILD_OPTIONS:-} | tr ' ' '\n' | sed -ne 's/parallel=/-j/p') + +set -x +export TMPDIR PYPY_USESSION_BASENAME +rm -rf "$TMPDIR" +mkdir "$TMPDIR" + +cd pypy/goal +$PYTHON -u $RPYTHON $RPYOPTS targetpypystandalone $TARGETOPTS 2>&1 +cd "$CURDIR" + +# PyPy captures all make output, so we call it ourselves (--source), to save +# memory and avoid timeouts: +make -C $C_SRC_DIR $SUB_MAKEFLAGS + +cp $C_SRC_DIR/pypy-c $C_SRC_DIR/libpypy-c.so pypy/goal |