Import pypy (7.2.0+dfsg-1)debian/7.2.0+dfsg-1debian

8 files changed, 681 insertions, 0 deletions

diff --git a/debian/scripts/build-cffi-modules.py b/debian/scripts/build-cffi-modules.py
new file mode 100755
index 0000000..f321aa3
--- /dev/null
+++ b/debian/scripts/build-cffi-modules.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+
+import os
+import sys
+sys.path.insert(0, '.')
+
+import py.path
+
+from pypy.tool.build_cffi_imports import create_cffi_import_libraries
+
+
+class FakeOptions(object):
+    def __getattr__(self, name):
+        # Build all the modules
+        if name.startswith('no_'):
+            return False
+
+        raise AttributeError()
+
+
+os.environ['LD_LIBRARY_PATH'] = 'pypy/goal'
+pypy_c = py.path.local('pypy/goal/pypy-c')
+options = FakeOptions()
+create_cffi_import_libraries(pypy_c, options, '.')
diff --git a/debian/scripts/cleanup-lib.sh b/debian/scripts/cleanup-lib.sh
new file mode 100755
index 0000000..a1d0e0e
--- /dev/null
+++ b/debian/scripts/cleanup-lib.sh
@@ -0,0 +1,32 @@
+#!/bin/sh
+
+set -euxf
+
+pkg=$1
+
+find debian/$pkg \
+     -name '*.pyc' -delete
+find debian/$pkg \
+     -name '__pycache__' -delete
+
+# Don't need these, and lintian will make a noise about them
+find debian/$pkg \( \
+     -name 'regen' \
+     -o -name '*.bat' \
+     -o -name 'fetch_*' \
+     -o -name '*.pickle' \
+     \) -delete
+
+# Remove empty directories, because pypyclean will
+find debian/$pkg/usr/lib/pypy/lib-python -type d -empty -delete
+
+# Nothing in the stdlib should be executable
+chmod -R a-x+X debian/$pkg/usr/lib/pypy/lib-python/
+if [ "$pkg" = "pypy-lib" ]; then
+	chmod -R a-x+X debian/$pkg/usr/lib/pypy/lib_pypy/
+fi
+
+# Fix interpreters
+find debian/$pkg \
+     -name '*.py' -print0 \
+     | xargs -0 sed -i -e '1s|^#!.*python.*|#!/usr/bin/pypy|'
diff --git a/debian/scripts/gen-backend-versions.py b/debian/scripts/gen-backend-versions.py
new file mode 100755
index 0000000..9dd5fa8
--- /dev/null
+++ b/debian/scripts/gen-backend-versions.py
@@ -0,0 +1,74 @@
+#!/usr/bin/python
+
+import codecs
+import os
+import re
+import subprocess
+
+
+def cffi_version():
+    with codecs.open('lib_pypy/cffi/__init__.py', encoding='UTF-8') as f:
+        for line in f:
+            m = re.match(r'^__version__ = "([0-9.]+)"$', line)
+            if m:
+                return m.group(1)
+    raise Exception('Version not found')
+
+
+def target_version():
+    with codecs.open('lib_pypy/cffi/recompiler.py', encoding='UTF-8') as f:
+        for line in f:
+            m = re.match(r'^VERSION_CHAR16CHAR32 = (0x[0-9A-F]+)$', line)
+            if m:
+                return int(m.group(1), 16)
+    raise Exception('Version not found')
+
+
+def backend_supported_versions():
+    versions = {}
+    with codecs.open('pypy/module/_cffi_backend/cffi1_module.py',
+                     encoding='UTF-8') as f:
+        for line in f:
+            m = re.match(r'^VERSION_(MIN|MAX) *= (0x[0-9A-F]+)$',
+                         line)
+            if m:
+                versions[m.group(1)] = int(m.group(2), 16)
+                if len(versions) == 2:
+                    return versions['MIN'], versions['MAX']
+    raise Exception('Versions not found')
+
+
+def pypy_abi():
+    if 'pypy' not in subprocess.check_output(('dh_listpackages',)).split():
+        return ''
+    soabi = subprocess.check_output((
+        'pypy/goal/pypy-c', '-c',
+        'import sysconfig; print sysconfig.get_config_var("SOABI")'))
+    return soabi.strip().replace('-', '-abi-')
+
+
+cffi = cffi_version()
+versions = backend_supported_versions()
+target = target_version()
+
+subst = {
+    'cffi': cffi,
+    'min': versions[0],
+    'max': versions[1],
+    'target': target,
+}
+with codecs.open('debian/pypy-lib.substvars', 'a', encoding='UTF-8') as f:
+    f.write('cffi:Provides=pypy-cffi (= {cffi}), '
+            'pypy-cffi-backend-api-min (= {min}), '
+            'pypy-cffi-backend-api-max (= {max}), '
+            'pypy-cffi-backend-api-{target}\n'.format(**subst))
+
+with codecs.open('debian/pypy.substvars', 'a', encoding='UTF-8') as f:
+    f.write('pypy-abi={}\n'.format(pypy_abi()))
+
+path = 'debian/pypy-lib/usr/share/pypy/dist'
+os.makedirs(path)
+with codecs.open(os.path.join(path, 'pypy-cffi'), 'w', encoding='UTF-8') as f:
+    f.write('cffi pypy-cffi-backend-api-min (<= {target}), '
+            'pypy-cffi-backend-api-max (>= {target})\n'
+            .format(**subst))
diff --git a/debian/scripts/multiarch-extensions.sh b/debian/scripts/multiarch-extensions.sh
new file mode 100755
index 0000000..d396eb4
--- /dev/null
+++ b/debian/scripts/multiarch-extensions.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# Rename libraries to fully-qualified multiarch filenames.
+
+set -euf
+
+# Skip if we built without multiarch (backport-friendly)
+if ! pypy/goal/pypy-c -c 'import sys; sys.exit(0 if hasattr(sys, "_multiarch") else 1)'; then
+	exit 0
+fi
+
+find lib_pypy -name '*.so' | while read extension; do
+	if echo "$extension" | grep -q '\.pypy-[0-9]*[a-z]*\.so'; then
+		dest="${extension%.so}-${DEB_HOST_MULTIARCH}.so"
+		mv "$extension" "$dest"
+	fi
+done
diff --git a/debian/scripts/pypyclean b/debian/scripts/pypyclean
new file mode 100755
index 0000000..83c8907
--- /dev/null
+++ b/debian/scripts/pypyclean
@@ -0,0 +1,200 @@
+#!/usr/bin/pypy
+
+import argparse
+import collections
+import itertools
+import os
+import shutil
+import subprocess
+import sys
+
+
+def abort(message):
+    print >> sys.stderr, message
+    sys.exit(1)
+
+
+def package_modules(package):
+    '''Iterate through all python modules in an installed Debian package'''
+    p = subprocess.Popen(('dpkg', '-L', package), stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE)
+    files, stderr = p.communicate()
+    if p.returncode != 0:
+        abort('Unable to list files in %s. Is it installed?' % package)
+
+    for fn in files.splitlines():
+        if fn.endswith('.py'):
+            if fn.startswith('/usr/share/doc/'):
+                continue
+            yield fn
+
+
+def installed_namespaces():
+    '''Return a dictionary of package: frozenset(namespaces)'''
+    ns_dir = '/usr/share/pypy/ns'
+    ns_by_pkg = {}
+    for pkg in os.listdir(ns_dir):
+        ns_file = os.path.join(ns_dir, pkg)
+        with open(ns_file) as f:
+            contents = f.read().decode('utf-8').strip()
+            namespaces = [line.strip() for line in contents.splitlines()]
+            ns_by_pkg[pkg] = frozenset(namespaces)
+    return ns_by_pkg
+
+
+def cleanup_namespaces(package, verbose):
+    '''Check if a namespace is still being used and, if not:
+    Remove the __init__.py.
+    Remove any pycs related to it.
+    '''
+    ns_by_pkg = installed_namespaces()
+    pkg_namespaces = ns_by_pkg.pop(package, None)
+    if not pkg_namespaces:
+        return
+    foreign_namespaces = reduce(lambda a, b: a | b, ns_by_pkg.values(), set())
+    orphan_namespaces = pkg_namespaces - foreign_namespaces
+    inits_to_remove = []
+    for namespace in orphan_namespaces:
+        init = os.path.join('/usr/lib/pypy/dist-packages',
+                            namespace.replace('.', '/'),
+                            '__init__.py')
+        if not os.path.exists(init):
+            print 'Missing namespace init: %s' % init
+            continue
+        if os.path.getsize(init) != 0:
+            print 'Non-empty init, ignoring: %s' % init
+            continue
+        inits_to_remove.append(init)
+
+    clean_modules(inits_to_remove, verbose)
+    for init in inits_to_remove:
+        os.unlink(init)
+
+
+def cleanup_package_modules(package, verbose):
+    '''Iterate through all python modules in an installed Debian package that
+    were in /usr/share/doc/, and previously byte-compiled by
+    pypy << 6.0.0+dfsg-2. See #904521
+
+    Can be removed once every pypy library has been re-uploaded, since this was
+    added.
+    '''
+    p = subprocess.Popen(('dpkg', '-L', package), stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE)
+    files, stderr = p.communicate()
+    if p.returncode != 0:
+        abort('Unable to list files in %s. Is it installed?' % package)
+
+    modules = [
+        fn for fn in files.splitlines()
+        if fn.startswith('/usr/share/doc/') and fn.endswith('.py')]
+
+    try:
+        clean_modules(modules, verbose)
+    except IOError as e:
+        if e.errno != errno.ENOENT:
+            raise
+
+
+def find_modules(root):
+    '''Iterate through all python modules in directory tree root'''
+    if os.path.isfile(root):
+        yield root
+        return
+
+    for dirpath, dirnames, filenames in os.walk(root):
+        for fn in filenames:
+            if fn.endswith('.py'):
+                yield os.path.join(dirpath, fn)
+
+
+def clean_modules(modules, verbose):
+    '''Remove all .pyc files for every module specified'''
+    clean = collections.defaultdict(list)
+    for module in modules:
+        dir_, basename = os.path.split(module)
+        clean[dir_].append(os.path.splitext(basename)[0])
+
+    for dir_, basenames in clean.iteritems():
+        pycache = os.path.join(dir_, '__pycache__')
+        if not os.path.exists(pycache):
+            continue
+
+        empty = True
+        for fn in os.listdir(pycache):
+            if fn.endswith('.pyc') and fn.rsplit('.', 2)[0] in basenames:
+                if verbose:
+                    print 'Removing %s' % os.path.join(pycache, fn)
+                os.unlink(os.path.join(pycache, fn))
+            else:
+                empty = False
+
+        if empty:
+            if verbose:
+                print 'Pruning %s' % pycache
+            os.rmdir(pycache)
+
+
+def clean_directories(directories, verbose):
+    '''Indiscriminately remove __pycache__ directories'''
+    for root in directories:
+        for dirpath, dirnames, filenames in os.walk(root):
+            for dir_ in dirnames:
+                if dir_ == '__pycache__':
+                    if verbose:
+                        print 'Removing %s' % os.path.join(dirpath, dir_)
+                    shutil.rmtree(os.path.join(dirpath, dir_))
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Remove byte-compiled files for a package')
+    parser.add_argument('-p', '--package', metavar='PACKAGE',
+                        action='append', default=[],
+                        help='Debian package to byte-compile '
+                             '(may be specified multiple times)')
+    parser.add_argument('directory', nargs='*',
+                        help='Directory tree (or file) to byte-compile')
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help='Be more verbose')
+    parser.add_argument('-q', '--quiet', action='store_true',
+                        help='Be quiet')
+    args = parser.parse_args()
+
+    if not (args.package or args.directory):
+        parser.error('Either a package or a directory must be specified')
+    if args.quiet and args.verbose:
+        parser.error('--quiet and --verbose cannot both be specified')
+
+    modules_p = set(itertools.chain(*(
+        package_modules(package) for package in args.package)))
+    modules_d = set(itertools.chain(*(
+        find_modules(dir_) for dir_ in args.directory)))
+
+    if args.package and args.directory:
+        modules = modules_d & modules_p
+    elif args.package:
+        modules = modules_p
+    else:
+        # Split files from directories, so that we can completely clean any
+        # specified directories.
+        modules = set()
+        directories = set()
+        for fn in args.directory:
+            if os.path.isfile(fn) and fn.endswith('.py'):
+                modules.add(fn)
+            else:
+                directories.add(fn)
+        clean_directories(directories, args.verbose)
+
+    clean_modules(modules, args.verbose)
+
+    for package in args.package:
+        cleanup_namespaces(package, args.verbose)
+        cleanup_package_modules(package, args.verbose)
+
+
+if __name__ == '__main__':
+    main()
+
+# vim: ft=python
diff --git a/debian/scripts/pypycompile b/debian/scripts/pypycompile
new file mode 100755
index 0000000..31abe2d
--- /dev/null
+++ b/debian/scripts/pypycompile
@@ -0,0 +1,129 @@
+#!/usr/bin/pypy
+
+import argparse
+import itertools
+import os
+import re
+import py_compile
+import subprocess
+import sys
+
+
+def abort(message):
+    print >> sys.stderr, message
+    sys.exit(1)
+
+
+def package_modules(package):
+    '''Iterate through all python modules in an installed Debian package'''
+    p = subprocess.Popen(('dpkg', '-L', package), stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE)
+    files, stderr = p.communicate()
+    if p.returncode != 0:
+        abort('Unable to list files in %s. Is it installed?' % package)
+
+    for fn in files.splitlines():
+        if fn.endswith('.py'):
+            if fn.startswith('/usr/share/doc/'):
+                continue
+            yield fn
+
+
+def find_modules(root):
+    '''Iterate through all python modules in directory tree root'''
+    if os.path.isfile(root):
+        yield root
+        return
+
+    for dirpath, dirnames, filenames in os.walk(root):
+        for fn in filenames:
+            if fn.endswith('.py'):
+                yield os.path.join(dirpath, fn)
+
+
+def generate_namespace_init(package, verbose):
+    '''Iterate through a package's ns file.
+    Create all necessary__init__.pys, and yield them.
+    '''
+    ns_file = os.path.join('/usr/share/pypy/ns', package)
+    if not os.path.exists(ns_file):
+        return
+    with open(ns_file) as f:
+        contents = f.read().decode('utf-8').strip()
+        namespaces = [line.strip() for line in contents.splitlines()]
+
+    for namespace in namespaces:
+        init = os.path.join('/usr/lib/pypy/dist-packages',
+                            namespace.replace('.', '/'),
+                            '__init__.py')
+        if verbose:
+            print 'Ensuring %s exists' % init
+        with open(init, 'w') as f:
+            pass
+        yield init
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Byte-compile Python source files in a package, for PyPy')
+    parser.add_argument('-p', '--package', metavar='PACKAGE',
+                        action='append', default=[],
+                        help='Debian package to byte-compile '
+                             '(may be specified multiple times)')
+    parser.add_argument('directory', nargs='*',
+                        help='Directory tree (or file) to byte-compile')
+    parser.add_argument('-X', '--exclude', metavar='REGEXPR',
+                        action='append', default=[], type=re.compile,
+                        help='Exclude items that match given REGEXPR '
+                             '(may be specified multiple times)')
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help='Be more verbose')
+    parser.add_argument('-q', '--quiet', action='store_true',
+                        help='Be quiet')
+    parser.add_argument('-V', metavar='VRANGE', dest='vrange',
+                        help=argparse.SUPPRESS)
+    parser.add_argument('-O', action='store_true', dest='pyo',
+                        help=argparse.SUPPRESS)
+    args = parser.parse_args()
+
+    if not (args.package or args.directory):
+        parser.error('Either a package or a directory must be specified')
+    if args.quiet and args.verbose:
+        parser.error('--quiet and --verbose cannot both be specified')
+
+    if args.vrange:
+        print >> sys.stderr, '-V is ignored in pypycompile'
+    if args.pyo:
+        print >> sys.stderr, '-O is ignored in pypycompile'
+
+    modules_p = set(itertools.chain(*(
+        package_modules(package) for package in args.package)))
+    modules_d = set(itertools.chain(*(
+        find_modules(dir_) for dir_ in args.directory)))
+
+    if args.package and args.directory:
+        modules = modules_d & modules_p
+    else:
+        modules = modules_d | modules_p
+
+    for package in args.package:
+        modules |= set(generate_namespace_init(package, verbose=args.verbose))
+
+    modules = filter(lambda module: not any(pattern.match(module)
+                                            for pattern in args.exclude),
+                     modules)
+
+    for module in modules:
+        if args.verbose:
+            print 'Byte-compiling %s' % module
+        try:
+            py_compile.compile(module, doraise=True)
+        except py_compile.PyCompileError as e:
+            if not args.quiet:
+                print >> sys.stderr, ('Failed to byte-compile %s: %s'
+                                      % (module, e.msg))
+
+if __name__ == '__main__':
+    main()
+
+# vim: ft=python
diff --git a/debian/scripts/timeout-inactive.py b/debian/scripts/timeout-inactive.py
new file mode 100755
index 0000000..d865fc8
--- /dev/null
+++ b/debian/scripts/timeout-inactive.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+
+import argparse
+import re
+import signal
+import subprocess
+import sys
+import threading
+
+
+def main():
+    p = argparse.ArgumentParser(
+        description='Run a program, and kill it if it is silent for too long.'
+                    'After the arguments, specify the program to run prefixed '
+                    'with -- .')
+    p.add_argument('--timeout', type=parse_timeout,
+                   help='Timeout (in seconds, if no unit specified')
+
+    our_args = sys.argv[1:]
+    cmd = []
+    if '--' in our_args:
+        index = our_args.index('--')
+        cmd = our_args[index + 1:]
+        our_args = our_args[:index]
+
+    args = p.parse_args(our_args)
+    if not cmd:
+        p.error('No command specified')
+
+    run_cmd(cmd, args.timeout)
+
+
+def parse_timeout(arg):
+    m = re.match(r'(\d+) *(s(ec(ond)?s?)?|m(in(ute)?s?)?|h((ou)?rs?)?)?', arg)
+    if not m:
+        raise argparse.ArgumentTypeError('Un-parseable time')
+    value = int(m.group(1))
+    unit = m.group(2)
+    if unit:
+        unit = unit[0]
+        if unit == 'm':
+            return value * 60
+        if unit == 'h':
+            return value * 60 * 60
+    return value
+
+
+def run_cmd(cmd, timeout):
+    p = subprocess.Popen(cmd, bufsize=1, stdout=subprocess.PIPE)
+    watchdog = Watchdog(timeout, p)
+    watchdog.start()
+    while True:
+        line = p.stdout.readline()
+        watchdog.event.set()
+        sys.stdout.write(line)
+        sys.stdout.flush()
+        if p.poll() != None:
+            sys.exit(p.returncode)
+
+
+class Watchdog(threading.Thread):
+    def __init__(self, timeout, process):
+        super(Watchdog, self).__init__()
+        self.daemon = True
+        self.timeout = timeout
+        self.process = process
+        self.event = threading.Event()
+
+    def run(self):
+        while True:
+            ev = self.event.wait(self.timeout)
+            if ev is True:
+                self.event.clear()
+            else:
+                sys.stderr.write(
+                    '\nTimed out after {} seconds of inactivity...\n'
+                    .format(self.timeout))
+                sys.stderr.flush()
+                self.process.send_signal(signal.SIGINT)
+                return
+
+
+if __name__ == '__main__':
+    main()
diff --git a/debian/scripts/translate.sh b/debian/scripts/translate.sh
new file mode 100755
index 0000000..ab01086
--- /dev/null
+++ b/debian/scripts/translate.sh
@@ -0,0 +1,121 @@
+#!/bin/sh
+# Determine the translation options and translate
+
+set -e -u
+
+usage() {
+	name=$(basename $0)
+	echo <<EOF
+Usage: $name [options...]"
+Options:
+  --python=PYTHON Use PYTHON to tranlate PyPy
+
+EOF
+}
+
+PYTHON=python
+
+while [ $# -ge 1 ]; do
+	case "$1" in
+	--python)
+		PYTHON="$2"
+		shift
+		;;
+	--help|-h)
+		usage
+		exit 0
+		;;
+	*)
+		echo "Unkown option: $1" >&2
+		usage
+		exit 1
+		;;
+	esac
+	shift
+done
+
+RPYOPTS="--batch --source"
+TARGETOPTS=""
+
+OPT=3
+if echo "$DEB_BUILD_OPTIONS" | egrep -q '\bnoopt\b'; then
+	OPT=0
+elif dpkg-architecture -iany-i386; then
+	OPT=jit
+	#RPYOPTS="$RPYOPTS --jit-backend=x86-without-sse2"
+elif dpkg-architecture -iany-amd64; then
+	# No JIT on x32
+	if ! dpkg-architecture -elinux-x32; then
+		OPT=jit
+	fi
+elif dpkg-architecture -iany-arm; then
+	# No JIT support for ARMv5
+	if ! dpkg-architecture -elinux-armel; then
+		OPT=jit
+	fi
+elif dpkg-architecture -iany-arm64; then
+	OPT=jit
+elif dpkg-architecture -iany-ppc64el || dpkg-architecture -iany-ppc64; then
+	OPT=jit
+elif dpkg-architecture -iany-s390x; then
+	OPT=jit
+fi
+RPYOPTS="$RPYOPTS --opt=$OPT --shared"
+
+if [ $OPT = 3 ]; then
+	RPYOPTS="$RPYOPTS --no-profopt"
+fi
+
+CONTINUATION=0
+if dpkg-architecture -iany-i386 \
+   || dpkg-architecture -iany-amd64 \
+   || dpkg-architecture -iany-arm \
+   || dpkg-architecture -iany-arm64 \
+   || dpkg-architecture -iany-mips64el \
+   || dpkg-architecture -iany-ppc64 \
+   || dpkg-architecture -iany-ppc64el \
+   || dpkg-architecture -iany-s390x; then
+	CONTINUATION=1
+fi
+if [ $CONTINUATION -eq 0 ]; then
+	TARGETOPTS="$TARGETOPTS --withoutmod-_continuation"
+fi
+
+if dpkg-architecture --is kfreebsd-any; then
+	# No support for F_GETPATH
+	TARGETOPTS="$TARGETOPTS --withoutmod-_vmprof"
+fi
+
+if dpkg-architecture -iany-s390x; then
+	# Target the oldest s390x supported by upstream
+	export CFLAGS="$CFLAGS -march=z10"
+fi
+
+if echo "$PYTHON" | grep -Fq pypy; then
+	if [ $(dpkg-architecture -q DEB_HOST_ARCH_BITS) -eq 32 ]; then
+		export PYPY_GC_MAX_DELTA=200MB
+		PYTHON="$PYTHON --jit loop_longevity=300"
+	fi
+fi
+
+CURDIR=$(pwd)
+RPYTHON=$CURDIR/rpython/bin/rpython
+TMPDIR=$CURDIR/build-tmp
+PYPY_USESSION_BASENAME=debian
+C_SRC_DIR=$TMPDIR/usession-$PYPY_USESSION_BASENAME-0/testing_1
+SUB_MAKEFLAGS=$(echo ${DEB_BUILD_OPTIONS:-} | tr ' ' '\n' | sed -ne 's/parallel=/-j/p')
+
+set -x
+export TMPDIR PYPY_USESSION_BASENAME
+rm -rf "$TMPDIR"
+mkdir "$TMPDIR"
+
+cd pypy/goal
+$PYTHON -u $RPYTHON $RPYOPTS targetpypystandalone $TARGETOPTS 2>&1
+cd "$CURDIR"
+
+# PyPy captures all make output, so we call it ourselves (--source), to save
+# memory and avoid timeouts:
+make -C $C_SRC_DIR $SUB_MAKEFLAGS
+
+cp $C_SRC_DIR/pypy-c $C_SRC_DIR/libpypy-c.so pypy/goal