path: root/debian/patches/pep3147-stdlib

From: Stefano Rivera <stefanor@debian.org>
Date: Sat, 7 Oct 2017 09:38:57 +0200
Subject: PEP3147 changes to lib-python

Backported from cpython's PEP3147 commit

Origin: cpython: http://hg.python.org/cpython/rev/7b69e630d237
Author: Barry Warsaw <barry@python.org>
Author: Stefano Rivera <stefanor@debian.org>
Last-Update: 2013-02-23
---
 lib-python/2.7/compileall.py                |  67 ++++++++-----
 lib-python/2.7/inspect.py                   |   1 +
 lib-python/2.7/py_compile.py                |  35 ++++---
 lib-python/2.7/pydoc.py                     |   3 +-
 lib-python/2.7/runpy.py                     |   2 +
 lib-python/2.7/site.py                      |  13 ++-
 lib-python/2.7/test/script_helper.py        |  23 ++---
 lib-python/2.7/test/test_cmd_line_script.py |  20 ++--
 lib-python/2.7/test/test_compileall.py      |  61 +++++++++++-
 lib-python/2.7/test/test_imp.py             | 142 ++++++++++++++++++++++++++++
 lib-python/2.7/test/test_import.py          |  65 +++++++------
 lib-python/2.7/test/test_pkg.py             |  20 ++--
 lib-python/2.7/test/test_pkgimport.py       |  20 ++--
 lib-python/2.7/test/test_py_compile.py      |   5 +-
 lib-python/2.7/test/test_pydoc.py           |   3 +-
 lib-python/2.7/test/test_runpy.py           |  23 +++--
 lib-python/2.7/test/test_site.py            |  47 ++++++---
 lib-python/2.7/test/test_support.py         |  48 ++++++++--
 lib-python/2.7/test/test_traceback.py       |   5 +-
 lib-python/2.7/test/test_zipfile.py         |   9 +-
 lib-python/2.7/test/test_zipimport.py       |   9 +-
 lib-python/2.7/zipfile.py                   |  43 ++++++---
 22 files changed, 499 insertions(+), 165 deletions(-)

diff --git a/lib-python/2.7/compileall.py b/lib-python/2.7/compileall.py
index 5cfa8be..1418be2 100644
--- a/lib-python/2.7/compileall.py
+++ b/lib-python/2.7/compileall.py
@@ -11,6 +11,7 @@ packages -- for now, you'll have to deal with packages separately.)
 See module py_compile for details of the actual byte-compilation.
 """
 import os
+import errno
 import sys
 import py_compile
 import struct
@@ -19,7 +20,7 @@ import imp
 __all__ = ["compile_dir","compile_file","compile_path"]
 
 def compile_dir(dir, maxlevels=10, ddir=None,
-                force=0, rx=None, quiet=0):
+                force=False, rx=None, quiet=False, legacy=False):
     """Byte-compile all modules in the given directory tree.
 
     Arguments (only dir is required):
@@ -28,8 +29,9 @@ def compile_dir(dir, maxlevels=10, ddir=None,
     maxlevels: maximum recursion level (default 10)
     ddir:      the directory that will be prepended to the path to the
                file as it is compiled into each byte-code file.
-    force:     if 1, force compilation, even if timestamps are up-to-date
-    quiet:     if 1, be quiet during compilation
+    force:     if True, force compilation, even if timestamps are up-to-date
+    quiet:     if True, be quiet during compilation
+    legacy:    if True, produce legacy pyc paths instead of PEP 3147 paths
     """
     if not quiet:
         print 'Listing', dir, '...'
@@ -47,18 +49,20 @@ def compile_dir(dir, maxlevels=10, ddir=None,
         else:
             dfile = None
         if not os.path.isdir(fullname):
-            if not compile_file(fullname, ddir, force, rx, quiet):
+            if not compile_file(fullname, ddir, force, rx, quiet, legacy):
                 success = 0
         elif maxlevels > 0 and \
              name != os.curdir and name != os.pardir and \
              os.path.isdir(fullname) and \
-             not os.path.islink(fullname):
+             not os.path.islink(fullname) and \
+             name != '__pycache__':
             if not compile_dir(fullname, maxlevels - 1, dfile, force, rx,
-                               quiet):
+                               quiet, legacy):
                 success = 0
     return success
 
-def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0):
+def compile_file(fullname, ddir=None, force=0, rx=None, quiet=False,
+                 legacy=False):
     """Byte-compile one file.
 
     Arguments (only fullname is required):
@@ -66,8 +70,9 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0):
     fullname:  the file to byte-compile
     ddir:      if given, the directory name compiled in to the
                byte-code file.
-    force:     if 1, force compilation, even if timestamps are up-to-date
-    quiet:     if 1, be quiet during compilation
+    force:     if True, force compilation, even if timestamps are up-to-date
+    quiet:     if True, be quiet during compilation
+    legacy:    if True, produce legacy pyc paths instead of PEP 3147 paths
     """
     success = 1
     name = os.path.basename(fullname)
@@ -82,11 +87,20 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0):
     if os.path.isfile(fullname):
         head, tail = name[:-3], name[-3:]
         if tail == '.py':
+            if legacy:
+                cfile = fullname + ('c' if __debug__ else 'o')
+            else:
+                cfile = imp.cache_from_source(fullname)
+                cache_dir = os.path.dirname(cfile)
+                try:
+                    os.mkdir(cache_dir)
+                except OSError, error:
+                    if error.errno != errno.EEXIST:
+                        raise
             if not force:
                 try:
                     mtime = int(os.stat(fullname).st_mtime)
                     expect = struct.pack('<4sl', imp.get_magic(), mtime)
-                    cfile = fullname + (__debug__ and 'c' or 'o')
                     with open(cfile, 'rb') as chandle:
                         actual = chandle.read(8)
                     if expect == actual:
@@ -96,7 +110,7 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0):
             if not quiet:
                 print 'Compiling', fullname, '...'
             try:
-                ok = py_compile.compile(fullname, None, dfile, True)
+                ok = py_compile.compile(fullname, cfile, dfile, True)
             except py_compile.PyCompileError,err:
                 if quiet:
                     print 'Compiling', fullname, '...'
@@ -110,15 +124,17 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0):
                     success = 0
     return success
 
-def compile_path(skip_curdir=1, maxlevels=0, force=0, quiet=0):
+def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=False,
+                 legacy=False):
     """Byte-compile all module on sys.path.
 
     Arguments (all optional):
 
     skip_curdir: if true, skip current directory (default true)
     maxlevels:   max recursion level (default 0)
-    force: as for compile_dir() (default 0)
-    quiet: as for compile_dir() (default 0)
+    force: as for compile_dir() (default False)
+    quiet: as for compile_dir() (default False)
+    legacy: as for compile_dir() (default False)
     """
     success = 1
     for dir in sys.path:
@@ -126,7 +142,8 @@ def compile_path(skip_curdir=1, maxlevels=0, force=0, quiet=0):
             print 'Skipping current directory'
         else:
             success = success and compile_dir(dir, maxlevels, None,
-                                              force, quiet=quiet)
+                                              force, quiet=quiet,
+                                              legacy=legacy)
     return success
 
 def expand_args(args, flist):
@@ -152,7 +169,7 @@ def main():
     """Script main program."""
     import getopt
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'lfqd:x:i:')
+        opts, args = getopt.getopt(sys.argv[1:], 'lfqd:x:i:b')
     except getopt.error, msg:
         print msg
         print "usage: python compileall.py [-l] [-f] [-q] [-d destdir] " \
@@ -177,23 +194,26 @@ def main():
         print "-i file: add all the files and directories listed in file to " \
               "the list considered for"
         print '         compilation; if "-", names are read from stdin'
+        print "-b: Produce legacy byte-compile file paths"
 
         sys.exit(2)
     maxlevels = 10
     ddir = None
-    force = 0
-    quiet = 0
+    force = False
+    quiet = False
     rx = None
     flist = None
+    legacy = False
     for o, a in opts:
         if o == '-l': maxlevels = 0
         if o == '-d': ddir = a
-        if o == '-f': force = 1
-        if o == '-q': quiet = 1
+        if o == '-f': force = True
+        if o == '-q': quiet = True
         if o == '-x':
             import re
             rx = re.compile(a)
         if o == '-i': flist = a
+        if o == '-b': legacy = True
     if ddir:
         if len(args) != 1 and not os.path.isdir(args[0]):
             print "-d destdir require exactly one directory argument"
@@ -210,13 +230,14 @@ def main():
                 for arg in args:
                     if os.path.isdir(arg):
                         if not compile_dir(arg, maxlevels, ddir,
-                                           force, rx, quiet):
+                                           force, rx, quiet, legacy):
                             success = 0
                     else:
-                        if not compile_file(arg, ddir, force, rx, quiet):
+                        if not compile_file(arg, ddir, force, rx,
+                                            quiet, legacy):
                             success = 0
         else:
-            success = compile_path()
+            success = compile_path(legacy=legacy)
     except KeyboardInterrupt:
         print "\n[interrupted]"
         success = 0
diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py
index 4335258..32d3ba3 100644
--- a/lib-python/2.7/inspect.py
+++ b/lib-python/2.7/inspect.py
@@ -56,6 +56,7 @@ def ismodule(object):
     """Return true if the object is a module.
 
     Module objects provide these attributes:
+        __cached__      pathname to byte compiled file
         __doc__         documentation string
         __file__        filename (missing for built-in modules)"""
     return isinstance(object, types.ModuleType)
diff --git a/lib-python/2.7/py_compile.py b/lib-python/2.7/py_compile.py
index 8334ed9..9d0ad16 100644
--- a/lib-python/2.7/py_compile.py
+++ b/lib-python/2.7/py_compile.py
@@ -4,6 +4,7 @@ This module has intimate knowledge of the format of .pyc files.
 """
 
 import __builtin__
+import errno
 import imp
 import marshal
 import os
@@ -71,20 +72,18 @@ def wr_long(f, x):
 def compile(file, cfile=None, dfile=None, doraise=False):
     """Byte-compile one Python source file to Python bytecode.
 
-    Arguments:
-
-    file:    source filename
-    cfile:   target filename; defaults to source with 'c' or 'o' appended
-             ('c' normally, 'o' in optimizing mode, giving .pyc or .pyo)
-    dfile:   purported filename; defaults to source (this is the filename
-             that will show up in error messages)
-    doraise: flag indicating whether or not an exception should be
-             raised when a compile error is found. If an exception
-             occurs and this flag is set to False, a string
-             indicating the nature of the exception will be printed,
-             and the function will return to the caller. If an
-             exception occurs and this flag is set to True, a
-             PyCompileError exception will be raised.
+    :param file: The source file name.
+    :param cfile: The target byte compiled file name.  When not given, this
+        defaults to the PEP 3147 location.
+    :param dfile: Purported file name, i.e. the file name that shows up in
+        error messages.  Defaults to the source file name.
+    :param doraise: Flag indicating whether or not an exception should be
+        raised when a compile error is found.  If an exception occurs and this
+        flag is set to False, a string indicating the nature of the exception
+        will be printed, and the function will return to the caller. If an
+        exception occurs and this flag is set to True, a PyCompileError
+        exception will be raised.
+    :return: Path to the resulting byte compiled file.
 
     Note that it isn't necessary to byte-compile Python modules for
     execution efficiency -- Python itself byte-compiles a module when
@@ -119,7 +118,12 @@ def compile(file, cfile=None, dfile=None, doraise=False):
             sys.stderr.write(py_exc.msg + '\n')
             return
     if cfile is None:
-        cfile = file + (__debug__ and 'c' or 'o')
+        cfile = imp.cache_from_source(file)
+    try:
+        os.mkdir(os.path.dirname(cfile))
+    except OSError, error:
+        if error.errno != errno.EEXIST:
+            raise
     with open(cfile, 'wb') as fc:
         fc.write('\0\0\0\0')
         wr_long(fc, timestamp)
@@ -127,6 +131,7 @@ def compile(file, cfile=None, dfile=None, doraise=False):
         fc.flush()
         fc.seek(0, 0)
         fc.write(MAGIC)
+    return cfile
 
 def main(args=None):
     """Compile several source files.
diff --git a/lib-python/2.7/pydoc.py b/lib-python/2.7/pydoc.py
index de9ce1c..14ff9bb 100755
--- a/lib-python/2.7/pydoc.py
+++ b/lib-python/2.7/pydoc.py
@@ -162,7 +162,8 @@ def visiblename(name, all=None, obj=None):
     """Decide whether to show documentation on a variable."""
     # Certain special names are redundant.
     _hidden_names = ('__builtins__', '__doc__', '__file__', '__path__',
-                     '__module__', '__name__', '__slots__', '__package__')
+                     '__module__', '__name__', '__slots__', '__package__',
+                     '__cached__')
     if name in _hidden_names: return 0
     # Private names are hidden, but special names are displayed.
     if name.startswith('__') and name.endswith('__'): return 1
diff --git a/lib-python/2.7/runpy.py b/lib-python/2.7/runpy.py
index ad4d077..9c436bb 100644
--- a/lib-python/2.7/runpy.py
+++ b/lib-python/2.7/runpy.py
@@ -67,6 +67,7 @@ def _run_code(code, run_globals, init_globals=None,
         run_globals.update(init_globals)
     run_globals.update(__name__ = mod_name,
                        __file__ = mod_fname,
+                       __cached__ = None,
                        __loader__ = mod_loader,
                        __package__ = pkg_name)
     exec code in run_globals
@@ -154,6 +155,7 @@ def _run_module_as_main(mod_name, alter_argv=True):
        At the very least, these variables in __main__ will be overwritten:
            __name__
            __file__
+           __cached__
            __loader__
            __package__
     """
diff --git a/lib-python/2.7/site.py b/lib-python/2.7/site.py
index 1a42691..37de98e 100644
--- a/lib-python/2.7/site.py
+++ b/lib-python/2.7/site.py
@@ -85,8 +85,8 @@ def makepath(*paths):
     return dir, os.path.normcase(dir)
 
 
-def abs__file__():
-    """Set all module' __file__ attribute to an absolute path"""
+def abs_paths():
+    """Set all module __file__ and __cached__ attributes to an absolute path"""
     for m in sys.modules.values():
         if hasattr(m, '__loader__'):
             continue   # don't mess with a PEP 302-supplied __file__
@@ -97,6 +97,13 @@ def abs__file__():
                 m.__file__ = new
         except (AttributeError, OSError):
             pass
+        try:
+            prev = m.__cached__
+            new = os.path.abspath(m.__cached__)
+            if prev != new:
+                m.__cached__ = new
+        except (AttributeError, OSError):
+            pass
 
 
 def removeduppaths():
@@ -543,7 +550,7 @@ def main():
     global ENABLE_USER_SITE
 
     import_builtin_stuff()
-    abs__file__()
+    abs_paths()
     known_paths = removeduppaths()
     if ENABLE_USER_SITE is None:
         ENABLE_USER_SITE = check_enableusersite()
diff --git a/lib-python/2.7/test/script_helper.py b/lib-python/2.7/test/script_helper.py
index 6be47bd..28a73dd 100644
--- a/lib-python/2.7/test/script_helper.py
+++ b/lib-python/2.7/test/script_helper.py
@@ -20,6 +20,9 @@ except ImportError:
 
 from test.test_support import strip_python_stderr
 
+from imp import source_from_cache
+from test.test_support import make_legacy_pyc
+
 # Executing the interpreter in a subprocess
 def _assert_python(expected_success, *args, **env_vars):
     cmd_line = [sys.executable]
@@ -111,20 +114,18 @@ def make_script(script_dir, script_basename, source):
     script_file.close()
     return script_name
 
-def compile_script(script_name):
-    py_compile.compile(script_name, doraise=True)
-    if __debug__:
-        compiled_name = script_name + 'c'
-    else:
-        compiled_name = script_name + 'o'
-    return compiled_name
-
 def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None):
     zip_filename = zip_basename+os.extsep+'zip'
     zip_name = os.path.join(zip_dir, zip_filename)
     zip_file = zipfile.ZipFile(zip_name, 'w')
     if name_in_zip is None:
-        name_in_zip = os.path.basename(script_name)
+        parts = script_name.split(os.sep)
+        if len(parts) >= 2 and parts[-2] == '__pycache__':
+            legacy_pyc = make_legacy_pyc(source_from_cache(script_name))
+            name_in_zip = os.path.basename(legacy_pyc)
+            script_name = legacy_pyc
+        else:
+            name_in_zip = os.path.basename(script_name)
     zip_file.write(script_name, name_in_zip)
     zip_file.close()
     #if test.test_support.verbose:
@@ -147,8 +148,8 @@ def make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename,
     script_name = make_script(zip_dir, script_basename, source)
     unlink.append(script_name)
     if compiled:
-        init_name = compile_script(init_name)
-        script_name = compile_script(script_name)
+        init_name = py_compile(init_name, doraise=True)
+        script_name = py_compile(script_name, doraise=True)
         unlink.extend((init_name, script_name))
     pkg_names = [os.sep.join([pkg_name]*i) for i in range(1, depth+1)]
     script_name_in_zip = os.path.join(pkg_names[-1], os.path.basename(script_name))
diff --git a/lib-python/2.7/test/test_cmd_line_script.py b/lib-python/2.7/test/test_cmd_line_script.py
index 2e12ff2..5895285 100644
--- a/lib-python/2.7/test/test_cmd_line_script.py
+++ b/lib-python/2.7/test/test_cmd_line_script.py
@@ -4,9 +4,10 @@ import contextlib
 import unittest
 import os
 import os.path
+import py_compile
 import test.test_support
 from test.script_helper import (run_python,
-                                temp_dir, make_script, compile_script,
+                                temp_dir, make_script,
                                 assert_python_failure, make_pkg,
                                 make_zip_script, make_zip_pkg)
 
@@ -33,6 +34,7 @@ assertEqual(result, ['Top level assignment', 'Lower level reference'])
 # Check population of magic variables
 assertEqual(__name__, '__main__')
 print '__file__==%r' % __file__
+assertEqual(__cached__, None)
 print '__package__==%r' % __package__
 # Check the sys module
 import sys
@@ -106,7 +108,7 @@ class CmdLineTest(unittest.TestCase):
     def test_script_compiled(self):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, 'script')
-            compiled_name = compile_script(script_name)
+            compiled_name = py_compile.compile(script_name, doraise=True)
             os.remove(script_name)
             self._check_script(compiled_name, compiled_name, compiled_name, None)
 
@@ -120,9 +122,10 @@ class CmdLineTest(unittest.TestCase):
             raise unittest.SkipTest("pypy won't load lone .pyc files")
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
-            compiled_name = compile_script(script_name)
+            compiled_name = py_compile.compile(script_name, doraise=True)
             os.remove(script_name)
-            self._check_script(script_dir, compiled_name, script_dir, '')
+            pyc_file = test.test_support.make_legacy_pyc(script_name)
+            self._check_script(script_dir, pyc_file, script_dir, '')
 
     def test_directory_error(self):
         with temp_dir() as script_dir:
@@ -138,7 +141,7 @@ class CmdLineTest(unittest.TestCase):
     def test_zipfile_compiled(self):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
-            compiled_name = compile_script(script_name)
+            compiled_name = py_compile.compile(script_name, doraise=True)
             zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name)
             self._check_script(zip_name, run_name, zip_name, '')
 
@@ -185,11 +188,12 @@ class CmdLineTest(unittest.TestCase):
             pkg_dir = os.path.join(script_dir, 'test_pkg')
             make_pkg(pkg_dir)
             script_name = _make_test_script(pkg_dir, '__main__')
-            compiled_name = compile_script(script_name)
+            compiled_name = py_compile.compile(script_name, doraise=True)
             os.remove(script_name)
+            pyc_file = test.test_support.make_legacy_pyc(script_name)
             launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg')
-            self._check_script(launch_name, compiled_name,
-                               compiled_name, 'test_pkg')
+            self._check_script(launch_name, pyc_file,
+                               pyc_file, 'test_pkg')
 
     def test_package_error(self):
         with temp_dir() as script_dir:
diff --git a/lib-python/2.7/test/test_compileall.py b/lib-python/2.7/test/test_compileall.py
index d3a26db..7fd44c5 100644
--- a/lib-python/2.7/test/test_compileall.py
+++ b/lib-python/2.7/test/test_compileall.py
@@ -4,6 +4,8 @@ import os
 import py_compile
 import shutil
 import struct
+import sys
+import subprocess
 import tempfile
 from test import test_support
 import unittest
@@ -14,11 +16,11 @@ class CompileallTests(unittest.TestCase):
     def setUp(self):
         self.directory = tempfile.mkdtemp()
         self.source_path = os.path.join(self.directory, '_test.py')
-        self.bc_path = self.source_path + ('c' if __debug__ else 'o')
+        self.bc_path = imp.cache_from_source(self.source_path)
         with open(self.source_path, 'w') as file:
             file.write('x = 123\n')
         self.source_path2 = os.path.join(self.directory, '_test2.py')
-        self.bc_path2 = self.source_path2 + ('c' if __debug__ else 'o')
+        self.bc_path2 = imp.cache_from_source(self.source_path2)
         shutil.copyfile(self.source_path, self.source_path2)
 
     def tearDown(self):
@@ -71,8 +73,61 @@ class CompileallTests(unittest.TestCase):
         os.unlink(self.bc_path)
         os.unlink(self.bc_path2)
 
+class CommandLineTests(unittest.TestCase):
+    """Test some aspects of compileall's CLI."""
+
+    def setUp(self):
+        self.addCleanup(self._cleanup)
+        self.directory = tempfile.mkdtemp()
+        self.pkgdir = os.path.join(self.directory, 'foo')
+        os.mkdir(self.pkgdir)
+        # Touch the __init__.py and a package module.
+        with open(os.path.join(self.pkgdir, '__init__.py'), 'w'):
+            pass
+        with open(os.path.join(self.pkgdir, 'bar.py'), 'w'):
+            pass
+        sys.path.insert(0, self.directory)
+
+    def _cleanup(self):
+        test_support.rmtree(self.directory)
+        assert sys.path[0] == self.directory, 'Missing path'
+        del sys.path[0]
+
+    def test_pep3147_paths(self):
+        # Ensure that the default behavior of compileall's CLI is to create
+        # PEP 3147 pyc/pyo files.
+        retcode = subprocess.call(
+            (sys.executable, '-m', 'compileall', '-q', self.pkgdir))
+        self.assertEqual(retcode, 0)
+        # Verify the __pycache__ directory contents.
+        cachedir = os.path.join(self.pkgdir, '__pycache__')
+        self.assertTrue(os.path.exists(cachedir))
+        ext = ('pyc' if __debug__ else 'pyo')
+        expected = sorted(base.format(imp.get_tag(), ext) for base in
+                          ('__init__.{}.{}', 'bar.{}.{}'))
+        self.assertEqual(sorted(os.listdir(cachedir)), expected)
+        # Make sure there are no .pyc files in the source directory.
+        self.assertFalse([pyc_file for pyc_file in os.listdir(self.pkgdir)
+                          if pyc_file.endswith(ext)])
+
+    def test_legacy_paths(self):
+        # Ensure that with the proper switch, compileall leaves legacy
+        # pyc/pyo files, and no __pycache__ directory.
+        retcode = subprocess.call(
+            (sys.executable, '-m', 'compileall', '-b', '-q', self.pkgdir))
+        self.assertEqual(retcode, 0)
+        # Verify the __pycache__ directory contents.
+        cachedir = os.path.join(self.pkgdir, '__pycache__')
+        self.assertFalse(os.path.exists(cachedir))
+        ext = ('pyc' if __debug__ else 'pyo')
+        expected = [base.format(ext) for base in ('__init__.{}', 'bar.{}')]
+        expected.extend(['__init__.py', 'bar.py'])
+        expected.sort()
+        self.assertEqual(sorted(os.listdir(self.pkgdir)), expected)
+
+
 def test_main():
-    test_support.run_unittest(CompileallTests)
+    test_support.run_unittest(CommandLineTests, CompileallTests)
 
 
 if __name__ == "__main__":
diff --git a/lib-python/2.7/test/test_imp.py b/lib-python/2.7/test/test_imp.py
index 1bdc47a..341d36b 100644
--- a/lib-python/2.7/test/test_imp.py
+++ b/lib-python/2.7/test/test_imp.py
@@ -1,4 +1,7 @@
 import imp
+import os
+import shutil
+import sys
 import unittest
 from test import test_support
 
@@ -70,8 +73,147 @@ class ReloadTests(unittest.TestCase):
             imp.reload(marshal)
 
 
+class PEP3147Tests(unittest.TestCase):
+    """Tests of PEP 3147."""
+
+    tag = imp.get_tag()
+
+    def test_cache_from_source(self):
+        # Given the path to a .py file, return the path to its PEP 3147
+        # defined .pyc file (i.e. under __pycache__).
+        self.assertEqual(
+            imp.cache_from_source('/foo/bar/baz/qux.py', True),
+            '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag))
+
+    def test_cache_from_source_optimized(self):
+        # Given the path to a .py file, return the path to its PEP 3147
+        # defined .pyo file (i.e. under __pycache__).
+        if test_support.check_impl_detail(pypy=True):
+            # PyPy doesn't support .pyo, so we expect .pyc
+            extension = 'pyc'
+        else:
+            extension = 'pyo'
+        self.assertEqual(
+            imp.cache_from_source('/foo/bar/baz/qux.py', False),
+            '/foo/bar/baz/__pycache__/qux.{}.{}'.format(self.tag, extension))
+
+    def test_cache_from_source_cwd(self):
+        self.assertEqual(imp.cache_from_source('foo.py', True),
+                         os.sep.join(('__pycache__',
+                                      'foo.{}.pyc'.format(self.tag))))
+
+    def test_cache_from_source_override(self):
+        # When debug_override is not None, it can be any true-ish or false-ish
+        # value.
+        if test_support.check_impl_detail(pypy=True):
+            # PyPy doesn't support .pyo, so we expect .pyc
+            extension = 'pyc'
+        else:
+            extension = 'pyo'
+        self.assertEqual(
+            imp.cache_from_source('/foo/bar/baz.py', []),
+            '/foo/bar/__pycache__/baz.{}.pyc'.format(self.tag))
+        self.assertEqual(
+            imp.cache_from_source('/foo/bar/baz.py', [17]),
+            '/foo/bar/__pycache__/baz.{}.{}'.format(self.tag, extension))
+        # However if the bool-ishness can't be determined, the exception
+        # propagates.
+        class Bearish:
+            def __bool__(self): raise RuntimeError
+
+        if test_support.check_impl_detail(pypy=True):
+            # Pypy doesn't even determine bool-ishness
+            try:
+                imp.cache_from_source('/foo/bar/baz.py', Bearish())
+            except RuntimeError:
+                pass
+        else:
+            self.assertRaises(
+                RuntimeError,
+                imp.cache_from_source, '/foo/bar/baz.py', Bearish())
+
+    @unittest.skipIf(os.altsep is None,
+                     'test meaningful only where os.altsep is defined')
+    def test_altsep_cache_from_source(self):
+        # Windows path and PEP 3147.
+        self.assertEqual(
+            imp.cache_from_source('\\foo\\bar\\baz\\qux.py', True),
+            '\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
+
+    @unittest.skipIf(os.altsep is None,
+                     'test meaningful only where os.altsep is defined')
+    def test_altsep_and_sep_cache_from_source(self):
+        # Windows path and PEP 3147 where altsep is right of sep.
+        self.assertEqual(
+            imp.cache_from_source('\\foo\\bar/baz\\qux.py', True),
+            '\\foo\\bar/baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
+
+    @unittest.skipIf(os.altsep is None,
+                     'test meaningful only where os.altsep is defined')
+    def test_sep_altsep_and_sep_cache_from_source(self):
+        # Windows path and PEP 3147 where sep is right of altsep.
+        self.assertEqual(
+            imp.cache_from_source('\\foo\\bar\\baz/qux.py', True),
+            '\\foo\\bar\\baz/__pycache__/qux.{}.pyc'.format(self.tag))
+
+    def test_source_from_cache(self):
+        # Given the path to a PEP 3147 defined .pyc file, return the path to
+        # its source.  This tests the good path.
+        self.assertEqual(imp.source_from_cache(
+            '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag)),
+            '/foo/bar/baz/qux.py')
+
+    def test_source_from_cache_bad_path(self):
+        # When the path to a pyc file is not in PEP 3147 format, a ValueError
+        # is raised.
+        self.assertRaises(
+            ValueError, imp.source_from_cache, '/foo/bar/bazqux.pyc')
+
+    def test_source_from_cache_no_slash(self):
+        # No slashes at all in path -> ValueError
+        self.assertRaises(
+            ValueError, imp.source_from_cache, 'foo.cpython-32.pyc')
+
+    def test_source_from_cache_too_few_dots(self):
+        # Too few dots in final path component -> ValueError
+        self.assertRaises(
+            ValueError, imp.source_from_cache, '__pycache__/foo.pyc')
+
+    def test_source_from_cache_too_many_dots(self):
+        # Too many dots in final path component -> ValueError
+        self.assertRaises(
+            ValueError, imp.source_from_cache,
+            '__pycache__/foo.cpython-32.foo.pyc')
+
+    def test_source_from_cache_no__pycache__(self):
+        # Another problem with the path -> ValueError
+        self.assertRaises(
+            ValueError, imp.source_from_cache,
+            '/foo/bar/foo.cpython-32.foo.pyc')
+
+    def test_package___file__(self):
+        # Test that a package's __file__ points to the right source directory.
+        os.mkdir('pep3147')
+        sys.path.insert(0, os.curdir)
+        def cleanup():
+            if sys.path[0] == os.curdir:
+                del sys.path[0]
+            shutil.rmtree('pep3147')
+        self.addCleanup(cleanup)
+        # Touch the __init__.py file.
+        with open('pep3147/__init__.py', 'w'):
+            pass
+        m = __import__('pep3147')
+        # Ensure we load the pyc file.
+        test_support.forget('pep3147')
+        m = __import__('pep3147')
+        self.assertEqual(m.__file__,
+                         os.sep.join(('.', 'pep3147', '__init__.py')))
+
+
 def test_main():
     tests = [
+        PEP3147Tests,
         ReloadTests,
         LockTests,
     ]
diff --git a/lib-python/2.7/test/test_import.py b/lib-python/2.7/test/test_import.py
index 716f8e0..0618068 100644
--- a/lib-python/2.7/test/test_import.py
+++ b/lib-python/2.7/test/test_import.py
@@ -13,7 +13,8 @@ import shutil
 
 from test.test_support import (unlink, TESTFN, unload, run_unittest, rmtree,
                                is_jython, check_warnings, EnvironmentVarGuard,
-                               impl_detail, check_impl_detail)
+                               impl_detail, check_impl_detail, forget,
+                               make_legacy_pyc, temp_umask)
 from test import symlink_support
 from test import script_helper
 
@@ -35,6 +36,11 @@ def chmod_files(name):
 def remove_files(name):
     for f in _files(name):
         unlink(f)
+    try:
+        shutil.rmtree('__pycache__')
+    except OSError, error:
+        if error.errno != errno.ENOENT:
+            raise
 
 
 class ImportTests(unittest.TestCase):
@@ -95,9 +101,9 @@ class ImportTests(unittest.TestCase):
             except ImportError, err:
                 self.fail("import from .pyc/.pyo failed: %s" % err)
             finally:
+                forget(TESTFN)
                 unlink(pyc)
                 unlink(pyo)
-                unload(TESTFN)
 
         sys.path.insert(0, os.curdir)
         try:
@@ -115,28 +121,26 @@ class ImportTests(unittest.TestCase):
     def test_execute_bit_not_copied(self):
         # Issue 6070: under posix .pyc files got their execute bit set if
         # the .py file had the execute bit set, but they aren't executable.
-        oldmask = os.umask(022)
-        sys.path.insert(0, os.curdir)
-        try:
-            fname = TESTFN + os.extsep + "py"
-            f = open(fname, 'w').close()
-            os.chmod(fname, (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
-                             stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
-            __import__(TESTFN)
-            fn = fname + 'c'
-            if not os.path.exists(fn):
-                fn = fname + 'o'
+        with temp_umask(022):
+            sys.path.insert(0, os.curdir)
+            try:
+                fname = TESTFN + os.extsep + "py"
+                f = open(fname, 'w').close()
+                os.chmod(fname, (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
+                                 stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
+                __import__(TESTFN)
+                fn = imp.cache_from_source(fname)
                 if not os.path.exists(fn):
                     self.fail("__import__ did not result in creation of "
                               "either a .pyc or .pyo file")
-            s = os.stat(fn)
-            self.assertEqual(stat.S_IMODE(s.st_mode),
-                             stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
-        finally:
-            os.umask(oldmask)
-            remove_files(TESTFN)
-            unload(TESTFN)
-            del sys.path[0]
+                    s = os.stat(fn)
+                    self.assertEqual(
+                        stat.S_IMODE(s.st_mode),
+                        stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
+            finally:
+                del sys.path[0]
+                remove_files(TESTFN)
+                unload(TESTFN)
 
     @unittest.skipIf(sys.dont_write_bytecode,
         "test meaningful only when writing bytecode")
@@ -205,12 +209,14 @@ class ImportTests(unittest.TestCase):
                 f.write('"",\n')
             f.write(']')
 
-        # Compile & remove .py file, we only need .pyc (or .pyo).
+        # Compile & remove .py file, we only need .pyc (or .pyo), but that
+        # must be relocated to the PEP 3147 bytecode-only location.
         with open(filename, 'r') as f:
             py_compile.compile(filename)
         if check_impl_detail(pypy=False):
             # pypy refuses to import a .pyc if the .py does not exist
             unlink(filename)
+        make_legacy_pyc(filename)
 
         # Need to be able to load from current dir.
         sys.path.append('')
@@ -345,7 +351,7 @@ class ImportTests(unittest.TestCase):
         sys.path.insert(0, os.curdir)
         try:
             source = TESTFN + ".py"
-            compiled = source + ('c' if __debug__ else 'o')
+            compiled = imp.cache_from_source(source)
             with open(source, 'w') as f:
                 pass
             try:
@@ -440,6 +446,7 @@ class PycRewritingTests(unittest.TestCase):
 import sys
 code_filename = sys._getframe().f_code.co_filename
 module_filename = __file__
+module_pyc_filename = __cached__
 constant = 1
 def func():
     pass
@@ -447,7 +454,7 @@ func_filename = func.func_code.co_filename
 """
     dir_name = os.path.abspath(TESTFN)
     file_name = os.path.join(dir_name, module_name) + os.extsep + "py"
-    compiled_name = file_name + ("c" if __debug__ else "o")
+    compiled_name = imp.cache_from_source(file_name)
 
     def setUp(self):
         self.sys_path = sys.path[:]
@@ -475,19 +482,22 @@ func_filename = func.func_code.co_filename
     def test_basics(self):
         mod = self.import_module()
         self.assertEqual(mod.module_filename, self.file_name)
+        self.assertEqual(mod.module_pyc_filename, self.compiled_name)
         self.assertEqual(mod.code_filename, self.file_name)
         self.assertEqual(mod.func_filename, self.file_name)
         del sys.modules[self.module_name]
         mod = self.import_module()
+        self.assertEqual(mod.module_filename, self.file_name)
         if not sys.dont_write_bytecode:
-            self.assertEqual(mod.module_filename, self.compiled_name)
+            self.assertEqual(mod.module_pyc_filename, self.compiled_name)
         self.assertEqual(mod.code_filename, self.file_name)
         self.assertEqual(mod.func_filename, self.file_name)
 
     def test_incorrect_code_name(self):
         py_compile.compile(self.file_name, dfile="another_module.py")
         mod = self.import_module()
-        self.assertEqual(mod.module_filename, self.compiled_name)
+        self.assertEqual(mod.module_filename, self.file_name)
+        self.assertEqual(mod.module_pyc_filename, self.compiled_name)
         self.assertEqual(mod.code_filename, self.file_name)
         self.assertEqual(mod.func_filename, self.file_name)
 
@@ -496,8 +506,9 @@ func_filename = func.func_code.co_filename
         target = "another_module.py"
         py_compile.compile(self.file_name, dfile=target)
         os.remove(self.file_name)
+        pyc_file = make_legacy_pyc(self.file_name)
         mod = self.import_module()
-        self.assertEqual(mod.module_filename, self.compiled_name)
+        self.assertEqual(mod.module_filename, pyc_file)
         self.assertEqual(mod.code_filename, target)
         self.assertEqual(mod.func_filename, target)
 
diff --git a/lib-python/2.7/test/test_pkg.py b/lib-python/2.7/test/test_pkg.py
index 5f1659b..853069a 100644
--- a/lib-python/2.7/test/test_pkg.py
+++ b/lib-python/2.7/test/test_pkg.py
@@ -194,14 +194,14 @@ class Test(unittest.TestCase):
 
         import t5
         self.assertEqual(fixdir(dir(t5)),
-                         ['__doc__', '__file__', '__name__',
+                         ['__cached__', '__doc__', '__file__', '__name__',
                           '__package__', '__path__', 'foo', 'string', 't5'])
         self.assertEqual(fixdir(dir(t5.foo)),
-                         ['__doc__', '__file__', '__name__', '__package__',
-                          'string'])
+                         ['__cached__', '__doc__', '__file__', '__name__',
+                          '__package__', 'string'])
         self.assertEqual(fixdir(dir(t5.string)),
-                         ['__doc__', '__file__', '__name__','__package__',
-                          'spam'])
+                         ['__cached__', '__doc__', '__file__', '__name__',
+                          '__package__', 'spam'])
 
     def test_6(self):
         hier = [
@@ -216,13 +216,13 @@ class Test(unittest.TestCase):
 
         import t6
         self.assertEqual(fixdir(dir(t6)),
-                         ['__all__', '__doc__', '__file__',
+                         ['__all__', '__cached__', '__doc__', '__file__',
                           '__name__', '__package__', '__path__'])
         s = """
             import t6
             from t6 import *
             self.assertEqual(fixdir(dir(t6)),
-                             ['__all__', '__doc__', '__file__',
+                             ['__all__', '__cached__', '__doc__', '__file__',
                               '__name__', '__package__', '__path__',
                               'eggs', 'ham', 'spam'])
             self.assertEqual(dir(), ['eggs', 'ham', 'self', 'spam', 't6'])
@@ -250,18 +250,18 @@ class Test(unittest.TestCase):
         t7, sub, subsub = None, None, None
         import t7 as tas
         self.assertEqual(fixdir(dir(tas)),
-                         ['__doc__', '__file__', '__name__',
+                         ['__cached__', '__doc__', '__file__', '__name__',
                           '__package__', '__path__'])
         self.assertFalse(t7)
         from t7 import sub as subpar
         self.assertEqual(fixdir(dir(subpar)),
-                         ['__doc__', '__file__', '__name__',
+                         ['__cached__', '__doc__', '__file__', '__name__',
                           '__package__', '__path__'])
         self.assertFalse(t7)
         self.assertFalse(sub)
         from t7.sub import subsub as subsubsub
         self.assertEqual(fixdir(dir(subsubsub)),
-                         ['__doc__', '__file__', '__name__',
+                         ['__cached__', '__doc__', '__file__', '__name__',
                          '__package__', '__path__', 'spam'])
         self.assertFalse(t7)
         self.assertFalse(sub)
diff --git a/lib-python/2.7/test/test_pkgimport.py b/lib-python/2.7/test/test_pkgimport.py
index 30d48cd..490060f 100644
--- a/lib-python/2.7/test/test_pkgimport.py
+++ b/lib-python/2.7/test/test_pkgimport.py
@@ -1,5 +1,6 @@
-import os, sys, string, random, tempfile, unittest
+import os, sys, shutil, string, random, tempfile, unittest
 
+from imp import cache_from_source
 from test.test_support import run_unittest
 
 class TestImport(unittest.TestCase):
@@ -27,22 +28,17 @@ class TestImport(unittest.TestCase):
         self.module_path = os.path.join(self.package_dir, 'foo'+os.extsep+'py')
 
     def tearDown(self):
-        for file in os.listdir(self.package_dir):
-            os.remove(os.path.join(self.package_dir, file))
-        os.rmdir(self.package_dir)
-        os.rmdir(self.test_dir)
+        shutil.rmtree(self.test_dir)
         self.assertNotEqual(sys.path.count(self.test_dir), 0)
         sys.path.remove(self.test_dir)
         self.remove_modules()
 
     def rewrite_file(self, contents):
-        for extension in "co":
-            compiled_path = self.module_path + extension
-            if os.path.exists(compiled_path):
-                os.remove(compiled_path)
-        f = open(self.module_path, 'w')
-        f.write(contents)
-        f.close()
+        compiled_path = cache_from_source(self.module_path)
+        if os.path.exists(compiled_path):
+            os.remove(compiled_path)
+        with open(self.module_path, 'w') as f:
+            f.write(contents)
 
     def test_package_import__semantics(self):
 
diff --git a/lib-python/2.7/test/test_py_compile.py b/lib-python/2.7/test/test_py_compile.py
index 5ec523a..eff87df 100644
--- a/lib-python/2.7/test/test_py_compile.py
+++ b/lib-python/2.7/test/test_py_compile.py
@@ -12,7 +12,7 @@ class PyCompileTests(unittest.TestCase):
     def setUp(self):
         self.directory = tempfile.mkdtemp()
         self.source_path = os.path.join(self.directory, '_test.py')
-        self.pyc_path = self.source_path + 'c'
+        self.pyc_path = imp.cache_from_source(self.source_path)
         self.cwd_drive = os.path.splitdrive(os.getcwd())[0]
         # In these tests we compute relative paths.  When using Windows, the
         # current working directory path and the 'self.source_path' might be
@@ -35,9 +35,10 @@ class PyCompileTests(unittest.TestCase):
         self.assertTrue(os.path.exists(self.pyc_path))
 
     def test_cwd(self):
+        pyc_file = imp.cache_from_source(os.path.basename(self.source_path))
         with support.change_cwd(self.directory):
             py_compile.compile(os.path.basename(self.source_path),
-                               os.path.basename(self.pyc_path))
+                               pyc_file)
         self.assertTrue(os.path.exists(self.pyc_path))
 
     def test_relative_path(self):
diff --git a/lib-python/2.7/test/test_pydoc.py b/lib-python/2.7/test/test_pydoc.py
index 0e9f5f3..58c5781 100644
--- a/lib-python/2.7/test/test_pydoc.py
+++ b/lib-python/2.7/test/test_pydoc.py
@@ -6,6 +6,7 @@ import re
 import py_compile
 import pydoc
 import contextlib
+import imp
 import inspect
 import keyword
 import pkgutil
@@ -418,7 +419,7 @@ foo = 1
     def test_synopsis_sourceless_empty_doc(self):
         with test.test_support.temp_cwd() as test_dir:
             init_path = os.path.join(test_dir, 'foomod42.py')
-            cached_path = os.path.join(test_dir, 'foomod42.pyc')
+            cached_path = imp.cache_from_source(init_path)
             with open(init_path, 'w') as fobj:
                 fobj.write("foo = 1")
             py_compile.compile(init_path)
diff --git a/lib-python/2.7/test/test_runpy.py b/lib-python/2.7/test/test_runpy.py
index b635c1d..a45f9be 100644
--- a/lib-python/2.7/test/test_runpy.py
+++ b/lib-python/2.7/test/test_runpy.py
@@ -5,8 +5,9 @@ import os.path
 import sys
 import re
 import tempfile
-from test.test_support import verbose, run_unittest, forget, check_impl_detail
-from test.script_helper import (temp_dir, make_script, compile_script,
+import py_compile
+from test.test_support import verbose, run_unittest, forget, check_impl_detail, make_legacy_pyc
+from test.script_helper import (temp_dir, make_script,
                                 make_pkg, make_zip_script, make_zip_pkg)
 
 if check_impl_detail(pypy=True):
@@ -50,6 +51,7 @@ class RunModuleCodeTest(unittest.TestCase):
         self.assertEqual(d["result"], self.expected_result)
         self.assertIs(d["__name__"], None)
         self.assertIs(d["__file__"], None)
+        self.assertIs(d["__cached__"], None)
         self.assertIs(d["__loader__"], None)
         self.assertIs(d["__package__"], None)
         self.assertIs(d["run_argv0"], saved_argv0)
@@ -78,6 +80,7 @@ class RunModuleCodeTest(unittest.TestCase):
         self.assertTrue(d2["run_name_in_sys_modules"])
         self.assertTrue(d2["module_in_sys_modules"])
         self.assertIs(d2["__file__"], file)
+        self.assertIs(d2["__cached__"], None)
         self.assertIs(d2["run_argv0"], file)
         self.assertIs(d2["__loader__"], loader)
         self.assertIs(d2["__package__"], package)
@@ -177,6 +180,7 @@ class RunModuleTest(unittest.TestCase):
                 __import__(mod_name)
                 os.remove(mod_fname)
                 if not sys.dont_write_bytecode:
+                    make_legacy_pyc(mod_fname)
                     if verbose: print "Running from compiled:", mod_name
                     d2 = run_module(mod_name) # Read from bytecode
                     self.assertIn("x", d2)
@@ -201,6 +205,7 @@ class RunModuleTest(unittest.TestCase):
                 __import__(mod_name)
                 os.remove(mod_fname)
                 if not sys.dont_write_bytecode:
+                    make_legacy_pyc(mod_fname)
                     if verbose: print "Running from compiled:", pkg_name
                     d2 = run_module(pkg_name) # Read from bytecode
                     self.assertIn("x", d2)
@@ -257,6 +262,7 @@ from ..uncle.cousin import nephew
                 __import__(mod_name)
                 os.remove(mod_fname)
                 if not sys.dont_write_bytecode:
+                    make_legacy_pyc(mod_fname)
                     if verbose: print "Running from compiled:", mod_name
                     d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
                     self.assertIn("__package__", d2)
@@ -348,6 +354,7 @@ argv0 = sys.argv[0]
         result = run_path(script_name)
         self.assertEqual(result["__name__"], expected_name)
         self.assertEqual(result["__file__"], expected_file)
+        self.assertEqual(result["__cached__"], None)
         self.assertIn("argv0", result)
         self.assertEqual(result["argv0"], expected_argv0)
         self.assertEqual(result["__package__"], expected_package)
@@ -367,7 +374,7 @@ argv0 = sys.argv[0]
         with temp_dir() as script_dir:
             mod_name = 'script'
             script_name = self._make_test_script(script_dir, mod_name)
-            compiled_name = compile_script(script_name)
+            compiled_name = py_compile.compile(script_name, doraise=True)
             os.remove(script_name)
             self._check_script(compiled_name, "<run_path>", compiled_name,
                                compiled_name, None)
@@ -385,9 +392,10 @@ argv0 = sys.argv[0]
         with temp_dir() as script_dir:
             mod_name = '__main__'
             script_name = self._make_test_script(script_dir, mod_name)
-            compiled_name = compile_script(script_name)
+            compiled_name = py_compile.compile(script_name, doraise=True)
             os.remove(script_name)
-            self._check_script(script_dir, "<run_path>", compiled_name,
+            legacy_pyc = make_legacy_pyc(script_name)
+            self._check_script(script_dir, "<run_path>", legacy_pyc,
                                script_dir, '')
 
     def test_directory_error(self):
@@ -408,8 +416,9 @@ argv0 = sys.argv[0]
         with temp_dir() as script_dir:
             mod_name = '__main__'
             script_name = self._make_test_script(script_dir, mod_name)
-            compiled_name = compile_script(script_name)
-            zip_name, fname = make_zip_script(script_dir, 'test_zip', compiled_name)
+            compiled_name = py_compile.compile(script_name, doraise=True)
+            zip_name, fname = make_zip_script(script_dir, 'test_zip',
+                                              compiled_name)
             self._check_script(zip_name, "<run_path>", fname, zip_name, '')
 
     def test_zipfile_error(self):
diff --git a/lib-python/2.7/test/test_site.py b/lib-python/2.7/test/test_site.py
index de3f28b..02331be 100644
--- a/lib-python/2.7/test/test_site.py
+++ b/lib-python/2.7/test/test_site.py
@@ -318,19 +318,40 @@ class ImportSideEffectTests(unittest.TestCase):
         """Restore sys.path"""
         sys.path[:] = self.sys_path
 
-    def test_abs__file__(self):
-        # Make sure all imported modules have their __file__ attribute
-        # as an absolute path.
-        # Handled by abs__file__()
-        site.abs__file__()
-        for module in (sys, os, __builtin__):
-            try:
-                self.assertTrue(os.path.isabs(module.__file__), repr(module))
-            except AttributeError:
-                continue
-        # We could try everything in sys.modules; however, when regrtest.py
-        # runs something like test_frozen before test_site, then we will
-        # be testing things loaded *after* test_site did path normalization
+    def test_abs_paths(self):
+        # Make sure all imported modules have their __file__ and __cached__
+        # attributes as absolute paths.  Arranging to put the Lib directory on
+        # PYTHONPATH would cause the os module to have a relative path for
+        # __file__ if abs_paths() does not get run.  sys and builtins (the
+        # only other modules imported before site.py runs) do not have
+        # __file__ or __cached__ because they are built-in.
+        parent = os.path.relpath(os.path.dirname(os.__file__))
+        env = os.environ.copy()
+        env['PYTHONPATH'] = parent
+        # We use uuid rather than os, as os isn't modified in pypy
+        # and so not in the same path as test_site
+        command = 'import uuid; print uuid.__file__, uuid.__cached__'
+        # First, prove that with -S (no 'import site'), the paths are
+        # relative.
+        proc = subprocess.Popen([sys.executable, '-S', '-c', command],
+                                env=env,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
+        stdout, stderr = proc.communicate()
+        self.assertEqual(proc.returncode, 0)
+        os__file__, os__cached__ = stdout.split()
+        self.assertFalse(os.path.isabs(os__file__))
+        self.assertFalse(os.path.isabs(os__cached__))
+        # Now, with 'import site', it works.
+        proc = subprocess.Popen([sys.executable, '-c', command],
+                                env=env,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
+        stdout, stderr = proc.communicate()
+        self.assertEqual(proc.returncode, 0)
+        os__file__, os__cached__ = stdout.split()
+        self.assertTrue(os.path.isabs(os__file__))
+        self.assertTrue(os.path.isabs(os__cached__))
 
     def test_no_duplicate_paths(self):
         # No duplicate paths should exist in sys.path
diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py
index 258b339..0e34980 100644
--- a/lib-python/2.7/test/test_support.py
+++ b/lib-python/2.7/test/test_support.py
@@ -18,6 +18,7 @@ import unittest
 import importlib
 import UserDict
 import re
+import imp
 import time
 import struct
 import sysconfig
@@ -36,7 +37,7 @@ __all__ = ["Error", "TestFailed", "ResourceDenied", "import_module",
            "open_urlresource", "check_warnings", "check_py3k_warnings",
            "CleanImport", "EnvironmentVarGuard", "captured_output",
            "captured_stdout", "TransientResource", "transient_internet",
-           "run_with_locale", "set_memlimit", "bigmemtest", "bigaddrspacetest",
+           "run_with_locale", "set_memlimit", "temp_umask", "bigmemtest", "bigaddrspacetest",
            "BasicTestRunner", "run_unittest", "run_doctest", "threading_setup",
            "threading_cleanup", "reap_threads", "start_threads", "cpython_only",
            "check_impl_detail", "get_attribute", "py3k_bytes",
@@ -292,16 +293,37 @@ def rmtree(path):
         if e.errno not in (errno.ENOENT, errno.ESRCH):
             raise
 
+def make_legacy_pyc(source):
+    """Move a PEP 3147 pyc/pyo file to its legacy pyc/pyo location.
+
+    The choice of .pyc or .pyo extension is done based on the __debug__ flag
+    value.
+
+    :param source: The file system path to the source file.  The source file
+        does not need to exist, however the PEP 3147 pyc file must exist.
+    :return: The file system path to the legacy pyc file.
+    """
+    pyc_file = imp.cache_from_source(source)
+    up_one = os.path.dirname(os.path.abspath(source))
+    legacy_pyc = os.path.join(up_one, source + ('c' if __debug__ else 'o'))
+    os.rename(pyc_file, legacy_pyc)
+    return legacy_pyc
+
 def forget(modname):
-    '''"Forget" a module was ever imported by removing it from sys.modules and
-    deleting any .pyc and .pyo files.'''
+    """'Forget' a module was ever imported.
+
+    This removes the module from sys.modules and deletes any PEP 3147 or
+    legacy .pyc and .pyo files.
+    """
     unload(modname)
     for dirname in sys.path:
-        unlink(os.path.join(dirname, modname + os.extsep + 'pyc'))
-        # Deleting the .pyo file cannot be within the 'try' for the .pyc since
-        # the chance exists that there is no .pyc (and thus the 'try' statement
-        # is exited) but there is a .pyo file.
-        unlink(os.path.join(dirname, modname + os.extsep + 'pyo'))
+        source = os.path.join(dirname, modname + os.extsep + 'py')
+        # It doesn't matter if they exist or not, unlink all possible
+        # combinations of PEP 3147 and legacy pyc and pyo files.
+        unlink(source + 'c')
+        unlink(source + 'o')
+        unlink(imp.cache_from_source(source, debug_override=True))
+        unlink(imp.cache_from_source(source, debug_override=False))
 
 # Check whether a gui is actually available
 def _is_gui_available():
@@ -765,6 +787,16 @@ def temp_cwd(name='tempcwd', quiet=False):
             rmtree(name)
 
 
+@contextlib.contextmanager
+def temp_umask(umask):
+    """Context manager that temporarily sets the process umask."""
+    oldmask = os.umask(umask)
+    try:
+        yield
+    finally:
+        os.umask(oldmask)
+
+
 def findfile(file, here=None, subdir=None):
     """Try to find a file on sys.path and the working directory.  If it is not
     found the argument passed to the function is returned (this does not
diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py
index c8ade06..7635579 100644
--- a/lib-python/2.7/test/test_traceback.py
+++ b/lib-python/2.7/test/test_traceback.py
@@ -1,6 +1,7 @@
 """Test cases for traceback module"""
 
 from StringIO import StringIO
+import shutil
 import sys
 import unittest
 from imp import reload
@@ -114,9 +115,7 @@ def test():
                 self.assertEqual(src, 'raise NotImplementedError')
         finally:
             sys.path[:] = savedpath
-            for f in os.listdir(testdir):
-                os.unlink(os.path.join(testdir, f))
-            os.rmdir(testdir)
+            shutil.rmtree(testdir)
 
         err = self.get_exception_format(self.syntax_error_bad_indentation2,
                                         IndentationError)
diff --git a/lib-python/2.7/test/test_zipfile.py b/lib-python/2.7/test/test_zipfile.py
index 9c63aeb..a084019 100644
--- a/lib-python/2.7/test/test_zipfile.py
+++ b/lib-python/2.7/test/test_zipfile.py
@@ -6,6 +6,7 @@ except ImportError:
 
 import os
 import io
+import imp
 import sys
 import time
 import struct
@@ -836,7 +837,13 @@ class PyZipFileTests(unittest.TestCase):
         with zipfile.PyZipFile(TemporaryFile(), "w") as zipfp:
             fn = __file__
             if fn.endswith('.pyc') or fn.endswith('.pyo'):
-                fn = fn[:-1]
+                path_split = fn.split(os.sep)
+                if os.altsep is not None:
+                    path_split.extend(fn.split(os.altsep))
+                if '__pycache__' in path_split:
+                    fn = imp.source_from_cache(fn)
+                else:
+                    fn = fn[:-1]
 
             zipfp.writepy(fn)
 
diff --git a/lib-python/2.7/test/test_zipimport.py b/lib-python/2.7/test/test_zipimport.py
index a66738a..da8c38e 100644
--- a/lib-python/2.7/test/test_zipimport.py
+++ b/lib-python/2.7/test/test_zipimport.py
@@ -43,17 +43,14 @@ NOW = time.time()
 test_pyc = make_pyc(test_co, NOW)
 
 
-if __debug__:
-    pyc_ext = ".pyc"
-else:
-    pyc_ext = ".pyo"
-
-
 TESTMOD = "ziptestmodule"
 TESTPACK = "ziptestpackage"
 TESTPACK2 = "ziptestpackage2"
 TEMP_ZIP = os.path.abspath("junk95142" + os.extsep + "zip")
 
+pyc_file = imp.cache_from_source(TESTMOD + '.py')
+pyc_ext = ('.pyc' if __debug__ else '.pyo')
+
 
 class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
 
diff --git a/lib-python/2.7/zipfile.py b/lib-python/2.7/zipfile.py
index a16d860..2198048 100644
--- a/lib-python/2.7/zipfile.py
+++ b/lib-python/2.7/zipfile.py
@@ -3,6 +3,7 @@ Read and write ZIP files.
 """
 import struct, os, time, sys, shutil
 import binascii, cStringIO, stat
+import imp
 import io
 import re
 import string
@@ -1457,22 +1458,42 @@ class PyZipFile(ZipFile):
         file_py  = pathname + ".py"
         file_pyc = pathname + ".pyc"
         file_pyo = pathname + ".pyo"
-        if os.path.isfile(file_pyo) and \
-                            os.stat(file_pyo).st_mtime >= os.stat(file_py).st_mtime:
-            fname = file_pyo    # Use .pyo file
-        elif not os.path.isfile(file_pyc) or \
-             os.stat(file_pyc).st_mtime < os.stat(file_py).st_mtime:
+        pycache_pyc = imp.cache_from_source(file_py, True)
+        pycache_pyo = imp.cache_from_source(file_py, False)
+        if (os.path.isfile(file_pyo) and
+            os.stat(file_pyo).st_mtime >= os.stat(file_py).st_mtime):
+            # Use .pyo file.
+            arcname = fname = file_pyo
+        elif (os.path.isfile(file_pyc) and
+              os.stat(file_pyc).st_mtime >= os.stat(file_py).st_mtime):
+            # Use .pyc file.
+            arcname = fname = file_pyc
+        elif (os.path.isfile(pycache_pyc) and
+              os.stat(pycache_pyc).st_mtime >= os.stat(file_py).st_mtime):
+            # Use the __pycache__/*.pyc file, but write it to the legacy pyc
+            # file name in the archive.
+            fname = pycache_pyc
+            arcname = file_pyc
+        elif (os.path.isfile(pycache_pyo) and
+              os.stat(pycache_pyo).st_mtime >= os.stat(file_py).st_mtime):
+            # Use the __pycache__/*.pyo file, but write it to the legacy pyo
+            # file name in the archive.
+            fname = pycache_pyo
+            arcname = file_pyo
+        else:
+            # Compile py into PEP 3147 pyc file.
             import py_compile
             if self.debug:
                 print "Compiling", file_py
             try:
-                py_compile.compile(file_py, file_pyc, None, True)
-            except py_compile.PyCompileError,err:
+                py_compile.compile(file_py, doraise=True)
+            except py_compile.PyCompileError, error:
                 print err.msg
-            fname = file_pyc
-        else:
-            fname = file_pyc
-        archivename = os.path.split(fname)[1]
+                fname = file_py
+            else:
+                fname = (pycache_pyc if __debug__ else pycache_pyo)
+                arcname = (file_pyc if __debug__ else file_pyo)
+        archivename = os.path.split(arcname)[1]
         if basename:
             archivename = "%s/%s" % (basename, archivename)
         return (fname, archivename)