From: Stefano Rivera Date: Sat, 7 Oct 2017 09:38:57 +0200 Subject: PEP3147 support Tests modified from Barry Warsaw's PEP3147 cpython support. Forwarded: no Last-Update: 2013-02-23 --- pypy/config/pypyoption.py | 5 + pypy/doc/interpreter.rst | 1 + pypy/interpreter/app_main.py | 1 + pypy/interpreter/main.py | 1 + pypy/interpreter/test/test_main.py | 11 + pypy/module/imp/importing.py | 81 +++++- pypy/module/imp/interp_imp.py | 12 + pypy/module/imp/moduledef.py | 3 + pypy/module/imp/test/test_app.py | 7 +- pypy/module/imp/test/test_import.py | 323 ++++++++++++++++++++++-- pypy/module/zipimport/test/test_undocumented.py | 23 +- 11 files changed, 430 insertions(+), 38 deletions(-) diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py index 9c627a9..52db19f 100644 --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -151,6 +151,11 @@ pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [ cmdline="--soabi", default=None), + StrOption("magic_tag", + "Tag to differentiate .pyc files for different Python interpreters", + cmdline="--magic_tag", + default=None), + BoolOption("honor__builtins__", "Honor the __builtins__ key of a module dictionary", default=False), diff --git a/pypy/doc/interpreter.rst b/pypy/doc/interpreter.rst index 57b5207..7f32dfd 100644 --- a/pypy/doc/interpreter.rst +++ b/pypy/doc/interpreter.rst @@ -239,6 +239,7 @@ attributes: * ``__doc__`` the docstring of the module * ``__file__`` the source filename from which this module was instantiated +* ``__cached__`` the filename for the byte-compiled cache of this module * ``__path__`` state used for relative imports Apart from the basic Module used for importing diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py index ec5dcab..d9f154d 100755 --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -742,6 +742,7 @@ def run_command_line(interactive, # on the command-line. filename = sys.argv[0] mainmodule.__file__ = filename + mainmodule.__cached__ = None sys.path.insert(0, sys.pypy_resolvedirof(filename)) # assume it's a pyc file only if its name says so. # CPython goes to great lengths to detect other cases diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py index e1141a5..24f58b7 100644 --- a/pypy/interpreter/main.py +++ b/pypy/interpreter/main.py @@ -43,6 +43,7 @@ def _run_eval_string(source, filename, space, eval): space.setitem(w_globals, space.newtext('__builtins__'), space.builtin) if filename is not None: space.setitem(w_globals, space.newtext('__file__'), space.newtext(filename)) + space.setitem(w_globals, space.newtext('__cached__'), space.w_None) retval = pycode.exec_code(space, w_globals, w_globals) if eval: diff --git a/pypy/interpreter/test/test_main.py b/pypy/interpreter/test/test_main.py index dc7e536..54eebe2 100644 --- a/pypy/interpreter/test/test_main.py +++ b/pypy/interpreter/test/test_main.py @@ -13,6 +13,12 @@ def main(): main() """ +test__file__code = """ +assert __file__ is not None +assert __cached__ is None +print len('hello world') +""" + # On module test we want to ensure that the called module __name__ is # '__main__' and argv is set as expected. testmodulecode = """ @@ -39,12 +45,14 @@ def checkoutput(space, expected_output, f, *args): assert capturefn.read(mode='rU') == expected_output testfn = udir.join('tmp_hello_world.py') +test__file__fn = udir.join('test__file__.py') testmodule = 'tmp_hello_module' testpackage = 'tmp_package' class TestMain: def setup_class(cls): testfn.write(testcode, 'w') + test__file__fn.write(test__file__code, 'w') udir.join(testmodule + '.py').write(testmodulecode, 'w') udir.ensure(testpackage, '__init__.py') udir.join(testpackage, testmodule + '.py').write(testmodulecode, 'w') @@ -78,3 +86,6 @@ class TestMain: testmodule, ['hello world']) checkoutput(self.space, testresultoutput, main.run_module, testpackage + '.' + testmodule, ['hello world']) + + def test__file__file(self): + checkoutput(self.space, testresultoutput, main.run_file, str(test__file__fn)) diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py index 24e5b31..12635cb 100644 --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -12,7 +12,7 @@ from pypy.interpreter.baseobjspace import W_Root, CannotHaveLock from pypy.interpreter.eval import Code from pypy.interpreter.pycode import PyCode from pypy.interpreter.streamutil import wrap_streamerror -from rpython.rlib import streamio, jit +from rpython.rlib import rstring, streamio, jit from rpython.rlib.streamio import StreamErrors from rpython.rlib.objectmodel import we_are_translated, specialize from pypy.module.sys.version import PYPY_VERSION @@ -40,6 +40,7 @@ SO = '.pyd' if _WIN32 else '.so' # split the two usages again. #DEFAULT_SOABI = 'pypy-%d%d' % PYPY_VERSION[:2] DEFAULT_SOABI = 'pypy-41' +DEFAULT_MAGIC_TAG = DEFAULT_SOABI @specialize.memo() def get_so_extension(space): @@ -605,6 +606,7 @@ def find_module(space, modulename, w_modulename, partname, w_path, def _prepare_module(space, w_mod, filename, pkgdir): space.sys.setmodule(w_mod) space.setattr(w_mod, space.newtext('__file__'), space.newtext(filename)) + space.setattr(w_mod, space.newtext('__cached__'), space.w_None) space.setattr(w_mod, space.newtext('__doc__'), space.w_None) if pkgdir is not None: space.setattr(w_mod, space.newtext('__path__'), space.newlist([space.newtext(pkgdir)])) @@ -885,6 +887,65 @@ def get_pyc_magic(space): return default_magic +def get_pyc_tag(space): + """Return the tag used in __pycache__ filenames""" + # XXX CPython testing hack: use the default + if not we_are_translated(): + return DEFAULT_MAGIC_TAG + + if space.config.objspace.magic_tag is not None: + magic_tag = space.config.objspace.magic_tag + else: + magic_tag = DEFAULT_MAGIC_TAG + return magic_tag + +def make_compiled_pathname(space, pathname): + """ + The PEP 3147 path to the byte-compiled file associated with the source path + """ + pathname = rstring.assert_str0(pathname) + + index = pathname.rfind(os.sep) + if index < 0: + pycachedir = '__pycache__' + basename = pathname + else: + pycachedir = pathname[:index + 1] + '__pycache__' + basename = pathname[index + 1:] + + index = basename.rfind('.') + if index > 0: + basename = basename[:index + 1] + + filename = basename + get_pyc_tag(space) + '.pyc' + cpathname = os.path.join(pycachedir, filename) + return cpathname + +def make_source_pathname(space, cpathname): + """ + Given the path to a PEP 3147 file name, return the associated source code + file path. + """ + cpathname = rstring.assert_str0(cpathname) + + index = cpathname.rfind(os.sep) + if index < 0: + raise OperationError(space.w_ValueError, space.newtext( + "Not a PEP 3147 pyc path: %s" % cpathname)) + pycachedir = cpathname[:index] + filename = cpathname[index + 1:] + + index = pycachedir.rfind(os.sep) + extension = '.' + get_pyc_tag(space) + '.pyc' + ext_index = len(filename) - len(extension) + if (index < 0 or pycachedir[index + 1:] != '__pycache__' + or not filename.endswith(extension) + or ext_index < 0): + raise OperationError(space.w_ValueError, space.newtext( + "Not a PEP 3147 pyc path: %s" % cpathname)) + basedir = pycachedir[:index] + basename = filename[:ext_index] + return os.path.join(basedir, basename + '.py') def parse_source_module(space, pathname, source): """ Parse a source file and return the corresponding code object """ @@ -927,7 +988,7 @@ def load_source_module(space, w_modulename, w_mod, pathname, source, fd, src_stat = os.fstat(fd) except OSError as e: raise wrap_oserror(space, e, pathname) # better report this error - cpathname = pathname + 'c' + cpathname = make_compiled_pathname(space, pathname) mtime = int(src_stat[stat.ST_MTIME]) mode = src_stat[stat.ST_MODE] stream = check_compiled_module(space, cpathname, mtime) @@ -939,7 +1000,7 @@ def load_source_module(space, w_modulename, w_mod, pathname, source, fd, _wrap_readall(space, stream)) finally: _close_ignore(stream) - space.setattr(w_mod, space.newtext('__file__'), space.newtext(cpathname)) + space.setattr(w_mod, space.newtext('__file__'), space.newtext(pathname)) else: code_w = parse_source_module(space, pathname, source) @@ -955,6 +1016,7 @@ def load_source_module(space, w_modulename, w_mod, pathname, source, fd, if optimize >= 2: code_w.remove_docstrings(space) + space.setattr(w_mod, space.newtext('__cached__'), space.newtext(cpathname)) update_code_filenames(space, code_w, pathname) return exec_code_module(space, w_mod, code_w, w_modulename, check_afterwards=check_afterwards) @@ -1117,6 +1179,19 @@ def write_compiled_module(space, co, cpathname, src_mode, src_mtime): raise #print "Problem while marshalling %s, skipping" % cpathname return + + # Create PEP3147 __pycache__ dir if necessary + index = cpathname.rfind(os.sep) + if index < 0: + return + pycachedir = cpathname[:index] + if not os.path.isdir(pycachedir): + mode = src_mode | 0755 + try: + os.mkdir(pycachedir, mode) + except OSError: + return + # # Careful here: we must not crash nor leave behind something that looks # too much like a valid pyc file but really isn't one. diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py index 09127ab..7d133f5 100644 --- a/pypy/module/imp/interp_imp.py +++ b/pypy/module/imp/interp_imp.py @@ -36,6 +36,18 @@ def get_magic(space): d = x & 0xff return space.newbytes(chr(a) + chr(b) + chr(c) + chr(d)) +def get_tag(space): + return space.newtext(importing.get_pyc_tag(space)) + +@unwrap_spec(path='fsencode') +def cache_from_source(space, path, w_debug_override=None): + # w_debug_override is ignored, pypy doesn't support __debug__ + return space.newtext(importing.make_compiled_pathname(space, path)) + +@unwrap_spec(path='fsencode') +def source_from_cache(space, path): + return space.newtext(importing.make_source_pathname(space, path)) + def get_file(space, w_file, filename, filemode): if space.is_none(w_file): try: diff --git a/pypy/module/imp/moduledef.py b/pypy/module/imp/moduledef.py index 39b577a..fb1023a 100644 --- a/pypy/module/imp/moduledef.py +++ b/pypy/module/imp/moduledef.py @@ -17,6 +17,9 @@ class Module(MixedModule): 'get_suffixes': 'interp_imp.get_suffixes', 'get_magic': 'interp_imp.get_magic', + 'get_tag': 'interp_imp.get_tag', + 'cache_from_source': 'interp_imp.cache_from_source', + 'source_from_cache': 'interp_imp.source_from_cache', 'find_module': 'interp_imp.find_module', 'load_module': 'interp_imp.load_module', 'load_source': 'interp_imp.load_source', diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py index f095168..70c0a00 100644 --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -150,6 +150,7 @@ class AppTestImpModule: def test_rewrite_pyc_check_code_name(self): # This one is adapted from cpython's Lib/test/test_import.py + from imp import cache_from_source from os import chmod from os.path import join from sys import modules, path @@ -159,6 +160,7 @@ class AppTestImpModule: import sys code_filename = sys._getframe().f_code.co_filename module_filename = __file__ + module_bytefilename = __cached__ constant = 1 def func(): pass @@ -170,7 +172,7 @@ class AppTestImpModule: file_name = join(dir_name, module_name + '.py') with open(file_name, "wb") as f: f.write(code) - compiled_name = file_name + ("c" if __debug__ else "o") + compiled_name = cache_from_source(file_name) chmod(file_name, 0777) # Setup @@ -188,7 +190,8 @@ class AppTestImpModule: try: # Ensure proper results assert mod != orig_module - assert mod.module_filename == compiled_name + assert mod.module_filename == file_name + assert mod.module_bytefilename == compiled_name assert mod.code_filename == file_name assert mod.func_filename == file_name finally: diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py index a7b92a0..e5851ea 100644 --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -8,7 +8,7 @@ from rpython.rlib import streamio from pypy.tool.option import make_config from pypy.tool.pytest.objspace import maketestobjspace import pytest -import sys, os +import shutil, sys, os import tempfile, marshal from pypy.module.imp import importing @@ -107,12 +107,18 @@ def setup_directory_structure(space): # create compiled/x.py and a corresponding pyc file p = setuppkg("compiled", x = "x = 84") + try: + p.mkdir('__pycache__') + except py.error.EEXIST: + pass + cpathname = p.join('__pycache__').join( + 'x.' + importing.get_pyc_tag(space) + '.pyc') if conftest.option.runappdirect: import marshal, stat, struct, imp code = py.code.Source(p.join("x.py").read()).compile() s3 = marshal.dumps(code) s2 = struct.pack(" ValueError + try: + importing.make_source_pathname(self.space, 'foo.%s.pyc' % self.tag) + except OperationError, e: + if not e.match(self.space, self.space.w_ValueError): + raise + else: + raise Exception("Should have raised ValueError") + + def test_make_source_pathname_too_few_dots(self): + # Too few dots in final path component -> ValueError + try: + importing.make_source_pathname(self.space, '__pycache__/foo.pyc') + except OperationError, e: + if not e.match(self.space, self.space.w_ValueError): + raise + else: + raise Exception("Should have raised ValueError") + + def test_make_source_pathname_too_many_dots(self): + # Too many dots in final path component -> ValueError + pathname = '__pycache__/foo.%s.foo.pyc' % self.tag + try: + importing.make_source_pathname(self.space, pathname) + except OperationError, e: + if not e.match(self.space, self.space.w_ValueError): + raise + else: + raise Exception("Should have raised ValueError") + + def test_make_source_pathname_no__pycache__(self): + # Another problem with the path -> ValueError + pathname = '/foo/bar/foo.%s.foo.pyc' % self.tag + try: + importing.make_source_pathname(self.space, pathname) + except OperationError, e: + if not e.match(self.space, self.space.w_ValueError): + raise + else: + raise Exception("Should have raised ValueError") + + +class AppTestPEP3147Pyc(object): + def test_package___file__(self): + import os, sys, shutil + # Test that a package's __file__ points to the right source directory. + try: + os.mkdir('pep3147') + sys.path.insert(0, os.curdir) + # Touch the __init__.py file. + with open('pep3147/__init__.py', 'w'): + pass + m = __import__('pep3147') + # Ensure we load the pyc file. + del sys.modules['pep3147'] + m = __import__('pep3147') + assert m.__file__ == os.sep.join(('.', 'pep3147', '__init__.py')) + finally: + if sys.path[0] == os.curdir: + del sys.path[0] + shutil.rmtree('pep3147') + + +class AppTestPycache(object): + # Test the various PEP 3147 related behaviors. + + def setup_class(cls): + space = cls.space + + cls.module = '_app_test_pycache' + cls.filename = cls.module + '.py' + cls.w_module = space.wrap(cls.module) + cls.w_filename = space.wrap(cls.filename) + cls.w_tag = space.wrap(importing.get_pyc_tag(space)) + + def setup_method(cls, method): + if os.path.exists('__pycache__'): + shutil.rmtree('__pycache__') + if os.path.exists(cls.filename): + os.unlink(cls.filename) + + with open(cls.filename, 'w') as fp: + print >> fp, '# This is a test file written by test_import.py' + + def teardown_method(cls, method): + if os.path.exists('__pycache__'): + shutil.rmtree('__pycache__') + if os.path.exists(cls.filename): + os.unlink(cls.filename) + + def test_import_pyc_path(self): + import sys, os + sys.path.insert(0, '.') + try: + assert not os.path.exists('__pycache__') + __import__(self.module) + assert os.path.exists('__pycache__') + assert os.path.exists(os.path.join( + '__pycache__', '%s.%s.pyc' % (self.module, self.tag))) + finally: + del sys.path[0] + sys.modules.pop(self.module, None) + + @pytest.mark.skipif('os.name != "posix"') + def test_unwritable_directory(self): + # When the umask causes the new __pycache__ directory to be + # unwritable, the import still succeeds but no .pyc file is written. + import os, sys + sys.path.insert(0, '.') + try: + oldmask = os.umask(0222) + try: + __import__(self.module) + finally: + os.umask(oldmask) + assert os.path.exists('__pycache__') + assert not os.path.exists(os.path.join( + '__pycache__', '%s.%s.pyc' % (self.module, self.tag))) + finally: + del sys.path[0] + sys.modules.pop(self.module, None) + + def test_missing_source(self): + # With PEP 3147 cache layout, removing the source but leaving the pyc + # file does not satisfy the import. + import imp, os, sys + sys.path.insert(0, '.') + try: + __import__(self.module) + pyc_file = imp.cache_from_source(self.filename) + assert os.path.exists(pyc_file) + os.remove(self.filename) + del sys.modules[self.module] + try: + __import__(self.module) + except ImportError: + pass + else: + raise "Expected ImportError to be raised" + finally: + del sys.path[0] + sys.modules.pop(self.module, None) + + def test___cached__(self): + # Modules now also have an __cached__ that points to the pyc file. + import imp, os, sys + sys.path.insert(0, '.') + try: + m = __import__(self.module) + pyc_file = imp.cache_from_source(self.filename) + assert m.__cached__ == os.path.join(os.curdir, pyc_file) + finally: + del sys.path[0] + sys.modules.pop(self.module, None) + + def test_package___cached__(self): + # Like test___cached__ but for packages. + import imp, os, shutil, sys + sys.path.insert(0, '.') + try: + os.mkdir('_test_pep3147') + # Touch the __init__.py + with open(os.path.join('_test_pep3147', '__init__.py'), 'w'): + pass + with open(os.path.join('_test_pep3147', 'foo.py'), 'w'): + pass + m = __import__('_test_pep3147.foo') + init_pyc = imp.cache_from_source( + os.path.join('_test_pep3147', '__init__.py')) + assert m.__cached__ == os.path.join(os.curdir, init_pyc) + foo_pyc = imp.cache_from_source(os.path.join('_test_pep3147', + 'foo.py')) + assert (sys.modules['_test_pep3147.foo'].__cached__ + == os.path.join(os.curdir, foo_pyc)) + finally: + shutil.rmtree('_test_pep3147') + del sys.path[0] + sys.modules.pop('_test_pep3147.foo', None) + sys.modules.pop('_test_pep3147', None) + + def test_package___cached___from_pyc(self): + # Like test___cached__ but ensuring __cached__ when imported from a + # PEP 3147 pyc file. + import imp, os, shutil, sys + sys.path.insert(0, '.') + try: + os.mkdir('_test_pep3147') + # Touch the __init__.py + with open(os.path.join('_test_pep3147', '__init__.py'), 'w'): + pass + with open(os.path.join('_test_pep3147', 'foo.py'), 'w'): + pass + m = __import__('_test_pep3147.foo') + del sys.modules['_test_pep3147.foo'] + del sys.modules['_test_pep3147'] + m = __import__('_test_pep3147.foo') + init_pyc = imp.cache_from_source( + os.path.join('_test_pep3147', '__init__.py')) + assert m.__cached__ == os.path.join(os.curdir, init_pyc) + foo_pyc = imp.cache_from_source(os.path.join('_test_pep3147', + 'foo.py')) + assert (sys.modules['_test_pep3147.foo'].__cached__ + == os.path.join(os.curdir, foo_pyc)) + finally: + shutil.rmtree('_test_pep3147') + del sys.path[0] + sys.modules.pop('_test_pep3147.foo', None) + sys.modules.pop('_test_pep3147', None) + def test_PYTHONPATH_takes_precedence(space): if sys.platform == "win32": @@ -1486,24 +1766,21 @@ class AppTestWriteBytecode(object): def test_default(self): import os.path from test_bytecode import a - assert a.__file__.endswith('a.py') - assert os.path.exists(a.__file__ + 'c') == (not self.sandbox) + assert os.path.exists(a.__cached__) == (not self.sandbox) def test_write_bytecode(self): import os.path import sys sys.dont_write_bytecode = False from test_bytecode import b - assert b.__file__.endswith('b.py') - assert os.path.exists(b.__file__ + 'c') + assert os.path.exists(b.__cached__) def test_dont_write_bytecode(self): import os.path import sys sys.dont_write_bytecode = True from test_bytecode import c - assert c.__file__.endswith('c.py') - assert not os.path.exists(c.__file__ + 'c') + assert not os.path.exists(c.__cached__) class AppTestWriteBytecodeSandbox(AppTestWriteBytecode): @@ -1523,25 +1800,21 @@ class _AppTestLonePycFileBase(object): def test_import_possibly_from_pyc(self): from compiled import x - assert x.__file__.endswith('x.pyc') + assert x.__file__.endswith('x.py') + assert x.__cached__.endswith('.pyc') try: from compiled import lone except ImportError: - assert not self.lonepycfiles, "should have found 'lone.pyc'" + assert not self.lonepycfiles, "should have found 'lone.TAG.pyc'" else: - assert self.lonepycfiles, "should not have found 'lone.pyc'" - assert lone.__file__.endswith('lone.pyc') + assert self.lonepycfiles, "should not have found 'lone.TAG.pyc'" + assert lone.__cached__.endswith('.pyc') class AppTestNoLonePycFile(_AppTestLonePycFileBase): spaceconfig = { "objspace.lonepycfiles": False } -class AppTestLonePycFile(_AppTestLonePycFileBase): - spaceconfig = { - "objspace.lonepycfiles": True - } - class AppTestMultithreadedImp(object): spaceconfig = dict(usemodules=['thread', 'time']) diff --git a/pypy/module/zipimport/test/test_undocumented.py b/pypy/module/zipimport/test/test_undocumented.py index 7271f20..44f47c7 100644 --- a/pypy/module/zipimport/test/test_undocumented.py +++ b/pypy/module/zipimport/test/test_undocumented.py @@ -31,12 +31,11 @@ class AppTestZipImport: Clears zipimport._zip_directory_cache. """ - import zipimport, os, shutil, zipfile, py_compile + import zipimport, os, shutil, zipfile, py_compile, imp example_code = 'attr = None' TESTFN = '@test' zipimport._zip_directory_cache.clear() zip_path = TESTFN + '.zip' - bytecode_suffix = 'c'# if __debug__ else 'o' zip_file = zipfile.ZipFile(zip_path, 'w') for path in created_paths: if os.sep in path: @@ -53,13 +52,13 @@ class AppTestZipImport: zip_file.write(code_path) if bytecode: py_compile.compile(code_path, doraise=True) - zip_file.write(code_path + bytecode_suffix) + bytecode_path = imp.cache_from_source(code_path) + zip_file.write(bytecode_path) zip_file.close() return os.path.abspath(zip_path) def w_cleanup_zipfile(self, created_paths): - import os, shutil - bytecode_suffix = 'c'# if __debug__ else 'o' + import os, shutil, imp zip_path = '@test.zip' for path in created_paths: if os.sep in path: @@ -67,9 +66,17 @@ class AppTestZipImport: if os.path.exists(directory): shutil.rmtree(directory) else: - for suffix in ('.py', '.py' + bytecode_suffix): - if os.path.exists(path + suffix): - os.unlink(path + suffix) + source_file = path + '.py' + if os.path.exists(source_file): + os.unlink(source_file) + bytecode_file = imp.cache_from_source(source_file) + if os.path.exists(bytecode_file): + os.unlink(bytecode_file) + try: + os.rmdir(os.path.dirname(bytecode_file)) + except OSError: + pass + os.unlink(zip_path) def test_inheritance(self):