Issue python#2377: Make importlib the implementation of __import__().

importlib._bootstrap is now frozen into Python/importlib.h and stored
as _frozen_importlib in sys.modules. Py_Initialize() loads the frozen
code along with sys and imp and then uses _frozen_importlib._install()
to set builtins.__import__() w/ _frozen_importlib.__import__().

Include/abstract.h

@@ -339,6 +339,10 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
 
      PyAPI_FUNC(PyObject *) PyObject_CallMethodObjArgs(PyObject *o,
                                                        PyObject *method, ...);
+     PyAPI_FUNC(PyObject *) _PyObject_CallMethodObjIdArgs(PyObject *o,
+                                               struct _Py_Identifier *method,
+                                               ...);
+
 
        /*
      Call the method named m of object o with a variable number of

Include/dictobject.h

@@ -109,6 +109,8 @@ PyAPI_DATA(PyTypeObject) PyDictValues_Type;
 PyAPI_FUNC(PyObject *) PyDict_New(void);
 PyAPI_FUNC(PyObject *) PyDict_GetItem(PyObject *mp, PyObject *key);
 PyAPI_FUNC(PyObject *) PyDict_GetItemWithError(PyObject *mp, PyObject *key);
+PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp,
+                                                  struct _Py_Identifier *key);
 PyAPI_FUNC(int) PyDict_SetItem(PyObject *mp, PyObject *key, PyObject *item);
 PyAPI_FUNC(int) PyDict_DelItem(PyObject *mp, PyObject *key);
 PyAPI_FUNC(void) PyDict_Clear(PyObject *mp);

Include/import.h

@@ -7,6 +7,9 @@
 extern "C" {
 #endif
 
+PyAPI_FUNC(void) _PyImportZip_Init(void);
+
+PyMODINIT_FUNC PyInit_imp(void);
 PyAPI_FUNC(long) PyImport_GetMagicNumber(void);
 PyAPI_FUNC(const char *) PyImport_GetMagicTag(void);
 PyAPI_FUNC(PyObject *) PyImport_ExecCodeModule(

Include/pystate.h

@@ -25,6 +25,7 @@ typedef struct _is {
     PyObject *modules_by_index;
     PyObject *sysdict;
     PyObject *builtins;
+    PyObject *importlib;
     PyObject *modules_reloading;
 
     PyObject *codec_search_path;
@@ -33,6 +34,7 @@ typedef struct _is {
     int codecs_initialized;
     int fscodec_initialized;
 
+
 #ifdef HAVE_DLOPEN
     int dlopenflags;
 #endif

Include/pythonrun.h

@@ -188,7 +188,7 @@ PyAPI_FUNC(const char *) _Py_hgversion(void);
 PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void);
 PyAPI_FUNC(PyObject *) _PySys_Init(void);
 PyAPI_FUNC(void) _PyImport_Init(void);
-PyAPI_FUNC(void) _PyExc_Init(void);
+PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod);
 PyAPI_FUNC(void) _PyImportHooks_Init(void);
 PyAPI_FUNC(int) _PyFrame_Init(void);
 PyAPI_FUNC(void) _PyFloat_Init(void);

Lib/importlib/_bootstrap.py

@@ -160,6 +160,13 @@ def _wrap(new, old):
 
 # Finder/loader utility code ##################################################
 
+def verbose_message(message, *args):
+    """Print the message to stderr if -v/PYTHONVERBOSE is turned on."""
+    if sys.flags.verbose:
+        if not message.startswith('#') and not message.startswith('import '):
+            message = '# ' + message
+        print(message.format(*args), file=sys.stderr)
+
 
 def set_package(fxn):
     """Set __package__ on the returned module."""
@@ -388,19 +395,24 @@ def _bytes_from_bytecode(self, fullname, data, bytecode_path, source_stats):
             raise ImportError("bad magic number in {}".format(fullname),
                               name=fullname, path=bytecode_path)
         elif len(raw_timestamp) != 4:
-            raise EOFError("bad timestamp in {}".format(fullname))
+            message = 'bad timestamp in {}'.format(fullname)
+            verbose_message(message)
+            raise EOFError(message)
         elif len(raw_size) != 4:
-            raise EOFError("bad size in {}".format(fullname))
+            message = 'bad size in {}'.format(fullname)
+            verbose_message(message)
+            raise EOFError(message)
         if source_stats is not None:
             try:
                 source_mtime = int(source_stats['mtime'])
             except KeyError:
                 pass
             else:
                 if _r_long(raw_timestamp) != source_mtime:
-                    raise ImportError(
-                        "bytecode is stale for {}".format(fullname),
-                        name=fullname, path=bytecode_path)
+                    message = 'bytecode is stale for {}'.format(fullname)
+                    verbose_message(message)
+                    raise ImportError(message, name=fullname,
+                                      path=bytecode_path)
             try:
                 source_size = source_stats['size'] & 0xFFFFFFFF
             except KeyError:
@@ -506,9 +518,13 @@ def get_code(self, fullname):
                     except (ImportError, EOFError):
                         pass
                     else:
+                        verbose_message('{} matches {}', bytecode_path,
+                                        source_path)
                         found = marshal.loads(bytes_data)
                         if isinstance(found, code_type):
                             imp._fix_co_filename(found, source_path)
+                            verbose_message('code object from {}',
+                                            bytecode_path)
                             return found
                         else:
                             msg = "Non-code object in {}"
@@ -517,6 +533,7 @@ def get_code(self, fullname):
         source_bytes = self.get_data(source_path)
         code_object = compile(source_bytes, source_path, 'exec',
                                 dont_inherit=True)
+        verbose_message('code object from {}', source_path)
         if (not sys.dont_write_bytecode and bytecode_path is not None and
             source_mtime is not None):
             # If e.g. Jython ever implements imp.cache_from_source to have
@@ -528,6 +545,7 @@ def get_code(self, fullname):
             data.extend(marshal.dumps(code_object))
             try:
                 self.set_data(bytecode_path, data)
+                verbose_message('wrote {!r}', bytecode_path)
             except NotImplementedError:
                 pass
         return code_object
@@ -596,6 +614,7 @@ def set_data(self, path, data):
                 return
         try:
             _write_atomic(path, data)
+            verbose_message('created {!r}', path)
         except (PermissionError, FileExistsError):
             # Don't worry if you can't write bytecode or someone is writing
             # it at the same time.
@@ -615,6 +634,7 @@ def get_code(self, fullname):
         bytes_data = self._bytes_from_bytecode(fullname, data, path, None)
         found = marshal.loads(bytes_data)
         if isinstance(found, code_type):
+            verbose_message('code object from {!r}', path)
             return found
         else:
             raise ImportError("Non-code object in {}".format(path),
@@ -644,7 +664,9 @@ def load_module(self, fullname):
         """Load an extension module."""
         is_reload = fullname in sys.modules
         try:
-            return imp.load_dynamic(fullname, self._path)
+            module = imp.load_dynamic(fullname, self._path)
+            verbose_message('extension module loaded from {!r}', self._path)
+            return module
         except:
             if not is_reload and fullname in sys.modules:
                 del sys.modules[fullname]
@@ -953,6 +975,7 @@ def _find_and_load(name, import_):
     elif name not in sys.modules:
         # The parent import may have already imported this module.
         loader.load_module(name)
+        verbose_message('import {!r} # {!r}', name, loader)
     # Backwards-compatibility; be nicer to skip the dict lookup.
     module = sys.modules[name]
     if parent:

Lib/importlib/test/import_/test_path.py

@@ -87,7 +87,7 @@ def test_path_importer_cache_empty_string(self):
 
 class DefaultPathFinderTests(unittest.TestCase):
 
-    """Test importlib._bootstrap._DefaultPathFinder."""
+    """Test _bootstrap._DefaultPathFinder."""
 
     def test_implicit_hooks(self):
         # Test that the implicit path hooks are used.

Lib/importlib/test/import_/util.py

@@ -1,6 +1,5 @@
 import functools
 import importlib
-import importlib._bootstrap
 import unittest
 
 

Lib/importlib/test/regrtest.py

@@ -13,16 +13,4 @@
 if __name__ == '__main__':
     __builtins__.__import__ = importlib.__import__
 
-    exclude = ['--exclude',
-                'test_frozen', # Does not expect __loader__ attribute
-                'test_pkg',  # Does not expect __loader__ attribute
-                'test_pydoc', # Does not expect __loader__ attribute
-              ]
-
-    # Switching on --exclude implies running all test but the ones listed, so
-    # only use it when one is not running an explicit test
-    if len(sys.argv) == 1:
-        # No programmatic way to specify tests to exclude
-        sys.argv.extend(exclude)
-
     regrtest.main(quiet=True, verbose2=True)

Lib/importlib/test/source/test_file_loader.py

@@ -1,5 +1,5 @@
+from ... import _bootstrap
 import importlib
-from importlib import _bootstrap
 from .. import abc
 from .. import util
 from . import util as source_util

Lib/importlib/test/source/test_finder.py

@@ -1,10 +1,11 @@
-from importlib import _bootstrap
 from .. import abc
 from . import util as source_util
-from test.support import make_legacy_pyc
-import os
+
+from importlib import _bootstrap
 import errno
+import os
 import py_compile
+from test.support import make_legacy_pyc
 import unittest
 import warnings
 

Lib/importlib/test/source/test_path_hook.py

@@ -1,5 +1,6 @@
-from importlib import _bootstrap
 from . import util as source_util
+
+from importlib import _bootstrap
 import unittest
 
 

Lib/importlib/test/source/test_source_encoding.py

@@ -1,6 +1,6 @@
-from importlib import _bootstrap
 from . import util as source_util
 
+from importlib import _bootstrap
 import codecs
 import re
 import sys
@@ -36,7 +36,7 @@ def run_test(self, source):
             with open(mapping[self.module_name], 'wb') as file:
                 file.write(source)
             loader = _bootstrap._SourceFileLoader(self.module_name,
-                                       mapping[self.module_name])
+                                                  mapping[self.module_name])
             return loader.load_module(self.module_name)
 
     def create_source(self, encoding):

Lib/importlib/test/util.py

@@ -35,7 +35,7 @@ def uncache(*names):
     for name in names:
         if name in ('sys', 'marshal', 'imp'):
             raise ValueError(
-                "cannot uncache {0} as it will break _importlib".format(name))
+                "cannot uncache {0}".format(name))
         try:
             del sys.modules[name]
         except KeyError:

Lib/os.py

@@ -42,6 +42,8 @@ def _get_exports_list(module):
     except AttributeError:
         return [n for n in dir(module) if n[0] != '_']
 
+# Any new dependencies of the os module and/or changes in path separator
+# requires updating importlib as well.
 if 'posix' in _names:
     name = 'posix'
     linesep = '\n'

Lib/pydoc.py

@@ -299,9 +299,8 @@ def safeimport(path, forceload=0, cache={}):
         elif exc is SyntaxError:
             # A SyntaxError occurred before we could execute the module.
             raise ErrorDuringImport(value.filename, info)
-        elif exc is ImportError and extract_tb(tb)[-1][2]=='safeimport':
-            # The import error occurred directly in this function,
-            # which means there is no such module in the path.
+        elif exc is ImportError and value.name == path:
+            # No such module in the path.
             return None
         else:
             # Some other error occurred during the importing process.

Lib/site.py

@@ -81,7 +81,8 @@ def makepath(*paths):
 def abs_paths():
     """Set all module __file__ and __cached__ attributes to an absolute path"""
     for m in set(sys.modules.values()):
-        if hasattr(m, '__loader__'):
+        if (getattr(getattr(m, '__loader__', None), '__module__', None) !=
+                '_frozen_importlib'):
             continue   # don't mess with a PEP 302-supplied __file__
         try:
             m.__file__ = os.path.abspath(m.__file__)

Lib/test/test_frozen.py

@@ -5,14 +5,22 @@
 import sys
 
 class FrozenTests(unittest.TestCase):
+
+    module_attrs = frozenset(['__builtins__', '__cached__', '__doc__',
+                              '__file__', '__loader__', '__name__',
+                              '__package__'])
+    package_attrs = frozenset(list(module_attrs) + ['__path__'])
+
     def test_frozen(self):
         with captured_stdout() as stdout:
             try:
                 import __hello__
             except ImportError as x:
                 self.fail("import __hello__ failed:" + str(x))
             self.assertEqual(__hello__.initialized, True)
-            self.assertEqual(len(dir(__hello__)), 7, dir(__hello__))
+            expect = set(self.module_attrs)
+            expect.add('initialized')
+            self.assertEqual(set(dir(__hello__)), expect)
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
         with captured_stdout() as stdout:
@@ -21,10 +29,13 @@ def test_frozen(self):
             except ImportError as x:
                 self.fail("import __phello__ failed:" + str(x))
             self.assertEqual(__phello__.initialized, True)
+            expect = set(self.package_attrs)
+            expect.add('initialized')
             if not "__phello__.spam" in sys.modules:
-                self.assertEqual(len(dir(__phello__)), 8, dir(__phello__))
+                self.assertEqual(set(dir(__phello__)), expect)
             else:
-                self.assertEqual(len(dir(__phello__)), 9, dir(__phello__))
+                expect.add('spam')
+                self.assertEqual(set(dir(__phello__)), expect)
             self.assertEqual(__phello__.__path__, [__phello__.__name__])
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
@@ -34,8 +45,13 @@ def test_frozen(self):
             except ImportError as x:
                 self.fail("import __phello__.spam failed:" + str(x))
             self.assertEqual(__phello__.spam.initialized, True)
-            self.assertEqual(len(dir(__phello__.spam)), 7)
-            self.assertEqual(len(dir(__phello__)), 9)
+            spam_expect = set(self.module_attrs)
+            spam_expect.add('initialized')
+            self.assertEqual(set(dir(__phello__.spam)), spam_expect)
+            phello_expect = set(self.package_attrs)
+            phello_expect.add('initialized')
+            phello_expect.add('spam')
+            self.assertEqual(set(dir(__phello__)), phello_expect)
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
         try:

Lib/test/test_import.py

@@ -73,6 +73,7 @@ def test_with_extension(ext):
 
             if TESTFN in sys.modules:
                 del sys.modules[TESTFN]
+            importlib.invalidate_caches()
             try:
                 try:
                     mod = __import__(TESTFN)
@@ -402,6 +403,7 @@ def test_module_without_source(self):
         py_compile.compile(self.file_name, dfile=target)
         os.remove(self.file_name)
         pyc_file = make_legacy_pyc(self.file_name)
+        importlib.invalidate_caches()
         mod = self.import_module()
         self.assertEqual(mod.module_filename, pyc_file)
         self.assertEqual(mod.code_filename, target)
@@ -509,7 +511,7 @@ def check_relative():
 
         # Check relative import fails with package set to a non-string
         ns = dict(__package__=object())
-        self.assertRaises(ValueError, check_relative)
+        self.assertRaises(TypeError, check_relative)
 
     def test_absolute_import_without_future(self):
         # If explicit relative import syntax is used, then do not try
@@ -644,6 +646,7 @@ def cleanup():
             pass
         unload('pep3147.foo')
         unload('pep3147')
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         init_pyc = imp.cache_from_source(
             os.path.join('pep3147', '__init__.py'))
@@ -666,9 +669,11 @@ def cleanup():
             pass
         with open(os.path.join('pep3147', 'foo.py'), 'w'):
             pass
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         unload('pep3147.foo')
         unload('pep3147')
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         init_pyc = imp.cache_from_source(
             os.path.join('pep3147', '__init__.py'))