Merge pull request #557 from python/feature/eager-imports

Restore the deferred imports to module scope

Author: jaraco

importlib_metadata/__init__.py

@@ -45,6 +45,39 @@
     SimplePath,
 )
 from .compat import py311
+from .compat.py314 import lazy_importer
+
+__lazy_modules__ = {
+    'csv',
+    'inspect',
+    'json',
+    # 'zipfile' in the stdlib, which needs no overlay
+    'zipp.compat.overlay',
+    f'{__name__}._adapters',
+}
+"""
+Imports to defer until first use (PEP 810).
+
+Each of them costs more than the path that reads it is traveled, so
+deferring keeps that cost off every other caller. Stating it here rather
+than importing inside those functions leaves the dependencies visible.
+
+Python 3.15 honors this declaration itself; earlier versions reach the
+same end through the context below.
+
+The adapters are imported by member rather than as ``from . import
+_adapters``, which names this package as the module to defer and which
+no emulation can honor from inside the package itself.
+"""
+
+with lazy_importer():
+    import csv
+    import inspect
+    import json
+
+    from zipp.compat.overlay import zipfile
+
+    from ._adapters import Message
 
 __all__ = [
     'Distribution',
@@ -599,10 +632,7 @@ def _has_metadata(self) -> bool:
 
     @staticmethod
     def _assemble_message(text: str) -> PackageMetadata:
-        # deferred for performance (python/cpython#109829)
-        from . import _adapters
-
-        return _adapters.Message(email.message_from_string(text))
+        return Message(email.message_from_string(text))
 
     def _ensure_metadata_present(self, text: str | None) -> str:
         if text is not None:
@@ -663,10 +693,6 @@ def make_file(name, hash=None, size_str=None):
 
         @pass_none
         def make_files(lines):
-            # Delay csv import, since Distribution.files is not as widely used
-            # as other parts of importlib.metadata
-            import csv
-
             return starmap(make_file, csv.reader(lines))
 
         @pass_none
@@ -788,9 +814,6 @@ def origin(self):
         return self._load_json('direct_url.json')
 
     def _load_json(self, filename):
-        # Deferred for performance (python/importlib_metadata#503)
-        import json
-
         return pass_none(json.loads)(
             self.read_text(filename),
             object_hook=lambda data: types.SimpleNamespace(**data),
@@ -908,9 +931,6 @@ def children(self):
         return []
 
     def zip_children(self):
-        # deferred for performance (python/importlib_metadata#502)
-        from zipp.compat.overlay import zipfile
-
         zip_path = zipfile.Path(self.root)
         names = zip_path.root.namelist()
         self.joinpath = zip_path.joinpath
@@ -1296,9 +1316,6 @@ def _get_toplevel_name(name: IPackagePath) -> str:
     >>> _get_toplevel_name(PackagePath('foo.dist-info'))
     'foo.dist-info'
     """
-    # Defer import of inspect for performance (python/cpython#118761)
-    import inspect
-
     return _topmost(name) or inspect.getmodulename(name) or str(name)
 
 

importlib_metadata/_compat/localize.py

@@ -6,12 +6,24 @@
 import importlib_metadata
 
 from ..compat import py312
+from ..compat.py314 import lazy_importer
 
-if TYPE_CHECKING:
-    import email.message
+__lazy_modules__ = {'importlib.metadata', 'importlib_metadata'}
+"""
+Imports to defer until first use (PEP 810).
+
+Only a foreign distribution reaches the code that reads these, so the
+cost belongs to that caller and not to every import of the package.
+"""
+
+with lazy_importer():
     import importlib.metadata
 
     from .. import _adapters
+
+if TYPE_CHECKING:
+    import email.message
+
     from .._meta import SimplePath
 
 
@@ -55,10 +67,6 @@ def dist(
     if isinstance(dist, importlib_metadata.Distribution):
         return dist
 
-    # only a foreign dist gets this far, so defer the cost of the import
-    # (python/cpython#109829)
-    import importlib.metadata
-
     if isinstance(dist, importlib.metadata.PathDistribution):
         return importlib_metadata.PathDistribution(cast('SimplePath', dist._path))
     # Warn on what the interface requires and the class omits, rather than
@@ -93,9 +101,6 @@ def message(
     >>> type(local)
     <class 'importlib_metadata._adapters.Message'>
     """
-    # deferred for performance (python/cpython#109829)
-    from .. import _adapters
-
     if isinstance(input, _adapters.Message):
         return input
     return _adapters.Message(input)

importlib_metadata/compat/py314.py

@@ -0,0 +1,44 @@
+import sys
+
+
+class nothing:
+    """
+    A context that does nothing, the interpreter deferring the imports itself.
+    """
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_info):
+        return False
+
+
+def backport():  # pragma: no cover
+    """
+    Borrow lazy_importer's emulation of PEP 810.
+    """
+    from lazy_importer import lazy_importer
+
+    return lazy_importer()
+
+
+lazy_importer = nothing if sys.version_info >= (3, 15) else backport
+"""
+Defer the imports in the block that follows, before 3.15 as after.
+
+Two properties of the emulation shape how a module may declare its lazy
+imports. Neither holds from 3.15, which defers them natively.
+
+A lazy name is bound to a proxy. Attribute access and calls reach the
+object it stands for, so ``csv.reader`` and ``_adapters.Message`` behave,
+but ``isinstance`` and anything else handed the object itself see the
+proxy and get the wrong answer in silence. Reach such an object through
+a module (``_adapters.Message``), or resolve the name with
+``lazy_importer.resolve`` before testing it.
+
+``from . import <submodule>`` names the package as the module to defer.
+That is fine from another module, but in the package's own ``__init__``
+the proxy stands for the module being executed and resolves itself
+through the globals entry that is the proxy, until the stack is
+exhausted. Import the member there instead.
+"""

mypy.ini

@@ -22,5 +22,9 @@ ignore_missing_imports = True
 [mypy-zipp.*]
 ignore_missing_imports = True
 
+# Not a dependency on Python 3.15 and later, which needs no backport
+[mypy-lazy_importer.*]
+ignore_missing_imports = True
+
 [mypy-test.support.*]
 ignore_missing_imports = True

newsfragments/554.feature.rst

@@ -0,0 +1,8 @@
+Restored ``csv``, ``inspect``, ``json``, ``zipfile``, ``_adapters``, and
+``importlib.metadata`` to module scope, where a reader can see what the
+module depends on, instead of hiding them in the functions that use them
+for performance. They are declared in ``__lazy_modules__`` and so stay
+deferred, and deferred alike on every supported Python: 3.15 honors the
+declaration natively (PEP 810) and earlier Pythons reach the same
+behavior through the ``lazy-importer`` backport, which is now a
+dependency there.

pyproject.toml

@@ -24,6 +24,8 @@ requires-python = ">=3.11"
 license = "Apache-2.0"
 dependencies = [
 	"zipp>=3.20",
+	# PEP 810 lazy imports, native from 3.15 (python/importlib_metadata#554)
+	"lazy-importer>=0.1; python_version < '3.15'",
 ]
 dynamic = ["version"]
 

tests/compat/test_localize.py

@@ -5,6 +5,7 @@
 
 import importlib.metadata as stdlib
 import pathlib
+import subprocess
 import sys
 import unittest
 import warnings
@@ -177,3 +178,38 @@ def metadata(self):
             warnings.simplefilter('always')
             assert localize.dist(hostile) is hostile
         assert not [str(warning.message) for warning in caught]
+
+
+class LazyAdaptersTests(unittest.TestCase):
+    """
+    Adapting a message withstands the adapters being imported lazily.
+    """
+
+    def test_adapted_message_passes_through(self):
+        """
+        An already adapted message survives localization unchanged.
+
+        ``localize.message`` tests the message with ``isinstance``,
+        reaching the class through a lazily imported module so that the
+        name resolves first. Were the class the lazy name itself, before
+        3.15 it would be a proxy, which answers as no match however the
+        object relates to it, and the message would be adapted a second
+        time. Run as a subprocess because reading a lazy name resolves it
+        for good, which any other test would do first.
+
+        Ref python/importlib_metadata#554.
+        """
+        script = (
+            'import email.message, importlib; '
+            'from importlib_metadata._compat import localize; '
+            'adapters = importlib.import_module("importlib_metadata._adapters"); '
+            'msg = adapters.Message(email.message.Message()); '
+            'print(localize.message(msg) is msg)'
+        )
+        proc = subprocess.run(
+            [sys.executable, '-c', script],
+            capture_output=True,
+            encoding='utf-8',
+            check=True,
+        )
+        assert proc.stdout.strip() == 'True', 'the message was adapted twice'

tests/test_main.py

@@ -3,6 +3,8 @@
 import io
 import pickle
 import re
+import subprocess
+import sys
 import unittest
 
 import pyfakefs.fake_filesystem_unittest as ffs
@@ -108,6 +110,36 @@ def test_resolve_without_attr(self):
         assert ep.load() is importlib_metadata
 
 
+class LazyImportTests(unittest.TestCase):
+    """
+    The imports at module scope cost no more than the deferred ones did.
+    """
+
+    def test_lazy_modules_stay_unloaded(self):
+        """
+        Importing the package loads none of its ``__lazy_modules__``.
+
+        ``importlib.abc`` goes first because the package imports it
+        eagerly and, on some Pythons, it reaches ``inspect`` through
+        ``importlib.resources``. Taking the snapshot after it leaves only
+        what the declaration itself governs.
+
+        Ref python/importlib_metadata#554.
+        """
+        script = (
+            'import sys, importlib.abc; before = frozenset(sys.modules); '
+            'import importlib_metadata as md; '
+            'print(*sorted(md.__lazy_modules__ & (sys.modules.keys() - before)))'
+        )
+        proc = subprocess.run(
+            [sys.executable, '-c', script],
+            capture_output=True,
+            encoding='utf-8',
+            check=True,
+        )
+        assert not proc.stdout.split(), f'imported eagerly: {proc.stdout.strip()}'
+
+
 class NameNormalizationTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
     @staticmethod
     def make_pkg(name):