diff --git a/Lib/importlib/resources/__init__.py b/Lib/importlib/resources/__init__.py index ec4441c91161188..723c9f9eb33ce1e 100644 --- a/Lib/importlib/resources/__init__.py +++ b/Lib/importlib/resources/__init__.py @@ -1,4 +1,11 @@ -"""Read resources contained within a package.""" +""" +Read resources contained within a package. + +This codebase is shared between importlib.resources in the stdlib +and importlib_resources in PyPI. See +https://github.com/python/importlib_metadata/wiki/Development-Methodology +for more detail. +""" from ._common import ( as_file, diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index d5381fb80d85e64..4e9014c45a056e2 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -66,10 +66,10 @@ def get_resource_reader(package: types.ModuleType) -> Optional[ResourceReader]: # zipimport.zipimporter does not support weak references, resulting in a # TypeError. That seems terrible. spec = package.__spec__ - reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore + reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore[union-attr] if reader is None: return None - return reader(spec.name) # type: ignore + return reader(spec.name) # type: ignore[union-attr] @functools.singledispatch @@ -93,12 +93,13 @@ def _infer_caller(): """ def is_this_file(frame_info): - return frame_info.filename == __file__ + return frame_info.filename == stack[0].filename def is_wrapper(frame_info): return frame_info.function == 'wrapper' - not_this_file = itertools.filterfalse(is_this_file, inspect.stack()) + stack = inspect.stack() + not_this_file = itertools.filterfalse(is_this_file, stack) # also exclude 'wrapper' due to singledispatch in the call stack callers = itertools.filterfalse(is_wrapper, not_this_file) return next(callers).frame diff --git a/Lib/importlib/resources/readers.py b/Lib/importlib/resources/readers.py index ccc5abbeb4e56ed..70fc7e2b9c0145b 100644 --- a/Lib/importlib/resources/readers.py +++ b/Lib/importlib/resources/readers.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import collections import contextlib import itertools @@ -6,6 +8,7 @@ import re import warnings import zipfile +from collections.abc import Iterator from . import abc @@ -135,27 +138,31 @@ class NamespaceReader(abc.TraversableResources): def __init__(self, namespace_path): if 'NamespacePath' not in str(namespace_path): raise ValueError('Invalid path') - self.path = MultiplexedPath(*map(self._resolve, namespace_path)) + self.path = MultiplexedPath(*filter(bool, map(self._resolve, namespace_path))) @classmethod - def _resolve(cls, path_str) -> abc.Traversable: + def _resolve(cls, path_str) -> abc.Traversable | None: r""" Given an item from a namespace path, resolve it to a Traversable. path_str might be a directory on the filesystem or a path to a zipfile plus the path within the zipfile, e.g. ``/foo/bar`` or ``/foo/baz.zip/inner_dir`` or ``foo\baz.zip\inner_dir\sub``. + + path_str might also be a sentinel used by editable packages to + trigger other behaviors (see python/importlib_resources#311). + In that case, return None. """ - (dir,) = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir()) - return dir + dirs = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir()) + return next(dirs, None) @classmethod - def _candidate_paths(cls, path_str): + def _candidate_paths(cls, path_str: str) -> Iterator[abc.Traversable]: yield pathlib.Path(path_str) yield from cls._resolve_zip_path(path_str) @staticmethod - def _resolve_zip_path(path_str): + def _resolve_zip_path(path_str: str): for match in reversed(list(re.finditer(r'[\\/]', path_str))): with contextlib.suppress( FileNotFoundError, diff --git a/Lib/importlib/resources/simple.py b/Lib/importlib/resources/simple.py index 96f117fec62c102..2e75299b13aabf3 100644 --- a/Lib/importlib/resources/simple.py +++ b/Lib/importlib/resources/simple.py @@ -77,7 +77,7 @@ class ResourceHandle(Traversable): def __init__(self, parent: ResourceContainer, name: str): self.parent = parent - self.name = name # type: ignore + self.name = name # type: ignore[misc] def is_file(self): return True diff --git a/Lib/test/test_importlib/resources/_path.py b/Lib/test/test_importlib/resources/_path.py index 1f97c96146960d4..b144628cb73c77d 100644 --- a/Lib/test/test_importlib/resources/_path.py +++ b/Lib/test/test_importlib/resources/_path.py @@ -2,15 +2,44 @@ import functools from typing import Dict, Union +from typing import runtime_checkable +from typing import Protocol #### -# from jaraco.path 3.4.1 +# from jaraco.path 3.7.1 -FilesSpec = Dict[str, Union[str, bytes, 'FilesSpec']] # type: ignore +class Symlink(str): + """ + A string indicating the target of a symlink. + """ + + +FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] + + +@runtime_checkable +class TreeMaker(Protocol): + def __truediv__(self, *args, **kwargs): ... # pragma: no cover + + def mkdir(self, **kwargs): ... # pragma: no cover + + def write_text(self, content, **kwargs): ... # pragma: no cover + + def write_bytes(self, content): ... # pragma: no cover -def build(spec: FilesSpec, prefix=pathlib.Path()): + def symlink_to(self, target): ... # pragma: no cover + + +def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: + return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore[return-value] + + +def build( + spec: FilesSpec, + prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore[assignment] +): """ Build a set of files/directories, as described by the spec. @@ -25,21 +54,25 @@ def build(spec: FilesSpec, prefix=pathlib.Path()): ... "__init__.py": "", ... }, ... "baz.py": "# Some code", - ... } + ... "bar.py": Symlink("baz.py"), + ... }, + ... "bing": Symlink("foo"), ... } >>> target = getfixture('tmp_path') >>> build(spec, target) >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') '# Some code' + >>> target.joinpath('bing/bar.py').read_text(encoding='utf-8') + '# Some code' """ for name, contents in spec.items(): - create(contents, pathlib.Path(prefix) / name) + create(contents, _ensure_tree_maker(prefix) / name) @functools.singledispatch def create(content: Union[str, bytes, FilesSpec], path): path.mkdir(exist_ok=True) - build(content, prefix=path) # type: ignore + build(content, prefix=path) # type: ignore[arg-type] @create.register @@ -52,5 +85,10 @@ def _(content: str, path): path.write_text(content, encoding='utf-8') +@create.register +def _(content: Symlink, path): + path.symlink_to(content) + + # end from jaraco.path #### diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 08b840834dfd4b3..f903d3bba85ebc9 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -1,3 +1,7 @@ +import os +import pathlib +import py_compile +import shutil import textwrap import unittest import warnings @@ -7,6 +11,7 @@ from importlib import resources from importlib.resources.abc import Traversable from . import util +from test.support import os_helper, import_helper @contextlib.contextmanager @@ -55,6 +60,26 @@ class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase): class OpenNamespaceTests(FilesTests, util.DiskSetup, unittest.TestCase): MODULE = 'namespacedata01' + def test_non_paths_in_dunder_path(self): + """ + Non-path items in a namespace package's ``__path__`` are ignored. + + As reported in python/importlib_resources#311, some tools + like Setuptools, when creating editable packages, will inject + non-paths into a namespace package's ``__path__``, a + sentinel like + ``__editable__.sample_namespace-1.0.finder.__path_hook__`` + to cause the ``PathEntryFinder`` to be called when searching + for packages. In that case, resources should still be loadable. + """ + import namespacedata01 + + namespacedata01.__path__.append( + '__editable__.sample_namespace-1.0.finder.__path_hook__' + ) + + resources.files(namespacedata01) + class OpenNamespaceZipTests(FilesTests, util.ZipSetup, unittest.TestCase): ZIP_MODULE = 'namespacedata01' @@ -81,7 +106,7 @@ def test_module_resources(self): """ A module can have resources found adjacent to the module. """ - import mod + import mod # type: ignore[import-not-found] actual = resources.files(mod).joinpath('res.txt').read_text(encoding='utf-8') assert actual == self.spec['res.txt'] @@ -97,8 +122,8 @@ class ModuleFilesZipTests(DirectSpec, util.ZipSetup, ModulesFiles, unittest.Test class ImplicitContextFiles: set_val = textwrap.dedent( - """ - import importlib.resources as res + f""" + import {resources.__name__} as res val = res.files().joinpath('res.txt').read_text(encoding='utf-8') """ ) @@ -108,6 +133,10 @@ class ImplicitContextFiles: 'submod.py': set_val, 'res.txt': 'resources are the best', }, + 'frozenpkg': { + '__init__.py': set_val.replace(resources.__name__, 'c_resources'), + 'res.txt': 'resources are the best', + }, } def test_implicit_files_package(self): @@ -122,6 +151,32 @@ def test_implicit_files_submodule(self): """ assert importlib.import_module('somepkg.submod').val == 'resources are the best' + def _compile_importlib(self): + """ + Make a compiled-only copy of the importlib resources package. + """ + bin_site = self.fixtures.enter_context(os_helper.temp_dir()) + c_resources = pathlib.Path(bin_site, 'c_resources') + sources = pathlib.Path(resources.__file__).parent + shutil.copytree(sources, c_resources, ignore=lambda *_: ['__pycache__']) + + for dirpath, _, filenames in os.walk(c_resources): + for filename in filenames: + source_path = pathlib.Path(dirpath) / filename + cfile = source_path.with_suffix('.pyc') + py_compile.compile(source_path, cfile) + pathlib.Path.unlink(source_path) + self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site)) + + def test_implicit_files_with_compiled_importlib(self): + """ + Caller detection works for compiled-only resources module. + + python/cpython#123085 + """ + self._compile_importlib() + assert importlib.import_module('frozenpkg').val == 'resources are the best' + class ImplicitContextFilesDiskTests( DirectSpec, util.DiskSetup, ImplicitContextFiles, unittest.TestCase diff --git a/Misc/NEWS.d/next/Library/2024-09-12-14-24-25.gh-issue-123987.7_OD1p.rst b/Misc/NEWS.d/next/Library/2024-09-12-14-24-25.gh-issue-123987.7_OD1p.rst new file mode 100644 index 000000000000000..b110900e7efd335 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-12-14-24-25.gh-issue-123987.7_OD1p.rst @@ -0,0 +1,3 @@ +Fixed issue in NamespaceReader where a non-path item in a namespace path, +such as a sentinel added by an editable installer, would break resource +loading. diff --git a/Misc/NEWS.d/next/Library/2024-09-12-14-26-05.gh-issue-123085.Naekav.rst b/Misc/NEWS.d/next/Library/2024-09-12-14-26-05.gh-issue-123085.Naekav.rst new file mode 100644 index 000000000000000..1ea9137c3d72647 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-12-14-26-05.gh-issue-123085.Naekav.rst @@ -0,0 +1,2 @@ +Fixed issue in bare ``importlib.resources.files()`` when the library is +available as compiled only (no source).