From fde74b033c111d940c28bcd2bd8fe2a16f3f7db6 Mon Sep 17 00:00:00 2001 From: Bruno FS Ciconelle Date: Tue, 3 Sep 2024 10:46:42 -0300 Subject: [PATCH] feature: add support for zipped sdists and pinned url dependencies Modified pybuild-deps internals to rely more on pip internals, which gave us both support for url dependencies and zip archived packages for "free". This should fix (or at least cover most of) issues #188 and #187. We will need to watch closely when pip releases new versions in order fix breaking changes in its internal APIs. --- .../compile_build_dependencies.py | 5 +- src/pybuild_deps/finder.py | 9 ++- src/pybuild_deps/parsers/requirements.py | 4 +- src/pybuild_deps/source.py | 80 ++++++++++--------- src/pybuild_deps/utils.py | 25 +++--- tests/test_main.py | 27 ++++++- tests/test_utils.py | 8 +- 7 files changed, 98 insertions(+), 60 deletions(-) diff --git a/src/pybuild_deps/compile_build_dependencies.py b/src/pybuild_deps/compile_build_dependencies.py index 354061f..c705f3c 100644 --- a/src/pybuild_deps/compile_build_dependencies.py +++ b/src/pybuild_deps/compile_build_dependencies.py @@ -167,7 +167,10 @@ def _find_build_dependencies( """Find build dependencies for a given ireq.""" ireq_version = get_version(ireq) for build_dep in find_build_dependencies( - ireq.name, ireq_version, raise_setuppy_parsing_exc=False + ireq.name, + ireq_version, + raise_setuppy_parsing_exc=False, + pip_session=self.repository.session, ): # The original 'find_build_dependencies' function is very naive by design. # It only returns a simple list of strings representing builds dependencies. diff --git a/src/pybuild_deps/finder.py b/src/pybuild_deps/finder.py index 1aca43b..f36f831 100644 --- a/src/pybuild_deps/finder.py +++ b/src/pybuild_deps/finder.py @@ -4,6 +4,8 @@ import tarfile +from pip._internal.network.session import PipSession + from pybuild_deps.parsers.setup_py import SetupPyParsingError from .logger import log @@ -12,7 +14,10 @@ def find_build_dependencies( - package_name, version, raise_setuppy_parsing_exc=True + package_name, + version, + raise_setuppy_parsing_exc=True, + pip_session: PipSession | None = None, ) -> list[str]: """Find build dependencies for a given package.""" file_parser_map = { @@ -21,7 +26,7 @@ def find_build_dependencies( "setup.py": parse_setup_py, } log.debug(f"retrieving source for package {package_name}=={version}") - source_path = get_package_source(package_name, version) + source_path = get_package_source(package_name, version, pip_session=pip_session) build_dependencies = [] with tarfile.open(fileobj=source_path.open("rb")) as tarball: for file_name, parser in file_parser_map.items(): diff --git a/src/pybuild_deps/parsers/requirements.py b/src/pybuild_deps/parsers/requirements.py index 10d86e8..040156a 100644 --- a/src/pybuild_deps/parsers/requirements.py +++ b/src/pybuild_deps/parsers/requirements.py @@ -14,7 +14,7 @@ ) from pybuild_deps.exceptions import PyBuildDepsError -from pybuild_deps.utils import is_pinned_requirement +from pybuild_deps.utils import is_supported_requirement def parse_requirements( @@ -31,7 +31,7 @@ def parse_requirements( filename, session, finder=finder, options=options, constraint=constraint ): ireq = install_req_from_parsed_requirement(parsed_req, isolated=isolated) - if not is_pinned_requirement(ireq): + if not is_supported_requirement(ireq): raise PyBuildDepsError( f"requirement '{ireq}' is not exact " "(pybuild-tools only supports pinned dependencies)." diff --git a/src/pybuild_deps/source.py b/src/pybuild_deps/source.py index c04afeb..111f971 100644 --- a/src/pybuild_deps/source.py +++ b/src/pybuild_deps/source.py @@ -7,15 +7,21 @@ from urllib.parse import urlparse import requests -from pip._internal.operations.prepare import unpack_vcs_link +from pip._internal.exceptions import InstallationError +from pip._internal.network.download import Downloader +from pip._internal.network.session import PipSession +from pip._internal.operations.prepare import unpack_url from pip._internal.req.constructors import install_req_from_req_string +from pip._internal.utils.temp_dir import global_tempdir_manager from pybuild_deps.constants import CACHE_PATH from pybuild_deps.exceptions import PyBuildDepsError -from pybuild_deps.utils import is_pinned_vcs +from pybuild_deps.utils import is_supported_requirement -def get_package_source(package_name: str, version: str) -> Path: +def get_package_source( + package_name: str, version: str, pip_session: PipSession | None = None +) -> Path: """Get source code for a given package.""" parsed_url = urlparse(version) is_url = all((parsed_url.scheme, parsed_url.netloc)) @@ -39,51 +45,51 @@ def get_package_source(package_name: str, version: str) -> Path: elif error_path.exists(): raise NotImplementedError() - if is_url: - # assume url is pointing to VCS - if it's not an error will be thrown later - return retrieve_and_save_source_from_vcs( - package_name, version, tarball_path=tarball_path, error_path=error_path - ) + url = version if is_url else get_source_url_from_pypi(package_name, version) - return retrieve_and_save_source_from_pypi( - package_name, version, tarball_path=tarball_path, error_path=error_path + return retrieve_and_save_source_from_url( + package_name, + url, + tarball_path=tarball_path, + error_path=error_path, + pip_session=pip_session, ) -def retrieve_and_save_source_from_pypi( +def retrieve_and_save_source_from_url( package_name: str, - version: str, + url: str, *, tarball_path: Path, error_path: Path, + pip_session: PipSession = None, ): - """Retrieve package source from pypi and store it in a cache.""" - source_url = get_source_url_from_pypi(package_name, version) - response = requests.get(source_url, timeout=10) - response.raise_for_status() - tarball_path.parent.mkdir(parents=True, exist_ok=True) - tarball_path.write_bytes(response.content) - return tarball_path - - -def retrieve_and_save_source_from_vcs( - package_name: str, - version: str, - *, - tarball_path: Path, - error_path: Path, -): - """Retrieve package source from VCS.""" - ireq = install_req_from_req_string(f"{package_name} @ {version}") - if not is_pinned_vcs(ireq): + """Retrieve package source from URL.""" + ireq = install_req_from_req_string(f"{package_name} @ {url}") + if not is_supported_requirement(ireq): raise PyBuildDepsError( - f"Unsupported requirement ({ireq.name} @ {ireq.link}). Url requirements " - "must use a VCS scheme like 'git+https'." + f"Unsupported requirement '{ireq.req}'. Requirement must be either pinned " + "(==), a vcs link with sha or a direct url." ) - tarball_path.parent.mkdir(parents=True, exist_ok=True) - with TemporaryDirectory() as tmp_dir, tarfile.open(tarball_path, "w") as tarball: - unpack_vcs_link(ireq.link, tmp_dir, verbosity=0) - tarball.add(tmp_dir, arcname=package_name) + + pip_session = pip_session or PipSession() + pip_downloader = Downloader(pip_session, "") + + with global_tempdir_manager(), TemporaryDirectory() as tmp_dir: + try: + unpack_url( + ireq.link, + tmp_dir, + download=pip_downloader, + verbosity=0, + ) + except InstallationError as err: + raise PyBuildDepsError( + f"Unable to unpack '{ireq.req}'. Is {ireq.link} a python package?" + ) from err + tarball_path.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(tarball_path, "w:gz") as tarball: + tarball.add(tmp_dir, arcname=package_name) return tarball_path diff --git a/src/pybuild_deps/utils.py b/src/pybuild_deps/utils.py index abb3421..81df8a2 100644 --- a/src/pybuild_deps/utils.py +++ b/src/pybuild_deps/utils.py @@ -10,22 +10,21 @@ def get_version(ireq: InstallRequirement): """Get version string from InstallRequirement.""" - if not is_pinned_requirement(ireq): - raise PyBuildDepsError( - f"requirement '{ireq}' is not exact " - "(pybuild-tools only supports pinned dependencies)." - ) - if ireq.link and ireq.link.is_vcs: + if not is_supported_requirement(ireq): + raise PyBuildDepsError(f"requirement '{ireq}' is not exact.") + if ireq.req.url: return ireq.req.url return next(iter(ireq.specifier)).version -def is_pinned_requirement(ireq: InstallRequirement): - """Returns True if requirement is pinned or vcs.""" - return _is_pinned_requirement(ireq) or is_pinned_vcs(ireq) +def is_supported_requirement(ireq: InstallRequirement): + """Returns True if requirement is pinned, vcs poiting to a SHA or a direct url.""" + return ( + _is_pinned_requirement(ireq) or _is_pinned_vcs(ireq) or _is_non_vcs_link(ireq) + ) -def is_pinned_vcs(ireq: InstallRequirement): +def _is_pinned_vcs(ireq: InstallRequirement): """Check if given ireq is a pinned vcs dependency.""" if not ireq.link: return False @@ -37,3 +36,9 @@ def is_pinned_vcs(ireq: InstallRequirement): # some_project.git@da39a3ee5e6b4b0d3255bfef95601890afd80709 # https://pip.pypa.io/en/latest/topics/vcs-support/ return parts == 2 + + +def _is_non_vcs_link(ireq: InstallRequirement): + if not ireq.link: + return False + return not ireq.link.is_vcs diff --git a/tests/test_main.py b/tests/test_main.py index 1320cb2..9d019b6 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -57,6 +57,27 @@ def test_main_succeeds(runner: CliRunner) -> None: "setuptools-rust>=0.11.4", ], ), + ( + "cryptography", + "git+https://github.com/pyca/cryptography@41.0.5", + [ + "setuptools>=61.0.0", + "wheel", + "cffi>=1.12; platform_python_implementation != 'PyPy'", + "setuptools-rust>=0.11.4", + ], + ), + ( + "cryptography", + "https://github.com/pyca/cryptography/archive/refs/tags/43.0.0.tar.gz", + [ + "maturin>=1,<2", + "cffi>=1.12; platform_python_implementation != 'PyPy'", + "setuptools", + ], + ), + ("azure-identity", "1.14.1", []), + ("debugpy", "1.8.5", ["wheel", "setuptools"]), ], ) def test_find_build_deps( @@ -68,7 +89,7 @@ def test_find_build_deps( assert result.exit_code == 0 assert result.stdout.splitlines() == expected_deps assert cache.exists() - # repeating the same test to cover a cached version + # repeating the same test to cover the cached version result = runner.invoke(main.cli, args=["find-build-deps", package_name, version]) assert result.exit_code == 0 assert result.stdout.splitlines() == expected_deps @@ -90,12 +111,12 @@ def test_find_build_deps( ( "some-package", "git+https://example.com", - "Unsupported requirement (some-package @ git+https://example.com). Url requirements must use a VCS scheme like 'git+https'.", # noqa: E501 + "Unsupported requirement 'some-package@ git+https://example.com'. Requirement must be either pinned (==), a vcs link with sha or a direct url.", # noqa: E501 ), ( "cryptography", "git+https://github.com/pyca/cryptography", - "Unsupported requirement (cryptography @ git+https://github.com/pyca/cryptography). Url requirements must use a VCS scheme like 'git+https'.", # noqa: E501 + "Unsupported requirement 'cryptography@ git+https://github.com/pyca/cryptography'. Requirement must be either pinned (==), a vcs link with sha or a direct url.", # noqa: E501 ), ], ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 125fdcc..9b8ced0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,7 +3,7 @@ import pytest from pip._internal.req.constructors import install_req_from_req_string -from pybuild_deps.utils import get_version, is_pinned_requirement +from pybuild_deps.utils import get_version, is_supported_requirement @pytest.mark.parametrize( @@ -16,7 +16,7 @@ def test_is_pinned_or_vcs(req): """Ensure pinned or vcs dependencies are properly detected.""" ireq = install_req_from_req_string(req) - assert is_pinned_requirement(ireq) + assert is_supported_requirement(ireq) @pytest.mark.parametrize( @@ -24,15 +24,13 @@ def test_is_pinned_or_vcs(req): [ "requests>1.2.3", "requests @ git+https://github.com/psf/requests", - "requests @ https://example.com", - "requests @ https://github.com/psf/requests@some-commit-sha", "requests", ], ) def test_not_pinned_or_vcs(req): """Negative test for 'is_pinned_or_vcs'.""" ireq = install_req_from_req_string(req) - assert not is_pinned_requirement(ireq) + assert not is_supported_requirement(ireq) def test_get_version_url():