feature: add support for zipped sdists and pinned url dependencies

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pybuild-deps"
-version = "0.3.0"
+version = "0.4.0"
 description = "A simple tool for detection of PEP-517 build dependencies."
 authors = ["Bruno Ciconelle <[email protected]>"]
 license = "GPL-3.0"

src/pybuild_deps/compile_build_dependencies.py

@@ -167,7 +167,10 @@ def _find_build_dependencies(
         """Find build dependencies for a given ireq."""
         ireq_version = get_version(ireq)
         for build_dep in find_build_dependencies(
-            ireq.name, ireq_version, raise_setuppy_parsing_exc=False
+            ireq.name,
+            ireq_version,
+            raise_setuppy_parsing_exc=False,
+            pip_session=self.repository.session,
         ):
             # The original 'find_build_dependencies' function is very naive by design.
             # It only returns a simple list of strings representing builds dependencies.

src/pybuild_deps/finder.py

@@ -4,6 +4,8 @@
 
 import tarfile
 
+from pip._internal.network.session import PipSession
+
 from pybuild_deps.parsers.setup_py import SetupPyParsingError
 
 from .logger import log
@@ -12,7 +14,10 @@
 
 
 def find_build_dependencies(
-    package_name, version, raise_setuppy_parsing_exc=True
+    package_name,
+    version,
+    raise_setuppy_parsing_exc=True,
+    pip_session: PipSession | None = None,
 ) -> list[str]:
     """Find build dependencies for a given package."""
     file_parser_map = {
@@ -21,7 +26,7 @@ def find_build_dependencies(
         "setup.py": parse_setup_py,
     }
     log.debug(f"retrieving source for package {package_name}=={version}")
-    source_path = get_package_source(package_name, version)
+    source_path = get_package_source(package_name, version, pip_session=pip_session)
     build_dependencies = []
     with tarfile.open(fileobj=source_path.open("rb")) as tarball:
         for file_name, parser in file_parser_map.items():

src/pybuild_deps/parsers/requirements.py

@@ -14,7 +14,7 @@
 )
 
 from pybuild_deps.exceptions import PyBuildDepsError
-from pybuild_deps.utils import is_pinned_requirement
+from pybuild_deps.utils import is_supported_requirement
 
 
 def parse_requirements(
@@ -31,7 +31,7 @@ def parse_requirements(
         filename, session, finder=finder, options=options, constraint=constraint
     ):
         ireq = install_req_from_parsed_requirement(parsed_req, isolated=isolated)
-        if not is_pinned_requirement(ireq):
+        if not is_supported_requirement(ireq):
             raise PyBuildDepsError(
                 f"requirement '{ireq}' is not exact "
                 "(pybuild-tools only supports pinned dependencies)."

src/pybuild_deps/source.py

@@ -1,21 +1,29 @@
 """Get source code for a given package."""
 
+from __future__ import annotations
+
 import logging
 import tarfile
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from urllib.parse import urlparse
 
 import requests
-from pip._internal.operations.prepare import unpack_vcs_link
+from pip._internal.exceptions import InstallationError
+from pip._internal.network.download import Downloader
+from pip._internal.network.session import PipSession
+from pip._internal.operations.prepare import unpack_url
 from pip._internal.req.constructors import install_req_from_req_string
+from pip._internal.utils.temp_dir import global_tempdir_manager
 
 from pybuild_deps.constants import CACHE_PATH
 from pybuild_deps.exceptions import PyBuildDepsError
-from pybuild_deps.utils import is_pinned_vcs
+from pybuild_deps.utils import is_supported_requirement
 
 
-def get_package_source(package_name: str, version: str) -> Path:
+def get_package_source(
+    package_name: str, version: str, pip_session: PipSession | None = None
+) -> Path:
     """Get source code for a given package."""
     parsed_url = urlparse(version)
     is_url = all((parsed_url.scheme, parsed_url.netloc))
@@ -39,51 +47,51 @@ def get_package_source(package_name: str, version: str) -> Path:
     elif error_path.exists():
         raise NotImplementedError()
 
-    if is_url:
-        # assume url is pointing to VCS - if it's not an error will be thrown later
-        return retrieve_and_save_source_from_vcs(
-            package_name, version, tarball_path=tarball_path, error_path=error_path
-        )
+    url = version if is_url else get_source_url_from_pypi(package_name, version)
 
-    return retrieve_and_save_source_from_pypi(
-        package_name, version, tarball_path=tarball_path, error_path=error_path
+    return retrieve_and_save_source_from_url(
+        package_name,
+        url,
+        tarball_path=tarball_path,
+        error_path=error_path,
+        pip_session=pip_session,
     )
 
 
-def retrieve_and_save_source_from_pypi(
-    package_name: str,
-    version: str,
-    *,
-    tarball_path: Path,
-    error_path: Path,
-):
-    """Retrieve package source from pypi and store it in a cache."""
-    source_url = get_source_url_from_pypi(package_name, version)
-    response = requests.get(source_url, timeout=10)
-    response.raise_for_status()
-    tarball_path.parent.mkdir(parents=True, exist_ok=True)
-    tarball_path.write_bytes(response.content)
-    return tarball_path
-
-
-def retrieve_and_save_source_from_vcs(
+def retrieve_and_save_source_from_url(
     package_name: str,
-    version: str,
+    url: str,
     *,
     tarball_path: Path,
     error_path: Path,
+    pip_session: PipSession = None,
 ):
-    """Retrieve package source from VCS."""
-    ireq = install_req_from_req_string(f"{package_name} @ {version}")
-    if not is_pinned_vcs(ireq):
+    """Retrieve package source from URL."""
+    ireq = install_req_from_req_string(f"{package_name} @ {url}")
+    if not is_supported_requirement(ireq):
         raise PyBuildDepsError(
-            f"Unsupported requirement ({ireq.name} @ {ireq.link}). Url requirements "
-            "must use a VCS scheme like 'git+https'."
+            f"Unsupported requirement '{ireq.req}'. Requirement must be either pinned "
+            "(==), a vcs link with sha or a direct url."
         )
-    tarball_path.parent.mkdir(parents=True, exist_ok=True)
-    with TemporaryDirectory() as tmp_dir, tarfile.open(tarball_path, "w") as tarball:
-        unpack_vcs_link(ireq.link, tmp_dir, verbosity=0)
-        tarball.add(tmp_dir, arcname=package_name)
+
+    pip_session = pip_session or PipSession()
+    pip_downloader = Downloader(pip_session, "")
+
+    with global_tempdir_manager(), TemporaryDirectory() as tmp_dir:
+        try:
+            unpack_url(
+                ireq.link,
+                tmp_dir,
+                download=pip_downloader,
+                verbosity=0,
+            )
+        except InstallationError as err:
+            raise PyBuildDepsError(
+                f"Unable to unpack '{ireq.req}'. Is '{ireq.link}' a python package?"
+            ) from err
+        tarball_path.parent.mkdir(parents=True, exist_ok=True)
+        with tarfile.open(tarball_path, "w:gz") as tarball:
+            tarball.add(tmp_dir, arcname=package_name)
     return tarball_path
 
 

src/pybuild_deps/utils.py

@@ -10,22 +10,21 @@
 
 def get_version(ireq: InstallRequirement):
     """Get version string from InstallRequirement."""
-    if not is_pinned_requirement(ireq):
-        raise PyBuildDepsError(
-            f"requirement '{ireq}' is not exact "
-            "(pybuild-tools only supports pinned dependencies)."
-        )
-    if ireq.link and ireq.link.is_vcs:
+    if not is_supported_requirement(ireq):
+        raise PyBuildDepsError(f"requirement '{ireq}' is not exact.")
+    if ireq.req.url:
         return ireq.req.url
     return next(iter(ireq.specifier)).version
 
 
-def is_pinned_requirement(ireq: InstallRequirement):
-    """Returns True if requirement is pinned or vcs."""
-    return _is_pinned_requirement(ireq) or is_pinned_vcs(ireq)
+def is_supported_requirement(ireq: InstallRequirement):
+    """Returns True if requirement is pinned, vcs poiting to a SHA or a direct url."""
+    return (
+        _is_pinned_requirement(ireq) or _is_pinned_vcs(ireq) or _is_non_vcs_link(ireq)
+    )
 
 
-def is_pinned_vcs(ireq: InstallRequirement):
+def _is_pinned_vcs(ireq: InstallRequirement):
     """Check if given ireq is a pinned vcs dependency."""
     if not ireq.link:
         return False
@@ -37,3 +36,9 @@ def is_pinned_vcs(ireq: InstallRequirement):
     # some_project.git@da39a3ee5e6b4b0d3255bfef95601890afd80709
     # https://pip.pypa.io/en/latest/topics/vcs-support/
     return parts == 2
+
+
+def _is_non_vcs_link(ireq: InstallRequirement):
+    if not ireq.link:
+        return False
+    return not ireq.link.is_vcs

tests/test_main.py

@@ -57,6 +57,27 @@ def test_main_succeeds(runner: CliRunner) -> None:
                 "setuptools-rust>=0.11.4",
             ],
         ),
+        (
+            "cryptography",
+            "git+https://github.com/pyca/[email protected]",
+            [
+                "setuptools>=61.0.0",
+                "wheel",
+                "cffi>=1.12; platform_python_implementation != 'PyPy'",
+                "setuptools-rust>=0.11.4",
+            ],
+        ),
+        (
+            "cryptography",
+            "https://github.com/pyca/cryptography/archive/refs/tags/43.0.0.tar.gz",
+            [
+                "maturin>=1,<2",
+                "cffi>=1.12; platform_python_implementation != 'PyPy'",
+                "setuptools",
+            ],
+        ),
+        ("azure-identity", "1.14.1", []),
+        ("debugpy", "1.8.5", ["wheel", "setuptools"]),
     ],
 )
 def test_find_build_deps(
@@ -68,7 +89,7 @@ def test_find_build_deps(
     assert result.exit_code == 0
     assert result.stdout.splitlines() == expected_deps
     assert cache.exists()
-    # repeating the same test to cover a cached version
+    # repeating the same test to cover the cached version
     result = runner.invoke(main.cli, args=["find-build-deps", package_name, version])
     assert result.exit_code == 0
     assert result.stdout.splitlines() == expected_deps
@@ -90,12 +111,17 @@ def test_find_build_deps(
         (
             "some-package",
             "git+https://example.com",
-            "Unsupported requirement (some-package @ git+https://example.com). Url requirements must use a VCS scheme like 'git+https'.",  # noqa: E501
+            "Unsupported requirement 'some-package@ git+https://example.com'. Requirement must be either pinned (==), a vcs link with sha or a direct url.",  # noqa: E501
+        ),
+        (
+            "some-package",
+            "https://example.com",
+            "Unable to unpack 'some-package@ https://example.com'. Is 'https://example.com' a python package?",  # noqa: E501
         ),
         (
             "cryptography",
             "git+https://github.com/pyca/cryptography",
-            "Unsupported requirement (cryptography @ git+https://github.com/pyca/cryptography). Url requirements must use a VCS scheme like 'git+https'.",  # noqa: E501
+            "Unsupported requirement 'cryptography@ git+https://github.com/pyca/cryptography'. Requirement must be either pinned (==), a vcs link with sha or a direct url.",  # noqa: E501
         ),
     ],
 )

tests/test_utils.py

@@ -3,7 +3,7 @@
 import pytest
 from pip._internal.req.constructors import install_req_from_req_string
 
-from pybuild_deps.utils import get_version, is_pinned_requirement
+from pybuild_deps.utils import get_version, is_supported_requirement
 
 
 @pytest.mark.parametrize(
@@ -16,23 +16,21 @@
 def test_is_pinned_or_vcs(req):
     """Ensure pinned or vcs dependencies are properly detected."""
     ireq = install_req_from_req_string(req)
-    assert is_pinned_requirement(ireq)
+    assert is_supported_requirement(ireq)
 
 
 @pytest.mark.parametrize(
     "req",
     [
         "requests>1.2.3",
         "requests @ git+https://github.com/psf/requests",
-        "requests @ https://example.com",
-        "requests @ https://github.com/psf/requests@some-commit-sha",
         "requests",
     ],
 )
 def test_not_pinned_or_vcs(req):
     """Negative test for 'is_pinned_or_vcs'."""
     ireq = install_req_from_req_string(req)
-    assert not is_pinned_requirement(ireq)
+    assert not is_supported_requirement(ireq)
 
 
 def test_get_version_url():