feature: Add 'compile' functionality

Combine `find-build-deps` with the powerful pip-compile (from pip-tools)
to craft a command that can recursively searchs for build dependencies
and generate a pinned requirements file of those.

README.md

@@ -19,13 +19,19 @@
 [pre-commit]: https://github.com/pre-commit/pre-commit
 [black]: https://github.com/psf/black
 
-## Features
+CLI tools to help dealing with python build dependencies. It aims to complement
+tools that can pin dependencies like `pip-tools` and `poetry`.
+For users relying exclusively on python wheels, those tools are more than enough.
+However, for users building applications from source, finding and pinning build dependencies
+is required for reproducible builds.
 
-- TODO
+`pybuild-tools` might be useful for developers that need to explicitly declare
+**all** dependencies for compliance reasons or supply chain concerns.
 
-## Requirements
+## Features
 
-- TODO
+- find build dependencies for a given python package
+- generate pinned build requirements from requirements.txt files.
 
 ## Installation
 

conftest.py

@@ -1,6 +1,18 @@
 """pytest configuration."""
 
 
+import pytest
+
+
 def pytest_configure(config):
     """Configure pytst session."""
     config.addinivalue_line("markers", "e2e: end to end tests.")
+
+
+@pytest.fixture
+def cache(mocker, tmp_path):
+    """Mock pybuild-deps cache."""
+    mocked_cache = tmp_path / "cache"
+    mocker.patch("pybuild_deps.constants.CACHE_PATH", mocked_cache)
+    mocker.patch("pybuild_deps.source.CACHE_PATH", mocked_cache)
+    yield mocked_cache

pyproject.toml

@@ -19,6 +19,7 @@ click = ">=8.0.1"
 tomli = { version = "^2.0.1", python = "<3.11" }
 xdg = "*"
 requests = "*"
+pip-tools = "^7.3.0"
 
 [tool.poetry.group.dev.dependencies]
 Pygments = ">=2.10.0"
@@ -40,6 +41,7 @@ xdoctest = { extras = ["colors"], version = ">=0.15.10" }
 myst-parser = { version = ">=0.16.1" }
 ruff = "^0.0.235"
 pytest-mock = "^3.10.0"
+ipykernel = "^6.25.2"
 
 [tool.poetry.scripts]
 pybuild-deps = "pybuild_deps.__main__:cli"
@@ -73,7 +75,7 @@ select = [
     "UP",  # pyupgrade
     "W",   # pycodestyle
 ]
-ignore = ["D203"]
+ignore = ["D203", "D107", "D212"]
 src = ["src", "tests"]
 
 [tool.ruff.per-file-ignores]

src/pybuild_deps/__main__.py

@@ -1,28 +1,33 @@
 """Command-line interface."""
-import logging
 
 import click
 
+from .finder import find_build_dependencies
+from .logger import log
+from .scripts import compile
+
 
 @click.group()
-@click.version_option()
-@click.option("--log-level", default="ERROR")
-def cli(log_level) -> None:
+@click.version_option(package_name="pybuild-deps")
+def cli() -> None:
     """Entrypoint for PyBuild Deps."""
-    logging.basicConfig(level=log_level)
+    log.as_library = False  # pragma: no cover
 
 
 @cli.command()
 @click.argument("package-name")
-@click.argument("version")
-def find_build_deps(package_name, version):
+@click.argument("package-version")
+@click.option("-v", "--verbose", count=True, help="Show more output")
+def find_build_deps(package_name, package_version, verbose):
     """Find build dependencies for given package."""
-    from pybuild_deps.find_build_dependencies import find_build_dependencies
+    log.verbosity = verbose
 
-    deps = find_build_dependencies(package_name=package_name, version=version)
+    deps = find_build_dependencies(package_name=package_name, version=package_version)
     for dep in deps:
         click.echo(dep)
 
 
+cli.add_command(compile.cli, "compile")
+
 if __name__ == "__main__":
     cli(prog_name="pybuild-deps")  # pragma: no cover

src/pybuild_deps/compile_build_dependencies.py

@@ -0,0 +1,73 @@
+"""
+compile build dependencies module.
+
+Heavily rely on pip-tools BacktrackingResolver and our own find_build_deps
+to recursively find all build dependencies and generate a pinned list
+of build dependencies.
+"""
+
+from __future__ import annotations
+
+from typing import Iterable
+
+from pip._internal.req import InstallRequirement
+from pip._internal.req.constructors import install_req_from_req_string
+from piptools.repositories import PyPIRepository
+from piptools.resolver import BacktrackingResolver
+from piptools.utils import (
+    is_pinned_requirement,
+)
+
+from .exceptions import PyBuildDepsError
+from .finder import find_build_dependencies
+
+
+def get_version(ireq: InstallRequirement):
+    """Get version string from InstallRequirement."""
+    if not is_pinned_requirement(ireq):
+        raise PyBuildDepsError(
+            f"requirement '{ireq}' is not exact "
+            "(pybuild-tools only supports pinned dependencies)."
+        )
+    return next(iter(ireq.specifier)).version
+
+
+class BuildDependencyCompiler:
+    """Resolve exact build dependencies."""
+
+    def __init__(self, repository: PyPIRepository) -> None:
+        self.repository = repository
+        self.resolver = None
+
+    def resolve(
+        self,
+        install_requirements: Iterable[InstallRequirement],
+        constraints: Iterable[InstallRequirement] | None = None,
+    ) -> set[InstallRequirement]:
+        """Resolve all build dependencies for a given set of dependencies."""
+        constraints: list[InstallRequirement] = list(constraints) if constraints else []
+        constraint_qty = len(constraints)
+        for req in install_requirements:
+            req_version = get_version(req)
+            raw_build_dependencies = find_build_dependencies(req.name, req_version)
+            for raw_build_req in raw_build_dependencies:
+                build_req = install_req_from_req_string(
+                    raw_build_req, comes_from=req.name
+                )
+                constraints.append(build_req)
+        # override resolver - we only want the latest and greatest
+        self.resolver = BacktrackingResolver(
+            constraints=constraints,
+            existing_constraints={},
+            repository=self.repository,
+            allow_unsafe=True,
+        )
+        build_dependencies = self.resolver.resolve()
+        # dependencies of build dependencies might have their own build dependencies,
+        # so let's recursively search for those.
+        while len(build_dependencies) != constraint_qty:
+            constraint_qty = len(build_dependencies)
+            build_dependencies = self.resolve(
+                build_dependencies, constraints=build_dependencies
+            )
+        return build_dependencies

src/pybuild_deps/constants.py

@@ -0,0 +1,7 @@
+"""constants for pybuild deps."""
+
+from pip._internal.utils.appdirs import user_cache_dir
+from xdg import xdg_cache_home
+
+CACHE_PATH = xdg_cache_home() / "pybuild-deps"
+PIP_CACHE_DIR = user_cache_dir("pybuild-deps")

src/pybuild_deps/exceptions.py

@@ -0,0 +1,5 @@
+"""custom exceptions for pybuild-deps."""
+
+
+class PyBuildDepsError(Exception):
+    """Custom exception for pybuild-deps."""

src/pybuild_deps/find_build_dependencies.py → src/pybuild_deps/finder.py

@@ -1,10 +1,10 @@
 """Find build dependencies of a python package."""
 
-import logging
 import tarfile
 
-from pybuild_deps.get_package_source import get_package_source
-from pybuild_deps.parsers import parse_pyproject_toml, parse_setup_cfg, parse_setup_py
+from .logger import log
+from .parsers import parse_pyproject_toml, parse_setup_cfg, parse_setup_py
+from .source import get_package_source
 
 
 def find_build_dependencies(package_name, version):
@@ -14,7 +14,7 @@ def find_build_dependencies(package_name, version):
         "setup.cfg": parse_setup_cfg,
         "setup.py": parse_setup_py,
     }
-    logging.info("retrieving source for package %s==%s", package_name, version)
+    log.debug(f"retrieving source for package {package_name}=={version}")
     source_path = get_package_source(package_name, version)
     build_dependencies = []
     with tarfile.open(fileobj=source_path.open("rb")) as tarball:
@@ -23,18 +23,12 @@ def find_build_dependencies(package_name, version):
             try:
                 file = tarball.extractfile(f"{root_dir}/{file_name}")
             except KeyError:
-                logging.info(
-                    "%s file not found for package %s==%s",
-                    file_name,
-                    package_name,
-                    version,
+                log.debug(
+                    f"{file_name} file not found for package {package_name}=={version}",
                 )
                 continue
-            logging.info(
-                "parsing file %s for package %s==%s",
-                file_name,
-                package_name,
-                version,
+            log.debug(
+                f"parsing file {file_name} for package {package_name}=={version}",
             )
             build_dependencies += parser(file.read().decode())
     return build_dependencies

src/pybuild_deps/logger.py

@@ -0,0 +1,74 @@
+"""custom logger for pybuild-deps."""
+# ruff: noqa: D102
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import click
+
+logging.basicConfig()
+_logger = logging.getLogger("pybuild-deps")
+
+
+class Logger:
+    """
+    Custom logger for pybuild-deps.
+
+    When invoked as a CLI, will use click to log messages. Otherwise
+    will use default python logger.
+    """
+
+    def __init__(self, verbosity: int = 0):
+        self._verbosity = verbosity
+        self.as_library = True
+
+    @property
+    def verbosity(self):
+        return self._verbosity
+
+    @verbosity.setter
+    def verbosity(self, value):
+        self._verbosity = value
+        if self._verbosity < 0:
+            _logger.setLevel(logging.WARNING)
+        if self._verbosity == 0:
+            _logger.setLevel(logging.INFO)
+        if self._verbosity >= 1:
+            _logger.setLevel(logging.DEBUG)
+
+    def log(self, level: int, message: str, *args: Any, **kwargs: Any) -> None:
+        if self.as_library:
+            _logger.log(level, message, *args, **kwargs)
+        else:
+            self._cli_log(level, message, args, kwargs)
+
+    def _cli_log(self, level, message, args, kwargs):
+        kwargs.setdefault("err", True)
+        if level >= logging.ERROR:
+            kwargs.setdefault("fg", "red")
+        elif level >= logging.WARNING:
+            kwargs.setdefault("fg", "yellow")
+        elif level >= logging.INFO and self.verbosity < 0:
+            return
+        elif level >= logging.DEBUG and self.verbosity < 1:
+            return
+        elif level <= logging.DEBUG and self.verbosity >= 1:
+            kwargs.setdefault("fg", "blue")
+        click.secho(message, *args, **kwargs)
+
+    def debug(self, message: str, *args: Any, **kwargs: Any) -> None:
+        self.log(logging.DEBUG, message, *args, **kwargs)
+
+    def info(self, message: str, *args: Any, **kwargs: Any) -> None:
+        self.log(logging.INFO, message, *args, **kwargs)
+
+    def warning(self, message: str, *args: Any, **kwargs: Any) -> None:
+        self.log(logging.WARNING, message, *args, **kwargs)
+
+    def error(self, message: str, *args: Any, **kwargs: Any) -> None:
+        self.log(logging.ERROR, message, *args, **kwargs)
+
+
+log = Logger()

src/pybuild_deps/parsers/__init__.py

@@ -8,6 +8,7 @@
     import tomli as toml
 
 
+from .requirements import parse_requirements
 from .setup_py import parse_setup_py  # imported here for convenience
 
 

src/pybuild_deps/parsers/requirements.py

@@ -0,0 +1,40 @@
+"""parser for requirement.txt files."""
+
+from __future__ import annotations
+
+import optparse
+from typing import Generator
+
+from pip._internal.index.package_finder import PackageFinder
+from pip._internal.network.session import PipSession
+from pip._internal.req import InstallRequirement
+from pip._internal.req import parse_requirements as _parse_requirements
+from pip._internal.req.constructors import (
+    install_req_from_parsed_requirement,
+)
+from piptools.utils import (
+    is_pinned_requirement,
+)
+
+from pybuild_deps.exceptions import PyBuildDepsError
+
+
+def parse_requirements(
+    filename: str,
+    session: PipSession,
+    finder: PackageFinder | None = None,
+    options: optparse.Values | None = None,
+    constraint: bool = False,
+    isolated: bool = False,
+) -> Generator[InstallRequirement]:
+    """Thin wrapper around pip's `parse_requirements`."""
+    for parsed_req in _parse_requirements(
+        filename, session, finder=finder, options=options, constraint=constraint
+    ):
+        ireq = install_req_from_parsed_requirement(parsed_req, isolated=isolated)
+        if not is_pinned_requirement(ireq):
+            raise PyBuildDepsError(
+                f"requirement '{ireq}' is not exact "
+                "(pybuild-tools only supports pinned dependencies)."
+            )
+        yield ireq