From 4da0d92f5806f9e2b50e4f8ac97b1775a217052e Mon Sep 17 00:00:00 2001 From: SJ Date: Thu, 9 May 2024 13:33:56 -0400 Subject: [PATCH] [mypyc] Using built-in `mypyc` and just showing fib example (#99) --- examples/python/hellofib/BUILD.pants | 15 ++ examples/python/hellofib/hellofib/lib.py | 12 + examples/python/hellofib/hellofib/main.py | 16 +- examples/python/hellofib/setup.py | 8 + examples/python/hellofib/setup.py.bak | 14 - pants-plugins/experimental/mypyc/BUILD | 1 - pants-plugins/experimental/mypyc/README.md | 73 +----- pants-plugins/experimental/mypyc/__init__.py | 0 pants-plugins/experimental/mypyc/register.py | 10 - pants-plugins/experimental/mypyc/rules.py | 240 ------------------ .../experimental/mypyc/target_types.py | 34 --- pants.toml | 2 +- 12 files changed, 40 insertions(+), 385 deletions(-) create mode 100644 examples/python/hellofib/hellofib/lib.py create mode 100644 examples/python/hellofib/setup.py delete mode 100644 examples/python/hellofib/setup.py.bak delete mode 100644 pants-plugins/experimental/mypyc/BUILD delete mode 100644 pants-plugins/experimental/mypyc/__init__.py delete mode 100644 pants-plugins/experimental/mypyc/register.py delete mode 100644 pants-plugins/experimental/mypyc/rules.py delete mode 100644 pants-plugins/experimental/mypyc/target_types.py diff --git a/examples/python/hellofib/BUILD.pants b/examples/python/hellofib/BUILD.pants index af3f054..b4b31c6 100644 --- a/examples/python/hellofib/BUILD.pants +++ b/examples/python/hellofib/BUILD.pants @@ -1,5 +1,20 @@ python_sources(name="libhellofib", sources=["**/*.py"]) +python_distribution( + name="hellofib-dist", + dependencies=[":libhellofib"], + provides=python_artifact(name="hellofib-dist", version="0.0.1"), + generate_setup=False, + sdist=False, + uses_mypyc=True, +) + +pex_binary( + name="hellofib-mypyc", + entry_point="hellofib.main", + dependencies=[":hellofib-dist"], +) + pex_binary( name="hellofib-pex", entry_point="hellofib.main", diff --git a/examples/python/hellofib/hellofib/lib.py b/examples/python/hellofib/hellofib/lib.py new file mode 100644 index 0000000..eb4c002 --- /dev/null +++ b/examples/python/hellofib/hellofib/lib.py @@ -0,0 +1,12 @@ +import time + + +def fib() -> None: + def fib(n: int) -> int: + if n <= 1: + return n + return fib(n - 2) + fib(n - 1) + + start_time = time.time() + fib(34) + print(f"Calculating fibs took {time.time() - start_time} seconds") diff --git a/examples/python/hellofib/hellofib/main.py b/examples/python/hellofib/hellofib/main.py index ab9a38f..9216363 100644 --- a/examples/python/hellofib/hellofib/main.py +++ b/examples/python/hellofib/hellofib/main.py @@ -1,18 +1,6 @@ -import time - - -def main() -> None: - def fib(n: int) -> int: - if n <= 1: - return n - return fib(n - 2) + fib(n - 1) - - start_time = time.time() - fib(34) - print(f"Calculating fibs took {time.time() - start_time} seconds") - +from hellofib.lib import fib # In .bzl config, setting python_config.run_module = "hellofib.main" should cause this to run as the entry point if __name__ == "__main__": print("Launching HelloFib from __main__") - main() + fib() diff --git a/examples/python/hellofib/setup.py b/examples/python/hellofib/setup.py new file mode 100644 index 0000000..56eab38 --- /dev/null +++ b/examples/python/hellofib/setup.py @@ -0,0 +1,8 @@ +from mypyc.build import mypycify # pants: no-infer-dep +from setuptools import setup + +setup( + name="hellofib", + packages=["hellofib"], + ext_modules=mypycify(["hellofib/__init__.py", "hellofib/lib.py"]), +) diff --git a/examples/python/hellofib/setup.py.bak b/examples/python/hellofib/setup.py.bak deleted file mode 100644 index a682176..0000000 --- a/examples/python/hellofib/setup.py.bak +++ /dev/null @@ -1,14 +0,0 @@ -from setuptools import setup - -from mypyc.build import mypycify - -setup( - name="hellofib", - packages=["hellofib"], - ext_modules=mypycify( - [ - "hellofib/__init__.py", - "hellofib/main.py", - ] - ), -) diff --git a/pants-plugins/experimental/mypyc/BUILD b/pants-plugins/experimental/mypyc/BUILD deleted file mode 100644 index db46e8d..0000000 --- a/pants-plugins/experimental/mypyc/BUILD +++ /dev/null @@ -1 +0,0 @@ -python_sources() diff --git a/pants-plugins/experimental/mypyc/README.md b/pants-plugins/experimental/mypyc/README.md index d88fc7c..c12f7cd 100644 --- a/pants-plugins/experimental/mypyc/README.md +++ b/pants-plugins/experimental/mypyc/README.md @@ -1,74 +1,5 @@ # pants-mypyc-plugin -## How to use +# Similar solution mainlined in 2.13 via [PR #15380](https://github.com/pantsbuild/pants/pull/15380) by Benjy -1. This plugin requires changes to the Pants source code, so you'll need to use the [pants_from_sources](https://www.pantsbuild.org/docs/running-pants-from-sources#running-pants-from-sources-in-other-repos) approach against https://github.com/sureshjoshi/pants/tree/mypyc-support - -2. Add `mypyc` support to the setuptools process: - - ```toml - # pants.toml - [setuptools] - extra_requirements = ["wheel", "mypy"] - lockfile = "build-support/setuptools.txt" - ``` - -3. In your `BUILD` file, use the new `mypyc_python_distribution` target (which is identical to `python_distribution` with a new name) - ```python - mypyc_python_distribution( - name="hellofib-dist", - dependencies=[":libhellofib"], - wheel=True, - sdist=False, - provides=setup_py( - name="hellofib-dist", - version="0.0.1", - description="A distribution for the hello fib library.", - ), - # Setting this True or False depends on the next step - generate_setup = True, - ) - ``` - -4. If you want to test using your own `setup.py`, place one in your source root and set `generate_setup = False` in the `BUILD` file - ```python - # setup.py - from setuptools import setup - - from mypyc.build import mypycify - - setup( - name="hellofib", - packages=["hellofib"], - ext_modules=mypycify( - [ - "hellofib/__init__.py", - "hellofib/main.py", - ] - ), - ) - ``` - -5. If you want the plugin to auto-generate your `setup.py`, set `generate_setup = True` in the `BUILD` file (or remove the line, since `True` is the default). The plugin will pass all of the source files from the dependencies into `mypycify` - -## Examples/Libraries to test - -All of the examples that can compile with `mypyc_python_distribution` have that target applied. There are some outstanding examples which fail when `mypy` tries to compile them. - -To quickly see which examples are supported, type the following: `./pants_from_sources filter --target-type=mypyc_python_distribution ::` - -For example, with `hellofib`: -```bash -./pants_from_sources --version -./pants_from_sources package hellofib:hellofib-dist - -pip install dist/hellofib-{whatever}-.whl --force-reinstall -python -c "from hellofib.main import main; main()" -``` - -## Next Steps - -1. Add support for multiple dependency targets (only tested/working with one `python_sources` dependency) -2. Handle use case where `ext_modules` are already specified in the SetupKwargs -3. Figure out better API for SetupPyContentRequest - it feels a bit hacky to expect a certain key from another method, where there might be a better, more holistic solution -4. ~Test on imported libraries and add libraries to pants deps~ +Refer to [examples/python/hellofib](https://github.com/sureshjoshi/pants-plugins/tree/main/examples/python/hellofib) for example usage (`pants run examples/python/hellofib:hellofib-mypyc`). diff --git a/pants-plugins/experimental/mypyc/__init__.py b/pants-plugins/experimental/mypyc/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pants-plugins/experimental/mypyc/register.py b/pants-plugins/experimental/mypyc/register.py deleted file mode 100644 index c93a91a..0000000 --- a/pants-plugins/experimental/mypyc/register.py +++ /dev/null @@ -1,10 +0,0 @@ -from experimental.mypyc.rules import rules as mypyc_rules -from experimental.mypyc.target_types import MyPycPythonDistribution - - -def rules(): - return (*mypyc_rules(),) - - -def target_types(): - return (MyPycPythonDistribution,) diff --git a/pants-plugins/experimental/mypyc/rules.py b/pants-plugins/experimental/mypyc/rules.py deleted file mode 100644 index dc1c3b6..0000000 --- a/pants-plugins/experimental/mypyc/rules.py +++ /dev/null @@ -1,240 +0,0 @@ -import logging -import os -from dataclasses import dataclass -from pathlib import Path - -from experimental.mypyc.target_types import MyPycPythonDistribution -from pants.backend.python.goals.setup_py import ( - SETUP_BOILERPLATE, - DistBuildChroot, - DistBuildChrootRequest, - DistBuildRequest, - DistBuildResult, - ExportedTarget, - NoDistTypeSelected, - PythonProvidesField, - SetupKwargs, - SetupKwargsRequest, - SetupPyContent, - SetupPyContentRequest, - WheelConfigSettingsField, - WheelField, -) -from pants.backend.python.subsystems.setup import PythonSetup -from pants.backend.python.target_types import PythonRequirementsField -from pants.backend.python.util_rules.dists import ( - BuildSystem, - Setuptools, - distutils_repr, -) -from pants.backend.python.util_rules.interpreter_constraints import ( - InterpreterConstraints, -) -from pants.backend.python.util_rules.pex import PexRequirements -from pants.backend.python.util_rules.python_sources import ( - PythonSourceFiles, - PythonSourceFilesRequest, -) -from pants.core.goals.package import BuiltPackage, BuiltPackageArtifact, PackageFieldSet -from pants.engine.fs import AddPrefix, Digest, Snapshot -from pants.engine.rules import Get, collect_rules, rule -from pants.engine.target import Target, TransitiveTargets, TransitiveTargetsRequest -from pants.engine.unions import UnionRule -from pants.util.frozendict import FrozenDict -from pants.util.logging import LogLevel - -logger = logging.getLogger(__name__) - - -MYPYC_SETUP_BOILERPLATE = """ -# DO NOT EDIT THIS FILE -- AUTOGENERATED BY PANTS -# Target: {target_address_spec} -# mypy: ignore-errors - -from setuptools import setup, Extension -from mypyc.build import mypycify - -setup(**{setup_kwargs_str}, ext_modules=mypycify({mypycify_files}),) -""" - - -@dataclass(frozen=True) -class MyPycSetupKwargsRequest(SetupKwargsRequest): - @classmethod - def is_applicable(cls, target: Target) -> bool: - return isinstance(target, MyPycPythonDistribution) - - -@rule(level=LogLevel.DEBUG) -async def mypyc_setup_kwargs(request: MyPycSetupKwargsRequest) -> SetupKwargs: - logger.info(f"mypyc_setup_kwargs: Running on requested target: {request.target}") - transitive_targets = await Get( - TransitiveTargets, - TransitiveTargetsRequest([request.target.address]), - ) - logger.info( - f"mypyc_setup_kwargs: Transitive targets of {request.target.address} : {transitive_targets}" - ) - - logger.info(f"mypyc_setup_kwargs: Dependencies {transitive_targets.dependencies}") - - python_source_files = ( - await Get( - PythonSourceFiles, - PythonSourceFilesRequest( - transitive_targets.closure, include_resources=False, include_files=False - ), - ), - ) - - # TODO: Handle multiple PythonSourceFiles - logger.debug( - f"mypyc_setup_kwargs: Internals of the retrieved PythonSourceFiles: {python_source_files[0]}" - ) - source_files = [ - Path(file) for file in python_source_files[0].source_files.snapshot.files - ] - # TODO: Handle multiple source roots - source_root = python_source_files[0].source_roots[0] - relative_source_files = [ - str(file.relative_to(source_root)) for file in source_files - ] - - kwargs = SetupKwargs( - { - **request.explicit_kwargs, - "mypycify_files": relative_source_files, - }, - address=request.target.address, - ) - logger.debug(f"mypyc_setup_kwargs: Resulting SetupKwargs: {kwargs.kwargs}") - return kwargs - - -class MyPycSetupPyContentRequest(SetupPyContentRequest): - @classmethod - def is_applicable(cls, target: Target) -> bool: - return isinstance(target, MyPycPythonDistribution) - - -@rule(level=LogLevel.DEBUG) -async def generate_setup_py_content( - request: MyPycSetupPyContentRequest, -) -> SetupPyContent: - setup_kwargs = request.finalized_setup_kwargs.kwargs - mypycify_files = setup_kwargs.pop("mypycify_files", []) - - template = MYPYC_SETUP_BOILERPLATE - if not mypycify_files: - logger.warning( - "generate_setup_py_content: No mypyc files were specified. Expecting a key named 'mypycify_files' with a list of source file paths" - ) - template = SETUP_BOILERPLATE - - # TODO: What happens if "ext_modules" was already specified in kwargs? Need to merge or error out as unsupported - content = template.format( - target_address_spec=request.target.address.spec, - setup_kwargs_str=distutils_repr(setup_kwargs), - mypycify_files=distutils_repr(mypycify_files), - ).encode() - logger.debug(f"generate_setup_py_content: Generating mypyc setup.py: {content}") - return SetupPyContent(content) - - -@dataclass(frozen=True) -class MyPycPythonDistributionFieldSet(PackageFieldSet): - required_fields = (PythonProvidesField,) - - provides: PythonProvidesField - - -@rule(level=LogLevel.DEBUG) -async def package_mypyc_python_dist( - field_set: MyPycPythonDistributionFieldSet, - python_setup: PythonSetup, - setuptools: Setuptools, -) -> BuiltPackage: - transitive_targets = await Get( - TransitiveTargets, TransitiveTargetsRequest([field_set.address]) - ) - exported_target = ExportedTarget(transitive_targets.roots[0]) - - # TODO: This will ignore the requirements/constraints files - requirements = [ - target.address.target_name - for target in transitive_targets.dependencies - if target.has_field(PythonRequirementsField) - ] - - dist_tgt = exported_target.target - wheel = dist_tgt.get(WheelField).value - # sdist = dist_tgt.get(SDistField).value - if not wheel: - raise NoDistTypeSelected( - f"In order to package {dist_tgt.address.spec}, {WheelField.alias!r} or must be `True`." - ) - - wheel_config_settings = dist_tgt.get(WheelConfigSettingsField).value or FrozenDict() - # sdist_config_settings = dist_tgt.get(SDistConfigSettingsField).value or FrozenDict() - - interpreter_constraints = InterpreterConstraints.create_from_targets( - transitive_targets.closure, python_setup - ) or InterpreterConstraints(python_setup.interpreter_constraints) - chroot = await Get( - DistBuildChroot, - DistBuildChrootRequest( - exported_target, - py2=interpreter_constraints.includes_python2(), - ), - ) - - # We prefix the entire chroot, and run with this prefix as the cwd, so that we can capture - # any changes setup made within it without also capturing other artifacts of the pex - # process invocation. - chroot_prefix = "chroot" - working_directory = os.path.join(chroot_prefix, chroot.working_directory) - prefixed_chroot = await Get(Digest, AddPrefix(chroot.digest, chroot_prefix)) - dist_snapshot = await Get(Snapshot, Digest, chroot.digest) - # build_system = await Get(BuildSystem, BuildSystemRequest(typer_digest, working_directory)) - - # TODO: Check if pyproject.toml exists? - build_system = BuildSystem( - requires=PexRequirements( - req_strings=( - *setuptools.all_requirements, - *requirements, - ) - ), - build_backend="setuptools.build_meta:__legacy__", - ) - - logger.info(f"Build system: {build_system}") - setup_py_result = await Get( - DistBuildResult, - DistBuildRequest( - build_system=build_system, - interpreter_constraints=interpreter_constraints, - build_wheel=wheel, - build_sdist=False, - input=prefixed_chroot, - working_directory=working_directory, - target_address_spec=exported_target.target.address.spec, - wheel_config_settings=wheel_config_settings, - # sdist_config_settings=sdist_config_settings, - ), - ) - - dist_snapshot = await Get(Snapshot, Digest, setup_py_result.output) - return BuiltPackage( - setup_py_result.output, - tuple(BuiltPackageArtifact(path) for path in dist_snapshot.files), - ) - - -def rules(): - return ( - *collect_rules(), - UnionRule(SetupKwargsRequest, MyPycSetupKwargsRequest), - UnionRule(SetupPyContentRequest, MyPycSetupPyContentRequest), - UnionRule(PackageFieldSet, MyPycPythonDistributionFieldSet), - ) diff --git a/pants-plugins/experimental/mypyc/target_types.py b/pants-plugins/experimental/mypyc/target_types.py deleted file mode 100644 index a6552d0..0000000 --- a/pants-plugins/experimental/mypyc/target_types.py +++ /dev/null @@ -1,34 +0,0 @@ -from pants.backend.python.target_types import ( - GenerateSetupField, - PythonDistributionEntryPointsField, - PythonProvidesField, - SDistConfigSettingsField, - SDistField, - WheelConfigSettingsField, - WheelField, -) -from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies, Target -from pants.util.docutil import doc_url - - -class MyPycPythonDistributionDependenciesField(Dependencies): - supports_transitive_excludes = True - - -class MyPycPythonDistribution(Target): - alias = "mypyc_python_distribution" - core_fields = ( - *COMMON_TARGET_FIELDS, - MyPycPythonDistributionDependenciesField, - PythonDistributionEntryPointsField, - PythonProvidesField, - GenerateSetupField, - WheelField, - SDistField, - WheelConfigSettingsField, - SDistConfigSettingsField, - ) - help = ( - "A publishable Python setuptools distribution (e.g. an sdist or wheel).\n\nSee " - f"{doc_url('python-distributions')}." - ) diff --git a/pants.toml b/pants.toml index 30cb137..49ca205 100644 --- a/pants.toml +++ b/pants.toml @@ -44,7 +44,7 @@ root_patterns = [ [python] enable_resolves = true -interpreter_constraints = [">=3.9"] +interpreter_constraints = ["==3.9.*"] tailor_pex_binary_targets = false [python.resolves]