diff --git a/docs/notes/2.23.x.md b/docs/notes/2.23.x.md index 315596f6a48..e510a5bbf9a 100644 --- a/docs/notes/2.23.x.md +++ b/docs/notes/2.23.x.md @@ -110,6 +110,8 @@ Deprecate the `--export-py-hermetic-scripts` option in favor of the new `--expor When using the `vcs_version` target, force `setuptools_scm` git operations to run in the local environment, so that the local git state is available to them. +When building function-as-a-service targets like `python_google_cloud_function`, `python_aws_lambda_function`, and `python_aws_lambda_layer`, the `complete_platforms` field may now be specified as either a `file` target or a `resource` target. + #### Terraform The default version of terraform has been updated from 1.7.1 to 1.9.0. diff --git a/src/python/pants/backend/google_cloud_function/python/rules_test.py b/src/python/pants/backend/google_cloud_function/python/rules_test.py index 8eaefa21cfd..0a27dccdfe4 100644 --- a/src/python/pants/backend/google_cloud_function/python/rules_test.py +++ b/src/python/pants/backend/google_cloud_function/python/rules_test.py @@ -7,6 +7,7 @@ import subprocess from io import BytesIO from textwrap import dedent +from typing import Literal from unittest.mock import Mock from zipfile import ZipFile @@ -40,6 +41,7 @@ FileTarget, RelocatedFiles, ResourcesGeneratorTarget, + ResourceTarget, ) from pants.core.target_types import rules as core_target_types_rules from pants.engine.addresses import Address @@ -68,6 +70,7 @@ def rule_runner() -> PythonRuleRunner: PythonRequirementTarget, PythonSourcesGeneratorTarget, RelocatedFiles, + ResourceTarget, ResourcesGeneratorTarget, ], ) @@ -191,34 +194,65 @@ def handler(event, context): @pytest.mark.parametrize( - ("ics", "runtime"), + ("ics", "runtime", "complete_platforms_target_type"), [ - pytest.param(["==3.7.*"], None, id="runtime inferred from ICs"), - pytest.param(None, "python37", id="runtime explicitly set"), + pytest.param(["==3.7.*"], None, None, id="runtime inferred from ICs"), + pytest.param(None, "python37", None, id="runtime explicitly set"), + pytest.param(["==3.7.*"], None, "file", id="complete platforms with file target"), + pytest.param(["==3.7.*"], None, "resource", id="complete platforms with resource target"), ], ) def test_create_hello_world_gcf( - ics: list[str] | None, runtime: None | str, rule_runner: PythonRuleRunner + ics: list[str] | None, + runtime: None | str, + complete_platforms_target_type: Literal["file", "resource"] | None, + rule_runner: PythonRuleRunner, + complete_platform: bytes, ) -> None: + if runtime: + assert ( + complete_platforms_target_type is None + ), "Cannot set both runtime and complete platforms!" + + complete_platforms_target_name = ( + f"complete_platforms_{complete_platforms_target_type}" + if complete_platforms_target_type + else "" + ) + complete_platforms_target_declaration = ( + f"""{complete_platforms_target_type}(name="{complete_platforms_target_name}", source="complete_platforms.json")\n""" + if complete_platforms_target_type + else "" + ) + runtime_declaration = ( + f'complete_platforms=[":{complete_platforms_target_name}"]' + if complete_platforms_target_type + else f"runtime={runtime!r}" + ) + rule_runner.write_files( { "src/python/foo/bar/hello_world.py": dedent( - """ + """\ import mureq def handler(event, context): print('Hello, World!') """ ), + "src/python/foo/bar/complete_platforms.json": complete_platform.decode(), "src/python/foo/bar/BUILD": dedent( - f""" + f"""\ python_requirement(name="mureq", requirements=["mureq==0.2"]) + python_sources(interpreter_constraints={ics!r}) + {complete_platforms_target_declaration} + python_google_cloud_function( name='gcf', handler='foo.bar.hello_world:handler', - runtime={runtime!r}, + {runtime_declaration}, type='event', ) """ @@ -226,13 +260,12 @@ def handler(event, context): } ) + extra_log_lines_base = tuple() if complete_platforms_target_type else (" Runtime: python37",) + expected_extra_log_lines = extra_log_lines_base + (" Handler: handler",) zip_file_relpath, content = create_python_google_cloud_function( rule_runner, Address("src/python/foo/bar", target_name="gcf"), - expected_extra_log_lines=( - " Runtime: python37", - " Handler: handler", - ), + expected_extra_log_lines=expected_extra_log_lines, ) assert "src.python.foo.bar/gcf.zip" == zip_file_relpath diff --git a/src/python/pants/backend/python/goals/package_pex_binary_integration_test.py b/src/python/pants/backend/python/goals/package_pex_binary_integration_test.py index ec4ddb6e332..1e56a40f470 100644 --- a/src/python/pants/backend/python/goals/package_pex_binary_integration_test.py +++ b/src/python/pants/backend/python/goals/package_pex_binary_integration_test.py @@ -291,7 +291,8 @@ def test_resolve_local_platforms(pex_executable: str, rule_runner: PythonRuleRun @skip_unless_python38_present -def test_complete_platforms(rule_runner: PythonRuleRunner) -> None: +@pytest.mark.parametrize("target_type", ["files", "resources"]) +def test_complete_platforms(rule_runner: PythonRuleRunner, target_type: str) -> None: linux_complete_platform = pkgutil.get_data(__name__, "platform-linux-py38.json") assert linux_complete_platform is not None @@ -303,9 +304,9 @@ def test_complete_platforms(rule_runner: PythonRuleRunner) -> None: "src/py/project/platform-linux-py38.json": linux_complete_platform, "src/py/project/platform-mac-py38.json": mac_complete_platform, "src/py/project/BUILD": dedent( - """\ + f"""\ python_requirement(name="p537", requirements=["p537==1.0.6"]) - files(name="platforms", sources=["platform*.json"]) + {target_type}(name="platforms", sources=["platform*.json"]) pex_binary( dependencies=[":p537"], complete_platforms=[":platforms"], diff --git a/src/python/pants/backend/python/target_types.py b/src/python/pants/backend/python/target_types.py index d9f880fd733..03eaf098a55 100644 --- a/src/python/pants/backend/python/target_types.py +++ b/src/python/pants/backend/python/target_types.py @@ -488,7 +488,7 @@ class PexCompletePlatformsField(SpecialCasedDependencies): You can give a list of multiple complete platforms to create a multiplatform PEX, meaning that the PEX will be executable in all of the supported environments. - Complete platforms should be addresses of `file` targets that point to files that contain + Complete platforms should be addresses of `file` or `resource` targets that point to files that contain complete platform JSON as described by Pex (https://pex.readthedocs.io/en/latest/buildingpex.html#complete-platform). diff --git a/src/python/pants/backend/python/util_rules/BUILD b/src/python/pants/backend/python/util_rules/BUILD index f07ac50ec86..6c2224f9d40 100644 --- a/src/python/pants/backend/python/util_rules/BUILD +++ b/src/python/pants/backend/python/util_rules/BUILD @@ -8,12 +8,13 @@ python_sources( ) resources(name="complete_platform_faas-test", sources=["complete_platform_faas-test*.json"]) +resource(name="complete_platform_pex_test", source="complete_platform_pex_test.json") python_tests( name="tests", overrides={ "local_dists_test.py": {"timeout": 120}, "pex_from_targets_test.py": {"timeout": 200}, - "pex_test.py": {"timeout": 600}, + "pex_test.py": {"timeout": 600, "dependencies": [":complete_platform_pex_test"]}, "package_dists_test.py": {"timeout": 150}, "vcs_versioning_test.py": {"timeout": 120}, "faas_test.py": {"dependencies": [":complete_platform_faas-test"]}, diff --git a/src/python/pants/backend/python/util_rules/complete_platform_pex_test.json b/src/python/pants/backend/python/util_rules/complete_platform_pex_test.json new file mode 100644 index 00000000000..9e26dfeeb6e --- /dev/null +++ b/src/python/pants/backend/python/util_rules/complete_platform_pex_test.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/python/pants/backend/python/util_rules/pex.py b/src/python/pants/backend/python/util_rules/pex.py index fc998944924..2790d03b928 100644 --- a/src/python/pants/backend/python/util_rules/pex.py +++ b/src/python/pants/backend/python/util_rules/pex.py @@ -51,7 +51,7 @@ validate_metadata, ) from pants.build_graph.address import Address -from pants.core.target_types import FileSourceField +from pants.core.target_types import FileSourceField, ResourceSourceField from pants.core.util_rules.environments import EnvironmentTarget from pants.core.util_rules.stripped_source_files import StrippedFileName, StrippedFileNameRequest from pants.core.util_rules.stripped_source_files import rules as stripped_source_rules @@ -136,7 +136,13 @@ async def digest_complete_platform_addresses( original_files_sources = await MultiGet( Get( HydratedSources, - HydrateSourcesRequest(tgt.get(SourcesField), for_sources_types=(FileSourceField,)), + HydrateSourcesRequest( + tgt.get(SourcesField), + for_sources_types=( + FileSourceField, + ResourceSourceField, + ), + ), ) for tgt in original_file_targets ) diff --git a/src/python/pants/backend/python/util_rules/pex_test.py b/src/python/pants/backend/python/util_rules/pex_test.py index 838dd01d783..08bd9ae2405 100644 --- a/src/python/pants/backend/python/util_rules/pex_test.py +++ b/src/python/pants/backend/python/util_rules/pex_test.py @@ -4,6 +4,7 @@ from __future__ import annotations import os.path +import pkgutil import re import shutil import textwrap @@ -19,7 +20,7 @@ from pants.backend.python.goals import lockfile from pants.backend.python.goals.lockfile import GeneratePythonLockfile from pants.backend.python.subsystems.setup import PythonSetup -from pants.backend.python.target_types import EntryPoint +from pants.backend.python.target_types import EntryPoint, PexCompletePlatformsField from pants.backend.python.util_rules import pex_test_utils from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.backend.python.util_rules.lockfile_metadata import PythonLockfileMetadata @@ -58,6 +59,7 @@ parse_requirements, ) from pants.core.goals.generate_lockfiles import GenerateLockfileResult +from pants.core.target_types import FileTarget, ResourceTarget from pants.core.util_rules.lockfile_metadata import InvalidLockfileError from pants.engine.fs import ( EMPTY_DIGEST, @@ -67,6 +69,7 @@ Directory, FileContent, ) +from pants.engine.internals.native_engine import Address from pants.engine.process import Process, ProcessCacheScope, ProcessResult from pants.option.global_options import GlobalOptions from pants.testutil.option_util import create_subsystem @@ -94,6 +97,11 @@ def rule_runner() -> RuleRunner: QueryRule(ProcessResult, (Process,)), QueryRule(PexResolveInfo, (Pex,)), QueryRule(PexResolveInfo, (VenvPex,)), + QueryRule(CompletePlatforms, (PexCompletePlatformsField,)), + ], + target_types=[ + ResourceTarget, + FileTarget, ], ) @@ -911,3 +919,29 @@ def test_lockfile_validation(rule_runner: RuleRunner) -> None: create_pex_and_get_all_data( rule_runner, requirements=EntireLockfile(_lockfile, ("ansicolors",)) ) + + +@pytest.mark.parametrize("target_type", ["file", "resource"]) +def test_digest_complete_platforms(rule_runner: RuleRunner, target_type: str) -> None: + # Read the complete_platforms content using pkgutil + complete_platforms_content = pkgutil.get_data(__name__, "complete_platform_pex_test.json") + assert complete_platforms_content is not None + + # Create a target with the complete platforms file + rule_runner.write_files( + { + "BUILD": f"{target_type}(name='complete_platforms', source='complete_platforms.json')", + "complete_platforms.json": complete_platforms_content, + } + ) + + # Get the CompletePlatforms object + target = rule_runner.get_target(Address("", target_name="complete_platforms")) + complete_platforms = rule_runner.request( + CompletePlatforms, + [PexCompletePlatformsField([":complete_platforms"], target.address)], + ) + + # Verify the result + assert len(complete_platforms) == 1 + assert complete_platforms.digest != EMPTY_DIGEST