Allow both resource and file targets for complete_platforms on …

…Python FaaS (#21169)

Closes #20805 - when specifying `complete_platforms` for FaaS targets,
we don't currently allow using `resource` targets for doing so. This
change makes it so that the build process should accept both `file` and
`resource` targets.

docs/notes/2.23.x.md

@@ -110,6 +110,8 @@ Deprecate the `--export-py-hermetic-scripts` option in favor of the new `--expor
 
 When using the `vcs_version` target, force `setuptools_scm` git operations to run in the local environment, so that the local git state is available to them.
 
+When building function-as-a-service targets like `python_google_cloud_function`, `python_aws_lambda_function`, and `python_aws_lambda_layer`, the `complete_platforms` field may now be specified as either a `file` target or a `resource` target.
+
 #### Terraform
 
 The default version of terraform has been updated from 1.7.1 to 1.9.0.

src/python/pants/backend/google_cloud_function/python/rules_test.py

@@ -7,6 +7,7 @@
 import subprocess
 from io import BytesIO
 from textwrap import dedent
+from typing import Literal
 from unittest.mock import Mock
 from zipfile import ZipFile
 
@@ -40,6 +41,7 @@
     FileTarget,
     RelocatedFiles,
     ResourcesGeneratorTarget,
+    ResourceTarget,
 )
 from pants.core.target_types import rules as core_target_types_rules
 from pants.engine.addresses import Address
@@ -68,6 +70,7 @@ def rule_runner() -> PythonRuleRunner:
             PythonRequirementTarget,
             PythonSourcesGeneratorTarget,
             RelocatedFiles,
+            ResourceTarget,
             ResourcesGeneratorTarget,
         ],
     )
@@ -191,48 +194,78 @@ def handler(event, context):
 
 
 @pytest.mark.parametrize(
-    ("ics", "runtime"),
+    ("ics", "runtime", "complete_platforms_target_type"),
     [
-        pytest.param(["==3.7.*"], None, id="runtime inferred from ICs"),
-        pytest.param(None, "python37", id="runtime explicitly set"),
+        pytest.param(["==3.7.*"], None, None, id="runtime inferred from ICs"),
+        pytest.param(None, "python37", None, id="runtime explicitly set"),
+        pytest.param(["==3.7.*"], None, "file", id="complete platforms with file target"),
+        pytest.param(["==3.7.*"], None, "resource", id="complete platforms with resource target"),
     ],
 )
 def test_create_hello_world_gcf(
-    ics: list[str] | None, runtime: None | str, rule_runner: PythonRuleRunner
+    ics: list[str] | None,
+    runtime: None | str,
+    complete_platforms_target_type: Literal["file", "resource"] | None,
+    rule_runner: PythonRuleRunner,
+    complete_platform: bytes,
 ) -> None:
+    if runtime:
+        assert (
+            complete_platforms_target_type is None
+        ), "Cannot set both runtime and complete platforms!"
+
+    complete_platforms_target_name = (
+        f"complete_platforms_{complete_platforms_target_type}"
+        if complete_platforms_target_type
+        else ""
+    )
+    complete_platforms_target_declaration = (
+        f"""{complete_platforms_target_type}(name="{complete_platforms_target_name}", source="complete_platforms.json")\n"""
+        if complete_platforms_target_type
+        else ""
+    )
+    runtime_declaration = (
+        f'complete_platforms=[":{complete_platforms_target_name}"]'
+        if complete_platforms_target_type
+        else f"runtime={runtime!r}"
+    )
+
     rule_runner.write_files(
         {
             "src/python/foo/bar/hello_world.py": dedent(
-                """
+                """\
                 import mureq
 
                 def handler(event, context):
                     print('Hello, World!')
                 """
             ),
+            "src/python/foo/bar/complete_platforms.json": complete_platform.decode(),
             "src/python/foo/bar/BUILD": dedent(
-                f"""
+                f"""\
                 python_requirement(name="mureq", requirements=["mureq==0.2"])
+
                 python_sources(interpreter_constraints={ics!r})
 
+                {complete_platforms_target_declaration}
+
                 python_google_cloud_function(
                     name='gcf',
                     handler='foo.bar.hello_world:handler',
-                    runtime={runtime!r},
+                    {runtime_declaration},
                     type='event',
                 )
                 """
             ),
         }
     )
 
+    extra_log_lines_base = tuple() if complete_platforms_target_type else ("    Runtime: python37",)
+    expected_extra_log_lines = extra_log_lines_base + ("    Handler: handler",)
     zip_file_relpath, content = create_python_google_cloud_function(
         rule_runner,
         Address("src/python/foo/bar", target_name="gcf"),
-        expected_extra_log_lines=(
-            "    Runtime: python37",
-            "    Handler: handler",
-        ),
+        expected_extra_log_lines=expected_extra_log_lines,
     )
     assert "src.python.foo.bar/gcf.zip" == zip_file_relpath
 

src/python/pants/backend/python/goals/package_pex_binary_integration_test.py

@@ -291,7 +291,8 @@ def test_resolve_local_platforms(pex_executable: str, rule_runner: PythonRuleRun
 
 
 @skip_unless_python38_present
-def test_complete_platforms(rule_runner: PythonRuleRunner) -> None:
+@pytest.mark.parametrize("target_type", ["files", "resources"])
+def test_complete_platforms(rule_runner: PythonRuleRunner, target_type: str) -> None:
     linux_complete_platform = pkgutil.get_data(__name__, "platform-linux-py38.json")
     assert linux_complete_platform is not None
 
@@ -303,9 +304,9 @@ def test_complete_platforms(rule_runner: PythonRuleRunner) -> None:
             "src/py/project/platform-linux-py38.json": linux_complete_platform,
             "src/py/project/platform-mac-py38.json": mac_complete_platform,
             "src/py/project/BUILD": dedent(
-                """\
+                f"""\
                 python_requirement(name="p537", requirements=["p537==1.0.6"])
-                files(name="platforms", sources=["platform*.json"])
+                {target_type}(name="platforms", sources=["platform*.json"])
                 pex_binary(
                     dependencies=[":p537"],
                     complete_platforms=[":platforms"],

src/python/pants/backend/python/target_types.py

@@ -488,7 +488,7 @@ class PexCompletePlatformsField(SpecialCasedDependencies):
         You can give a list of multiple complete platforms to create a multiplatform PEX,
         meaning that the PEX will be executable in all of the supported environments.
 
-        Complete platforms should be addresses of `file` targets that point to files that contain
+        Complete platforms should be addresses of `file` or `resource` targets that point to files that contain
         complete platform JSON as described by Pex
         (https://pex.readthedocs.io/en/latest/buildingpex.html#complete-platform).
 

src/python/pants/backend/python/util_rules/BUILD

@@ -8,12 +8,13 @@ python_sources(
 )
 
 resources(name="complete_platform_faas-test", sources=["complete_platform_faas-test*.json"])
+resource(name="complete_platform_pex_test", source="complete_platform_pex_test.json")
 python_tests(
     name="tests",
     overrides={
         "local_dists_test.py": {"timeout": 120},
         "pex_from_targets_test.py": {"timeout": 200},
-        "pex_test.py": {"timeout": 600},
+        "pex_test.py": {"timeout": 600, "dependencies": [":complete_platform_pex_test"]},
         "package_dists_test.py": {"timeout": 150},
         "vcs_versioning_test.py": {"timeout": 120},
         "faas_test.py": {"dependencies": [":complete_platform_faas-test"]},

src/python/pants/backend/python/util_rules/complete_platform_pex_test.json

@@ -0,0 +1 @@
+{}

src/python/pants/backend/python/util_rules/pex.py

@@ -51,7 +51,7 @@
     validate_metadata,
 )
 from pants.build_graph.address import Address
-from pants.core.target_types import FileSourceField
+from pants.core.target_types import FileSourceField, ResourceSourceField
 from pants.core.util_rules.environments import EnvironmentTarget
 from pants.core.util_rules.stripped_source_files import StrippedFileName, StrippedFileNameRequest
 from pants.core.util_rules.stripped_source_files import rules as stripped_source_rules
@@ -136,7 +136,13 @@ async def digest_complete_platform_addresses(
     original_files_sources = await MultiGet(
         Get(
             HydratedSources,
-            HydrateSourcesRequest(tgt.get(SourcesField), for_sources_types=(FileSourceField,)),
+            HydrateSourcesRequest(
+                tgt.get(SourcesField),
+                for_sources_types=(
+                    FileSourceField,
+                    ResourceSourceField,
+                ),
+            ),
         )
         for tgt in original_file_targets
     )

src/python/pants/backend/python/util_rules/pex_test.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import os.path
+import pkgutil
 import re
 import shutil
 import textwrap
@@ -19,7 +20,7 @@
 from pants.backend.python.goals import lockfile
 from pants.backend.python.goals.lockfile import GeneratePythonLockfile
 from pants.backend.python.subsystems.setup import PythonSetup
-from pants.backend.python.target_types import EntryPoint
+from pants.backend.python.target_types import EntryPoint, PexCompletePlatformsField
 from pants.backend.python.util_rules import pex_test_utils
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.lockfile_metadata import PythonLockfileMetadata
@@ -58,6 +59,7 @@
     parse_requirements,
 )
 from pants.core.goals.generate_lockfiles import GenerateLockfileResult
+from pants.core.target_types import FileTarget, ResourceTarget
 from pants.core.util_rules.lockfile_metadata import InvalidLockfileError
 from pants.engine.fs import (
     EMPTY_DIGEST,
@@ -67,6 +69,7 @@
     Directory,
     FileContent,
 )
+from pants.engine.internals.native_engine import Address
 from pants.engine.process import Process, ProcessCacheScope, ProcessResult
 from pants.option.global_options import GlobalOptions
 from pants.testutil.option_util import create_subsystem
@@ -94,6 +97,11 @@ def rule_runner() -> RuleRunner:
             QueryRule(ProcessResult, (Process,)),
             QueryRule(PexResolveInfo, (Pex,)),
             QueryRule(PexResolveInfo, (VenvPex,)),
+            QueryRule(CompletePlatforms, (PexCompletePlatformsField,)),
+        ],
+        target_types=[
+            ResourceTarget,
+            FileTarget,
         ],
     )
 
@@ -911,3 +919,29 @@ def test_lockfile_validation(rule_runner: RuleRunner) -> None:
         create_pex_and_get_all_data(
             rule_runner, requirements=EntireLockfile(_lockfile, ("ansicolors",))
         )
+
+
+@pytest.mark.parametrize("target_type", ["file", "resource"])
+def test_digest_complete_platforms(rule_runner: RuleRunner, target_type: str) -> None:
+    # Read the complete_platforms content using pkgutil
+    complete_platforms_content = pkgutil.get_data(__name__, "complete_platform_pex_test.json")
+    assert complete_platforms_content is not None
+
+    # Create a target with the complete platforms file
+    rule_runner.write_files(
+        {
+            "BUILD": f"{target_type}(name='complete_platforms', source='complete_platforms.json')",
+            "complete_platforms.json": complete_platforms_content,
+        }
+    )
+
+    # Get the CompletePlatforms object
+    target = rule_runner.get_target(Address("", target_name="complete_platforms"))
+    complete_platforms = rule_runner.request(
+        CompletePlatforms,
+        [PexCompletePlatformsField([":complete_platforms"], target.address)],
+    )
+
+    # Verify the result
+    assert len(complete_platforms) == 1
+    assert complete_platforms.digest != EMPTY_DIGEST