feat(consume): Add consume cache, and --input flag accepts versioned release names

docs/CHANGELOG.md

@@ -88,6 +88,8 @@ Release tarball changes:
 - 🔀 Change `--dist` flag to the default value, `load`, for better parallelism handling during test filling ([#1118](https://github.com/ethereum/execution-spec-tests/pull/1118)).
 - 🔀 Refactor framework code to use the [`ethereum-rlp`](https://pypi.org/project/ethereum-rlp/) package instead of `ethereum.rlp`, previously available in ethereum/execution-specs ([#1180](https://github.com/ethereum/execution-spec-tests/pull/1180)).
 - 🔀 Update EELS / execution-specs EEST dependency to [99238233](https://github.com/ethereum/execution-specs/commit/9923823367b5586228e590537d47aa9cc4c6a206) for EEST framework libraries and test case generation ([#1181](https://github.com/ethereum/execution-spec-tests/pull/1181)).
+- ✨ Add the `consume cache` command to cache fixtures before running consume commands ([#1044](https://github.com/ethereum/execution-spec-tests/pull/1044)).
+- ✨ The `--input` flag of the consume commands now supports parsing of tagged release names in the format `<RELEASE_NAME>@<RELEASE_VERSION>` ([#1044](https://github.com/ethereum/execution-spec-tests/pull/1044)).
 
 ### 🔧 EVM Tools
 
@@ -119,6 +121,7 @@ Release tarball changes:
 - Remove redundant tests within stable and develop fixture releases, moving them to a separate legacy release ([#788](https://github.com/ethereum/execution-spec-tests/pull/788)).
 - Ruff now replaces Flake8, Isort and Black resulting in significant changes to the entire code base including its usage ([#922](https://github.com/ethereum/execution-spec-tests/pull/922)).
 - `state_test`, `blockchain_test` and `blockchain_test_engine` fixtures now contain a `config` field, which contains an object that contains a `blobSchedule` field. On the `blockchain_test` and `blockchain_test_engine` fixtures, the object also contains a duplicate of the `network` root field. The root's `network` field will be eventually deprecated ([#1040](https://github.com/ethereum/execution-spec-tests/pull/1040)).
+- `latest-stable-release` and `latest-develop-release` keywords for the `--input` flag in consume commands have been replaced with `stable@latest` and `develop@latest` respectively ([#1044](https://github.com/ethereum/execution-spec-tests/pull/1044)).
 
 ## [v3.0.0](https://github.com/ethereum/execution-spec-tests/releases/tag/v3.0.0) - 2024-07-22
 

docs/consuming_tests/index.md

@@ -30,7 +30,7 @@ The @ethereum/execution-spec-tests repository provides [releases](https://github
 
 | Release Artifact               | Consumer | Fork/feature scope |
 | ------------------------------ | -------- | ------------------ |
-| `fixtures.tar.gz`              | Clients  | All tests until the last stable fork ("must pass") |
+| `fixtures_stable.tar.gz`       | Clients  | All tests until the last stable fork ("must pass") |
 | `fixtures_develop.tar.gz`      | Clients  | All tests until the last development fork |
 
 ## Obtaining the Most Recent Release Artifacts

pytest-framework.ini

@@ -10,8 +10,10 @@ markers =
 addopts = 
     -p pytester
     -p pytest_plugins.eels_resolver
-    --ignore=src/pytest_plugins/consume/
+    --ignore=src/pytest_plugins/consume/test_cache.py
+    --ignore=src/pytest_plugins/consume/direct/
     --ignore=src/pytest_plugins/consume/direct/test_via_direct.py
+    --ignore=src/pytest_plugins/consume/hive_simulators/
     --ignore=src/pytest_plugins/consume/hive_simulators/engine/test_via_engine.py
     --ignore=src/pytest_plugins/consume/hive_simulators/rlp/test_via_rlp.py
     --ignore=src/pytest_plugins/execute/test_recover.py

src/cli/pytest_commands/consume.py

@@ -57,6 +57,12 @@ def get_command_paths(command_name: str, is_hive: bool) -> List[Path]:
     return command_paths
 
 
+@click.group(context_settings={"help_option_names": ["-h", "--help"]})
+def consume() -> None:
+    """Consume command to aid client consumption of test fixtures."""
+    pass
+
+
 def consume_command(is_hive: bool = False) -> Callable[[Callable[..., Any]], click.Command]:
     """Generate a consume sub-command."""
 
@@ -93,16 +99,6 @@ def decorator(func: Callable[..., Any]) -> click.Command:
     return decorator
 
 
-@click.group(
-    context_settings={
-        "help_option_names": ["-h", "--help"],
-    }
-)
-def consume() -> None:
-    """Consume command to aid client consumption of test fixtures."""
-    pass
-
-
 @consume_command(is_hive=False)
 def direct() -> None:
     """Clients consume directly via the `blocktest` interface."""
@@ -125,3 +121,14 @@ def engine() -> None:
 def hive() -> None:
     """Client consumes via all available hive methods (rlp, engine)."""
     pass
+
+
+@consume.command(
+    context_settings={"ignore_unknown_options": True},
+)
+@common_click_options
+def cache(pytest_args: List[str], **kwargs) -> None:
+    """Consume command to cache test fixtures."""
+    args = handle_consume_command_flags(pytest_args, is_hive=False)
+    args += ["src/pytest_plugins/consume/test_cache.py"]
+    sys.exit(pytest.main(args))

src/pytest_plugins/consume/consume.py

@@ -1,12 +1,13 @@
 """A pytest plugin providing common functionality for consuming test fixtures."""
 
-import os
 import sys
 import tarfile
+from io import BytesIO
 from pathlib import Path
-from typing import Literal, Union
+from typing import List, Literal, Union
 from urllib.parse import urlparse
 
+import platformdirs
 import pytest
 import requests
 import rich
@@ -15,9 +16,13 @@
 from ethereum_test_fixtures.consume import TestCases
 from ethereum_test_tools.utility.versioning import get_current_commit_hash_or_tag
 
-cached_downloads_directory = Path("./cached_downloads")
+from .releases import ReleaseTag, get_release_url
 
-JsonSource = Union[Path, Literal["stdin"]]
+CACHED_DOWNLOADS_DIRECTORY = (
+    Path(platformdirs.user_cache_dir("ethereum-execution-spec-tests")) / "cached_downloads"
+)
+
+FixturesSource = Union[Path, Literal["stdin"]]
 
 
 def default_input_directory() -> str:
@@ -57,11 +62,7 @@ def download_and_extract(url: str, base_directory: Path) -> Path:
     response = requests.get(url)
     response.raise_for_status()
 
-    archive_path = extract_to / filename
-    with open(archive_path, "wb") as file:
-        file.write(response.content)
-
-    with tarfile.open(archive_path, "r:gz") as tar:
+    with tarfile.open(fileobj=BytesIO(response.content), mode="r:gz") as tar:  # noqa: SC200
         tar.extractall(path=extract_to)
 
     return extract_to / "fixtures"
@@ -74,15 +75,28 @@ def pytest_addoption(parser):  # noqa: D103
     consume_group.addoption(
         "--input",
         action="store",
-        dest="fixture_source",
-        default=default_input_directory(),
+        dest="fixtures_source",
+        default=None,
         help=(
             "Specify the JSON test fixtures source. Can be a local directory, a URL pointing to a "
-            " fixtures.tar.gz archive, or one of the special keywords: 'stdin', "
-            "'latest-stable', 'latest-develop'. "
+            " fixtures.tar.gz archive, a release name and version in the form of `[email protected]` "
+            "(`stable` and `develop` are valid release names, and `latest` is a valid version), "
+            "or the special keyword 'stdin'. "
             f"Defaults to the following local directory: '{default_input_directory()}'."
         ),
     )
+    consume_group.addoption(
+        "--cache-folder",
+        action="store",
+        dest="fixture_cache_folder",
+        default=CACHED_DOWNLOADS_DIRECTORY,
+        help=(
+            "Specify the path where the downloaded fixtures are cached. "
+            f"Defaults to the following directory: '{CACHED_DOWNLOADS_DIRECTORY}'."
+        ),
+    )
+    if "cache" in sys.argv:
+        return
     consume_group.addoption(
         "--fork",
         action="store",
@@ -112,48 +126,51 @@ def pytest_configure(config):  # noqa: D103
     called before the pytest-html plugin's pytest_configure to ensure that
     it uses the modified `htmlpath` option.
     """
-    input_flag = any(arg.startswith("--input") for arg in config.invocation_params.args)
-    input_source = config.getoption("fixture_source")
-
-    if input_flag and input_source == "stdin":
+    fixtures_source = config.getoption("fixtures_source")
+    if "cache" in sys.argv and not config.getoption("fixtures_source"):
+        pytest.exit("The --input flag is required when using the cache command.")
+    config.fixture_source_flags = ["--input", fixtures_source]
+
+    if fixtures_source is None:
+        config.fixture_source_flags = []
+        fixtures_source = default_input_directory()
+    elif fixtures_source == "stdin":
         config.test_cases = TestCases.from_stream(sys.stdin)
+        config.fixtures_real_source = "stdin"
+        config.fixtures_source = "stdin"
         return
+    elif ReleaseTag.is_release_string(fixtures_source):
+        fixtures_source = get_release_url(fixtures_source)
 
-    latest_base_url = "https://github.com/ethereum/execution-spec-tests/releases/latest/download"
-    if input_source == "latest-stable-release" or input_source == "latest-stable":
-        input_source = f"{latest_base_url}/fixtures_stable.tar.gz"
-    if input_source == "latest-develop-release" or input_source == "latest-develop":
-        input_source = f"{latest_base_url}/fixtures_develop.tar.gz"
-
-    if is_url(input_source):
+    config.fixtures_real_source = fixtures_source
+    if is_url(fixtures_source):
+        cached_downloads_directory = Path(config.getoption("fixture_cache_folder"))
         cached_downloads_directory.mkdir(parents=True, exist_ok=True)
-        input_source = download_and_extract(input_source, cached_downloads_directory)
-        config.option.fixture_source = input_source
+        fixtures_source = download_and_extract(fixtures_source, cached_downloads_directory)
 
-    input_source = Path(input_source)
-    if not input_source.exists():
-        pytest.exit(f"Specified fixture directory '{input_source}' does not exist.")
-    if not any(input_source.glob("**/*.json")):
+    fixtures_source = Path(fixtures_source)
+    config.fixtures_source = fixtures_source
+    if not fixtures_source.exists():
+        pytest.exit(f"Specified fixture directory '{fixtures_source}' does not exist.")
+    if not any(fixtures_source.glob("**/*.json")):
         pytest.exit(
-            f"Specified fixture directory '{input_source}' does not contain any JSON files."
+            f"Specified fixture directory '{fixtures_source}' does not contain any JSON files."
         )
 
-    index_file = input_source / ".meta" / "index.json"
+    index_file = fixtures_source / ".meta" / "index.json"
     index_file.parent.mkdir(parents=True, exist_ok=True)
     if not index_file.exists():
         rich.print(f"Generating index file [bold cyan]{index_file}[/]...")
         generate_fixtures_index(
-            input_source, quiet_mode=False, force_flag=False, disable_infer_format=False
+            fixtures_source, quiet_mode=False, force_flag=False, disable_infer_format=False
         )
     config.test_cases = TestCases.from_index_file(index_file)
 
-    if config.option.collectonly:
+    if config.option.collectonly or "cache" in sys.argv:
         return
     if not config.getoption("disable_html") and config.getoption("htmlpath") is None:
         # generate an html report by default, unless explicitly disabled
-        config.option.htmlpath = os.path.join(
-            config.getoption("fixture_source"), default_html_report_file_path()
-        )
+        config.option.htmlpath = Path(default_html_report_file_path())
 
 
 def pytest_html_report_title(report):
@@ -163,20 +180,29 @@ def pytest_html_report_title(report):
 
 def pytest_report_header(config):  # noqa: D103
     consume_version = f"consume commit: {get_current_commit_hash_or_tag()}"
-    input_source = f"fixtures: {config.getoption('fixture_source')}"
-    return [consume_version, input_source]
+    fixtures_real_source = f"fixtures: {config.fixtures_real_source}"
+    return [consume_version, fixtures_real_source]
 
 
-@pytest.fixture(scope="function")
-def fixture_source(request) -> JsonSource:  # noqa: D103
-    return request.config.getoption("fixture_source")
+@pytest.fixture(scope="session")
+def fixture_source_flags(request) -> List[str]:
+    """Return the input flags used to specify the JSON test fixtures source."""
+    return request.config.fixture_source_flags
+
+
+@pytest.fixture(scope="session")
+def fixtures_source(request) -> FixturesSource:  # noqa: D103
+    return request.config.fixtures_source
 
 
 def pytest_generate_tests(metafunc):
     """
     Generate test cases for every test fixture in all the JSON fixture files
     within the specified fixtures directory, or read from stdin if the directory is 'stdin'.
     """
+    if "cache" in sys.argv:
+        return
+
     fork = metafunc.config.getoption("single_fork")
     metafunc.parametrize(
         "test_case",

src/pytest_plugins/consume/direct/conftest.py

@@ -17,6 +17,8 @@
 from ethereum_test_fixtures.consume import TestCaseIndexFile, TestCaseStream
 from ethereum_test_fixtures.file import Fixtures
 
+from ..consume import FixturesSource
+
 
 def pytest_addoption(parser):  # noqa: D103
     consume_group = parser.getgroup(
@@ -96,13 +98,13 @@ def test_dump_dir(
 
 
 @pytest.fixture
-def fixture_path(test_case: TestCaseIndexFile | TestCaseStream, fixture_source):
+def fixture_path(test_case: TestCaseIndexFile | TestCaseStream, fixtures_source: FixturesSource):
     """
     Path to the current JSON fixture file.
 
     If the fixture source is stdin, the fixture is written to a temporary json file.
     """
-    if fixture_source == "stdin":
+    if fixtures_source == "stdin":
         assert isinstance(test_case, TestCaseStream)
         temp_dir = tempfile.TemporaryDirectory()
         fixture_path = Path(temp_dir.name) / f"{test_case.id.replace('/', '_')}.json"
@@ -113,7 +115,7 @@ def fixture_path(test_case: TestCaseIndexFile | TestCaseStream, fixture_source):
         temp_dir.cleanup()
     else:
         assert isinstance(test_case, TestCaseIndexFile)
-        yield fixture_source / test_case.json_path
+        yield fixtures_source / test_case.json_path
 
 
 @pytest.fixture(scope="function")