tests: adapt benchmark fixture after pytest refactoring

pyproject.toml

@@ -459,6 +459,7 @@ disable = [ # Any rule listed here can be disabled: https://github.com/astral-sh
     "line-too-long",
     "unused-variable",
     "redefined-builtin",
+    "global-statement",
     "abstract-class-instantiated", # Overlap with https://mypy.readthedocs.io/en/stable/error_code_list.html#check-instantiation-of-abstract-classes-abstract
     "unexpected-keyword-arg", # Overlap with https://mypy.readthedocs.io/en/stable/error_code_list.html#check-arguments-in-calls-call-arg and other rules
     "no-value-for-parameter" # Overlap with https://mypy.readthedocs.io/en/stable/error_code_list.html#check-arguments-in-calls-call-arg

tests/benchmark/conftest.py

@@ -10,8 +10,6 @@
 from _pytest.terminal import TerminalReporter
 
 from anta.catalog import AntaCatalog
-from anta.device import AsyncEOSDevice
-from anta.inventory import AntaInventory
 
 from .utils import AntaMockEnvironment
 
@@ -20,32 +18,21 @@
 TEST_CASE_COUNT = None
 
 
-@pytest.fixture(scope="session")
-def catalog() -> AntaCatalog:
-    """Fixture that generate an ANTA catalog from unit test data. Also configure respx to mock eAPI responses."""
-    global TEST_CASE_COUNT  # noqa: PLW0603  pylint: disable=global-statement
+@pytest.fixture(name="anta_mock_env", scope="session")  # We want this fixture to have a scope set to session to avoid reparsing all the unit tests data.
+def anta_mock_env_fixture() -> AntaMockEnvironment:
+    """Return an AntaMockEnvironment for this test session. Also configure respx to mock eAPI responses."""
+    global TEST_CASE_COUNT  # noqa: PLW0603
     eapi_route = respx.post(path="/command-api", headers={"Content-Type": "application/json-rpc"})
     env = AntaMockEnvironment()
-    TEST_CASE_COUNT = len(env.catalog.tests)
+    TEST_CASE_COUNT = env.tests_count
     eapi_route.side_effect = env.eapi_response
-    return env.catalog
-
-
-@pytest.fixture
-def inventory(request: pytest.FixtureRequest) -> AntaInventory:
-    """Generate an ANTA inventory."""
-    inv = AntaInventory()
-    for i in range(request.param["count"]):
-        inv.add_device(
-            AsyncEOSDevice(
-                host=f"device-{i}.avd.arista.com",
-                username="admin",
-                password="admin",  # noqa: S106
-                name=f"device-{i}",
-                disable_cache=(not request.param["cache"]),
-            )
-        )
-    return inv
+    return env
+
+
+@pytest.fixture  # This fixture should have a scope set to function as the indexing result is stored in this object
+def catalog(anta_mock_env: AntaMockEnvironment) -> AntaCatalog:
+    """Fixture that return an ANTA catalog from the AntaMockEnvironment of this test session."""
+    return anta_mock_env.catalog
 
 
 def pytest_terminal_summary(terminalreporter: TerminalReporter) -> None:

tests/benchmark/test_anta.py

@@ -5,7 +5,7 @@
 
 import asyncio
 import logging
-from unittest.mock import AsyncMock, Mock, patch
+from unittest.mock import patch
 
 import pytest
 import respx
@@ -25,8 +25,8 @@
 @pytest.mark.parametrize(
     "inventory",
     [
-        pytest.param({"count": 1, "cache": False}, id="1 device"),
-        pytest.param({"count": 2, "cache": False}, id="2 devices"),
+        pytest.param({"count": 1, "disable_cache": True, "reachable": False}, id="1 device"),
+        pytest.param({"count": 2, "disable_cache": True, "reachable": False}, id="2 devices"),
     ],
     indirect=True,
 )
@@ -45,17 +45,22 @@ def bench() -> ResultManager:
     logging.disable(logging.NOTSET)
     if len(manager.results) != 0:
         pytest.fail("ANTA Dry-Run mode should not return any result", pytrace=False)
+    if catalog.final_tests_count != len(inventory) * len(catalog.tests):
+        pytest.fail(f"Expected {len(inventory) * len(catalog.tests)} selected tests but got {catalog.final_tests_count}", pytrace=False)
+    bench_info = (
+        "\n--- ANTA NRFU Dry-Run Benchmark Information ---\n" f"Selected tests: {catalog.final_tests_count}\n" "-----------------------------------------------"
+    )
+    logger.info(bench_info)
 
 
 @pytest.mark.parametrize(
     "inventory",
     [
-        pytest.param({"count": 1, "cache": False}, id="1 device"),
-        pytest.param({"count": 2, "cache": False}, id="2 devices"),
+        pytest.param({"count": 1, "disable_cache": True}, id="1 device"),
+        pytest.param({"count": 2, "disable_cache": True}, id="2 devices"),
     ],
     indirect=True,
 )
-@patch("asyncio.open_connection", AsyncMock(spec=asyncio.open_connection, return_value=(Mock(), Mock())))  # We just want all devices to be reachable
 @patch("anta.models.AntaTest.collect", collect)
 @patch("anta.device.AntaDevice.collect_commands", collect_commands)
 @respx.mock  # Mock eAPI responses

tests/benchmark/utils.py

@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 import asyncio
+import copy
 import importlib
 import json
 import pkgutil
@@ -60,7 +61,16 @@ class AntaMockEnvironment:  # pylint: disable=too-few-public-methods
     """
 
     def __init__(self) -> None:
-        self.catalog, self.eos_data_catalog = self._generate_catalog()
+        self._catalog, self.eos_data_catalog = self._generate_catalog()
+        self.tests_count = len(self._catalog.tests)
+
+    @property
+    def catalog(self) -> AntaCatalog:
+        """AntaMockEnvironment object will always return a new AntaCatalog object based on the initial parsing.
+
+        This is because AntaCatalog objects store indexes when tests are run and we want a new object each time a test is run.
+        """
+        return copy.deepcopy(self._catalog)
 
     def _generate_catalog(self) -> tuple[AntaCatalog, dict[tuple[str, str], list[dict[str, Any]]]]:
         """Generate the `catalog` and `eos_data_catalog` attributes."""