From 6484f72946f94d75a7fa02d1a4533e5412fa1b01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Wed, 25 Sep 2024 11:33:35 +0200 Subject: [PATCH] tests: adapt benchmark fixture after pytest refactoring --- pyproject.toml | 1 + tests/benchmark/conftest.py | 37 ++++++++++++------------------------ tests/benchmark/test_anta.py | 17 +++++++++++------ tests/benchmark/utils.py | 12 +++++++++++- 4 files changed, 35 insertions(+), 32 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 57dcc17bd..0ae244704 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -459,6 +459,7 @@ disable = [ # Any rule listed here can be disabled: https://github.com/astral-sh "line-too-long", "unused-variable", "redefined-builtin", + "global-statement", "abstract-class-instantiated", # Overlap with https://mypy.readthedocs.io/en/stable/error_code_list.html#check-instantiation-of-abstract-classes-abstract "unexpected-keyword-arg", # Overlap with https://mypy.readthedocs.io/en/stable/error_code_list.html#check-arguments-in-calls-call-arg and other rules "no-value-for-parameter" # Overlap with https://mypy.readthedocs.io/en/stable/error_code_list.html#check-arguments-in-calls-call-arg diff --git a/tests/benchmark/conftest.py b/tests/benchmark/conftest.py index 1f9ee3696..c07cc99c2 100644 --- a/tests/benchmark/conftest.py +++ b/tests/benchmark/conftest.py @@ -10,8 +10,6 @@ from _pytest.terminal import TerminalReporter from anta.catalog import AntaCatalog -from anta.device import AsyncEOSDevice -from anta.inventory import AntaInventory from .utils import AntaMockEnvironment @@ -20,32 +18,21 @@ TEST_CASE_COUNT = None -@pytest.fixture(scope="session") -def catalog() -> AntaCatalog: - """Fixture that generate an ANTA catalog from unit test data. Also configure respx to mock eAPI responses.""" - global TEST_CASE_COUNT # noqa: PLW0603 pylint: disable=global-statement +@pytest.fixture(name="anta_mock_env", scope="session") # We want this fixture to have a scope set to session to avoid reparsing all the unit tests data. +def anta_mock_env_fixture() -> AntaMockEnvironment: + """Return an AntaMockEnvironment for this test session. Also configure respx to mock eAPI responses.""" + global TEST_CASE_COUNT # noqa: PLW0603 eapi_route = respx.post(path="/command-api", headers={"Content-Type": "application/json-rpc"}) env = AntaMockEnvironment() - TEST_CASE_COUNT = len(env.catalog.tests) + TEST_CASE_COUNT = env.tests_count eapi_route.side_effect = env.eapi_response - return env.catalog - - -@pytest.fixture -def inventory(request: pytest.FixtureRequest) -> AntaInventory: - """Generate an ANTA inventory.""" - inv = AntaInventory() - for i in range(request.param["count"]): - inv.add_device( - AsyncEOSDevice( - host=f"device-{i}.avd.arista.com", - username="admin", - password="admin", # noqa: S106 - name=f"device-{i}", - disable_cache=(not request.param["cache"]), - ) - ) - return inv + return env + + +@pytest.fixture # This fixture should have a scope set to function as the indexing result is stored in this object +def catalog(anta_mock_env: AntaMockEnvironment) -> AntaCatalog: + """Fixture that return an ANTA catalog from the AntaMockEnvironment of this test session.""" + return anta_mock_env.catalog def pytest_terminal_summary(terminalreporter: TerminalReporter) -> None: diff --git a/tests/benchmark/test_anta.py b/tests/benchmark/test_anta.py index 93b5a8553..3f4577f8a 100644 --- a/tests/benchmark/test_anta.py +++ b/tests/benchmark/test_anta.py @@ -5,7 +5,7 @@ import asyncio import logging -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import patch import pytest import respx @@ -25,8 +25,8 @@ @pytest.mark.parametrize( "inventory", [ - pytest.param({"count": 1, "cache": False}, id="1 device"), - pytest.param({"count": 2, "cache": False}, id="2 devices"), + pytest.param({"count": 1, "disable_cache": True, "reachable": False}, id="1 device"), + pytest.param({"count": 2, "disable_cache": True, "reachable": False}, id="2 devices"), ], indirect=True, ) @@ -45,17 +45,22 @@ def bench() -> ResultManager: logging.disable(logging.NOTSET) if len(manager.results) != 0: pytest.fail("ANTA Dry-Run mode should not return any result", pytrace=False) + if catalog.final_tests_count != len(inventory) * len(catalog.tests): + pytest.fail(f"Expected {len(inventory) * len(catalog.tests)} selected tests but got {catalog.final_tests_count}", pytrace=False) + bench_info = ( + "\n--- ANTA NRFU Dry-Run Benchmark Information ---\n" f"Selected tests: {catalog.final_tests_count}\n" "-----------------------------------------------" + ) + logger.info(bench_info) @pytest.mark.parametrize( "inventory", [ - pytest.param({"count": 1, "cache": False}, id="1 device"), - pytest.param({"count": 2, "cache": False}, id="2 devices"), + pytest.param({"count": 1, "disable_cache": True}, id="1 device"), + pytest.param({"count": 2, "disable_cache": True}, id="2 devices"), ], indirect=True, ) -@patch("asyncio.open_connection", AsyncMock(spec=asyncio.open_connection, return_value=(Mock(), Mock()))) # We just want all devices to be reachable @patch("anta.models.AntaTest.collect", collect) @patch("anta.device.AntaDevice.collect_commands", collect_commands) @respx.mock # Mock eAPI responses diff --git a/tests/benchmark/utils.py b/tests/benchmark/utils.py index 27cc10b91..c59400130 100644 --- a/tests/benchmark/utils.py +++ b/tests/benchmark/utils.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +import copy import importlib import json import pkgutil @@ -60,7 +61,16 @@ class AntaMockEnvironment: # pylint: disable=too-few-public-methods """ def __init__(self) -> None: - self.catalog, self.eos_data_catalog = self._generate_catalog() + self._catalog, self.eos_data_catalog = self._generate_catalog() + self.tests_count = len(self._catalog.tests) + + @property + def catalog(self) -> AntaCatalog: + """AntaMockEnvironment object will always return a new AntaCatalog object based on the initial parsing. + + This is because AntaCatalog objects store indexes when tests are run and we want a new object each time a test is run. + """ + return copy.deepcopy(self._catalog) def _generate_catalog(self) -> tuple[AntaCatalog, dict[tuple[str, str], list[dict[str, Any]]]]: """Generate the `catalog` and `eos_data_catalog` attributes."""