From f268f0fff208bc8f5c33a2e792aede487cbb4ad9 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Fri, 8 Mar 2024 11:48:31 +0100 Subject: [PATCH 1/2] [uss_qualifier] fragments for dss0210 op-intent ref. synchronization PR fixes: reword checks in fragment, adapt validator, add cleanup_op_intent --- monitoring/monitorlib/schema_validation.py | 4 +- .../astm/utm/dss/fragments/oir/crud/create.md | 21 + .../astm/utm/dss/fragments/oir/crud/delete.md | 20 + .../astm/utm/dss/fragments/oir/crud/read.md | 27 + .../astm/utm/dss/fragments/oir/crud/update.md | 20 + .../astm/utm/dss/fragments/oir/sync.md | 33 ++ .../dss/fragments/oir/validate/correctness.md | 53 ++ .../utm/dss/fragments/oir/validate/mutated.md | 14 + .../dss/fragments/oir/validate/non_mutated.md | 14 + .../astm/utm/dss/test_step_fragments.py | 50 ++ .../astm/utm/dss/validators/oir_validator.py | 516 ++++++++++++++++++ 11 files changed, 770 insertions(+), 2 deletions(-) create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/delete.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/read.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/sync.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/validate/correctness.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/validate/mutated.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/validate/non_mutated.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/oir_validator.py diff --git a/monitoring/monitorlib/schema_validation.py b/monitoring/monitorlib/schema_validation.py index 72bffa250c..3d61fd2507 100644 --- a/monitoring/monitorlib/schema_validation.py +++ b/monitoring/monitorlib/schema_validation.py @@ -1,13 +1,13 @@ import os.path from dataclasses import dataclass +from datetime import datetime from enum import Enum from pathlib import Path -from typing import List, Dict, Type, TypeVar +from typing import List, Dict, Type import bc_jsonpath_ng import jsonschema.validators import yaml - from implicitdict import ImplicitDict from implicitdict.jsonschema import SchemaVars, make_json_schema diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md new file mode 100644 index 0000000000..7f340ba94b --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md @@ -0,0 +1,21 @@ +# Create operational intent reference test step fragment + +This test step fragment validates that operational intent references can be created + +## 🛑 Create operational intent reference query succeeds check + +As per **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**, the DSS API must allow callers to create an operational intent reference with either one or both of the +start and end time missing, provided all the required parameters are valid. + +## 🛑 Create operational intent reference response content is correct check + +A successful operational intent reference creation query is expected to return a body, the content of which reflects the created operational intent reference. +If the content of the response does not correspond to what was requested, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +This check will usually be performing a series of sub-checks from the [validate](../validate) fragments. + +## ⚠️ Create operational intent reference response format conforms to spec check + +The response to a successful operational intent reference creation query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. + +If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/delete.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/delete.md new file mode 100644 index 0000000000..eab08126c8 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/delete.md @@ -0,0 +1,20 @@ +# Delete operational intent reference test step fragment + +This test step fragment validates that operational intent references can be deleted + +## 🛑 Delete operational intent reference query succeeds check + +A query to delete an operational intent reference, by its owner and when the correct OVN is provided, should succeed, otherwise the DSS is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +## 🛑 Delete operational intent reference response content is correct check + +A successful operational intent reference deletion query is expected to return a body, the content of which reflects the operational intent reference at the moment of deletion. +If the content of the response does not correspond to what was requested, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +This check will usually be performing a series of sub-checks from the [validate](../validate) fragments. + +## ⚠️ Delete operational intent reference response format conforms to spec check + +The response to a successful operational intent reference deletion query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. + +If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/read.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/read.md new file mode 100644 index 0000000000..3d06d6f69a --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/read.md @@ -0,0 +1,27 @@ +# Read operational intent reference test step fragment + +This test step fragment validates that operational intent references can be read + +## 🛑 Get operational intent reference by ID check + +If an operational intent reference cannot be queried using its ID, the DSS is failing to meet **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +## ⚠️ Get operational intent reference response format conforms to spec check + +The response to a successful get operational intent reference query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. + +If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +## 🛑 Successful operational intent reference search query check + +If the DSS fails to let us search in the area for which the OIR was created, it is failing to meet **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +## 🛑 Created operational intent reference is in search results check + +If the existing operational intent reference is not returned in a search that covers the area it was created for, the DSS is not properly implementing **[astm.f3548.v21.DSS0005,2](../../../../../../../requirements/astm/f3548/v21.md)**. + +## ⚠️ Search operational intent reference response format conforms to spec check + +The response to a successful operational intent reference search query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. + +If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md new file mode 100644 index 0000000000..e3e66d3e00 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md @@ -0,0 +1,20 @@ +# Update operational intent reference test step fragment + +This test step fragment validates that operational intent references can be updated. + +## 🛑 Mutate operational intent reference query succeeds check + +As per **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**, the DSS API must allow callers to mutate an operational intent reference. + +## 🛑 Mutate operational intent reference response content is correct check + +A successful operational intent reference mutation query is expected to return a well-defined body, the content of which reflects the updated operational intent reference. +If the content of the response does not correspond to what was requested, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +This check will usually be performing a series of sub-checks from the [validate](../validate) fragments. + +## ⚠️ Mutate operational intent reference response format conforms to spec check + +The response to a successful operational intent reference mutation query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. + +If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/sync.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/sync.md new file mode 100644 index 0000000000..131d0a28a1 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/sync.md @@ -0,0 +1,33 @@ +# Synchronize operational intent reference test step fragment + +This test step fragment validates that operational intent references are properly synchronized across a set of DSS instances. + +## 🛑 Operational intent reference can be found at every DSS check + +If the previously created or mutated operational intent reference cannot be found at a DSS, either one of the instances at which the operational intent reference was created or the one that was queried, +may be failing to implement **[astm.f3548.v21.DSS0210,2a](../../../../../../requirements/astm/f3548/v21.md)**. + +## 🛑 Propagated operational intent reference contains the correct manager check + +If the operational intent reference returned by a DSS to which the operational intent reference was synchronized to does not contain the correct manager, +either one of the instances at which the operational intent reference was created or the one that was queried, may be failing to implement **[astm.f3548.v21.DSS0210,2b](../../../../../../requirements/astm/f3548/v21.md)**. + +## 🛑 Propagated operational intent reference contains the correct USS base URL check + +If the operational intent reference returned by a DSS to which the operational intent reference was synchronized to does not contain the correct USS base URL, +either one of the instances at which the operational intent reference was created or the one that was queried, may be failing to implement **[astm.f3548.v21.DSS0210,2c](../../../../../../requirements/astm/f3548/v21.md)**. + +## 🛑 Propagated operational intent reference contains the correct state check + +If the operational intent reference returned by a DSS to which the operational intent reference was synchronized to does not contain the correct state, +either one of the instances at which the operational intent reference was created or the one that was queried, may be failing to implement **[astm.f3548.v21.DSS0210,2d](../../../../../../requirements/astm/f3548/v21.md)**. + +## 🛑 Propagated operational intent reference contains the correct start time check + +If the operational intent reference returned by a DSS to which the operational intent reference was synchronized to does not contain the correct start time, +either one of the instances at which the operational intent reference was created or the one that was queried, may be failing to implement **[astm.f3548.v21.DSS0210,2f](../../../../../../requirements/astm/f3548/v21.md)**. + +## 🛑 Propagated operational intent reference contains the correct end time check + +If the operational intent reference returned by a DSS to which the operational intent reference was synchronized to does not contain the correct end time, +either one of the instances at which the operational intent reference was created or the one that was queried, may be failing to implement **[astm.f3548.v21.DSS0210,2f](../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/validate/correctness.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/validate/correctness.md new file mode 100644 index 0000000000..597be17494 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/validate/correctness.md @@ -0,0 +1,53 @@ +# Validate the content of an operational intent reference test step fragment + +This test step fragment attempts to validate the content of a single operational intent reference returned by the DSS. + +Fields that require different handling based on if the operational intent reference was mutated or not are covered + +The code for these checks lives in the [oir_validator.py](../../../validators/oir_validator.py) class. + +## ⚠️ Returned operational intent reference ID is correct check + +If the returned operational intent reference ID does not correspond to the one specified in the creation parameters, +**[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)** is not respected. + +## ⚠️ Returned operational intent reference has a manager check + +If the returned operational intent reference has no manager defined, **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)** is not respected. + +## ⚠️ Returned operational intent reference manager is correct check + +The returned manager must correspond to the identity of the client that created the operational intent at the DSS, +otherwise the DSS is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +## ⚠️ Returned operational intent reference state is correct check + +The returned state must be the same as the provided one, otherwise the DSS is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +## ⚠️ Returned operational intent reference has an USS base URL check + +If the returned operational intent reference has no USS base URL defined, **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)** is not respected. + +## ⚠️ Returned operational intent reference base URL is correct check + +The returned USS base URL must be prefixed with the USS base URL that was provided at operational intent reference creation, otherwise the DSS is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +## ⚠️ Returned operational intent reference has a start time check + +If the returned operational intent reference has no start time defined, **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)** is not respected. + +## ⚠️ Returned start time is correct check + +The returned start time must be the same as the provided one, otherwise the DSS is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +## ⚠️ Returned operational intent reference has an end time check + +Operational intent references need a defined end time in order to limit their duration: if the DSS omits to set the end time, it will be in violation of **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +## ⚠️ Returned end time is correct check + +The returned end time must be the same as the provided one, otherwise the DSS is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +## ⚠️ Returned operational intent reference has a version check + +If the returned operational intent reference has no version defined, **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)** is not respected. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/validate/mutated.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/validate/mutated.md new file mode 100644 index 0000000000..635a01dcf7 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/validate/mutated.md @@ -0,0 +1,14 @@ +# Validate mutated operational intent reference test step fragment + +This test step fragment attempts to validate a single operational intent reference returned by the DSS, +usually after it has been mutated. + +The code for these checks lives in the [oir_validator.py](../../../validators/oir_validator.py) class. + +## ⚠️ Mutated operational intent reference version is updated check + +Following a mutation, the DSS needs to update the operational intent reference version, otherwise it is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +## ⚠️ Mutated operational intent reference OVN is updated check + +Following a mutation, the DSS needs to update the operational intent reference OVN, otherwise it is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/validate/non_mutated.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/validate/non_mutated.md new file mode 100644 index 0000000000..f0b4687501 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/validate/non_mutated.md @@ -0,0 +1,14 @@ +# Validate non-mutated operational intent reference test step fragment + +This test step fragment attempts to validate a single operational intent reference returned by the DSS, +usually after it has been created or to confirm it has not been mutated by an action. + +The code for these checks lives in the [oir_validator.py](../../../validators/oir_validator.py) class. + +## ⚠️ Non-mutated operational intent reference keeps the same version check + +If the version of the operational intent reference is updated without there having been any mutation of the operational intent reference, the DSS is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +## ⚠️ Non-mutated operational intent reference keeps the same OVN check + +If the OVN of the operational intent reference is updated without there having been any mutation of the operational intent reference, the DSS is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py index 1cf30767cf..743ec9df6b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py @@ -2,6 +2,7 @@ import loguru +from monitoring.monitorlib.fetch import QueryError from monitoring.monitorlib.mutate.scd import MutatedSubscription from monitoring.monitorlib import fetch from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance @@ -92,3 +93,52 @@ def cleanup_active_subs( for sub_id in query.subscriptions.keys(): cleanup_sub(scenario, dss, sub_id) + + +def cleanup_active_oirs( + scenario: TestScenarioType, + dss: DSSInstance, + volume: Volume4D, + manager_identity: str, +) -> None: + with scenario.check( + "Operational intent references can be searched for", [dss.participant_id] + ) as check: + try: + oirs, query = dss.find_op_intent(volume) + except QueryError as qe: + scenario.record_queries(qe.queries) + check.record_failed( + summary="Failed to query operational intent references", + details=f"Failed to query operational intent references: got response code {qe.queries[0].status_code}", + query_timestamps=[qe.queries[0].request.timestamp], + ) + return + + for oir in oirs: + if oir.manager == manager_identity: + remove_op_intent(scenario, dss, oir.id, oir.ovn) + + +def cleanup_op_intent( + scenario: TestScenarioType, dss: DSSInstance, oi_id: EntityID +) -> None: + """Remove the specified operational intent reference from the DSS, if it exists.""" + + with scenario.check( + "Operational intent references can be queried by ID", [dss.participant_id] + ) as check: + try: + oir, q = dss.get_op_intent_reference(oi_id) + except fetch.QueryError as e: + scenario.record_queries(e.queries) + if e.cause_status_code != 404: + check.record_failed( + summary="OIR Get query returned code different from 200 or 404", + details=e.msg, + query_timestamps=e.query_timestamps, + ) + else: + return + + remove_op_intent(scenario, dss, oi_id, oir.ovn) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/oir_validator.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/oir_validator.py new file mode 100644 index 0000000000..2b073f2c70 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/oir_validator.py @@ -0,0 +1,516 @@ +from datetime import datetime +from typing import Optional, List + +from implicitdict import ImplicitDict +from uas_standards.astm.f3548.v21.api import ( + PutOperationalIntentReferenceParameters, + EntityID, + OperationalIntentReference, + ChangeOperationalIntentReferenceResponse, + EntityOVN, + GetOperationalIntentReferenceResponse, + QueryOperationalIntentReferenceResponse, +) + +from monitoring.monitorlib import schema_validation, fetch +from monitoring.monitorlib.schema_validation import F3548_21 +from monitoring.uss_qualifier.scenarios.scenario import PendingCheck, TestScenario + +TIME_TOLERANCE_SEC = 1 +"""tolerance when comparing created vs returned timestamps""" + + +class OIRValidator: + """ + Wraps the validation logic for an operational intent reference that was returned by a DSS + + It will compare the provided OIR with the parameters specified at its creation. + """ + + _main_check: PendingCheck + """ + The overarching check corresponding to the general validation of an OIR. + This check will be failed if any of the sub-checks carried out by this validator fail. + """ + + _scenario: TestScenario + """ + Scenario in which this validator is being used. Will be used to register checks. + """ + + _oir_params: Optional[PutOperationalIntentReferenceParameters] + _pid: List[str] + """Participant ID(s) to use for the checks""" + + def __init__( + self, + main_check: PendingCheck, + scenario: TestScenario, + expected_manager: str, + participant_id: List[str], + oir_params: Optional[PutOperationalIntentReferenceParameters], + ): + self._main_check = main_check + self._scenario = scenario + self._pid = participant_id + self._oir_params = oir_params + self._expected_manager = expected_manager + self._expected_start = oir_params.extents[0].time_start.value.datetime + self._expected_end = oir_params.extents[-1].time_end.value.datetime + + def _fail_sub_check( + self, sub_check: PendingCheck, summary: str, details: str, t_dss: datetime + ) -> None: + """ + Fail the passed sub check with the passed summary and details, and fail + the main check with the passed details. + + Note that this method should only be used to fail sub-checks related to the CONTENT of the OIR, + but not its FORMAT, as the main-check should only be pertaining to the content. + + The provided timestamp is forwarded into the query_timestamps of the check failure. + """ + sub_check.record_failed( + summary=summary, + details=details, + query_timestamps=[t_dss], + ) + + self._main_check.record_failed( + summary=f"Invalid OIR returned by the DSS: {summary}", + details=details, + query_timestamps=[t_dss], + ) + + def _validate_oir( + self, + expected_entity_id: EntityID, + dss_oir: OperationalIntentReference, + t_dss: datetime, + previous_version: Optional[int], + expected_version: Optional[int], + previous_ovn: Optional[str], + expected_ovn: Optional[str], + ) -> None: + """ + Args: + expected_entity_id: the ID we expect to find in the entity + dss_oir: the OIR returned by the DSS + t_dss: timestamp of the query to the DSS for failure reporting + previous_ovn: previous OVN of the entity, if we are verifying a mutation + expected_ovn: expected OVN of the entity, if we are verifying a read query + previous_version: previous version of the entity, if we are verifying a mutation + expected_version: expected version of the entity, if we are verifying a read query + """ + + with self._scenario.check( + "Returned operational intent reference ID is correct", self._pid + ) as check: + if dss_oir.id != expected_entity_id: + self._fail_sub_check( + check, + summary=f"Returned OIR ID is incorrect", + details=f"Expected OIR ID {expected_entity_id}, got {dss_oir.id}", + t_dss=t_dss, + ) + + with self._scenario.check( + "Returned operational intent reference has a manager", self._pid + ) as check: + # Check for empty string. None should have failed the schema check earlier + if not dss_oir.manager: + self._fail_sub_check( + check, + summary="No OIR manager was specified", + details=f"Expected: {self._expected_manager}, got an empty or undefined string", + t_dss=t_dss, + ) + + with self._scenario.check( + "Returned operational intent reference manager is correct", self._pid + ) as check: + if dss_oir.manager != self._expected_manager: + self._fail_sub_check( + check, + summary="Returned manager is incorrect", + details=f"Expected. {self._expected_manager}, got {dss_oir.manager}", + t_dss=t_dss, + ) + + with self._scenario.check( + "Returned operational intent reference has an USS base URL", self._pid + ) as check: + # If uss_base_url is not present, or it is None or Empty, we should fail: + if "uss_base_url" not in dss_oir or not dss_oir.uss_base_url: + self._fail_sub_check( + check, + summary="Returned OIR has no USS base URL", + details="The OIR returned by the DSS has no USS base URL when it should have one", + t_dss=t_dss, + ) + + with self._scenario.check( + "Returned operational intent reference base URL is correct", self._pid + ) as check: + if dss_oir.uss_base_url != self._oir_params.uss_base_url: + self._fail_sub_check( + check, + summary="Returned USS Base URL does not match provided one", + details=f"Provided: {self._oir_params.uss_base_url}, Returned: {dss_oir.uss_base_url}", + t_dss=t_dss, + ) + + with self._scenario.check( + "Returned operational intent reference has a start time", self._pid + ) as check: + if "time_start" not in dss_oir or dss_oir.time_start is None: + self._fail_sub_check( + check, + summary="Returned OIR has no start time", + details="The operational intent reference returned by the DSS has no start time when it should have one", + t_dss=t_dss, + ) + + with self._scenario.check( + "Returned operational intent reference has an end time", self._pid + ) as check: + if "time_end" not in dss_oir or dss_oir.time_end is None: + self._fail_sub_check( + check, + summary="Returned OIR has no end time", + details="The operational intent reference returned by the DSS has no end time when it should have one", + t_dss=t_dss, + ) + + with self._scenario.check("Returned start time is correct", self._pid) as check: + if ( + abs( + dss_oir.time_start.value.datetime - self._expected_start + ).total_seconds() + > TIME_TOLERANCE_SEC + ): + self._fail_sub_check( + check, + summary="Returned start time does not match provided one", + details=f"Provided: {self._oir_params.start_time}, Returned: {dss_oir.time_start}", + t_dss=t_dss, + ) + + with self._scenario.check("Returned end time is correct", self._pid) as check: + if ( + abs( + dss_oir.time_end.value.datetime - self._expected_end + ).total_seconds() + > TIME_TOLERANCE_SEC + ): + self._fail_sub_check( + check, + summary="Returned end time does not match provided one", + details=f"Provided: {self._oir_params.end_time}, Returned: {dss_oir.time_end}", + t_dss=t_dss, + ) + + # If the previous OVN is not None, we are dealing with a mutation: + if previous_ovn is not None: + with self._scenario.check( + "Mutated operational intent reference OVN is updated", self._pid + ) as check: + if dss_oir.ovn == previous_ovn: + self._fail_sub_check( + check, + summary="Returned OIR OVN was not updated", + details=f"Expected OVN to be different from {previous_ovn}, but it was not", + t_dss=t_dss, + ) + + if expected_ovn is not None: + with self._scenario.check( + "Non-mutated operational intent reference keeps the same OVN", self._pid + ) as check: + if dss_oir.ovn != expected_ovn: + self._fail_sub_check( + check, + summary="Returned OIR OVN was updated", + details=f"Expected OVN to be {expected_ovn}, Returned: {dss_oir.ovn}", + t_dss=t_dss, + ) + + # If the previous version is not None, we are dealing with a mutation: + if previous_version is not None: + with self._scenario.check( + "Mutated operational intent reference version is updated", self._pid + ) as check: + # TODO confirm that a mutation should imply a version update + if dss_oir.version == previous_version: + self._fail_sub_check( + check, + summary="Returned OIR version was not updated", + details=f"Expected version to be different from {previous_ovn}, but it was not", + t_dss=t_dss, + ) + + # TODO version _might_ get incremented due to changes caused outside of the uss_qualifier + # and we should probably check if it is equal or higher. + if expected_version is not None: + with self._scenario.check( + "Non-mutated operational intent reference keeps the same version", + self._pid, + ) as check: + if dss_oir.version != expected_version: + self._fail_sub_check( + check, + summary="Returned OIR version was updated", + details=f"Expected version to be {expected_ovn}, Returned: {dss_oir.version}", + t_dss=t_dss, + ) + + # TODO add check for: + # - state + # - subscription ID of the OIR (based on passed parameters, if these were set) + + def _validate_put_oir_response_schema( + self, oir_query: fetch.Query, t_dss: datetime, action: str + ) -> bool: + """Validate response bodies for creation and mutation of OIRs. + Returns 'False' if the schema validation failed, 'True' otherwise. + """ + + check_name = ( + "Create operational intent reference response format conforms to spec" + if action == "create" + else "Mutate operational intent reference response format conforms to spec" + ) + + with self._scenario.check(check_name, self._pid) as check: + errors = schema_validation.validate( + F3548_21.OpenAPIPath, + F3548_21.ChangeOperationalIntentReferenceResponse, + oir_query.response.json, + ) + if errors: + _fail_with_schema_errors(check, errors, t_dss) + return False + + return True + + def validate_created_oir( + self, expected_oir_id: EntityID, new_oir: fetch.Query + ) -> None: + """Validate an OIR that was just explicitly created, meaning + we don't have a previous version to compare to, and we expect it to not be an implicit one.""" + + t_dss = new_oir.request.timestamp + + # Validate the response schema + if not self._validate_put_oir_response_schema(new_oir, t_dss, "create"): + return + + # Expected to pass given that we validated the JSON against the schema + parsed_resp = ImplicitDict.parse( + new_oir.response.json, ChangeOperationalIntentReferenceResponse + ) + + oir: OperationalIntentReference = parsed_resp.operational_intent_reference + + # Validate the OIR itself + self._validate_oir( + expected_entity_id=expected_oir_id, + dss_oir=oir, + t_dss=t_dss, + previous_version=None, + expected_version=None, + previous_ovn=None, + expected_ovn=None, + ) + + def validate_mutated_oir( + self, + expected_oir_id: EntityID, + mutated_oir: fetch.Query, + previous_ovn: str, + previous_version: int, + ) -> None: + """Validate an OIR that was just mutated, meaning we have a previous version and OVN to compare to. + Callers must specify if this is an implicit OIR or not.""" + t_dss = mutated_oir.request.timestamp + + # Validate the response schema + if not self._validate_put_oir_response_schema(mutated_oir, t_dss, "mutate"): + return + + oir = ImplicitDict.parse( + mutated_oir.response.json, ChangeOperationalIntentReferenceResponse + ).operational_intent_reference + + # Validate the OIR itself + self._validate_oir( + expected_entity_id=expected_oir_id, + dss_oir=oir, + t_dss=t_dss, + previous_version=previous_version, + expected_version=None, + previous_ovn=previous_ovn, + expected_ovn=None, + ) + + def validate_fetched_oir( + self, + expected_oir_id: EntityID, + fetched_oir: fetch.Query, + expected_version: int, + expected_ovn: EntityOVN, + ) -> None: + """Validate an OIR that was directly queried by its ID.""" + + t_dss = fetched_oir.request.timestamp + + # Validate the response schema + with self._scenario.check( + "Get operational intent reference response format conforms to spec", + self._pid, + ) as check: + errors = schema_validation.validate( + F3548_21.OpenAPIPath, + F3548_21.GetOperationalIntentReferenceResponse, + fetched_oir.response.json, + ) + if errors: + _fail_with_schema_errors(check, errors, t_dss) + + parsed_resp = fetched_oir.parse_json_result( + GetOperationalIntentReferenceResponse + ) + # Validate the OIR itself + self._validate_oir( + expected_entity_id=expected_oir_id, + dss_oir=parsed_resp.operational_intent_reference, + t_dss=t_dss, + previous_version=None, + expected_version=expected_version, + previous_ovn=None, + expected_ovn=expected_ovn, + ) + + def validate_searched_oir( + self, + expected_oir_id: EntityID, + search_response: fetch.Query, + expected_ovn: str, + expected_version: int, + ) -> None: + """Validate an OIR that was retrieved through search. + Note that the callers need to pass the entire response from the DSS, as the schema check + will be performed on the entire response, not just the OIR itself. + However, only the expected OIR is checked for the correctness of its contents.""" + + t_dss = search_response.request.timestamp + + # Validate the response schema + self.validate_searched_oir_format(search_response, t_dss) + + resp_parsed = search_response.parse_json_result( + QueryOperationalIntentReferenceResponse + ) + + by_id = {oir: oir.id for oir in resp_parsed.operational_intent_references} + + with self._scenario.check( + "Created operational intent reference is in search results", self._pid + ) as check: + if expected_oir_id not in by_id: + self._fail_sub_check( + check, + summary="Created OIR is not present in search results", + details=f"The OIR with ID {expected_oir_id} was expected to be found in the search results, but these only contained the following entities: {by_id.keys()}", + t_dss=t_dss, + ) + # Depending on the severity defined in the documentation, the above might not raise an exception, + # and we should still stop here if the check failed. + return + + oir = by_id[expected_oir_id] + + # Validate the OIR itself + self._validate_oir( + expected_entity_id=expected_oir_id, + dss_oir=oir, + t_dss=t_dss, + previous_ovn=None, + expected_ovn=expected_ovn, + previous_version=None, + expected_version=expected_version, + ) + + def validate_searched_oir_format( + self, search_response: fetch.Query, t_dss: datetime + ) -> None: + # Validate the response schema + with self._scenario.check( + "Search operational intent reference response format conforms to spec", + self._pid, + ) as check: + errors = schema_validation.validate( + F3548_21.OpenAPIPath, + F3548_21.QueryOperationalIntentReferenceResponse, + search_response.response.json, + ) + if errors: + _fail_with_schema_errors(check, errors, t_dss) + + def validate_deleted_oir( + self, + expected_oir_id: EntityID, + deleted_oir: fetch.Query, + expected_ovn: str, + expected_version: int, + ) -> None: + + t_dss = deleted_oir.request.timestamp + + # Validate the response schema + with self._scenario.check( + "Delete operational intent reference response format conforms to spec", + self._pid, + ) as check: + errors = schema_validation.validate( + F3548_21.OpenAPIPath, + F3548_21.ChangeOperationalIntentReferenceResponse, + deleted_oir.response.json, + ) + if errors: + _fail_with_schema_errors(check, errors, t_dss) + + oir_resp = deleted_oir.parse_json_result( + ChangeOperationalIntentReferenceResponse + ) + + # Validate the OIR itself + self._validate_oir( + expected_entity_id=expected_oir_id, + dss_oir=oir_resp.operational_intent_reference, + t_dss=t_dss, + previous_ovn=None, + expected_ovn=expected_ovn, + previous_version=None, + expected_version=expected_version, + ) + + +def _fail_with_schema_errors( + check: PendingCheck, + errors: List[schema_validation.ValidationError], + t_dss: datetime, +) -> None: + """ + Fail the passed check with the passed schema validation errors. + Note: + The main check IS NOT failed: + The main check pertains to the CONTENT of the response but not its FORMAT. + """ + details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) + check.record_failed( + summary="Response format was invalid", + details="Found the following schema validation errors in the DSS response:\n" + + details, + query_timestamps=[t_dss], + ) From c87e3d7b2c06f56ec529438cc7f7a20f57c1d9bc Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Tue, 19 Mar 2024 10:08:54 +0100 Subject: [PATCH 2/2] latest PR comments --- .../astm/utm/dss/fragments/oir/crud/create.md | 12 ++++++------ .../astm/utm/dss/fragments/oir/crud/delete.md | 12 ++++++------ .../astm/utm/dss/fragments/oir/crud/read.md | 12 ++++++------ .../astm/utm/dss/fragments/oir/crud/update.md | 12 ++++++------ .../scenarios/astm/utm/dss/fragments/oir/sync.md | 10 +++++----- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md index 7f340ba94b..b6594e25d3 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md @@ -7,15 +7,15 @@ This test step fragment validates that operational intent references can be crea As per **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**, the DSS API must allow callers to create an operational intent reference with either one or both of the start and end time missing, provided all the required parameters are valid. +## 🛑 Create operational intent reference response format conforms to spec check + +The response to a successful operational intent reference creation query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. + +If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + ## 🛑 Create operational intent reference response content is correct check A successful operational intent reference creation query is expected to return a body, the content of which reflects the created operational intent reference. If the content of the response does not correspond to what was requested, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. This check will usually be performing a series of sub-checks from the [validate](../validate) fragments. - -## ⚠️ Create operational intent reference response format conforms to spec check - -The response to a successful operational intent reference creation query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. - -If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/delete.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/delete.md index eab08126c8..aff39aa345 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/delete.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/delete.md @@ -6,15 +6,15 @@ This test step fragment validates that operational intent references can be dele A query to delete an operational intent reference, by its owner and when the correct OVN is provided, should succeed, otherwise the DSS is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. +## 🛑 Delete operational intent reference response format conforms to spec check + +The response to a successful operational intent reference deletion query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. + +If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + ## 🛑 Delete operational intent reference response content is correct check A successful operational intent reference deletion query is expected to return a body, the content of which reflects the operational intent reference at the moment of deletion. If the content of the response does not correspond to what was requested, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. This check will usually be performing a series of sub-checks from the [validate](../validate) fragments. - -## ⚠️ Delete operational intent reference response format conforms to spec check - -The response to a successful operational intent reference deletion query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. - -If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/read.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/read.md index 3d06d6f69a..d85f363282 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/read.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/read.md @@ -6,7 +6,7 @@ This test step fragment validates that operational intent references can be read If an operational intent reference cannot be queried using its ID, the DSS is failing to meet **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. -## ⚠️ Get operational intent reference response format conforms to spec check +## 🛑 Get operational intent reference response format conforms to spec check The response to a successful get operational intent reference query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. @@ -16,12 +16,12 @@ If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../ If the DSS fails to let us search in the area for which the OIR was created, it is failing to meet **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Created operational intent reference is in search results check - -If the existing operational intent reference is not returned in a search that covers the area it was created for, the DSS is not properly implementing **[astm.f3548.v21.DSS0005,2](../../../../../../../requirements/astm/f3548/v21.md)**. - -## ⚠️ Search operational intent reference response format conforms to spec check +## 🛑 Search operational intent reference response format conforms to spec check The response to a successful operational intent reference search query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + +## 🛑 Created operational intent reference is in search results check + +If the existing operational intent reference is not returned in a search that covers the area it was created for, the DSS is not properly implementing **[astm.f3548.v21.DSS0005,2](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md index e3e66d3e00..c65141c1e0 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md @@ -6,15 +6,15 @@ This test step fragment validates that operational intent references can be upda As per **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**, the DSS API must allow callers to mutate an operational intent reference. +## 🛑 Mutate operational intent reference response format conforms to spec check + +The response to a successful operational intent reference mutation query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. + +If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. + ## 🛑 Mutate operational intent reference response content is correct check A successful operational intent reference mutation query is expected to return a well-defined body, the content of which reflects the updated operational intent reference. If the content of the response does not correspond to what was requested, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. This check will usually be performing a series of sub-checks from the [validate](../validate) fragments. - -## ⚠️ Mutate operational intent reference response format conforms to spec check - -The response to a successful operational intent reference mutation query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. - -If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/sync.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/sync.md index 131d0a28a1..d6623f85e4 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/sync.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/sync.md @@ -7,27 +7,27 @@ This test step fragment validates that operational intent references are properl If the previously created or mutated operational intent reference cannot be found at a DSS, either one of the instances at which the operational intent reference was created or the one that was queried, may be failing to implement **[astm.f3548.v21.DSS0210,2a](../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Propagated operational intent reference contains the correct manager check +## ⚠️ Propagated operational intent reference contains the correct manager check If the operational intent reference returned by a DSS to which the operational intent reference was synchronized to does not contain the correct manager, either one of the instances at which the operational intent reference was created or the one that was queried, may be failing to implement **[astm.f3548.v21.DSS0210,2b](../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Propagated operational intent reference contains the correct USS base URL check +## ⚠️ Propagated operational intent reference contains the correct USS base URL check If the operational intent reference returned by a DSS to which the operational intent reference was synchronized to does not contain the correct USS base URL, either one of the instances at which the operational intent reference was created or the one that was queried, may be failing to implement **[astm.f3548.v21.DSS0210,2c](../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Propagated operational intent reference contains the correct state check +## ⚠️ Propagated operational intent reference contains the correct state check If the operational intent reference returned by a DSS to which the operational intent reference was synchronized to does not contain the correct state, either one of the instances at which the operational intent reference was created or the one that was queried, may be failing to implement **[astm.f3548.v21.DSS0210,2d](../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Propagated operational intent reference contains the correct start time check +## ⚠️ Propagated operational intent reference contains the correct start time check If the operational intent reference returned by a DSS to which the operational intent reference was synchronized to does not contain the correct start time, either one of the instances at which the operational intent reference was created or the one that was queried, may be failing to implement **[astm.f3548.v21.DSS0210,2f](../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Propagated operational intent reference contains the correct end time check +## ⚠️ Propagated operational intent reference contains the correct end time check If the operational intent reference returned by a DSS to which the operational intent reference was synchronized to does not contain the correct end time, either one of the instances at which the operational intent reference was created or the one that was queried, may be failing to implement **[astm.f3548.v21.DSS0210,2f](../../../../../../requirements/astm/f3548/v21.md)**.