diff --git a/monitoring/monitorlib/clients/flight_planning/client.py b/monitoring/monitorlib/clients/flight_planning/client.py index 0975b0eda4..e9a575ba68 100644 --- a/monitoring/monitorlib/clients/flight_planning/client.py +++ b/monitoring/monitorlib/clients/flight_planning/client.py @@ -93,3 +93,12 @@ def clear_area(self, area: Volume4D) -> TestPreparationActivityResponse: * PlanningActivityError """ raise NotImplementedError() + + @abstractmethod + def get_base_url(self) -> str: + """ + Get the base_url associated with this FlightPlannerClient + Returns: + + """ + raise NotImplementedError diff --git a/monitoring/monitorlib/clients/flight_planning/client_scd.py b/monitoring/monitorlib/clients/flight_planning/client_scd.py index 499e23ff27..8345a56c58 100644 --- a/monitoring/monitorlib/clients/flight_planning/client_scd.py +++ b/monitoring/monitorlib/clients/flight_planning/client_scd.py @@ -260,3 +260,6 @@ def clear_area(self, area: Volume4D) -> TestPreparationActivityResponse: errors = [f"[{resp.outcome.timestamp}]: {resp.outcome.message}"] return TestPreparationActivityResponse(errors=errors, queries=[query]) + + def get_base_url(self): + return self._session.get_prefix_url() diff --git a/monitoring/monitorlib/clients/flight_planning/client_v1.py b/monitoring/monitorlib/clients/flight_planning/client_v1.py index 8f37cf77c6..e7c99c9c65 100644 --- a/monitoring/monitorlib/clients/flight_planning/client_v1.py +++ b/monitoring/monitorlib/clients/flight_planning/client_v1.py @@ -222,3 +222,6 @@ def clear_area(self, area: Volume4D) -> TestPreparationActivityResponse: errors = [resp.outcome.message] return TestPreparationActivityResponse(errors=errors, queries=[query]) + + def get_base_url(self): + return self._session.get_prefix_url() diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index 404fe1ed4b..340ad38347 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -3,6 +3,7 @@ import os import traceback import uuid +import jwt from typing import Dict, Optional, List, Union from enum import Enum @@ -290,6 +291,15 @@ def error_message(self) -> Optional[str]: else None ) + def get_client_sub(self): + headers = self.request.headers + if "Authorization" in headers: + token = headers.get("Authorization").split(" ")[1] + payload = jwt.decode( + token, algorithms="RS256", options={"verify_signature": False} + ) + return payload["sub"] + class QueryError(RuntimeError): """Error encountered when interacting with a server in the UTM ecosystem.""" diff --git a/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py b/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py index 76c29fbdb6..aabb03bc50 100644 --- a/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py +++ b/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py @@ -1,9 +1,8 @@ from __future__ import annotations - import uuid -from typing import Tuple, List, Optional -from urllib.parse import urlparse +from typing import Tuple, List, Dict, Optional +from urllib.parse import urlparse from implicitdict import ImplicitDict from monitoring.monitorlib import infrastructure, fetch @@ -127,6 +126,29 @@ def get_full_op_intent( op_intent_ref: OperationalIntentReference, uss_participant_id: Optional[str] = None, ) -> Tuple[OperationalIntent, fetch.Query]: + result, query = self.get_full_op_intent_without_validation( + op_intent_ref, + uss_participant_id, + ) + if query.status_code != 200: + result = None + else: + result = ImplicitDict.parse( + query.response.json, GetOperationalIntentDetailsResponse + ).operational_intent + return result, query + + def get_full_op_intent_without_validation( + self, + op_intent_ref: OperationalIntentReference, + uss_participant_id: Optional[str] = None, + ) -> Tuple[Dict, fetch.Query]: + """ + GET OperationalIntent without validating, as invalid data expected for negative tests + + Returns: + returns the response json when query is successful + """ op = OPERATIONS[OperationID.GetOperationalIntentDetails] query = fetch.query_and_describe( self.client, @@ -136,12 +158,10 @@ def get_full_op_intent( uss_participant_id, scope=SCOPE_SC, ) - if query.status_code != 200: - result = None - else: - result = ImplicitDict.parse( - query.response.json, GetOperationalIntentDetailsResponse - ).operational_intent + result = None + if query.status_code == 200: + result = query.response.json + return result, query def put_op_intent( diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index f6c3c67eb8..4fbb2cdfb4 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -35,12 +35,14 @@ Flight 2 should be successfully planned by the control USS. ### [Validate flight 2 sharing test step](../validate_shared_operational_intent.md) Validate that flight 2 is planned -### Precondition - check tested_uss has no subscription in flight 2 area test step +### Check for notification to tested_uss due to subscription in flight 2 area test step In order to run this test scenario, we need tested_uss to trigger GET operational intent request to mock_uss. -So we need to make sure that there is no subscription of tested_uss, that would trigger notification of flight 2 to tested_uss. -No notification pushed by control_uss to tested_uss, will ensure that tested_uss will make a GET request to control_uss for flight 2 details, -while planning a nearby flight. -If a notification is sent to tested_uss, the precondition for running this scenario will not be satisfied. +But, if there is a subscription by tested_uss, that would trigger notification of flight 2 to tested_uss. +Some USSes will not make a GET request to control_uss for flight 2 details, while planning a nearby flight, +if they got a notification for flight2. Hence, if a USS didn't make a GET request, we will only fail it if didn't get +a notification, or else, a precondition for the test will not be met. Some USSes might make a GET request despite getting +a notification, but as it would not be clear whether invalid information through notification or GET request was used for planning, +the test will be not be continued. ### [Tested_uss plans flight 1 test step](../../../flight_planning/plan_flight_intent.md) The test driver attempts to plan flight 1 via the tested USS. It checks if any conflicts with flight 2 @@ -49,7 +51,7 @@ which is of equal priority and came first. ### [Validate flight 1 sharing test step](../validate_shared_operational_intent.md) Validate flight 1 is planned. -### [Validate flight2 GET interaction test step](test_steps/validate_get_operational_intent.md) +### [Validate flight2 GET interaction, if no notification test step](test_steps/validate_get_operational_intent.md) Validate that tested_uss makes a GET request for obtaining details of flight 2 from control_uss. In a previous step (Precondition - check tested_uss has no subscription in flight 2 area), we ensured that no notification of flight 2 was sent to tested_uss. Hence, tested_uss will need to make a GET request to obtain flight 2 details. @@ -71,12 +73,14 @@ The control_uss, which is mock_uss is instructed to share invalid data with othe ### [Validate flight 2 shared operational intent with invalid data test step](test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md) Validate that flight 2 is shared with invalid data as a modified behavior is injected by uss_qualifier for a negative test. -### Precondition - check tested_uss has no subscription in flight 2 area test step +### Check for notification to tested_uss due to subscription in flight 2 area test step In order to run this test scenario, we need tested_uss to trigger GET operational intent request to mock_uss. -So we need to make sure that there is no subscription of tested_uss, that would trigger notification of flight 2 to tested_uss. -No notification pushed by control_uss to tested_uss, will ensure that tested_uss will make a GET request to control_uss for flight 2 details, -while planning a nearby flight. -If a notification is sent to tested_uss, the precondition for running this scenario will not be satisfied. +But, if there is a subscription by tested_uss, that would trigger notification of flight 2 to tested_uss. +Some USSes will not make a GET request to control_uss for flight 2 details, while planning a nearby flight, +if they got a notification for flight2. Hence, if a USS didn't make a GET request, we will only fail it if didn't get +a notification, or else, a precondition for the test will not be met. Some USSes might make a GET request despite getting +a notification, but as it would not be clear whether invalid information through notification or GET request was used for planning, +the test will be not be continued. ### [Tested_uss attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) The test driver attempts to plan the flight 1 via the tested_uss. It checks if any conflicts with flight 2 @@ -85,7 +89,7 @@ which is of equal priority and came first. ### [Validate flight 1 not shared by tested_uss test step](../validate_not_shared_operational_intent.md) Validate flight 1 is not shared with DSS, as plan failed. -### [Validate flight 2 GET interaction test step](test_steps/validate_get_operational_intent.md) +### [Validate flight2 GET interaction, if no notification test step](test_steps/validate_get_operational_intent.md) Validate that tested_uss makes a GET request for obtaining details of flight 2 from control_uss. In a previous step (Precondition - check tested_uss has no subscription in flight 2 area), we ensured that no notification of flight 2 was sent to tested_uss. Hence, tested_uss will need to make a GET request to obtain flight 2 details. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py index 65f7e485eb..9e5a341c72 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py @@ -1,11 +1,10 @@ from typing import Optional, Dict - from monitoring.monitorlib.clients.flight_planning.flight_info_template import ( FlightInfoTemplate, ) -from monitoring.monitorlib.temporal import TimeDuringTest, Time +from monitoring.monitorlib.temporal import TimeDuringTest import arrow - +from monitoring.monitorlib.temporal import Time from monitoring.monitorlib.clients.flight_planning.client import FlightPlannerClient from monitoring.uss_qualifier.resources.astm.f3548.v21 import DSSInstanceResource from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance @@ -24,8 +23,16 @@ from monitoring.uss_qualifier.scenarios.astm.utm.data_exchange_validation.test_steps.invalid_op_test_steps import ( plan_flight_intent_expect_failed, ) -from monitoring.uss_qualifier.scenarios.astm.utm.test_steps import OpIntentValidator - +from monitoring.uss_qualifier.scenarios.astm.utm.test_steps import ( + OpIntentValidator, + OpIntentValidationFailureType, +) +from monitoring.uss_qualifier.scenarios.astm.utm.data_exchange_validation.test_steps.expected_interactions_test_steps import ( + expect_interuss_post_interactions, + expect_get_requests_to_mock_uss_when_no_notification, + expect_no_interuss_post_interactions, + check_any_notification, +) from monitoring.monitorlib.clients.mock_uss.mock_uss_scd_injection_api import ( MockUssFlightBehavior, ) @@ -138,6 +145,7 @@ def _tested_uss_plans_deconflicted_flight_near_existing_flight( ): times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime) flight_2 = self.flight_2.resolve(times) + with OpIntentValidator( self, self.control_uss_client, @@ -145,6 +153,7 @@ def _tested_uss_plans_deconflicted_flight_near_existing_flight( "Validate flight 2 sharing", self._intents_extent, ) as validator: + planning_time = Time(arrow.utcnow().datetime) _, self.flight_2_id = plan_flight( self, "Control_uss plans flight 2", @@ -152,12 +161,16 @@ def _tested_uss_plans_deconflicted_flight_near_existing_flight( flight_2, ) - validator.expect_shared(flight_2) + flight_2_oi_ref = validator.expect_shared(flight_2) self.begin_test_step( - "Precondition - check tested_uss has no subscription in flight 2 area" + "Check for notification to tested_uss due to subscription in flight 2 area" + ) + tested_uss_notified = check_any_notification( + self, + self.control_uss, + planning_time, ) - # ToDo - Add the test step details self.end_test_step() times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime) @@ -170,24 +183,36 @@ def _tested_uss_plans_deconflicted_flight_near_existing_flight( "Validate flight 1 sharing", self._intents_extent, ) as validator: - _, self.flight_1_id = plan_flight( + planning_time = Time(arrow.utcnow().datetime) + plan_res, self.flight_1_id = plan_flight( self, "Tested_uss plans flight 1", self.tested_uss_client, flight_1, ) - validator.expect_shared( flight_1, ) + if not tested_uss_notified: + expect_get_requests_to_mock_uss_when_no_notification( + self, + self.control_uss, + planning_time, + self.control_uss.base_url, + flight_2_oi_ref.id, + self.tested_uss_client.participant_id, + "Validate flight2 GET interaction, if no notification", + ) - self.begin_test_step("Validate flight2 GET interaction") - # ToDo - Add the test step details - self.end_test_step() - - self.begin_test_step("Validate flight1 Notification sent to Control_uss") - # ToDo - Add the test step details - self.end_test_step() + expect_interuss_post_interactions( + self, + self.control_uss, + planning_time, + self.control_uss.base_url, + self.tested_uss_client.participant_id, + plan_res.queries[0].request.timestamp, + "Validate flight1 Notification sent to Control_uss", + ) delete_flight( self, "Delete tested_uss flight", self.tested_uss_client, self.flight_1_id @@ -202,58 +227,85 @@ def _tested_uss_unable_to_plan_flight_near_invalid_shared_existing_flight( times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime) flight_info = self.flight_2.resolve(times) + modify_field1 = "state" + modify_field2 = "priority" # Modifying the request with invalid data behavior = MockUssFlightBehavior( modify_sharing_methods=["GET", "POST"], modify_fields={ - "reference": {"state": "Flying"}, - "details": {"priority": -1}, + "reference": {modify_field1: "Flying"}, + "details": {modify_field2: 1.2}, }, ) additional_fields = {"behavior": behavior} - - _, self.flight_2_id = plan_flight( + with OpIntentValidator( self, - "Control_uss plans flight 2, sharing invalid operational intent data", self.control_uss_client, - flight_info, - additional_fields, - ) + self.dss, + "Validate flight 2 shared operational intent with invalid data", + self._intents_extent, + ) as validator: + planning_time = Time(arrow.utcnow().datetime) + _, self.flight_2_id = plan_flight( + self, + "Control_uss plans flight 2, sharing invalid operational intent data", + self.control_uss_client, + flight_info, + additional_fields, + ) + flight_2_oi_ref = validator.expect_shared_with_invalid_data( + flight_info, + validation_failure_type=OpIntentValidationFailureType.DataFormat, + invalid_fields=[modify_field1, modify_field2], + ) self.begin_test_step( - "Validate flight 2 shared operational intent with invalid data" + "Check for notification to tested_uss due to subscription in flight 2 area" ) - # ToDo - Add the test step details - self.end_test_step() - - self.begin_test_step( - "Precondition - check tested_uss has no subscription in flight 2 area" + tested_uss_notified = check_any_notification( + self, + self.control_uss, + planning_time, ) - # ToDo - Add the test step details self.end_test_step() times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime) flight_1 = self.flight_1.resolve(times) - - _, self.flight_1_id = plan_flight_intent_expect_failed( + with OpIntentValidator( self, - "Tested_uss attempts to plan flight 1, expect failure", self.tested_uss_client, - flight_1, - ) - - self.begin_test_step("Validate flight 1 not shared by tested_uss") - # ToDo - Add the test step details - self.end_test_step() + self.dss, + "Validate flight 1 not shared by tested_uss", + self._intents_extent, + ) as validator: + planning_time = Time(arrow.utcnow().datetime) + _, self.flight_1_id = plan_flight_intent_expect_failed( + self, + "Tested_uss attempts to plan flight 1, expect failure", + self.tested_uss_client, + flight_1, + ) + validator.expect_not_shared() - self.begin_test_step("Validate flight 2 GET interaction") - # ToDo - Add the test step details - self.end_test_step() + if not tested_uss_notified: + expect_get_requests_to_mock_uss_when_no_notification( + self, + self.control_uss, + planning_time, + self.control_uss.base_url, + flight_2_oi_ref.id, + self.tested_uss_client.participant_id, + "Validate flight2 GET interaction, if no notification", + ) - self.begin_test_step("Validate flight 1 Notification not sent to Control_uss") - # ToDo - Add the test step details - self.end_test_step() + expect_no_interuss_post_interactions( + self, + self.control_uss, + planning_time, + self.tested_uss_client.participant_id, + "Validate flight 1 Notification not sent to Control_uss", + ) delete_flight( self, "Delete Control_uss flight", self.control_uss_client, self.flight_2_id diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/constants.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/constants.py new file mode 100644 index 0000000000..96d10437c3 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/constants.py @@ -0,0 +1,7 @@ +MaxTimeToWaitForSubscriptionNotificationSeconds = 7 +""" +This constant is used for waiting to check notifications were sent to mock_uss for an entity created +by another USS, due to subscription in that area. +The details of usage of this constant are in ./validate_notification_operational_intent.md +and ./validate_no_notification_operational_intent.md +""" diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py new file mode 100644 index 0000000000..0278488416 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import datetime +from typing import List, Tuple, Optional +import time + +from monitoring.monitorlib.fetch import QueryError, Query +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.scenarios.scenario import TestScenarioType +from monitoring.uss_qualifier.resources.interuss.mock_uss.client import MockUSSClient +from implicitdict import StringBasedDateTime +from loguru import logger +from monitoring.monitorlib.clients.mock_uss.interactions import Interaction +from monitoring.uss_qualifier.scenarios.astm.utm.data_exchange_validation.test_steps.constants import ( + MaxTimeToWaitForSubscriptionNotificationSeconds as max_wait_time, +) + +# Interval to wait for checking notification received +WAIT_INTERVAL = 1 + + +def expect_interuss_post_interactions( + scenario: TestScenarioType, + mock_uss: MockUSSClient, + st: StringBasedDateTime, + posted_to_url: str, + participant_id: str, + plan_request_time: datetime.datetime, + test_step: str, +): + """ + This step checks if a notification was sent to a subscribed USS, from time 'st' to now + Args: + posted_to_url: url of the subscribed USS + participant_id: id of the participant responsible to send the notification + plan_request_time: timestamp of the flight plan query that would lead to sending notification + + """ + scenario.begin_test_step(test_step) + + # Check for 'notification found' will be done periodically by waiting for a duration till max_wait_time + time_waited = 0 + duration = 0 + while time_waited <= max_wait_time: + time.sleep(duration) + interactions, query = _get_interuss_interactions_with_check( + scenario, + mock_uss, + st, + ) + found = _any_oi_notification_in_interactions(interactions, posted_to_url) + time_waited += duration + if found: + logger.debug(f"Waited for {time_waited} to check notifications.") + break + # wait for WAIT_INTERVAL till max_wait_time reached + duration = min(WAIT_INTERVAL, max_wait_time - time_waited) + + with scenario.check("Expect Notification sent", [participant_id]) as check: + if not found: + check.record_failed( + summary=f"Notification to {posted_to_url} not sent", + severity=Severity.Medium, + details=f"Notification to {posted_to_url} not sent even though DSS instructed the planning USS to notify due to subscription.", + query_timestamps=[plan_request_time, query.request.timestamp], + ) + scenario.end_test_step() + + +def expect_no_interuss_post_interactions( + scenario: TestScenarioType, + mock_uss: MockUSSClient, + st: StringBasedDateTime, + participant_id: str, + test_step: str, +): + """ + This step checks no notification was sent to any USS as no DSS entity was created, from time 'st' to now + Args: + participant_id: id of the participant responsible to send the notification + """ + scenario.begin_test_step(test_step) + + # Wait for next MaxTimeToWaitForSubscriptionNotificationSeconds duration to capture any notification + time.sleep(max_wait_time) + interactions, query = _get_interuss_interactions_with_check( + scenario, + mock_uss, + st, + ) + found = _any_oi_notification_in_interactions(interactions) + with scenario.check("Expect Notification not sent", [participant_id]) as check: + if found: + check.record_failed( + summary=f"Notification was wrongly sent for an entity not created.", + severity=Severity.Medium, + details=f"Notification was wrongly sent for an entity not created.", + query_timestamps=[query.request.timestamp], + ) + scenario.end_test_step() + + +def expect_get_requests_to_mock_uss_when_no_notification( + scenario: TestScenarioType, + mock_uss: MockUSSClient, + st: StringBasedDateTime, + mock_uss_base_url: str, + id: str, + participant_id: str, + test_step: str, +): + """ + This step checks a GET request was made to mock_uss for an existing entity, from time 'st' to now + Args: + mock_uss_base_url: url of the mock_uss that is managing the entity + id: entity id + participant_id: id of the participant responsible to send GET request + + """ + scenario.begin_test_step(test_step) + interactions, query = _get_interuss_interactions_with_check(scenario, mock_uss, st) + logger.debug(f"Checking for GET request to {mock_uss_base_url} for id {id}") + get_requested = False + for interaction in interactions: + method = interaction.query.request.method + url = interaction.query.request.url + if method == "GET" and url.startswith(mock_uss_base_url) and id in url: + get_requested = True + break + if not get_requested: + with scenario.check( + "Expect GET request when no notification", [participant_id] + ) as check: + check.record_failed( + summary=f"No GET request received at {mock_uss_base_url} for {id} ", + severity=Severity.High, + details=f"No GET request received at {mock_uss_base_url} for {id}. A planning USS in the area should have sent a reques to get the intent details.", + query_timestamps=[query.request.timestamp], + ) + scenario.end_test_step() + + +def _get_interuss_interactions_with_check( + scenario: TestScenarioType, + mock_uss: MockUSSClient, + st: StringBasedDateTime, +) -> Tuple[List[Interaction], Query]: + """ + Method to get interuss interactions with a scenario check from mock_uss from time 'st' to now. + """ + with scenario.check( + "MockUSS interactions request", [mock_uss.participant_id] + ) as check: + try: + interactions, query = _get_interuss_interactions( + mock_uss, + st, + ) + scenario.record_query(query) + return interactions, query + except QueryError as e: + for q in e.queries: + scenario.record_query(q) + check.record_failed( + summary=f"Error from mock_uss when attempting to get interactions from_time {st}", + severity=Severity.High, + details=f"{str(e)}\n\nStack trace:\n{e.stacktrace}", + query_timestamps=[q.request.timestamp for q in e.queries], + ) + + +def _get_interuss_interactions( + mock_uss: MockUSSClient, + st: StringBasedDateTime, +) -> Tuple[List[Interaction], Query]: + """ + Method to get interuss interactions from mock_uss from time 'st' to now. + """ + all_interactions, query = mock_uss.get_interactions(st) + exclude_sub = mock_uss.session.auth_adapter.get_sub() + + def is_uss_interaction(interaction: Interaction, excl_sub: str) -> bool: + sub = interaction.query.get_client_sub() + if sub: + if sub == excl_sub: + return False + else: + return True + else: + logger.error(f"Interaction received without Authorization : {interaction}") + return False + + interuss_interactions = [] + for interaction in all_interactions: + if is_uss_interaction(interaction, exclude_sub): + interuss_interactions.append(interaction) + + return interuss_interactions, query + + +def check_any_notification( + scenario: TestScenarioType, + mock_uss: MockUSSClient, + st: StringBasedDateTime, +) -> bool: + """ + This method helps check any notification have been sent, to or from mock_uss. + + Returns: True if any notification found, otherwise False + """ + interactions, query = _get_interuss_interactions( + mock_uss, + st, + ) + scenario.record_query(query) + return _any_oi_notification_in_interactions(interactions) + + +def _any_oi_notification_in_interactions( + interactions: List[Interaction], recipient_base_url: Optional[str] = None +) -> bool: + """ + Checks if there is any POST request made to 'recipient_base_url', and returns True if found. + If 'recipient_base_url' is None, any POST request found returns True. + """ + for interaction in interactions: + method = interaction.query.request.method + url = interaction.query.request.url + if method == "POST": + if recipient_base_url is None or url.startswith(recipient_base_url): + return True + return False diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py index 4dd4ba7e3b..ed65613779 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py @@ -1,23 +1,9 @@ from typing import Optional, Tuple -from implicitdict import ImplicitDict -from uas_standards.astm.f3548.v21.api import ( - OperationalIntentState, - OperationalIntentReference, - GetOperationalIntentDetailsResponse, -) -from loguru import logger -from uas_standards.interuss.automated_testing.scd.v1.api import ( - InjectFlightRequest, - InjectFlightResponse, -) -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.flight_planning.flight_planner import ( FlightPlannerClient, ) -from monitoring.uss_qualifier.scenarios.astm.utm.test_steps import OpIntentValidator from monitoring.uss_qualifier.scenarios.flight_planning.test_steps import ( submit_flight, - expect_flight_intent_state, ) from monitoring.uss_qualifier.scenarios.scenario import TestScenarioType from monitoring.monitorlib.clients.flight_planning.flight_info import FlightInfo diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_get_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_get_operational_intent.md index ae8ac1bcd1..82e6c5c709 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_get_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_get_operational_intent.md @@ -5,6 +5,6 @@ This step verifies that a USS makes a GET request to get the intent_details of a ## MockUSS interactions request check **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. -## Expect GET request check +## Expect GET request when no notification check **[astm.f3548.v21.SCD0035](../../../../../requirements/astm/f3548/v21.md)** SCD0035 needs a USS to verify before transitioning to Accepted that it does not conflict with a type of operational intent, and the only way to have verified this is by knowing all operational intent details, and (from previous checks of no notifications) the only way to know the operational intent details of flight is to have requested them via a GET details interaction. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md index b953bf983e..d05c766676 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md @@ -8,3 +8,13 @@ This step verifies when a flight is not created, it is also not notified by chec ## Expect Notification not sent check **[interuss.f3548.notification_requirements.NoDssEntityNoNotification](../../../../../requirements/interuss/f3548/notification_requirements.md)** + +As per the above requirement, the notification should not be sent by a USS about an entity that could not be created in DSS +to any USS. To verify that notification was indeed not sent, we need to wait and check up to a threshold to get confidence +that USS did not send notification. +The max duration for sending a notification in [SCD0085](../../../../../requirements/astm/f3548/v21.md) is MaxRespondToSubscriptionNotification(5) seconds. +However, this duration is from time start - Receipt of subscription notification from DSS, which does not exist for this check. +In this check we use time start when the test driver asked the USS to plan the failed flight. +When checking notification not sent, we should wait for the same duration that is used for when checking notification sent. +[Expect Notification sent](./validate_notification_operational_intent.md). +So, we plan to use [MaxTimeToWaitForSubscriptionNotificationSeconds](./constants.py) (7 seconds). diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md index f2e8515b5f..48c83c8e34 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md @@ -6,7 +6,40 @@ This step verifies that, when creating or modifying an operational intent, a USS **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. ## Expect Notification sent check -**[astm.f3548.v21.SCD0085](../../../../../requirements/astm/f3548/v21.md)** +As per **[astm.f3548.v21.SCD0085](../../../../../requirements/astm/f3548/v21.md)**, the notification should be sent by a +USS about its operational intent to the subscribing USS in no more than MaxRespondToSubscriptionNotification (5) seconds, +95 percent of the time. +To verify that notification was indeed sent for this check, waiting up to MaxRespondToSubscriptionNotificationSeconds gets us +95 percent confidence in declaring the USS non-compliant if notification is not received. +To ensure the notifications sent are not missed for a test case, we can pick a threshold that gives +a very high (e.g., 99 percent per test) confidence of non-compliance. We can make conservative assumptions +about the distribution of the delays. If we assume that the notification delays have a normal distribution +with 95 percentile at 5 seconds, then with the standard deviation of 3.04, we get the 99 percentile at 7.07 seconds. +Hence, for test cases that check notification sent for an operational intent, we will wait for notifications till threshold +[MaxTimeToWaitForSubscriptionNotificationSeconds](./constants.py) (rounding to 7 seconds). + +####Note - +As per **[astm.f3548.v21.SCD0085](../../../../../requirements/astm/f3548/v21.md)**, MaxRespondToSubscriptionNotification +is measured from time_start - Receipt of subscription notification from DSS - +till time_end - Entity details sent to subscribing USS. +To make sure the test driver gives enough time for notifications to be received by mock_uss, +it marks the time to get interactions from mock_uss as - the time test driver initiates the plan. +The sequence of events is - +1. Test driver initiates plan to tested_uss. t0 +2. Tested_uss shares the plan with DSS and receives DSS response. t_time_start. +3. Tested_uss responds to test driver with Completed. +4. Test driver checks for shared operational_intent in DSS and checks its retrievable. t1 +5. Test driver waits for MaxTimeToWaitForSubscriptionNotificationSeconds. +6. Test driver retrieves interactions from mock_uss. t1 + MaxTimeToWaitForSubscriptionNotificationSeconds +7. Test driver should find the notification in these interactions to declare that USS sent notifications. + +We know from above that waiting from t_time_start for MaxTimeToWaitForSubscriptionNotificationSeconds would +give us 99% confidence that we receive the notifications. But, test_driver doesn't have access to t_time_start. +So, it starts waiting from a point of time after the t_time_start that is t1. +This ensures that test driver waits for a long enough duration before getting the interactions. Hence, we get +a high confidence that the test driver correctly verifies if a notification was sent by tested_uss. + + ## Notification data is valid check **[astm.f3548.v21.SCD0085](../../../../../requirements/astm/f3548/v21.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md index ec660b0741..46b70a3e15 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md @@ -6,7 +6,7 @@ This step verifies that a created flight is shared properly per ASTM F3548-21 by **[astm.f3548.v21.DSS0005](../../../../../requirements/astm/f3548/v21.md)** -## Operational intent shared with DSS check +## Operational intent shared correctly check If a reference to the operational intent for the flight is not found in the DSS, this check will fail per **[astm.f3548.v21.USS0005](../../../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.OPIN0025](../../../../../requirements/astm/f3548/v21.md)**. @@ -16,8 +16,10 @@ If the operational intent details for the flight cannot be retrieved from the US ## Invalid data in Operational intent details shared by Mock USS for negative test check -Mock USS shares operational intent details response for the negative test case as per [the GetOperationalIntentDetailsResponse schema of the OpenAPI specification](https://github.com/astm-utm/Protocol/blob/v1.0.0/utm.yaml#L1120). -If the operational intent details from mock_uss is valid, this check will fail. It would mean mock_uss is not behaving correctly. **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. +Mock USS shares operational intent details with specified invalid data in response for the negative test case as per +[the GetOperationalIntentDetailsResponse schema of the OpenAPI specification](https://github.com/astm-utm/Protocol/blob/v1.0.0/utm.yaml#L1120). +If the operational intent details from mock_uss does not contain the specified invalid data, this check will fail. +It would mean mock_uss is not behaving correctly. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.md b/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.md index bff29b4a02..5ab857df78 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.md @@ -10,6 +10,10 @@ This scenario prepares flight planner systems for execution of controlled test s FlightPlannersResource listing all USSs undergoing planning tests so that they can be checked for readiness and instructed to remove any existing flights from the area in this scenario. +### mock_uss + +(Optional) MockUSSResource is checked for readiness and instructed to remove any existing flights from the area in this scenario. + ### dss DSSInstanceResource to check for lingering operational intents after the area has been cleared. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py b/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py index 49f0443fc6..9a87b87273 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py @@ -9,6 +9,9 @@ from monitoring.uss_qualifier.scenarios.flight_planning.prep_planners import ( PrepareFlightPlanners as GenericPrepareFlightPlanners, ) +from monitoring.uss_qualifier.resources.interuss.mock_uss.client import ( + MockUSSResource, +) class PrepareFlightPlanners(GenericPrepareFlightPlanners): @@ -19,6 +22,7 @@ def __init__( flight_planners: FlightPlannersResource, dss: DSSInstanceResource, flight_intents: FlightIntentsResource, + mock_uss: Optional[MockUSSResource] = None, flight_intents2: Optional[FlightIntentsResource] = None, flight_intents3: Optional[FlightIntentsResource] = None, flight_intents4: Optional[FlightIntentsResource] = None, @@ -26,6 +30,7 @@ def __init__( super(PrepareFlightPlanners, self).__init__( flight_planners, flight_intents, + mock_uss, flight_intents2, flight_intents3, flight_intents4, diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py index 1290ecfb99..532dcb632e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py @@ -1,7 +1,8 @@ from __future__ import annotations -from typing import List, Optional, Union - +from enum import Enum +from typing import List, Optional, Union, Set +from implicitdict import ImplicitDict from monitoring.monitorlib import schema_validation, fetch from monitoring.monitorlib.clients.flight_planning.client import FlightPlannerClient from monitoring.monitorlib.geotemporal import Volume4DCollection @@ -9,8 +10,8 @@ OperationalIntentState, Volume4D, OperationalIntentReference, + GetOperationalIntentDetailsResponse, ) - from monitoring.monitorlib.clients.flight_planning.flight_info import ( UasState, AirspaceUsageState, @@ -30,6 +31,11 @@ ) from uas_standards.interuss.automated_testing.scd.v1.api import InjectFlightRequest +OI_DATA_FORMAT = "Operational intent details data format" +OI_CORRECT_DETAILS = "Correct operational intent details" +OFF_NOM_VOLS = "Off-nominal volumes" +VERTICES = "Vertices" + class OpIntentValidator(object): """ @@ -159,6 +165,180 @@ def expect_shared( :returns: the shared operational intent reference. None if skipped because not found. """ + oi_ref = self._operational_intent_shared_check(flight_intent, skip_if_not_found) + + oi_full, oi_full_query = self._dss.get_full_op_intent( + oi_ref, self._flight_planner.participant_id + ) + self._scenario.record_query(oi_full_query) + self._operational_intent_retrievable_check(oi_full_query, oi_ref.id) + + validation_failures = self._evaluate_op_intent_validation(oi_full_query) + + with self._scenario.check( + OI_DATA_FORMAT, + [self._flight_planner.participant_id], + ) as check: + data_format_fail = ( + self._expected_validation_failure_found( + validation_failures, OpIntentValidationFailureType.DataFormat + ) + if validation_failures + else None + ) + if data_format_fail: + errors = data_format_fail.errors + check.record_failed( + summary="Operational intent details response failed schema validation", + severity=Severity.Medium, + details="The response received from querying operational intent details failed validation against the required OpenAPI schema:\n" + + "\n".join( + f"At {e.json_path} in the response: {e.message}" for e in errors + ), + query_timestamps=[oi_full_query.request.timestamp], + ) + + with self._scenario.check( + OI_CORRECT_DETAILS, [self._flight_planner.participant_id] + ) as check: + priority = ( + flight_intent.operational_intent.priority + if isinstance(flight_intent, InjectFlightRequest) + else flight_intent.astm_f3548_21.priority + ) + if isinstance(flight_intent, InjectFlightRequest): + priority = flight_intent.operational_intent.priority + vols = Volume4DCollection.from_interuss_scd_api( + flight_intent.operational_intent.volumes + + flight_intent.operational_intent.off_nominal_volumes + ) + elif isinstance(flight_intent, FlightInfo): + priority = flight_intent.astm_f3548_21.priority + vols = flight_intent.basic_information.area + + error_text = validate_op_intent_details( + oi_full.details, + priority, + vols.bounding_volume.to_f3548v21(), + ) + if error_text: + check.record_failed( + summary="Operational intent details do not match user flight intent", + severity=Severity.High, + details=error_text, + query_timestamps=[oi_full_query.request.timestamp], + ) + + with self._scenario.check( + OFF_NOM_VOLS, [self._flight_planner.participant_id] + ) as check: + off_nom_vol_fail = ( + self._expected_validation_failure_found( + validation_failures, + OpIntentValidationFailureType.NominalWithOffNominalVolumes, + ) + if validation_failures + else None + ) + if off_nom_vol_fail: + check.record_failed( + summary="Accepted or Activated operational intents are not allowed off-nominal volumes", + severity=Severity.Medium, + details=off_nom_vol_fail.error_text, + query_timestamps=[oi_full_query.request.timestamp], + ) + + with self._scenario.check( + VERTICES, [self._flight_planner.participant_id] + ) as check: + vertices_fail = ( + self._expected_validation_failure_found( + validation_failures, OpIntentValidationFailureType.VertexCount + ) + if validation_failures + else None + ) + if vertices_fail: + check.record_failed( + summary="Too many vertices", + severity=Severity.Medium, + details=vertices_fail.error_text, + query_timestamps=[oi_full_query.request.timestamp], + ) + + self._scenario.end_test_step() + return oi_full.reference + + def expect_shared_with_invalid_data( + self, + flight_intent: Union[InjectFlightRequest, FlightInfo], + validation_failure_type: OpIntentValidationFailureType, + invalid_fields: Optional[List], + skip_if_not_found: bool = False, + ) -> Optional[OperationalIntentReference]: + """Validate that operational intent information was shared with dss, + but when shared with other USSes, it is expected to have specified invalid data. + + This function implements the test step described in validate_sharing_operational_intent_but_with_invalid_interuss_data. + + :param skip_if_not_found: set to True to skip the execution of the checks if the operational intent was not found while it should have been modified. + :param validation_failure_type: specific type of validation failure expected + :param invalid_fields: Optional list of invalid fields to expect when validation_failure_type is OI_DATA_FORMAT + + :returns: the shared operational intent reference. None if skipped because not found. + """ + + oi_ref = self._operational_intent_shared_check(flight_intent, skip_if_not_found) + + goidr_json, oi_full_query = self._dss.get_full_op_intent_without_validation( + oi_ref, self._flight_planner.participant_id + ) + + self._scenario.record_query(oi_full_query) + self._operational_intent_retrievable_check(oi_full_query, oi_ref.id) + + validation_failures = self._evaluate_op_intent_validation(oi_full_query) + expected_validation_failure_found = self._expected_validation_failure_found( + validation_failures, validation_failure_type, invalid_fields + ) + + # validation errors expected check + with self._scenario.check( + "Invalid data in Operational intent details shared by Mock USS for negative test", + [self._flight_planner.participant_id], + ) as check: + if not expected_validation_failure_found: + check.record_failed( + summary="This negative test case requires specific invalid data shared with other USS in Operational intent details ", + severity=Severity.High, + details=f"Data shared by Mock USS with other USSes did not have the specified invalid data, as expected for test case.", + query_timestamps=[oi_full_query.request.timestamp], + ) + + self._scenario.end_test_step() + return oi_ref + + def _operational_intent_retrievable_check( + self, oi_full_query: fetch.Query, ref_id: str + ): + with self._scenario.check( + "Operational intent details retrievable", + [self._flight_planner.participant_id], + ) as check: + if oi_full_query.status_code != 200: + check.record_failed( + summary="Operational intent details could not be retrieved from USS", + severity=Severity.High, + details=f"Received status code {oi_full_query.status_code} from {self._flight_planner.participant_id} when querying for details of operational intent {ref_id}", + query_timestamps=[oi_full_query.request.timestamp], + ) + + def _operational_intent_shared_check( + self, + flight_intent: Union[InjectFlightRequest | FlightInfo], + skip_if_not_found: bool, + ) -> OperationalIntentReference: + self._begin_step() with self._scenario.check( @@ -250,112 +430,152 @@ def expect_shared( ) oi_ref = self._new_oi_ref - oi_full, oi_full_query = self._dss.get_full_op_intent( - oi_ref, self._flight_planner.participant_id + return oi_ref + + def _evaluate_op_intent_validation( + self, oi_full_query: fetch.Query + ) -> Set[OpIntentValidationFailure]: + """Evaluates the validation failures in operational intent received""" + + validation_failures = set() + errors = schema_validation.validate( + schema_validation.F3548_21.OpenAPIPath, + schema_validation.F3548_21.GetOperationalIntentDetailsResponse, + oi_full_query.response.json, ) - self._scenario.record_query(oi_full_query) - with self._scenario.check( - "Operational intent details retrievable", - [self._flight_planner.participant_id], - ) as check: - if oi_full_query.status_code != 200: - check.record_failed( - summary="Operational intent details could not be retrieved from USS", - severity=Severity.High, - details=f"Received status code {oi_full_query.status_code} from {self._flight_planner.participant_id} when querying for details of operational intent {oi_ref.id}", - query_timestamps=[oi_full_query.request.timestamp], + if errors: + validation_failures.add( + OpIntentValidationFailure( + validation_failure_type=OpIntentValidationFailureType.DataFormat, + errors=errors, ) - - with self._scenario.check( - "Operational intent details data format", - [self._flight_planner.participant_id], - ) as check: - errors = schema_validation.validate( - schema_validation.F3548_21.OpenAPIPath, - schema_validation.F3548_21.GetOperationalIntentDetailsResponse, - oi_full_query.response.json, ) - if errors: - check.record_failed( - summary="Operational intent details response failed schema validation", - severity=Severity.Medium, - details="The response received from querying operational intent details failed validation against the required OpenAPI schema:\n" - + "\n".join( - f"At {e.json_path} in the response: {e.message}" for e in errors - ), - query_timestamps=[oi_full_query.request.timestamp], + else: + try: + goidr = ImplicitDict.parse( + oi_full_query.response.json, GetOperationalIntentDetailsResponse ) + oi_full = goidr.operational_intent + + if ( + oi_full.reference.state == OperationalIntentState.Accepted + or oi_full.reference.state == OperationalIntentState.Activated + ) and oi_full.details.get("off_nominal_volumes", None): + details = f"Operational intent {oi_full.reference.id} had {len(oi_full.details.off_nominal_volumes)} off-nominal volumes in wrong state - {oi_full.reference.state}" + validation_failures.add( + OpIntentValidationFailure( + validation_failure_type=OpIntentValidationFailureType.NominalWithOffNominalVolumes, + error_text=details, + ) + ) - with self._scenario.check( - "Correct operational intent details", [self._flight_planner.participant_id] - ) as check: - priority = ( - flight_intent.operational_intent.priority - if isinstance(flight_intent, InjectFlightRequest) - else flight_intent.astm_f3548_21.priority - ) - if isinstance(flight_intent, InjectFlightRequest): - priority = flight_intent.operational_intent.priority - vols = Volume4DCollection.from_interuss_scd_api( - flight_intent.operational_intent.volumes - + flight_intent.operational_intent.off_nominal_volumes - ) - elif isinstance(flight_intent, FlightInfo): - priority = flight_intent.astm_f3548_21.priority - vols = flight_intent.basic_information.area + def volume_vertices(v4): + if "outline_circle" in v4.volume: + return 1 + if "outline_polygon" in v4.volume: + return len(v4.volume.outline_polygon.vertices) - error_text = validate_op_intent_details( - oi_full.details, - priority, - vols.bounding_volume.to_f3548v21(), - ) - if error_text: - check.record_failed( - summary="Operational intent details do not match user flight intent", - severity=Severity.High, - details=error_text, - query_timestamps=[oi_full_query.request.timestamp], + all_volumes = oi_full.details.get("volumes", []) + oi_full.details.get( + "off_nominal_volumes", [] ) + n_vertices = sum(volume_vertices(v) for v in all_volumes) - with self._scenario.check( - "Off-nominal volumes", [self._flight_planner.participant_id] - ) as check: - if ( - oi_full.reference.state == OperationalIntentState.Accepted - or oi_full.reference.state == OperationalIntentState.Activated - ) and oi_full.details.get("off_nominal_volumes", None): - check.record_failed( - summary="Accepted or Activated operational intents are not allowed off-nominal volumes", - severity=Severity.Medium, - details=f"Operational intent {oi_full.reference.id} was {oi_full.reference.state} and had {len(oi_full.details.off_nominal_volumes)} off-nominal volumes", - query_timestamps=[oi_full_query.request.timestamp], + if n_vertices > 10000: + details = ( + f"Operational intent {oi_full.reference.id} had too many total vertices - {n_vertices}", + ) + validation_failures.add( + validation_failure_type=OpIntentValidationFailureType.VertexCount, + error_text=details, + ) + except (KeyError, ValueError) as e: + validation_failures.add( + validation_failure_type=OpIntentValidationFailureType.DataFormat, + error_text=e, ) - with self._scenario.check( - "Vertices", [self._flight_planner.participant_id] - ) as check: - - def volume_vertices(v4): - if "outline_circle" in v4.volume: - return 1 - if "outline_polygon" in v4.volume: - return len(v4.volume.outline_polygon.vertices) + return validation_failures - all_volumes = oi_full.details.get("volumes", []) + oi_full.details.get( - "off_nominal_volumes", [] - ) - n_vertices = sum(volume_vertices(v) for v in all_volumes) + def _expected_validation_failure_found( + self, + validation_failures: Set[OpIntentValidationFailure], + expected_validation_type: OpIntentValidationFailureType, + expected_invalid_fields: Optional[List[str]], + ) -> OpIntentValidationFailure: + """ + Checks if expected validation type is in validation failures + Args: + expected_invalid_fields: If provided with expected_validation_type OI_DATA_FORMAT, check is made for the fields. - if n_vertices > 10000: - check.record_failed( - summary="Too many vertices", - severity=Severity.Medium, - details=f"Operational intent {oi_full.reference.id} had {n_vertices} vertices total", - query_timestamps=[oi_full_query.request.timestamp], - ) + Returns: + Returns the expected validation failure if found, or else None + """ + failure_found: OpIntentValidationFailure = None + for failure in validation_failures: + if failure.validation_failure_type == expected_validation_type: + failure_found = failure - self._scenario.end_test_step() - return oi_ref + if failure_found: + if ( + expected_validation_type == OpIntentValidationFailureType.DataFormat + and expected_invalid_fields + ): + errors = failure_found.errors + + def expected_fields_in_errors( + fields: List[str], + errors: List[schema_validation.ValidationError], + ) -> bool: + all_found = True + for field in fields: + field_in_error = False + for error in errors: + if field in error.json_path: + field_in_error = True + break + all_found = all_found and field_in_error + return all_found + + if not expected_fields_in_errors(expected_invalid_fields, errors): + failure_found = None + + return failure_found + + +class OpIntentValidationFailureType(str, Enum): + DataFormat = "DataFormat" + """The operational intent did not validate against the canonical JSON Schema.""" + + NominalWithOffNominalVolumes = "NominalWithOffNominalVolumes" + """The operational intent was nominal, but it specified off-nominal volumes.""" + + VertexCount = "VertexCount" + """The operational intent had too many vertices.""" + + +class OpIntentValidationFailure(ImplicitDict): + validation_failure_type: OpIntentValidationFailureType + + error_text: Optional[str] = None + """Any error_text returned after validation check""" + + errors: Optional[List[schema_validation.ValidationError]] = None + """Any errors returned after validation check""" + + def __hash__(self): + return hash((self.validation_failure_type, self.error_text, str(self.errors))) + + def __eq__(self, other): + if isinstance(other, OpIntentValidationFailure): + return ( + self.validation_failure_type, + self.error_text, + str(self.errors), + ) == ( + other.validation_failure_type, + other.error_text, + str(other.errors), + ) def set_uss_available( diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/prep_planners.md b/monitoring/uss_qualifier/scenarios/flight_planning/prep_planners.md index 3a25e1dc85..aa05d2f774 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/prep_planners.md +++ b/monitoring/uss_qualifier/scenarios/flight_planning/prep_planners.md @@ -10,6 +10,10 @@ This scenario prepares flight planner systems for execution of controlled test s FlightPlannersResource listing all USSs undergoing planning tests so that they can be checked for readiness and instructed to remove any existing flights from the area in this scenario. +### mock_uss + +(Optional) MockUSSResource is checked for readiness and instructed to remove any existing flights from the area in this scenario. + ### flight_intents FlightIntentsResource containing flight intents that will be used in subsequent tests, so all planners should be instructed to clear any area involved with any of these intents of flights it manages. diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/prep_planners.py b/monitoring/uss_qualifier/scenarios/flight_planning/prep_planners.py index d7dac87e2b..f767574feb 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/prep_planners.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/prep_planners.py @@ -16,6 +16,9 @@ FlightIntentsResource, ) from monitoring.uss_qualifier.scenarios.scenario import TestScenario +from monitoring.uss_qualifier.resources.interuss.mock_uss.client import ( + MockUSSResource, +) MAX_TEST_DURATION = timedelta(minutes=15) """The maximum time the tests depending on the area being clear might last.""" @@ -29,6 +32,7 @@ def __init__( self, flight_planners: FlightPlannersResource, flight_intents: FlightIntentsResource, + mock_uss: Optional[MockUSSResource] = None, flight_intents2: Optional[FlightIntentsResource] = None, flight_intents3: Optional[FlightIntentsResource] = None, flight_intents4: Optional[FlightIntentsResource] = None, @@ -59,6 +63,10 @@ def __init__( self.flight_planners = { fp.participant_id: fp.client for fp in flight_planners.flight_planners } + if mock_uss is not None: + self.flight_planners.update( + {mock_uss.mock_uss.participant_id: mock_uss.mock_uss.flight_planner} + ) def run(self, context): self.begin_test_scenario(context) diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml index 4b44a2febe..96fab43374 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml @@ -38,6 +38,7 @@ actions: scenario_type: scenarios.astm.utm.PrepareFlightPlanners resources: flight_planners: flight_planners + mock_uss: mock_uss dss: dss flight_intents: invalid_flight_intents flight_intents2: priority_preemption_flights? diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.yaml b/monitoring/uss_qualifier/suites/uspace/flight_auth.yaml index 5d57aa329e..dd57cdd671 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.yaml +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.yaml @@ -31,6 +31,7 @@ actions: resources: flight_planners: flight_planners flight_intents: invalid_flight_auth_flights + mock_uss: mock_uss on_failure: Abort - action_generator: generator_type: action_generators.flight_planning.FlightPlannerCombinations