From 4e814aed4a741147ca5034aed0a6c5f0fc33f9c4 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Wed, 13 Dec 2023 15:48:15 +0100 Subject: [PATCH] [uss_qualifier] f3548 - GEN0300 verify interactions with interoperability test instance (#397) [uss_qualifier] gen0300 confirm that the test harness works --- .../scenarios/astm/utm/aggregate_checks.md | 14 ++++ .../scenarios/astm/utm/aggregate_checks.py | 79 +++++++++++++++++-- .../conflict_higher_priority.md | 1 + .../uss_qualifier/suites/astm/utm/f3548_21.md | 7 +- .../suites/faa/uft/message_signing.md | 7 +- .../suites/uspace/flight_auth.md | 7 +- .../suites/uspace/required_services.md | 7 +- 7 files changed, 113 insertions(+), 9 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md index 959526a49e..5ea938f54b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md @@ -20,3 +20,17 @@ planners provided as resource are used to determine and evaluate the 95th percen If the 95th percentile of the requests durations is higher than the threshold `MaxRespondToOIDetailsRequest` (1 second), this check will fail per **[astm.f3548.v21.SCD0075](../../../requirements/astm/f3548/v21.md)**. + +## Interoperability test instance is available test case + +### Interoperability test instance is available test step + +This step verifies that interactions with the interoperability test instances happened and where at least partly successful. + +#### Interoperability test instance is available check + +This check ensures that interactions with the interoperability test instance that each USS must provide are possible. + +If all interactions fail, or if no test instance can be reached, the USS is failing to meet **[astm.f3548.v21.GEN0300](../../../requirements/astm/f3548/v21.md)**. + +If no interaction with a test instance was found, this check is skipped. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py index bd4dc50afd..79657c3d36 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py @@ -1,19 +1,17 @@ from typing import List, Dict +from uas_standards.astm.f3548.v21 import constants + from monitoring.monitorlib import fetch from monitoring.monitorlib.fetch import evaluation, QueryType from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.configurations.configuration import ParticipantID from monitoring.uss_qualifier.resources.flight_planning import FlightPlannersResource -from monitoring.uss_qualifier.suites.suite import ExecutionContext - -from uas_standards.astm.f3548.v21 import constants - from monitoring.uss_qualifier.scenarios.scenario import TestScenario +from monitoring.uss_qualifier.suites.suite import ExecutionContext class AggregateChecks(TestScenario): - _queries: List[fetch.Query] _attributed_queries: Dict[ParticipantID, Dict[QueryType, List[fetch.Query]]] = {} @@ -76,6 +74,14 @@ def run(self, context: ExecutionContext): self.end_test_step() self.end_test_case() + self.begin_test_case("Interoperability test instance is available") + self.begin_test_step("Interoperability test instance is available") + + self._confirm_test_harness_queries_work() + + self.end_test_step() + self.end_test_case() + self.end_test_scenario() def _op_intent_details_step(self): @@ -118,3 +124,66 @@ def _op_intent_details_step(self): f"{participant}/{QueryType.F3548v21USSGetOperationalIntentDetails}", f"checked performances on {len(durations)} queries, 95th percentile: {p95}s", ) + + def _confirm_test_harness_queries_work(self): + """ + For each different type of call to the interoperability test instance, + we look for at least one successful query. + """ + for participant, queries_by_type in self._attributed_queries.items(): + self._validate_participant_test_interop_instance( + participant, queries_by_type + ) + + def _validate_participant_test_interop_instance( + self, + participant_id: str, + participant_queries: dict[QueryType, List[fetch.Query]], + ): + # Keep track of how many interactions we've found for this participant + # if there is None the condition is not met + test_interactions = 0 + success_by_type: Dict[QueryType, bool] = {} + for query_type, queries in participant_queries.items(): + if _is_interop_test_interaction(query_type): + test_interactions += len(queries) + success_by_type[query_type] = False + for query in queries: + if 200 <= query.response.status_code < 300: + success_by_type[query_type] = True + break + + self.record_note( + "test_interop_interactions", + f"Found {test_interactions} interactions with interoperability test instance for {participant_id}", + ) + if test_interactions == 0: + self.record_note( + "test_interop_check_skipped", + f"Skipping check for {participant_id} because no interactions with " + f"interoperability test instance were found", + ) + # If no interactions are observed, we can't determine if the test instance is available + # and the step here. + return + + with self.check( + "Interoperability test instance is available", [participant_id] + ) as check: + for query_type, success in success_by_type.items(): + if not success: + check.record_failed( + summary=f"No successful {query_type} interaction with interoperability test instance", + severity=Severity.Medium, + details=f"Found no successful {query_type} interaction with interoperability test instance, " + f"indicating that the test instance is either not available or not properly implemented.", + ) + + +def _is_interop_test_interaction(query_type: QueryType): + return ( + query_type == QueryType.InterUSSFlightPlanningV1GetStatus + or query_type == QueryType.InterUSSFlightPlanningV1ClearArea + or query_type == QueryType.InterUSSFlightPlanningV1UpsertFlightPlan + or query_type == QueryType.InterUSSFlightPlanningV1DeleteFlightPlan + ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md index fec8fa805a..5acf0a836a 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md @@ -211,6 +211,7 @@ Because the modification attempt was invalid, either Flight 1 should not have be original accepted request), or it should have been removed (because the USS rejected the replacement plan provided). + ## Cleanup ### Successful flight deletion check **[interuss.automated_testing.flight_planning.DeleteFlightSuccess](../../../../../requirements/interuss/automated_testing/flight_planning.md)** diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index 3b22fdfe11..4dd60ecf53 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -31,7 +31,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Access Control
Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted
Validation of operational intents @@ -46,6 +46,11 @@ Implemented ASTM F3548-21 UTM DSS interoperability + + GEN0300 + Implemented + ASTM F3548 UTM aggregate checks + GEN0310 Implemented diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md index 5eb1335141..972a811bd8 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md @@ -18,7 +18,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Access Control
Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted
Validation of operational intents @@ -33,6 +33,11 @@ Implemented ASTM F3548-21 UTM DSS interoperability + + GEN0300 + Implemented + ASTM F3548 UTM aggregate checks + GEN0310 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md index 8256df3cd4..a8956efd0e 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.md +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -19,7 +19,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Access Control
Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted
Validation of operational intents @@ -34,6 +34,11 @@ Implemented ASTM F3548-21 UTM DSS interoperability + + GEN0300 + Implemented + ASTM F3548 UTM aggregate checks + GEN0310 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index ccc007b6f5..e96c85f8f7 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -449,7 +449,7 @@ ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation
ASTM NetRID DSS: Token Validation - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Access Control
Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted
Validation of operational intents @@ -464,6 +464,11 @@ Implemented ASTM F3548-21 UTM DSS interoperability + + GEN0300 + Implemented + ASTM F3548 UTM aggregate checks + GEN0310 Implemented