[uss_qualifier] gen0300 confirm that the test harness works

monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md

@@ -20,3 +20,17 @@ planners provided as resource are used to determine and evaluate the 95th percen
 
 If the 95th percentile of the requests durations is higher than the threshold `MaxRespondToOIDetailsRequest` (1 second),
 this check will fail per **[astm.f3548.v21.SCD0075](../../../requirements/astm/f3548/v21.md)**.
+
+## Interoperability test instance is available test case
+
+### Interoperability test instance is available test step
+
+This step verifies that interactions with the interoperability test instances happened and where at least partly successful.
+
+#### Interoperability test instance is available check
+
+This check ensures that interactions with the interoperability test instance that each USS must provide are possible.
+
+If all interactions fail, or if no test instance can be reached, the USS is failing to meet **[astm.f3548.v21.GEN0300](../../../requirements/astm/f3548/v21.md)**.
+
+If no interaction with a test instance was found, this check is skipped.

monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py

@@ -1,19 +1,17 @@
 from typing import List, Dict
 
+from uas_standards.astm.f3548.v21 import constants
+
 from monitoring.monitorlib import fetch
 from monitoring.monitorlib.fetch import evaluation, QueryType
 from monitoring.uss_qualifier.common_data_definitions import Severity
 from monitoring.uss_qualifier.configurations.configuration import ParticipantID
 from monitoring.uss_qualifier.resources.flight_planning import FlightPlannersResource
-from monitoring.uss_qualifier.suites.suite import ExecutionContext
-
-from uas_standards.astm.f3548.v21 import constants
-
 from monitoring.uss_qualifier.scenarios.scenario import TestScenario
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
 
 
 class AggregateChecks(TestScenario):
-
     _queries: List[fetch.Query]
     _attributed_queries: Dict[ParticipantID, Dict[QueryType, List[fetch.Query]]] = {}
 
@@ -76,6 +74,14 @@ def run(self, context: ExecutionContext):
         self.end_test_step()
         self.end_test_case()
 
+        self.begin_test_case("Interoperability test instance is available")
+        self.begin_test_step("Interoperability test instance is available")
+
+        self._confirm_test_harness_queries_work()
+
+        self.end_test_step()
+        self.end_test_case()
+
         self.end_test_scenario()
 
     def _op_intent_details_step(self):
@@ -118,3 +124,73 @@ def _op_intent_details_step(self):
                 f"{participant}/{QueryType.F3548v21USSGetOperationalIntentDetails}",
                 f"checked performances on {len(durations)} queries, 95th percentile: {p95}s",
             )
+
+    def _confirm_test_harness_queries_work(self):
+        """
+        For each different type of call to the interoperability test instance,
+        we look for at least one successful query.
+        """
+        for participant, queries_by_type in self._attributed_queries.items():
+            self._validate_participant_test_interop_instance(
+                participant, queries_by_type
+            )
+
+    def _validate_participant_test_interop_instance(
+        self,
+        participant_id: str,
+        participant_queries: dict[QueryType, List[fetch.Query]],
+    ):
+        # Keep track of how many interactions we've found for this participant
+        # if there is None the condition is not met
+        test_interactions = 0
+        success_by_type: Dict[QueryType, bool] = {}
+        for query_type, queries in participant_queries.items():
+            if _is_interop_test_interaction(query_type):
+                test_interactions += len(queries)
+                success_by_type[query_type] = False
+                for query in queries:
+                    if 200 <= query.response.status_code < 300:
+                        success_by_type[query_type] = True
+                        break
+
+        self.record_note(
+            "test_interop_interactions",
+            f"Found {test_interactions} interactions with interoperability test instance for {participant_id}",
+        )
+        if test_interactions == 0:
+            self.record_note(
+                "test_interop_check_skipped",
+                f"Skipping check for {participant_id} because no interactions with "
+                f"interoperability test instance were found",
+            )
+            return
+
+        with self.check(
+            "Interoperability test instance is available", [participant_id]
+        ) as check:
+            if test_interactions == 0:
+                # TODO remove once finished with development
+                check.record_failed(
+                    summary="No interactions with interoperability test instance",
+                    severity=Severity.Medium,
+                    participants=[participant_id],
+                    details="Found no interaction with interoperability test instance",
+                )
+
+            for query_type, success in success_by_type.items():
+                if not success:
+                    check.record_failed(
+                        summary=f"No successful {query_type} interaction with interoperability test instance",
+                        severity=Severity.Medium,
+                        details=f"Found no successful {query_type} interaction with interoperability test instance, "
+                        f"indicating that the test instance is either not available or not properly implemented.",
+                    )
+
+
+def _is_interop_test_interaction(query_type: QueryType):
+    return (
+        query_type == QueryType.InterUSSFlightPlanningV1GetStatus
+        or query_type == QueryType.InterUSSFlightPlanningV1ClearArea
+        or query_type == QueryType.InterUSSFlightPlanningV1UpsertFlightPlan
+        or query_type == QueryType.InterUSSFlightPlanningV1DeleteFlightPlan
+    )

monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md

@@ -211,6 +211,7 @@ Because the modification attempt was invalid, either Flight 1 should not have be
 original accepted request), or it should have been removed (because the USS rejected the replacement plan provided).
 
 
+
 ## Cleanup
 ### Successful flight deletion check
 **[interuss.automated_testing.flight_planning.DeleteFlightSuccess](../../../../../requirements/interuss/automated_testing/flight_planning.md)**

monitoring/uss_qualifier/suites/astm/utm/f3548_21.md

@@ -29,7 +29,7 @@
     <th><a href="../../README.md#checked-in">Checked in</a></th>
   </tr>
   <tr>
-    <td rowspan="23" style="vertical-align:top;"><a href="../../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
+    <td rowspan="24" style="vertical-align:top;"><a href="../../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
     <td><a href="../../../requirements/astm/f3548/v21.md">DSS0005</a></td>
     <td>Implemented</td>
     <td><a href="../../../scenarios/astm/utm/prep_planners.md">ASTM F3548 flight planners preparation</a><br><a href="../../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md">Data Validation of GET operational intents by USS</a><br><a href="../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md">Nominal planning: conflict with higher priority</a><br><a href="../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md">Nominal planning: not permitted conflict with equal priority</a><br><a href="../../../scenarios/astm/utm/off_nominal_planning/down_uss.md">Off-Nominal planning: down USS</a><br><a href="../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md">Validation of operational intents</a></td>
@@ -44,6 +44,11 @@
     <td>Implemented</td>
     <td><a href="../../../scenarios/astm/utm/dss_interoperability.md">ASTM F3548-21 UTM DSS interoperability</a></td>
   </tr>
+  <tr>
+    <td><a href="../../../requirements/astm/f3548/v21.md">GEN0300</a></td>
+    <td>Implemented</td>
+    <td><a href="../../../scenarios/astm/utm/aggregate_checks.md">ASTM F3548 UTM aggregate checks</a></td>
+  </tr>
   <tr>
     <td><a href="../../../requirements/astm/f3548/v21.md">GEN0310</a></td>
     <td>Implemented</td>

monitoring/uss_qualifier/suites/faa/uft/message_signing.md

@@ -18,7 +18,7 @@
     <th><a href="../../README.md#checked-in">Checked in</a></th>
   </tr>
   <tr>
-    <td rowspan="23" style="vertical-align:top;"><a href="../../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
+    <td rowspan="24" style="vertical-align:top;"><a href="../../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
     <td><a href="../../../requirements/astm/f3548/v21.md">DSS0005</a></td>
     <td>Implemented</td>
     <td><a href="../../../scenarios/astm/utm/prep_planners.md">ASTM F3548 flight planners preparation</a><br><a href="../../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md">Data Validation of GET operational intents by USS</a><br><a href="../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md">Nominal planning: conflict with higher priority</a><br><a href="../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md">Nominal planning: not permitted conflict with equal priority</a><br><a href="../../../scenarios/astm/utm/off_nominal_planning/down_uss.md">Off-Nominal planning: down USS</a><br><a href="../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md">Validation of operational intents</a></td>
@@ -33,6 +33,11 @@
     <td>Implemented</td>
     <td><a href="../../../scenarios/astm/utm/dss_interoperability.md">ASTM F3548-21 UTM DSS interoperability</a></td>
   </tr>
+  <tr>
+    <td><a href="../../../requirements/astm/f3548/v21.md">GEN0300</a></td>
+    <td>Implemented</td>
+    <td><a href="../../../scenarios/astm/utm/aggregate_checks.md">ASTM F3548 UTM aggregate checks</a></td>
+  </tr>
   <tr>
     <td><a href="../../../requirements/astm/f3548/v21.md">GEN0310</a></td>
     <td>Implemented</td>

monitoring/uss_qualifier/suites/uspace/flight_auth.md

@@ -19,7 +19,7 @@
     <th><a href="../README.md#checked-in">Checked in</a></th>
   </tr>
   <tr>
-    <td rowspan="23" style="vertical-align:top;"><a href="../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
+    <td rowspan="24" style="vertical-align:top;"><a href="../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
     <td><a href="../../requirements/astm/f3548/v21.md">DSS0005</a></td>
     <td>Implemented</td>
     <td><a href="../../scenarios/astm/utm/prep_planners.md">ASTM F3548 flight planners preparation</a><br><a href="../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md">Data Validation of GET operational intents by USS</a><br><a href="../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md">Nominal planning: conflict with higher priority</a><br><a href="../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md">Nominal planning: not permitted conflict with equal priority</a><br><a href="../../scenarios/astm/utm/off_nominal_planning/down_uss.md">Off-Nominal planning: down USS</a><br><a href="../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md">Validation of operational intents</a></td>
@@ -34,6 +34,11 @@
     <td>Implemented</td>
     <td><a href="../../scenarios/astm/utm/dss_interoperability.md">ASTM F3548-21 UTM DSS interoperability</a></td>
   </tr>
+  <tr>
+    <td><a href="../../requirements/astm/f3548/v21.md">GEN0300</a></td>
+    <td>Implemented</td>
+    <td><a href="../../scenarios/astm/utm/aggregate_checks.md">ASTM F3548 UTM aggregate checks</a></td>
+  </tr>
   <tr>
     <td><a href="../../requirements/astm/f3548/v21.md">GEN0310</a></td>
     <td>Implemented</td>

monitoring/uss_qualifier/suites/uspace/required_services.md

@@ -449,7 +449,7 @@
     <td><a href="../../scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md">ASTM NetRID DSS: Concurrent Requests</a><br><a href="../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.md">ASTM NetRID DSS: ISA Subscription Interactions</a><br><a href="../../scenarios/astm/netrid/v22a/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../scenarios/astm/netrid/v22a/dss/isa_validation.md">ASTM NetRID DSS: Submitted ISA Validations</a><br><a href="../../scenarios/astm/netrid/v22a/dss/subscription_simple.md">ASTM NetRID DSS: Subscription Simple</a><br><a href="../../scenarios/astm/netrid/v22a/dss/subscription_validation.md">ASTM NetRID DSS: Subscription Validation</a><br><a href="../../scenarios/astm/netrid/v22a/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a></td>
   </tr>
   <tr>
-    <td rowspan="23" style="vertical-align:top;"><a href="../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
+    <td rowspan="24" style="vertical-align:top;"><a href="../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
     <td><a href="../../requirements/astm/f3548/v21.md">DSS0005</a></td>
     <td>Implemented</td>
     <td><a href="../../scenarios/astm/utm/prep_planners.md">ASTM F3548 flight planners preparation</a><br><a href="../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md">Data Validation of GET operational intents by USS</a><br><a href="../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md">Nominal planning: conflict with higher priority</a><br><a href="../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md">Nominal planning: not permitted conflict with equal priority</a><br><a href="../../scenarios/astm/utm/off_nominal_planning/down_uss.md">Off-Nominal planning: down USS</a><br><a href="../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md">Validation of operational intents</a></td>
@@ -464,6 +464,11 @@
     <td>Implemented</td>
     <td><a href="../../scenarios/astm/utm/dss_interoperability.md">ASTM F3548-21 UTM DSS interoperability</a></td>
   </tr>
+  <tr>
+    <td><a href="../../requirements/astm/f3548/v21.md">GEN0300</a></td>
+    <td>Implemented</td>
+    <td><a href="../../scenarios/astm/utm/aggregate_checks.md">ASTM F3548 UTM aggregate checks</a></td>
+  </tr>
   <tr>
     <td><a href="../../requirements/astm/f3548/v21.md">GEN0310</a></td>
     <td>Implemented</td>