diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py index 2b4ec2666e..aaa242d3c9 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py @@ -168,15 +168,17 @@ def _attempt_invalid(self): "Validate flight intent too far ahead of time not planned", self._intents_extent, ) as validator: + self.begin_test_step("Attempt to plan flight intent too far ahead of time") submit_flight_intent( self, - "Attempt to plan flight intent too far ahead of time", "Incorrectly planned", {InjectFlightResponseResult.Rejected}, {InjectFlightResponseResult.Failed: "Failure"}, self.tested_uss, self.invalid_too_far_away.request, ) + self.end_test_step() + validator.expect_not_shared() def _validate_ended_cancellation(self): @@ -222,9 +224,9 @@ def _validate_precision_intersection(self): "Validate conflicting flight not planned", self._intents_extent, ) as validator: + self.begin_test_step("Attempt to plan flight conflicting by a tiny overlap") submit_flight_intent( self, - "Attempt to plan flight conflicting by a tiny overlap", "Incorrectly planned", { InjectFlightResponseResult.ConflictWithFlight, @@ -234,6 +236,8 @@ def _validate_precision_intersection(self): self.tested_uss, self.valid_conflict_tiny_overlap.request, ) + self.end_test_step() + validator.expect_not_shared() def cleanup(self): diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py index 5c897ec6ff..4e24bbabbb 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py @@ -402,9 +402,9 @@ def _modify_activated_flight_preexisting_conflict( self._intents_extent, flight_2_oi_ref, ) as validator: + self.begin_test_step("Declare Flight 2 non-conforming") resp_flight_2, _ = submit_flight_intent( self, - "Declare Flight 2 non-conforming", "Successful transition to non-conforming state", { InjectFlightResponseResult.ReadyToFly, @@ -415,6 +415,8 @@ def _modify_activated_flight_preexisting_conflict( self.flight2_nonconforming.request, self.flight2_id, ) + self.end_test_step() + if resp_flight_2.result == InjectFlightResponseResult.NotSupported: msg = f"{self.control_uss.config.participant_id} does not support the transition to a Nonconforming state; execution of the scenario was stopped without failure" self.record_note("Control USS does not support CMSA role", msg) @@ -430,9 +432,11 @@ def _modify_activated_flight_preexisting_conflict( self._intents_extent, flight_1_oi_ref, ) as validator: + self.begin_test_step( + "Attempt to modify activated Flight 1 in conflict with nonconforming Flight 2" + ) resp_flight_1, _ = submit_flight_intent( self, - "Attempt to modify activated Flight 1 in conflict with nonconforming Flight 2", "Successful modification or rejection", { InjectFlightResponseResult.ReadyToFly, @@ -443,6 +447,7 @@ def _modify_activated_flight_preexisting_conflict( self.flight1m_activated.request, self.flight1_id, ) + self.end_test_step() if resp_flight_1.result == InjectFlightResponseResult.ReadyToFly: validator.expect_shared(self.flight1m_activated.request) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.py b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.py index ca54f54ca3..647f4f4be4 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.py @@ -228,40 +228,45 @@ def _plan_flight_conflict_planned(self): "Validate Flight 1 status", self._intents_extent, ) as validator: - expected_results = { - InjectFlightResponseResult.Planned, - # the following two results are considered expected in order to fail another check as low severity - InjectFlightResponseResult.Rejected, - InjectFlightResponseResult.ConflictWithFlight, - } - failed_checks = { - InjectFlightResponseResult.Failed: "Failure", - InjectFlightResponseResult.Rejected: ( - "Rejected planning", - Severity.Low, - ), - InjectFlightResponseResult.ConflictWithFlight: ( - "Rejected planning", - Severity.Low, - ), - } - + self.begin_test_step("Tested USS attempts to plan Flight 1") resp, flight_id = submit_flight_intent( self, - "Tested USS attempts to plan Flight 1", "Successful planning", - expected_results, - failed_checks, + { + InjectFlightResponseResult.Planned, + # the following two results are considered expected in order to fail another check as low severity + InjectFlightResponseResult.Rejected, + InjectFlightResponseResult.ConflictWithFlight, + }, + { + InjectFlightResponseResult.Failed: "Failure", + }, self.tested_uss, self.flight1_planned.request, ) if resp.result == InjectFlightResponseResult.Planned: + self.end_test_step() validator.expect_shared(self.flight1_planned.request) elif ( resp.result == InjectFlightResponseResult.Rejected or resp.result == InjectFlightResponseResult.ConflictWithFlight ): + with self.check( + "Rejected planning", [self.tested_uss.participant_id] + ) as check: + check_details = ( + f"{self.tested_uss.participant_id} indicated {resp.result}" + + f' with notes "{resp.notes}"' + if "notes" in resp and resp.notes + else " with no notes" + ) + check.record_failed( + summary="Warning (not a failure): planning got rejected, USS may have been more conservative", + severity=Severity.Low, + details=check_details, + ) + self.end_test_step() validator.expect_not_shared() def _clear_op_intents(self): diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py index 39097f6ee6..6cb4095653 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py @@ -102,9 +102,9 @@ def _plan_flight_conflict_activated(self) -> OperationalIntentReference: "Validate high-priority Flight 2 not shared", self._intents_extent, ) as validator: + self.begin_test_step("Tested USS attempts to plan high-priority Flight 2") submit_flight_intent( self, - "Tested USS attempts to plan high-priority Flight 2", "Incorrectly planned", { InjectFlightResponseResult.Rejected, @@ -116,6 +116,8 @@ def _plan_flight_conflict_activated(self) -> OperationalIntentReference: self.tested_uss, self.flight2_planned.request, ) + self.end_test_step() + validator.expect_not_shared() # Restore virtual USS availability at DSS test step @@ -150,9 +152,9 @@ def _plan_flight_conflict_nonconforming( "Validate high-priority Flight 2 not shared", self._intents_extent, ) as validator: + self.begin_test_step("Tested USS attempts to plan high-priority Flight 2") submit_flight_intent( self, - "Tested USS attempts to plan high-priority Flight 2", "Incorrectly planned", { InjectFlightResponseResult.Rejected, @@ -164,6 +166,8 @@ def _plan_flight_conflict_nonconforming( self.tested_uss, self.flight2_planned.request, ) + self.end_test_step() + validator.expect_not_shared() # Restore virtual USS availability at DSS test step @@ -196,9 +200,9 @@ def _plan_flight_conflict_contingent(self, oi_ref: OperationalIntentReference): "Validate high-priority Flight 2 not shared", self._intents_extent, ) as validator: + self.begin_test_step("Tested USS attempts to plan high-priority Flight 2") submit_flight_intent( self, - "Tested USS attempts to plan high-priority Flight 2", "Incorrectly planned", { InjectFlightResponseResult.Rejected, @@ -210,4 +214,6 @@ def _plan_flight_conflict_contingent(self, oi_ref: OperationalIntentReference): self.tested_uss, self.flight2_planned.request, ) + self.end_test_step() + validator.expect_not_shared() diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py b/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py index 76271c3092..52f2357930 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py @@ -34,9 +34,9 @@ def plan_priority_conflict_flight_intent( flight_intent, OperationalIntentState.Accepted, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, _ = submit_flight_intent( scenario, - test_step, "Incorrectly planned", { InjectFlightResponseResult.ConflictWithFlight, @@ -45,7 +45,10 @@ def plan_priority_conflict_flight_intent( {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, - )[0] + ) + + scenario.end_test_step() + return resp def modify_planned_priority_conflict_flight_intent( @@ -66,9 +69,9 @@ def modify_planned_priority_conflict_flight_intent( flight_intent, OperationalIntentState.Accepted, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, _ = submit_flight_intent( scenario, - test_step, "Incorrectly modified", { InjectFlightResponseResult.ConflictWithFlight, @@ -78,7 +81,10 @@ def modify_planned_priority_conflict_flight_intent( flight_planner, flight_intent, flight_id, - )[0] + ) + + scenario.end_test_step() + return resp def activate_priority_conflict_flight_intent( @@ -99,9 +105,9 @@ def activate_priority_conflict_flight_intent( flight_intent, OperationalIntentState.Activated, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, _ = submit_flight_intent( scenario, - test_step, "Incorrectly activated", { InjectFlightResponseResult.ConflictWithFlight, @@ -111,7 +117,10 @@ def activate_priority_conflict_flight_intent( flight_planner, flight_intent, flight_id, - )[0] + ) + + scenario.end_test_step() + return resp def modify_activated_priority_conflict_flight_intent( @@ -132,9 +141,9 @@ def modify_activated_priority_conflict_flight_intent( flight_intent, OperationalIntentState.Activated, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, _ = submit_flight_intent( scenario, - test_step, "Incorrectly modified", { InjectFlightResponseResult.ConflictWithFlight, @@ -144,7 +153,10 @@ def modify_activated_priority_conflict_flight_intent( flight_planner, flight_intent, flight_id, - )[0] + ) + + scenario.end_test_step() + return resp def plan_conflict_flight_intent( @@ -164,9 +176,9 @@ def plan_conflict_flight_intent( flight_intent, OperationalIntentState.Accepted, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, _ = submit_flight_intent( scenario, - test_step, "Incorrectly planned", { InjectFlightResponseResult.ConflictWithFlight, @@ -175,7 +187,10 @@ def plan_conflict_flight_intent( {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, - )[0] + ) + + scenario.end_test_step() + return resp def modify_planned_conflict_flight_intent( @@ -196,9 +211,9 @@ def modify_planned_conflict_flight_intent( flight_intent, OperationalIntentState.Accepted, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, _ = submit_flight_intent( scenario, - test_step, "Incorrectly modified", { InjectFlightResponseResult.ConflictWithFlight, @@ -208,7 +223,10 @@ def modify_planned_conflict_flight_intent( flight_planner, flight_intent, flight_id, - )[0] + ) + + scenario.end_test_step() + return resp def activate_conflict_flight_intent( @@ -229,9 +247,9 @@ def activate_conflict_flight_intent( flight_intent, OperationalIntentState.Activated, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, _ = submit_flight_intent( scenario, - test_step, "Incorrectly activated", { InjectFlightResponseResult.ConflictWithFlight, @@ -241,7 +259,10 @@ def activate_conflict_flight_intent( flight_planner, flight_intent, flight_id, - )[0] + ) + + scenario.end_test_step() + return resp def modify_activated_conflict_flight_intent( @@ -262,9 +283,9 @@ def modify_activated_conflict_flight_intent( flight_intent, OperationalIntentState.Activated, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, _ = submit_flight_intent( scenario, - test_step, "Incorrectly modified", { InjectFlightResponseResult.ConflictWithFlight, @@ -274,7 +295,10 @@ def modify_activated_conflict_flight_intent( flight_planner, flight_intent, flight_id, - )[0] + ) + + scenario.end_test_step() + return resp def plan_permitted_conflict_flight_intent( @@ -296,9 +320,9 @@ def plan_permitted_conflict_flight_intent( flight_intent, OperationalIntentState.Accepted, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, flight_id = submit_flight_intent( scenario, - test_step, "Successful planning", {InjectFlightResponseResult.Planned}, {InjectFlightResponseResult.Failed: "Failure"}, @@ -306,6 +330,9 @@ def plan_permitted_conflict_flight_intent( flight_intent, ) + scenario.end_test_step() + return resp, flight_id + def modify_planned_permitted_conflict_flight_intent( scenario: TestScenarioType, @@ -325,16 +352,19 @@ def modify_planned_permitted_conflict_flight_intent( flight_intent, OperationalIntentState.Accepted, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, _ = submit_flight_intent( scenario, - test_step, "Successful modification", {InjectFlightResponseResult.Planned}, {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, - )[0] + ) + + scenario.end_test_step() + return resp def activate_permitted_conflict_flight_intent( @@ -355,16 +385,19 @@ def activate_permitted_conflict_flight_intent( flight_intent, OperationalIntentState.Activated, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, _ = submit_flight_intent( scenario, - test_step, "Successful activation", {InjectFlightResponseResult.ReadyToFly}, {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, - )[0] + ) + + scenario.end_test_step() + return resp def modify_activated_permitted_conflict_flight_intent( @@ -385,13 +418,16 @@ def modify_activated_permitted_conflict_flight_intent( flight_intent, OperationalIntentState.Activated, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, _ = submit_flight_intent( scenario, - test_step, "Successful modification", {InjectFlightResponseResult.ReadyToFly}, {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, - )[0] + ) + + scenario.end_test_step() + return resp diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py index 38f6a19c40..08d3f40976 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py @@ -42,7 +42,7 @@ def expect_flight_intent_state( if flight_intent.operational_intent.state != expected_state: function_name = str(inspect.stack()[1][3]) raise ValueError( - f"Error in test data: operational intent state for {function_name} during test step '{test_step}' in scenario '{scenario.documentation.name}' is expected to be `Accepted`, but got `{flight_intent.operational_intent.state}` instead" + f"Error in test data: operational intent state for {function_name} during test step '{test_step}' in scenario '{scenario.documentation.name}' is expected to be `{expected_state}`, but got `{flight_intent.operational_intent.state}` instead" ) @@ -66,9 +66,9 @@ def plan_flight_intent( flight_intent, OperationalIntentState.Accepted, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, flight_id = submit_flight_intent( scenario, - test_step, "Successful planning", {InjectFlightResponseResult.Planned}, {InjectFlightResponseResult.Failed: "Failure"}, @@ -76,6 +76,9 @@ def plan_flight_intent( flight_intent, ) + scenario.end_test_step() + return resp, flight_id + def activate_flight_intent( scenario: TestScenarioType, @@ -95,16 +98,19 @@ def activate_flight_intent( flight_intent, OperationalIntentState.Activated, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, _ = submit_flight_intent( scenario, - test_step, "Successful activation", {InjectFlightResponseResult.ReadyToFly}, {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, - )[0] + ) + + scenario.end_test_step() + return resp def modify_planned_flight_intent( @@ -125,16 +131,19 @@ def modify_planned_flight_intent( flight_intent, OperationalIntentState.Accepted, scenario, test_step ) - return submit_flight_intent( + scenario.begin_test_step(test_step) + resp, _ = submit_flight_intent( scenario, - test_step, "Successful modification", {InjectFlightResponseResult.Planned}, {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, - )[0] + ) + + scenario.end_test_step() + return resp def modify_activated_flight_intent( @@ -157,47 +166,67 @@ def modify_activated_flight_intent( flight_intent, OperationalIntentState.Activated, scenario, test_step ) + scenario.begin_test_step(test_step) if preexisting_conflict: - expected_results = { - InjectFlightResponseResult.ReadyToFly, - InjectFlightResponseResult.NotSupported, - # the following two results are considered expected in order to fail another check as low severity - InjectFlightResponseResult.Rejected, - InjectFlightResponseResult.ConflictWithFlight, - } - failed_checks = { - InjectFlightResponseResult.Failed: "Failure", - InjectFlightResponseResult.Rejected: ( - "Rejected modification", - Severity.Low, - ), - InjectFlightResponseResult.ConflictWithFlight: ( - "Rejected modification", - Severity.Low, - ), - } + resp, flight_id = submit_flight_intent( + scenario, + "Successful modification", + { + InjectFlightResponseResult.ReadyToFly, + InjectFlightResponseResult.NotSupported, + # the following two results are considered expected in order to fail another check as low severity + InjectFlightResponseResult.Rejected, + InjectFlightResponseResult.ConflictWithFlight, + }, + { + InjectFlightResponseResult.Failed: "Failure", + }, + flight_planner, + flight_intent, + flight_id, + ) + + with scenario.check( + "Rejected modification", [flight_planner.participant_id] + ) as check: + if ( + resp.result == InjectFlightResponseResult.Rejected + or resp.result == InjectFlightResponseResult.ConflictWithFlight + ): + check_details = ( + f"{flight_planner.participant_id} indicated {resp.result}" + ) + check_details += ( + f' with notes "{resp.notes}"' + if "notes" in resp and resp.notes + else " with no notes" + ) + check.record_failed( + summary="Warning (not a failure): modification got rejected but a pre-existing conflict was present", + severity=Severity.Low, + details=check_details, + ) + else: - expected_results = {InjectFlightResponseResult.ReadyToFly} - failed_checks = {InjectFlightResponseResult.Failed: "Failure"} + resp, flight_id = submit_flight_intent( + scenario, + "Successful modification", + {InjectFlightResponseResult.ReadyToFly}, + {InjectFlightResponseResult.Failed: "Failure"}, + flight_planner, + flight_intent, + flight_id, + ) - return submit_flight_intent( - scenario, - test_step, - "Successful modification", - expected_results, - failed_checks, - flight_planner, - flight_intent, - flight_id, - )[0] + scenario.end_test_step() + return resp def submit_flight_intent( scenario: TestScenarioType, - test_step: str, success_check: str, expected_results: Set[InjectFlightResponseResult], - failed_checks: Dict[InjectFlightResponseResult, Union[str, Tuple[str, Severity]]], + failed_checks: Dict[InjectFlightResponseResult, str], flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: Optional[str] = None, @@ -214,7 +243,11 @@ def submit_flight_intent( * The injection response. * The ID of the injected flight if it is returned, None otherwise. """ - scenario.begin_test_step(test_step) + if expected_results.intersection(failed_checks.keys()): + raise ValueError( + f"expected and unexpected results overlap: {expected_results.intersection(failed_checks.keys())}" + ) + with scenario.check(success_check, [flight_planner.participant_id]) as check: try: resp, query, flight_id = flight_planner.request_flight( @@ -230,40 +263,34 @@ def submit_flight_intent( query_timestamps=[q.request.timestamp for q in e.queries], ) scenario.record_query(query) - notes_suffix = f': "{resp.notes}"' if "notes" in resp and resp.notes else "" - - for unexpected_result, failed_test_check in failed_checks.items(): - if isinstance(failed_test_check, str): - check_name = failed_test_check - check_severity = Severity.High - else: - check_name, check_severity = failed_test_check - - with scenario.check( - check_name, [flight_planner.participant_id] - ) as specific_failed_check: - if resp.result == unexpected_result: - specific_failed_check.record_failed( - summary=f"Flight unexpectedly {resp.result}", - severity=check_severity, - details=f'{flight_planner.participant_id} indicated {resp.result} rather than the expected {" or ".join(expected_results)}{notes_suffix}', - query_timestamps=[query.request.timestamp], - ) + check_details = ( + f'{flight_planner.participant_id} indicated {resp.result} rather than the expected {" or ".join(expected_results)}' + + f' with notes "{resp.notes}"' + if "notes" in resp and resp.notes + else " with no notes" + ) - if resp.result in expected_results: - scenario.end_test_step() - return resp, flight_id - else: + if resp.result not in expected_results: check.record_failed( summary=f"Flight unexpectedly {resp.result}", severity=Severity.High, - details=f'{flight_planner.participant_id} indicated {resp.result} rather than the expected {" or ".join(expected_results)}{notes_suffix}', + details=check_details, query_timestamps=[query.request.timestamp], ) - raise RuntimeError( - "Error with submission of flight intent, but a High Severity issue didn't interrupt execution" - ) + for failed_result, failed_check_name in failed_checks.items(): + with scenario.check( + failed_check_name, [flight_planner.participant_id] + ) as check: + if resp.result == failed_result: + check.record_failed( + summary=f"Flight unexpectedly {resp.result}", + severity=Severity.High, + details=check_details, + query_timestamps=[query.request.timestamp], + ) + + return resp, flight_id def delete_flight_intent(