[uss_qualifier/flight_planning/submit_flight_intent] Factor out test …

…step boundaries and specific failed checks (#411)

monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py

@@ -168,15 +168,17 @@ def _attempt_invalid(self):
             "Validate flight intent too far ahead of time not planned",
             self._intents_extent,
         ) as validator:
+            self.begin_test_step("Attempt to plan flight intent too far ahead of time")
             submit_flight_intent(
                 self,
-                "Attempt to plan flight intent too far ahead of time",
                 "Incorrectly planned",
                 {InjectFlightResponseResult.Rejected},
                 {InjectFlightResponseResult.Failed: "Failure"},
                 self.tested_uss,
                 self.invalid_too_far_away.request,
             )
+            self.end_test_step()
+
             validator.expect_not_shared()
 
     def _validate_ended_cancellation(self):
@@ -222,9 +224,9 @@ def _validate_precision_intersection(self):
             "Validate conflicting flight not planned",
             self._intents_extent,
         ) as validator:
+            self.begin_test_step("Attempt to plan flight conflicting by a tiny overlap")
             submit_flight_intent(
                 self,
-                "Attempt to plan flight conflicting by a tiny overlap",
                 "Incorrectly planned",
                 {
                     InjectFlightResponseResult.ConflictWithFlight,
@@ -234,6 +236,8 @@ def _validate_precision_intersection(self):
                 self.tested_uss,
                 self.valid_conflict_tiny_overlap.request,
             )
+            self.end_test_step()
+
             validator.expect_not_shared()
 
     def cleanup(self):

monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py

@@ -402,9 +402,9 @@ def _modify_activated_flight_preexisting_conflict(
             self._intents_extent,
             flight_2_oi_ref,
         ) as validator:
+            self.begin_test_step("Declare Flight 2 non-conforming")
             resp_flight_2, _ = submit_flight_intent(
                 self,
-                "Declare Flight 2 non-conforming",
                 "Successful transition to non-conforming state",
                 {
                     InjectFlightResponseResult.ReadyToFly,
@@ -415,6 +415,8 @@ def _modify_activated_flight_preexisting_conflict(
                 self.flight2_nonconforming.request,
                 self.flight2_id,
             )
+            self.end_test_step()
+
             if resp_flight_2.result == InjectFlightResponseResult.NotSupported:
                 msg = f"{self.control_uss.config.participant_id} does not support the transition to a Nonconforming state; execution of the scenario was stopped without failure"
                 self.record_note("Control USS does not support CMSA role", msg)
@@ -430,9 +432,11 @@ def _modify_activated_flight_preexisting_conflict(
             self._intents_extent,
             flight_1_oi_ref,
         ) as validator:
+            self.begin_test_step(
+                "Attempt to modify activated Flight 1 in conflict with nonconforming Flight 2"
+            )
             resp_flight_1, _ = submit_flight_intent(
                 self,
-                "Attempt to modify activated Flight 1 in conflict with nonconforming Flight 2",
                 "Successful modification or rejection",
                 {
                     InjectFlightResponseResult.ReadyToFly,
@@ -443,6 +447,7 @@ def _modify_activated_flight_preexisting_conflict(
                 self.flight1m_activated.request,
                 self.flight1_id,
             )
+            self.end_test_step()
 
             if resp_flight_1.result == InjectFlightResponseResult.ReadyToFly:
                 validator.expect_shared(self.flight1m_activated.request)

monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.py

@@ -228,40 +228,45 @@ def _plan_flight_conflict_planned(self):
             "Validate Flight 1 status",
             self._intents_extent,
         ) as validator:
-            expected_results = {
-                InjectFlightResponseResult.Planned,
-                # the following two results are considered expected in order to fail another check as low severity
-                InjectFlightResponseResult.Rejected,
-                InjectFlightResponseResult.ConflictWithFlight,
-            }
-            failed_checks = {
-                InjectFlightResponseResult.Failed: "Failure",
-                InjectFlightResponseResult.Rejected: (
-                    "Rejected planning",
-                    Severity.Low,
-                ),
-                InjectFlightResponseResult.ConflictWithFlight: (
-                    "Rejected planning",
-                    Severity.Low,
-                ),
-            }
-
+            self.begin_test_step("Tested USS attempts to plan Flight 1")
             resp, flight_id = submit_flight_intent(
                 self,
-                "Tested USS attempts to plan Flight 1",
                 "Successful planning",
-                expected_results,
-                failed_checks,
+                {
+                    InjectFlightResponseResult.Planned,
+                    # the following two results are considered expected in order to fail another check as low severity
+                    InjectFlightResponseResult.Rejected,
+                    InjectFlightResponseResult.ConflictWithFlight,
+                },
+                {
+                    InjectFlightResponseResult.Failed: "Failure",
+                },
                 self.tested_uss,
                 self.flight1_planned.request,
             )
 
             if resp.result == InjectFlightResponseResult.Planned:
+                self.end_test_step()
                 validator.expect_shared(self.flight1_planned.request)
             elif (
                 resp.result == InjectFlightResponseResult.Rejected
                 or resp.result == InjectFlightResponseResult.ConflictWithFlight
             ):
+                with self.check(
+                    "Rejected planning", [self.tested_uss.participant_id]
+                ) as check:
+                    check_details = (
+                        f"{self.tested_uss.participant_id} indicated {resp.result}"
+                        + f' with notes "{resp.notes}"'
+                        if "notes" in resp and resp.notes
+                        else " with no notes"
+                    )
+                    check.record_failed(
+                        summary="Warning (not a failure): planning got rejected, USS may have been more conservative",
+                        severity=Severity.Low,
+                        details=check_details,
+                    )
+                self.end_test_step()
                 validator.expect_not_shared()
 
     def _clear_op_intents(self):

monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py

@@ -102,9 +102,9 @@ def _plan_flight_conflict_activated(self) -> OperationalIntentReference:
             "Validate high-priority Flight 2 not shared",
             self._intents_extent,
         ) as validator:
+            self.begin_test_step("Tested USS attempts to plan high-priority Flight 2")
             submit_flight_intent(
                 self,
-                "Tested USS attempts to plan high-priority Flight 2",
                 "Incorrectly planned",
                 {
                     InjectFlightResponseResult.Rejected,
@@ -116,6 +116,8 @@ def _plan_flight_conflict_activated(self) -> OperationalIntentReference:
                 self.tested_uss,
                 self.flight2_planned.request,
             )
+            self.end_test_step()
+
             validator.expect_not_shared()
 
         # Restore virtual USS availability at DSS test step
@@ -150,9 +152,9 @@ def _plan_flight_conflict_nonconforming(
             "Validate high-priority Flight 2 not shared",
             self._intents_extent,
         ) as validator:
+            self.begin_test_step("Tested USS attempts to plan high-priority Flight 2")
             submit_flight_intent(
                 self,
-                "Tested USS attempts to plan high-priority Flight 2",
                 "Incorrectly planned",
                 {
                     InjectFlightResponseResult.Rejected,
@@ -164,6 +166,8 @@ def _plan_flight_conflict_nonconforming(
                 self.tested_uss,
                 self.flight2_planned.request,
             )
+            self.end_test_step()
+
             validator.expect_not_shared()
 
         # Restore virtual USS availability at DSS test step
@@ -196,9 +200,9 @@ def _plan_flight_conflict_contingent(self, oi_ref: OperationalIntentReference):
             "Validate high-priority Flight 2 not shared",
             self._intents_extent,
         ) as validator:
+            self.begin_test_step("Tested USS attempts to plan high-priority Flight 2")
             submit_flight_intent(
                 self,
-                "Tested USS attempts to plan high-priority Flight 2",
                 "Incorrectly planned",
                 {
                     InjectFlightResponseResult.Rejected,
@@ -210,4 +214,6 @@ def _plan_flight_conflict_contingent(self, oi_ref: OperationalIntentReference):
                 self.tested_uss,
                 self.flight2_planned.request,
             )
+            self.end_test_step()
+
             validator.expect_not_shared()