[uss_qualifier] Optimize RID tests (#261)

* Add scenario durations to sequence view artifact

* Optimize RID tests

* Use unattributed participant constant in templates

* Fix merge issue, improve error message

.github/workflows/ci.yml

@@ -60,18 +60,6 @@ jobs:
         cd monitoring/uss_qualifier
         make test
 
-  uss_qualifier_F3411-19-test:
-    name: uss_qualifier F3411-19 tests
-    uses: ./.github/workflows/monitoring-test.yml
-    with:
-      name: uss_qualifier_F3411-19
-      script: |
-        export CONFIG_NAME=configurations.dev.netrid_v19 \
-          USS_QUALIFIER_STOP_FAST=true
-
-        cd monitoring/uss_qualifier
-        make test
-
   prober-test:
     name: prober tests
     uses: ./.github/workflows/monitoring-test.yml
@@ -83,7 +71,7 @@ jobs:
 
   publish-gh-pages:
     name: Publish GitHub Pages
-    needs: [hygiene-tests, monitorlib-test, mock_uss-test, uss_qualifier-test, uss_qualifier_F3411-19-test, prober-test]
+    needs: [hygiene-tests, monitorlib-test, mock_uss-test, uss_qualifier-test, prober-test]
     if: ${{ always() && contains(join(needs.*.result, ','), 'success') }}
     runs-on: ubuntu-latest
     permissions:

monitoring/uss_qualifier/configurations/dev/library/resources.yaml

@@ -92,6 +92,15 @@ net_rid_sims:
             kml_source:
                 kml_file:
                     path: file://./test_data/usa/kentland/rid.kml
+            flight_start_delay: 5s
+    foca_flights_data:
+      $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json
+      resource_type: resources.netrid.FlightDataResource
+      specification:
+        kml_source:
+          kml_file:
+            path: file://./test_data/che/rid/foca.kml
+        flight_start_delay: 5s
 
 flight_auth:
     $ref: '#/f3548_che'

monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml

@@ -21,5 +21,22 @@ v1:
             report_path: output/report_netrid_v19.json
         tested_roles:
             report_path: output/tested_roles_netrid_v19
+        tested_requirements:
+          output_path: output/tested_requirements_f3411v19
+          requirement_collections:
+            sp_dp_dss:
+              requirement_sets:
+                - astm.f3411.v19.service_provider#Tested by automated tests
+                - astm.f3411.v19.display_provider#Automated verification
+                - astm.f3411.v19.dss_provider
+            sp_dss:
+              requirement_sets:
+                - astm.f3411.v19.service_provider#Tested by automated tests
+                - astm.f3411.v19.dss_provider
+          participant_requirements:
+            uss1: sp_dp_dss
+            uss2: sp_dss
+        sequence_view:
+          output_path: output/sequence_netrid_v19
     validation:
       $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml

@@ -33,9 +33,10 @@ v1:
                   requirement_sets:
                       - astm.f3411.v22a.service_provider#Mandatory requirements
                       - astm.f3411.v22a.dss_provider
-                      - astm.f3548.v21.scd#Automated verification
             participant_requirements:
                 uss1: sp_dp_dss
                 uss2: sp_dss
+        sequence_view:
+            output_path: output/sequence_netrid_v22a
     validation:
       $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/uspace.yaml

@@ -23,7 +23,7 @@ v1:
           scd_dss: scd_dss
           scd_dss_instances: scd_dss_instances
 
-          flights_data: adjacent_circular_flights_data
+          flights_data: foca_flights_data
           service_providers: netrid_service_providers_v22a
           observers: netrid_observers_v22a
           evaluation_configuration: netrid_observation_evaluation_configuration

monitoring/uss_qualifier/reports/sequence_view.py

@@ -1,4 +1,6 @@
 from __future__ import annotations
+
+import math
 import os
 from dataclasses import dataclass
 from datetime import datetime
@@ -15,9 +17,8 @@
 from monitoring.uss_qualifier.configurations.configuration import (
     ParticipantID,
     SequenceViewConfiguration,
-    TestConfiguration,
 )
-from monitoring.uss_qualifier.fileio import load_content, load_dict_with_references
+from monitoring.uss_qualifier.fileio import load_dict_with_references
 from monitoring.uss_qualifier.reports import jinja_env
 from monitoring.uss_qualifier.reports.report import (
     TestRunReport,
@@ -37,6 +38,9 @@
 from monitoring.uss_qualifier.suites.definitions import ActionType, TestSuiteDefinition
 
 
+UNATTRIBUTED_PARTICIPANT = "unattributed"
+
+
 class NoteEvent(ImplicitDict):
     key: str
     message: str
@@ -144,14 +148,17 @@ def rows(self) -> int:
 
 @dataclass
 class TestedParticipant(object):
-    has_failures: bool
+    has_failures: bool = False
+    has_successes: bool = False
+    has_queries: bool = False
 
 
 class TestedScenario(ImplicitDict):
     type: TestScenarioTypeName
     name: str
     url: str
     scenario_index: int
+    duration: str
     epochs: List[Epoch]
     participants: Dict[ParticipantID, TestedParticipant]
 
@@ -276,20 +283,28 @@ def append_notes(new_notes):
             for passed_check in step.passed_checks:
                 events.append(Event(passed_check=passed_check))
                 all_events.append(events[-1])
-                for pid in passed_check.participants:
-                    p = scenario_participants.get(
-                        pid, TestedParticipant(has_failures=False)
-                    )
+                participants = (
+                    passed_check.participants
+                    if passed_check.participants
+                    else [UNATTRIBUTED_PARTICIPANT]
+                )
+                for pid in participants:
+                    p = scenario_participants.get(pid, TestedParticipant())
+                    p.has_successes = True
                     scenario_participants[pid] = p
             if "queries" in step and step.queries:
                 for query in step.queries:
                     events.append(Event(query=query))
                     all_events.append(events[-1])
-                    if "server_id" in query and query.server_id:
-                        p = scenario_participants.get(
-                            query.server_id, TestedParticipant(has_failures=False)
-                        )
-                        scenario_participants[query.server_id] = p
+                    participant_id = (
+                        query.server_id
+                        if "server_id" in query and query.server_id
+                        else UNATTRIBUTED_PARTICIPANT
+                    )
+                    p = scenario_participants.get(participant_id, TestedParticipant())
+                    p.has_queries = True
+                    scenario_participants[participant_id] = p
+
             for failed_check in step.failed_checks:
                 query_events = []
                 if (
@@ -312,10 +327,13 @@ def append_notes(new_notes):
                     Event(failed_check=failed_check, query_events=query_events)
                 )
                 all_events.append(events[-1])
-                for pid in failed_check.participants:
-                    p = scenario_participants.get(
-                        pid, TestedParticipant(has_failures=True)
-                    )
+                participants = (
+                    failed_check.participants
+                    if failed_check.participants
+                    else [UNATTRIBUTED_PARTICIPANT]
+                )
+                for pid in participants:
+                    p = scenario_participants.get(pid, TestedParticipant())
                     p.has_failures = True
                     scenario_participants[pid] = p
             if "notes" in report and report.notes:
@@ -380,12 +398,23 @@ def append_notes(new_notes):
         else:
             post_notes = {}
         if post_notes:
+            latest_step_time = max(v.timestamp.datetime for v in post_notes.values())
             append_notes(post_notes)
 
+    if "end_time" in report and report.end_time:
+        latest_step_time = report.end_time.datetime
+
+    dt_s = round((latest_step_time - report.start_time.datetime).total_seconds())
+    dt_m = math.floor(dt_s / 60)
+    dt_s -= dt_m * 60
+    padding = "0" if dt_s < 10 else ""
+    duration = f"{dt_m}:{padding}{dt_s}"
+
     scenario = TestedScenario(
         type=report.scenario_type,
         name=report.name,
         url=report.documentation_url,
+        duration=duration,
         epochs=epochs,
         scenario_index=indexer.scenario_index,
         participants=scenario_participants,
@@ -573,6 +602,9 @@ def _generate_scenario_pages(
     if node.node_type == ActionNodeType.Scenario:
         all_participants = list(node.scenario.participants)
         all_participants.sort()
+        if UNATTRIBUTED_PARTICIPANT in all_participants:
+            all_participants.remove(UNATTRIBUTED_PARTICIPANT)
+            all_participants.append(UNATTRIBUTED_PARTICIPANT)
         scenario_file = os.path.join(
             config.output_path, f"s{node.scenario.scenario_index}.html"
         )
@@ -584,6 +616,7 @@ def _generate_scenario_pages(
                     all_participants=all_participants,
                     EpochType=EpochType,
                     EventType=EventType,
+                    UNATTRIBUTED_PARTICIPANT=UNATTRIBUTED_PARTICIPANT,
                     len=len,
                     str=str,
                 )
@@ -606,6 +639,9 @@ def generate_sequence_view(
     max_suite_cols = max(len(r.suite_cells) for r in overview_rows)
     all_participants = _enumerate_all_participants(node)
     all_participants.sort()
+    if UNATTRIBUTED_PARTICIPANT in all_participants:
+        all_participants.remove(UNATTRIBUTED_PARTICIPANT)
+        all_participants.append(UNATTRIBUTED_PARTICIPANT)
     overview_file = os.path.join(config.output_path, "index.html")
     template = jinja_env.get_template("sequence_view/overview.html")
     with open(overview_file, "w") as f:
@@ -617,6 +653,7 @@ def generate_sequence_view(
                 max_suite_cols=max_suite_cols,
                 all_participants=all_participants,
                 ActionNodeType=ActionNodeType,
+                UNATTRIBUTED_PARTICIPANT=UNATTRIBUTED_PARTICIPANT,
                 len=len,
             )
         )

monitoring/uss_qualifier/reports/templates/sequence_view/overview.html

@@ -124,7 +124,7 @@ <h3>{{ "Baseline" if is_baseline else "Environment" }}</h3>
         {% set current = namespace(address="v1.test_run.resources.resource_declarations") %}
         {% for resource_id, resource_dec in report.configuration.resources.resource_declarations.items() %}
           {% set current.address = current.address + "." + resource_id %}
-          {% if (current.address in report.configuration.non_baseline_inputs) != is_baseline %}
+          {% if ("non_baseline_inputs" in report.configuration and report.configuration.non_baseline_inputs and (current.address in report.configuration.non_baseline_inputs) != is_baseline) or (("non_baseline_inputs" not in report.configuration or not report.configuration.non_baseline_inputs) and is_baseline) %}
             <li id="resource_{{ resource_id }}"><code>{{ resource_id }}</code> ({{ resource_dec.resource_type }})
               <ul>
                 {% if resource_dec.dependencies %}
@@ -156,8 +156,15 @@ <h2>Scenarios executed</h2>
       {% endif %}
       <th>Scenario</th>
       {% for participant_id in all_participants %}
-        <th>{{ participant_id }}</th>
+        <th>
+          {% if participant_id != UNATTRIBUTED_PARTICIPANT %}
+            {{ participant_id }}
+          {% else %}
+            &lt;None&gt;
+          {% endif %}
+        </th>
       {% endfor %}
+      <th>Time</th>
     </tr>
     {% for row in overview_rows %}
       <tr>
@@ -185,13 +192,22 @@ <h2>Scenarios executed</h2>
           {% if row.scenario_node and participant_id in row.scenario_node.scenario.participants %}
             {% if row.scenario_node.scenario.participants[participant_id].has_failures %}
               <td class="fail_result">&#x274C;</td>
-            {% else %}
+            {% elif row.scenario_node.scenario.participants[participant_id].has_successes %}
               <td class="pass_result">&#x2705;</td>
+            {% elif row.scenario_node.scenario.participants[participant_id].has_queries %}
+              <td>&#x1F310;</td>
+            {% else %}
+              <td>???</td>
             {% endif %}
           {% else %}
             <td></td>
           {% endif %}
         {% endfor %}
+        <td>
+          {% if row.scenario_node %}
+            {{ row.scenario_node.scenario.duration }}
+          {% endif %}
+        </td>
       </tr>
     {% endfor %}
   </table>

monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html

@@ -81,7 +81,13 @@ <h3>{{ test_scenario.type }}</h3>
       <th>Step</th>
       <th colspan="3">Event</th>
       {% for participant_id in all_participants %}
-        <th>{{ participant_id }}</th>
+        <th>
+          {% if participant_id != UNATTRIBUTED_PARTICIPANT %}
+            {{ participant_id }}
+          {% else %}
+            &lt;None&gt;
+          {% endif %}
+        </th>
       {% endfor %}
     </tr>
 
@@ -118,7 +124,7 @@ <h3>{{ test_scenario.type }}</h3>
                   {{ event.passed_check.name }}
                 </td>
                 {% for participant_id in all_participants %}
-                  {% if participant_id in event.passed_check.participants %}
+                  {% if (participant_id != UNATTRIBUTED_PARTICIPANT and participant_id in event.passed_check.participants) or (participant_id == UNATTRIBUTED_PARTICIPANT and not event.passed_check.participants) %}
                     <td class="pass_result">&#x2705;</td>
                   {% else %}
                     <td></td>
@@ -145,7 +151,7 @@ <h3>{{ test_scenario.type }}</h3>
                   {% endif %}
                 </td>
                 {% for participant_id in all_participants %}
-                  {% if participant_id in event.failed_check.participants %}
+                  {% if (participant_id != UNATTRIBUTED_PARTICIPANT and participant_id in event.failed_check.participants) or (participant_id == UNATTRIBUTED_PARTICIPANT and not event.failed_check.participants) %}
                     <td class="fail_result">&#x274C;</td>
                   {% else %}
                     <td></td>
@@ -160,7 +166,7 @@ <h3>{{ test_scenario.type }}</h3>
                   {% set collapsible.queries = collapsible.queries + [query_id] %}
                 </td>
                 {% for participant_id in all_participants %}
-                  {% if participant_id == event.query.get("server_id", None) %}
+                  {% if (participant_id != UNATTRIBUTED_PARTICIPANT and participant_id == event.query.get("server_id", None)) or (participant_id == UNATTRIBUTED_PARTICIPANT and not event.query.get("server_id", None)) %}
                     <td>&#x1F310;</td>
                   {% else %}
                     <td></td>

monitoring/uss_qualifier/resources/netrid/flight_data_resources.py

@@ -1,3 +1,4 @@
+import json
 from datetime import timedelta
 from typing import List, Optional
 import uuid
@@ -49,7 +50,8 @@ def __init__(self, specification: FlightDataSpecification):
             )
         else:
             raise ValueError(
-                "A source of flight data was not identified in the specification for a FlightDataSpecification"
+                "A source of flight data was not identified in the specification for a FlightDataSpecification:\n"
+                + json.dumps(specification, indent=2)
             )
         self._flight_start_delay = specification.flight_start_delay.timedelta
 

monitoring/uss_qualifier/run_locally.sh

@@ -34,9 +34,9 @@ configurations.dev.general_flight_auth,\
 configurations.dev.f3548,\
 configurations.dev.f3548_self_contained,\
 configurations.dev.netrid_v22a,\
+configurations.dev.netrid_v19,\
 configurations.dev.uspace"
 fi
-# TODO: Add configurations.dev.netrid_v19
 
 echo "Running configuration(s): ${CONFIG_NAME}"
 

monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py

@@ -247,6 +247,8 @@ def inject_flights(
             )
             raise RuntimeError("High-severity issue did not abort test scenario")
 
+        start_time = None
+        end_time = None
         for flight in injections:
             injected_flights.append(
                 InjectedFlight(
@@ -256,6 +258,19 @@ def inject_flights(
                     query_timestamp=query.request.timestamp,
                 )
             )
+            earliest_time = min(t.timestamp.datetime for t in flight.telemetry)
+            latest_time = max(t.timestamp.datetime for t in flight.telemetry)
+            if start_time is None or earliest_time < start_time:
+                start_time = earliest_time
+            if end_time is None or latest_time > end_time:
+                end_time = latest_time
+        now = arrow.utcnow().datetime
+        dt0 = (start_time - now).total_seconds()
+        dt1 = (end_time - now).total_seconds()
+        test_scenario.record_note(
+            f"{test_id} time range",
+            f"Injected flights start {dt0:.1f} seconds from now and end {dt1:.1f} seconds from now",
+        )
 
     # Make sure the injected flights can be identified correctly by the test harness
     with test_scenario.check("Identifiable flights") as check: