[uss_qualifier] attach participant id to net0210 check, fix Issue #244

monitoring/monitorlib/fetch/rid.py

@@ -1,15 +1,16 @@
 from __future__ import annotations
+
 import datetime
 from typing import Dict, List, Optional, Any, Union
 
-from implicitdict import ImplicitDict, StringBasedDateTime
 import s2sphere
-from uas_standards.astm.f3411 import v19, v22a
 import uas_standards.astm.f3411.v19.api
 import uas_standards.astm.f3411.v19.constants
 import uas_standards.astm.f3411.v22a.api
 import uas_standards.astm.f3411.v22a.constants
 import yaml
+from implicitdict import ImplicitDict, StringBasedDateTime
+from uas_standards.astm.f3411 import v19, v22a
 from uas_standards.astm.f3411.v22a.api import RIDHeight
 from yaml.representer import Representer
 
@@ -1163,7 +1164,7 @@ def all_flights(
     session: UTMClientSession,
     dss_base_url: str = "",
     enhanced_details: bool = False,
-    server_id: Optional[str] = None,
+    dss_server_id: Optional[str] = None,
 ) -> FetchedFlights:
     t = datetime.datetime.utcnow()
     isa_list = isas(
@@ -1173,7 +1174,7 @@ def all_flights(
         rid_version,
         session,
         dss_base_url,
-        server_id=server_id,
+        server_id=dss_server_id,
     )
 
     uss_flight_queries: Dict[str, FetchedUSSFlights] = {}
@@ -1185,7 +1186,9 @@ def all_flights(
             include_recent_positions,
             rid_version,
             session,
-            server_id=server_id,
+            # Note that we have no clue at this point which participant the flights_url is for,
+            # this can only be determined later by comparing injected and observed flights.
+            server_id=None,
         )
         uss_flight_queries[flights_url] = flights_for_url
 
@@ -1197,7 +1200,7 @@ def all_flights(
                     enhanced_details,
                     rid_version,
                     session,
-                    server_id=server_id,
+                    server_id=None,
                 )
                 uss_flight_details_queries[flight.id] = details
 

monitoring/uss_qualifier/resources/netrid/service_providers.py

@@ -41,25 +41,27 @@ class NetRIDServiceProvidersSpecification(ImplicitDict):
 
 class NetRIDServiceProvider(object):
     participant_id: str
-    base_url: str
-    client: infrastructure.UTMClientSession
+    injection_base_url: str
+    flights_injection_client: infrastructure.UTMClientSession
     local_debug: bool
 
     def __init__(
         self,
         participant_id: str,
-        base_url: str,
+        injection_base_url: str,
         auth_adapter: infrastructure.AuthAdapter,
         local_debug: bool,
     ):
         self.participant_id = participant_id
-        self.base_url = base_url
-        self.client = infrastructure.UTMClientSession(base_url, auth_adapter)
+        self.injection_base_url = injection_base_url
+        self.flights_injection_client = infrastructure.UTMClientSession(
+            injection_base_url, auth_adapter
+        )
         self.local_debug = local_debug
 
     def submit_test(self, request: CreateTestParameters, test_id: str) -> fetch.Query:
         return fetch.query_and_describe(
-            self.client,
+            self.flights_injection_client,
             "PUT",
             url=f"/tests/{test_id}",
             json=request,
@@ -69,7 +71,7 @@ def submit_test(self, request: CreateTestParameters, test_id: str) -> fetch.Quer
 
     def delete_test(self, test_id: str, version: str) -> fetch.Query:
         return fetch.query_and_describe(
-            self.client,
+            self.flights_injection_client,
             "DELETE",
             url=f"/tests/{test_id}/{version}",
             scope=SCOPE_RID_QUALIFIER_INJECT,
@@ -87,10 +89,10 @@ def __init__(
     ):
         self.service_providers = [
             NetRIDServiceProvider(
-                s.participant_id,
-                s.injection_base_url,
-                auth_adapter.adapter,
-                s.get("local_debug", False),
+                participant_id=s.participant_id,
+                injection_base_url=s.injection_base_url,
+                auth_adapter=auth_adapter.adapter,
+                local_debug=s.get("local_debug", False),
             )
             for s in specification.service_providers
         ]

monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py

@@ -51,8 +51,9 @@ def __init__(
 
         # identify SPs and observers by their base URL
         self._participants_by_base_url.update(
-            {sp.base_url: sp.participant_id for sp in self._service_providers}
+            {sp.injection_base_url: sp.participant_id for sp in self._service_providers}
         )
+
         self._participants_by_base_url.update(
             {dp.base_url: dp.participant_id for dp in self._observers}
         )
@@ -101,7 +102,7 @@ def run(self):
         for sp in self._service_providers:
             self.record_note(
                 "service_providers",
-                f"configured service providers: {sp.participant_id} - {sp.base_url}",
+                f"configured service providers: {sp.participant_id} - {sp.injection_base_url}",
             )
 
         for o in self._observers:

monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py

@@ -1,42 +1,33 @@
 import time
 import traceback
-import uuid
-from typing import List
+from typing import List, Dict
 
 import arrow
 import s2sphere
-from implicitdict import ImplicitDict
 from loguru import logger
 from requests.exceptions import RequestException
-from uas_standards.interuss.automated_testing.rid.v1.injection import ChangeTestResponse
 
 from monitoring.monitorlib import fetch
 from monitoring.monitorlib.fetch import rid
+from monitoring.monitorlib.fetch.rid import FetchedFlights
 from monitoring.monitorlib.infrastructure import UTMClientSession
 from monitoring.monitorlib.rid import RIDVersion
-from monitoring.monitorlib.rid_automated_testing.injection_api import (
-    CreateTestParameters,
-)
-from monitoring.monitorlib.rid_automated_testing.injection_api import TestFlight
 from monitoring.uss_qualifier.common_data_definitions import Severity
 from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstancesResource
 from monitoring.uss_qualifier.resources.netrid import (
     FlightDataResource,
     NetRIDServiceProviders,
     EvaluationConfigurationResource,
 )
-from monitoring.uss_qualifier.scenarios.astm.netrid import display_data_evaluator
 from monitoring.uss_qualifier.scenarios.astm.netrid.common import nominal_behavior
-from monitoring.uss_qualifier.scenarios.astm.netrid.injected_flight_collection import (
-    InjectedFlightCollection,
+from monitoring.uss_qualifier.scenarios.astm.netrid.display_data_evaluator import (
+    DPObservedFlight,
+    _make_flight_mapping,
 )
 from monitoring.uss_qualifier.scenarios.astm.netrid.injection import (
     InjectedFlight,
     InjectedTest,
 )
-from monitoring.uss_qualifier.scenarios.astm.netrid.virtual_observer import (
-    VirtualObserver,
-)
 from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario
 
 
@@ -167,67 +158,78 @@ def _evaluate_and_test_authentication(
         no flights were yet returned by the authenticated queries.
         """
 
-        with self.check("Missing credentials") as check:
-            # We grab all flights from the SP's. This is authenticated
-            # and is expected to succeed
-            sp_observation = rid.all_flights(
-                rect,
-                include_recent_positions=True,
-                get_details=True,
-                rid_version=self._rid_version,
-                session=self._dss.client,
-                server_id=self._dss.participant_id,
-            )
-            # We fish out the queries that were used to grab the flights from the SP,
-            # and attempt to re-query without credentials. This should fail.
+        # We grab all flights from the SP's (which we know how to reach by first querying the DSS).
+        # This is authenticated and is expected to succeed
+        sp_observation = rid.all_flights(
+            rect,
+            include_recent_positions=True,
+            get_details=True,
+            rid_version=self._rid_version,
+            session=self._dss.client,
+            dss_server_id=self._dss.participant_id,
+        )
 
-            unauthenticated_session = UTMClientSession(
-                prefix_url=self._dss.client.get_prefix_url(),
-                auth_adapter=None,
-                timeout_seconds=self._dss.client.timeout_seconds,
-            )
+        # Set the participant ID on the SP queries wherever possible
+        _attribute_sp_queries_to_participant_id(self._injected_flights, sp_observation)
 
-            queries_to_repeat = list(sp_observation.uss_flight_queries.values()) + list(
-                sp_observation.uss_flight_details_queries.values()
-            )
+        # We fish out the queries that were used to grab the flights from the SP,
+        # and attempt to re-query without credentials. This should fail.
+        unauthenticated_session = UTMClientSession(
+            prefix_url=self._dss.client.get_prefix_url(),
+            auth_adapter=None,
+            timeout_seconds=self._dss.client.timeout_seconds,
+        )
 
-            if len(queries_to_repeat) == 0:
-                logger.debug("no flights queries to repeat at this point.")
-                return False
+        queries_to_repeat = list(sp_observation.uss_flight_queries.values()) + list(
+            sp_observation.uss_flight_details_queries.values()
+        )
 
-            logger.debug(
-                f"about to repeat {len(queries_to_repeat)} flights queries without credentials"
-            )
+        if len(queries_to_repeat) == 0:
+            logger.debug("no flights queries to repeat at this point.")
+            return False
 
-            # Attempt to re-query the flights and flight details URLs:
-            for fq in queries_to_repeat:
-                failed_q = fetch.query_and_describe(
-                    client=unauthenticated_session,
-                    verb=fq.query.request.method,
-                    url=fq.query.request.url,
-                    json=fq.query.request.json,
-                    data=fq.query.request.body,
-                    server_id=self._dss.participant_id,
-                )
-                logger.info(
-                    f"Repeating query to {fq.query.request.url} without credentials"
-                )
-                server_id = fq.query.get("server_id", "unknown")
+        logger.debug(
+            f"about to repeat {len(queries_to_repeat)} flights queries without credentials"
+        )
+
+        # Attempt to re-query the flights and flight details URLs:
+        for fq in queries_to_repeat:
+            sp_server_id = fq.query.get("server_id", "unknown")
+            # We may well have queried SP's for real flights that happen
+            # to be present in the area in which we inject flights;
+            if sp_server_id == "unknown":
+                # TODO we could skip these once development on the qualifier has stopped,
+                # however, unattributed queries are likely the cause of a bug,
+                # and we want to find out.
+                logger.error(f"got unattributed SP query to: {fq.query.request.url}")
+
+            failed_q = fetch.query_and_describe(
+                client=unauthenticated_session,
+                verb=fq.query.request.method,
+                url=fq.query.request.url,
+                json=fq.query.request.json,
+                data=fq.query.request.body,
+                server_id=sp_server_id,
+            )
+            logger.info(
+                f"Repeating query to {fq.query.request.url} without credentials"
+            )
+            with self.check("Missing credentials", [sp_server_id]) as check:
                 if failed_q.response.code not in [401, 403]:
                     check.record_failed(
                         "unauthenticated request was fulfilled",
-                        participants=[server_id],
+                        participants=[sp_server_id],
                         severity=Severity.MEDIUM,
                         details=f"queried flights on {fq.query.request.url} with no credentials, expected a failure but got a success reply",
                     )
                 else:
                     logger.info(
-                        f"participant with id {server_id} properly authenticated the request"
+                        f"participant with id {sp_server_id} properly authenticated the request"
                     )
                 # Keep track of the failed queries, too
                 self.record_query(failed_q)
 
-            return True
+        return True
 
     def cleanup(self):
         self.begin_cleanup()
@@ -263,3 +265,22 @@ def cleanup(self):
                     details=f"While trying to delete a test flight from {sp.participant_id}, encountered error:\n{stacktrace}",
                 )
         self.end_cleanup()
+
+
+def _attribute_sp_queries_to_participant_id(
+    injected_flights: List[InjectedFlight],
+    fetched_flights: FetchedFlights,
+):
+    observed_flights = []
+    for uss_query in fetched_flights.uss_flight_queries.values():
+        for f in range(len(uss_query.flights)):
+            observed_flights.append(DPObservedFlight(query=uss_query, flight=f))
+
+    mapping_by_injection_id = _make_flight_mapping(injected_flights, observed_flights)
+
+    for telemetry_mapping in mapping_by_injection_id.values():
+        # For flights that were mapped to an injection ID,
+        # update the observation queries with the participant id for future use in the aggregate checks
+        telemetry_mapping.observed_flight.query.set_server_id(
+            telemetry_mapping.injected_flight.uss_participant_id
+        )

monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py

@@ -239,11 +239,14 @@ def evaluate_system_instantaneously(
                 get_details=True,
                 rid_version=self._rid_version,
                 session=self._dss.client,
-                server_id=self._dss.participant_id,
+                dss_server_id=self._dss.participant_id,
             )
             for q in sp_observation.queries:
                 self._test_scenario.record_query(q)
 
+            # Evaluate observations
+            # (Note this also takes care of setting the server_id to the relevant
+            # participant_id on queries where possible)
             self._evaluate_sp_observation(sp_observation, rect)
 
             step_report = self._test_scenario.end_test_step()
@@ -344,6 +347,8 @@ def _evaluate_normal_observation(
         mapping_by_injection_id = _make_flight_mapping(
             self._injected_flights, observation.flights
         )
+        # TODO check if we need to set some server ids on observation
+        #  queries here (if we have unattributed queries this might be a source)
 
         self._evaluate_flight_presence(
             observer.participant_id,

monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md

@@ -48,7 +48,7 @@ It then repeats the exact same request while omitting the credentials, and expec
 
 #### Missing credentials check
 
-This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v19.NET0500](../../../../requirements/astm/f3411/v19.md)**,
+This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v19.NET0210](../../../../requirements/astm/f3411/v19.md)**,
 and that requests for existing flights that are executed with missing or incorrect credentials fail.
 
 ## Cleanup

monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md

@@ -48,7 +48,7 @@ It then repeats the exact same request while omitting the credentials, and expec
 
 #### Missing credentials check
 
-This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v22a.NET0500](../../../../requirements/astm/f3411/v22a.md)**,
+This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v22a.NET0210](../../../../requirements/astm/f3411/v22a.md)**,
 and that requests for existing flights that are executed with missing or incorrect credentials fail.
 
 ## Cleanup

monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md

@@ -21,7 +21,7 @@
     <th><a href="../../README.md#checked-in">Checked in</a></th>
   </tr>
   <tr>
-    <td rowspan="52" style="vertical-align:top;"><a href="../../../requirements/astm/f3411/v19.md">astm<br>.f3411<br>.v19</a></td>
+    <td rowspan="53" style="vertical-align:top;"><a href="../../../requirements/astm/f3411/v19.md">astm<br>.f3411<br>.v19</a></td>
     <td><a href="../../../requirements/astm/f3411/v19.md">A2-6-1,1a</a></td>
     <td>Implemented</td>
     <td><a href="../../../scenarios/astm/netrid/v19/dss_interoperability.md">ASTM F3411-19 NetRID DSS interoperability</a></td>
@@ -186,6 +186,11 @@
     <td>TODO</td>
     <td><a href="../../../scenarios/astm/netrid/v19/operator_interactions.md">ASTM NetRID: Operator interactions</a></td>
   </tr>
+  <tr>
+    <td><a href="../../../requirements/astm/f3411/v19.md">NET0210</a></td>
+    <td>Implemented</td>
+    <td><a href="../../../scenarios/astm/netrid/v19/misbehavior.md">ASTM NetRID SP clients misbehavior handling</a></td>
+  </tr>
   <tr>
     <td><a href="../../../requirements/astm/f3411/v19.md">NET0220</a></td>
     <td>Implemented</td>