From e12193c351f67bc90dfe0dca1e604a50a75c7ac8 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Thu, 12 Oct 2023 10:04:24 +0200 Subject: [PATCH] [uss_qualifier] attach participant id to net0210 check, fix Issue #244 --- monitoring/monitorlib/fetch/rid.py | 15 +- .../resources/netrid/service_providers.py | 24 +-- .../astm/netrid/common/aggregate_checks.py | 5 +- .../astm/netrid/common/misbehavior.py | 139 ++++++++++-------- .../astm/netrid/display_data_evaluator.py | 13 +- .../scenarios/astm/netrid/v19/misbehavior.md | 2 +- .../scenarios/astm/netrid/v22a/misbehavior.md | 2 +- .../suites/astm/netrid/f3411_19.md | 7 +- .../suites/astm/netrid/f3411_22a.md | 7 +- .../suites/uspace/network_identification.md | 7 +- .../suites/uspace/required_services.md | 7 +- 11 files changed, 141 insertions(+), 87 deletions(-) diff --git a/monitoring/monitorlib/fetch/rid.py b/monitoring/monitorlib/fetch/rid.py index 8d3867af11..8baeca8814 100644 --- a/monitoring/monitorlib/fetch/rid.py +++ b/monitoring/monitorlib/fetch/rid.py @@ -1,15 +1,16 @@ from __future__ import annotations + import datetime from typing import Dict, List, Optional, Any, Union -from implicitdict import ImplicitDict, StringBasedDateTime import s2sphere -from uas_standards.astm.f3411 import v19, v22a import uas_standards.astm.f3411.v19.api import uas_standards.astm.f3411.v19.constants import uas_standards.astm.f3411.v22a.api import uas_standards.astm.f3411.v22a.constants import yaml +from implicitdict import ImplicitDict, StringBasedDateTime +from uas_standards.astm.f3411 import v19, v22a from uas_standards.astm.f3411.v22a.api import RIDHeight from yaml.representer import Representer @@ -1163,7 +1164,7 @@ def all_flights( session: UTMClientSession, dss_base_url: str = "", enhanced_details: bool = False, - server_id: Optional[str] = None, + dss_server_id: Optional[str] = None, ) -> FetchedFlights: t = datetime.datetime.utcnow() isa_list = isas( @@ -1173,7 +1174,7 @@ def all_flights( rid_version, session, dss_base_url, - server_id=server_id, + server_id=dss_server_id, ) uss_flight_queries: Dict[str, FetchedUSSFlights] = {} @@ -1185,7 +1186,9 @@ def all_flights( include_recent_positions, rid_version, session, - server_id=server_id, + # Note that we have no clue at this point which participant the flights_url is for, + # this can only be determined later by comparing injected and observed flights. + server_id=None, ) uss_flight_queries[flights_url] = flights_for_url @@ -1197,7 +1200,7 @@ def all_flights( enhanced_details, rid_version, session, - server_id=server_id, + server_id=None, ) uss_flight_details_queries[flight.id] = details diff --git a/monitoring/uss_qualifier/resources/netrid/service_providers.py b/monitoring/uss_qualifier/resources/netrid/service_providers.py index 76c02a981f..b7d6b4528b 100644 --- a/monitoring/uss_qualifier/resources/netrid/service_providers.py +++ b/monitoring/uss_qualifier/resources/netrid/service_providers.py @@ -41,25 +41,27 @@ class NetRIDServiceProvidersSpecification(ImplicitDict): class NetRIDServiceProvider(object): participant_id: str - base_url: str - client: infrastructure.UTMClientSession + injection_base_url: str + flights_injection_client: infrastructure.UTMClientSession local_debug: bool def __init__( self, participant_id: str, - base_url: str, + injection_base_url: str, auth_adapter: infrastructure.AuthAdapter, local_debug: bool, ): self.participant_id = participant_id - self.base_url = base_url - self.client = infrastructure.UTMClientSession(base_url, auth_adapter) + self.injection_base_url = injection_base_url + self.flights_injection_client = infrastructure.UTMClientSession( + injection_base_url, auth_adapter + ) self.local_debug = local_debug def submit_test(self, request: CreateTestParameters, test_id: str) -> fetch.Query: return fetch.query_and_describe( - self.client, + self.flights_injection_client, "PUT", url=f"/tests/{test_id}", json=request, @@ -69,7 +71,7 @@ def submit_test(self, request: CreateTestParameters, test_id: str) -> fetch.Quer def delete_test(self, test_id: str, version: str) -> fetch.Query: return fetch.query_and_describe( - self.client, + self.flights_injection_client, "DELETE", url=f"/tests/{test_id}/{version}", scope=SCOPE_RID_QUALIFIER_INJECT, @@ -87,10 +89,10 @@ def __init__( ): self.service_providers = [ NetRIDServiceProvider( - s.participant_id, - s.injection_base_url, - auth_adapter.adapter, - s.get("local_debug", False), + participant_id=s.participant_id, + injection_base_url=s.injection_base_url, + auth_adapter=auth_adapter.adapter, + local_debug=s.get("local_debug", False), ) for s in specification.service_providers ] diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py index 9edd3b416e..01196fc52e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py @@ -51,8 +51,9 @@ def __init__( # identify SPs and observers by their base URL self._participants_by_base_url.update( - {sp.base_url: sp.participant_id for sp in self._service_providers} + {sp.injection_base_url: sp.participant_id for sp in self._service_providers} ) + self._participants_by_base_url.update( {dp.base_url: dp.participant_id for dp in self._observers} ) @@ -101,7 +102,7 @@ def run(self): for sp in self._service_providers: self.record_note( "service_providers", - f"configured service providers: {sp.participant_id} - {sp.base_url}", + f"configured service providers: {sp.participant_id} - {sp.injection_base_url}", ) for o in self._observers: diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py index a6fada8caf..e89377589c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py @@ -1,23 +1,17 @@ import time import traceback -import uuid -from typing import List +from typing import List, Dict import arrow import s2sphere -from implicitdict import ImplicitDict from loguru import logger from requests.exceptions import RequestException -from uas_standards.interuss.automated_testing.rid.v1.injection import ChangeTestResponse from monitoring.monitorlib import fetch from monitoring.monitorlib.fetch import rid +from monitoring.monitorlib.fetch.rid import FetchedFlights from monitoring.monitorlib.infrastructure import UTMClientSession from monitoring.monitorlib.rid import RIDVersion -from monitoring.monitorlib.rid_automated_testing.injection_api import ( - CreateTestParameters, -) -from monitoring.monitorlib.rid_automated_testing.injection_api import TestFlight from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstancesResource from monitoring.uss_qualifier.resources.netrid import ( @@ -25,18 +19,15 @@ NetRIDServiceProviders, EvaluationConfigurationResource, ) -from monitoring.uss_qualifier.scenarios.astm.netrid import display_data_evaluator from monitoring.uss_qualifier.scenarios.astm.netrid.common import nominal_behavior -from monitoring.uss_qualifier.scenarios.astm.netrid.injected_flight_collection import ( - InjectedFlightCollection, +from monitoring.uss_qualifier.scenarios.astm.netrid.display_data_evaluator import ( + DPObservedFlight, + map_observations_to_injected_flights, ) from monitoring.uss_qualifier.scenarios.astm.netrid.injection import ( InjectedFlight, InjectedTest, ) -from monitoring.uss_qualifier.scenarios.astm.netrid.virtual_observer import ( - VirtualObserver, -) from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario @@ -167,67 +158,78 @@ def _evaluate_and_test_authentication( no flights were yet returned by the authenticated queries. """ - with self.check("Missing credentials") as check: - # We grab all flights from the SP's. This is authenticated - # and is expected to succeed - sp_observation = rid.all_flights( - rect, - include_recent_positions=True, - get_details=True, - rid_version=self._rid_version, - session=self._dss.client, - server_id=self._dss.participant_id, - ) - # We fish out the queries that were used to grab the flights from the SP, - # and attempt to re-query without credentials. This should fail. + # We grab all flights from the SP's (which we know how to reach by first querying the DSS). + # This is authenticated and is expected to succeed + sp_observation = rid.all_flights( + rect, + include_recent_positions=True, + get_details=True, + rid_version=self._rid_version, + session=self._dss.client, + dss_server_id=self._dss.participant_id, + ) - unauthenticated_session = UTMClientSession( - prefix_url=self._dss.client.get_prefix_url(), - auth_adapter=None, - timeout_seconds=self._dss.client.timeout_seconds, - ) + # Set the participant ID on the SP queries wherever possible + _attribute_sp_queries_to_participant_id(self._injected_flights, sp_observation) - queries_to_repeat = list(sp_observation.uss_flight_queries.values()) + list( - sp_observation.uss_flight_details_queries.values() - ) + # We fish out the queries that were used to grab the flights from the SP, + # and attempt to re-query without credentials. This should fail. + unauthenticated_session = UTMClientSession( + prefix_url=self._dss.client.get_prefix_url(), + auth_adapter=None, + timeout_seconds=self._dss.client.timeout_seconds, + ) - if len(queries_to_repeat) == 0: - logger.debug("no flights queries to repeat at this point.") - return False + queries_to_repeat = list(sp_observation.uss_flight_queries.values()) + list( + sp_observation.uss_flight_details_queries.values() + ) - logger.debug( - f"about to repeat {len(queries_to_repeat)} flights queries without credentials" - ) + if len(queries_to_repeat) == 0: + logger.debug("no flights queries to repeat at this point.") + return False - # Attempt to re-query the flights and flight details URLs: - for fq in queries_to_repeat: - failed_q = fetch.query_and_describe( - client=unauthenticated_session, - verb=fq.query.request.method, - url=fq.query.request.url, - json=fq.query.request.json, - data=fq.query.request.body, - server_id=self._dss.participant_id, - ) - logger.info( - f"Repeating query to {fq.query.request.url} without credentials" - ) - server_id = fq.query.get("server_id", "unknown") + logger.debug( + f"about to repeat {len(queries_to_repeat)} flights queries without credentials" + ) + + # Attempt to re-query the flights and flight details URLs: + for fq in queries_to_repeat: + sp_server_id = fq.query.get("server_id", "unknown") + # We may well have queried SP's for real flights that happen + # to be present in the area in which we inject flights; + if sp_server_id == "unknown": + # TODO we could skip these once development on the qualifier has stopped, + # however, unattributed queries are likely the cause of a bug, + # and we want to find out. + logger.error(f"got unattributed SP query to: {fq.query.request.url}") + + failed_q = fetch.query_and_describe( + client=unauthenticated_session, + verb=fq.query.request.method, + url=fq.query.request.url, + json=fq.query.request.json, + data=fq.query.request.body, + server_id=sp_server_id, + ) + logger.info( + f"Repeating query to {fq.query.request.url} without credentials" + ) + with self.check("Missing credentials", [sp_server_id]) as check: if failed_q.response.code not in [401, 403]: check.record_failed( "unauthenticated request was fulfilled", - participants=[server_id], + participants=[sp_server_id], severity=Severity.MEDIUM, details=f"queried flights on {fq.query.request.url} with no credentials, expected a failure but got a success reply", ) else: logger.info( - f"participant with id {server_id} properly authenticated the request" + f"participant with id {sp_server_id} properly authenticated the request" ) # Keep track of the failed queries, too self.record_query(failed_q) - return True + return True def cleanup(self): self.begin_cleanup() @@ -263,3 +265,24 @@ def cleanup(self): details=f"While trying to delete a test flight from {sp.participant_id}, encountered error:\n{stacktrace}", ) self.end_cleanup() + + +def _attribute_sp_queries_to_participant_id( + injected_flights: List[InjectedFlight], + fetched_flights: FetchedFlights, +): + observed_flights = [] + for uss_query in fetched_flights.uss_flight_queries.values(): + for f in range(len(uss_query.flights)): + observed_flights.append(DPObservedFlight(query=uss_query, flight=f)) + + mapping_by_injection_id = map_observations_to_injected_flights( + injected_flights, observed_flights + ) + + for telemetry_mapping in mapping_by_injection_id.values(): + # For flights that were mapped to an injection ID, + # update the observation queries with the participant id for future use in the aggregate checks + telemetry_mapping.observed_flight.query.set_server_id( + telemetry_mapping.injected_flight.uss_participant_id + ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index c64257741f..89aa0536b5 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -114,7 +114,7 @@ class TelemetryMapping(object): observed_flight: ObservationType -def _make_flight_mapping( +def map_observations_to_injected_flights( injected_flights: List[InjectedFlight], observed_flights: List[ObservationType], ) -> Dict[str, TelemetryMapping]: @@ -239,11 +239,14 @@ def evaluate_system_instantaneously( get_details=True, rid_version=self._rid_version, session=self._dss.client, - server_id=self._dss.participant_id, + dss_server_id=self._dss.participant_id, ) for q in sp_observation.queries: self._test_scenario.record_query(q) + # Evaluate observations + # (Note this also takes care of setting the server_id to the relevant + # participant_id on queries where possible) self._evaluate_sp_observation(sp_observation, rect) step_report = self._test_scenario.end_test_step() @@ -341,9 +344,11 @@ def _evaluate_normal_observation( query_timestamps=[query.request.timestamp], ) - mapping_by_injection_id = _make_flight_mapping( + mapping_by_injection_id = map_observations_to_injected_flights( self._injected_flights, observation.flights ) + # TODO check if we need to set some server ids on observation + # queries here (if we have unattributed queries this might be a source) self._evaluate_flight_presence( observer.participant_id, @@ -839,7 +844,7 @@ def _evaluate_sp_observation( for uss_query in sp_observation.uss_flight_queries.values(): for f in range(len(uss_query.flights)): observed_flights.append(DPObservedFlight(query=uss_query, flight=f)) - mapping_by_injection_id = _make_flight_mapping( + mapping_by_injection_id = map_observations_to_injected_flights( self._injected_flights, observed_flights ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md index 0602647f31..537364b92f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md @@ -48,7 +48,7 @@ It then repeats the exact same request while omitting the credentials, and expec #### Missing credentials check -This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v19.NET0500](../../../../requirements/astm/f3411/v19.md)**, +This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v19.NET0210](../../../../requirements/astm/f3411/v19.md)**, and that requests for existing flights that are executed with missing or incorrect credentials fail. ## Cleanup diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md index d94a021a73..5089d9d23c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md @@ -48,7 +48,7 @@ It then repeats the exact same request while omitting the credentials, and expec #### Missing credentials check -This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v22a.NET0500](../../../../requirements/astm/f3411/v22a.md)**, +This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v22a.NET0210](../../../../requirements/astm/f3411/v22a.md)**, and that requests for existing flights that are executed with missing or incorrect credentials fail. ## Cleanup diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 4a00bf06f7..c354bb768e 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -21,7 +21,7 @@ Checked in - astm
.f3411
.v19 + astm
.f3411
.v19 A2-6-1,1a Implemented ASTM F3411-19 NetRID DSS interoperability @@ -186,6 +186,11 @@ TODO ASTM NetRID: Operator interactions + + NET0210 + Implemented + ASTM NetRID SP clients misbehavior handling + NET0220 Implemented diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index b0d5e2fbc9..d605e67cea 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -21,7 +21,7 @@ Checked in - astm
.f3411
.v22a + astm
.f3411
.v22a A2-6-1,1a Implemented ASTM F3411-22a NetRID DSS interoperability @@ -186,6 +186,11 @@ TODO ASTM NetRID: Operator interactions + + NET0210 + Implemented + ASTM NetRID SP clients misbehavior handling + NET0220 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index ddee175bc9..b9390d4cf8 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -16,7 +16,7 @@ Checked in - astm
.f3411
.v22a + astm
.f3411
.v22a A2-6-1,1a Implemented ASTM F3411-22a NetRID DSS interoperability @@ -181,6 +181,11 @@ TODO ASTM NetRID: Operator interactions + + NET0210 + Implemented + ASTM NetRID SP clients misbehavior handling + NET0220 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index 7997e33f9c..6250f43cf0 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -17,7 +17,7 @@ Checked in - astm
.f3411
.v22a + astm
.f3411
.v22a A2-6-1,1a Implemented ASTM F3411-22a NetRID DSS interoperability @@ -182,6 +182,11 @@ TODO ASTM NetRID: Operator interactions + + NET0210 + Implemented + ASTM NetRID SP clients misbehavior handling + NET0220 Implemented