From e539259436b1eef56bd8f5fdaa6f72e99cc1a758 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Wed, 20 Mar 2024 15:58:18 +0100 Subject: [PATCH] PR comments --- monitoring/prober/infrastructure.py | 2 +- .../authentication_validation.md | 8 +++ .../authentication_validation.py | 5 +- .../dss/authentication/oir_api_validator.py | 54 ++++++++++++------- .../astm/utm/dss/fragments/oir/crud/create.md | 6 +-- .../dss/fragments/oir/crud/create_format.md | 9 ++++ .../astm/utm/dss/fragments/oir/crud/update.md | 6 +-- .../dss/fragments/oir/crud/update_format.md | 9 ++++ .../astm/utm/dss/test_step_fragments.py | 2 +- .../suites/astm/utm/dss_probing.md | 2 +- .../uss_qualifier/suites/astm/utm/f3548_21.md | 2 +- .../suites/faa/uft/message_signing.md | 2 +- .../suites/interuss/dss/all_tests.md | 2 +- .../suites/uspace/flight_auth.md | 2 +- .../suites/uspace/required_services.md | 2 +- 15 files changed, 76 insertions(+), 37 deletions(-) create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create_format.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update_format.md diff --git a/monitoring/prober/infrastructure.py b/monitoring/prober/infrastructure.py index 348dd28670..95e3aac299 100644 --- a/monitoring/prober/infrastructure.py +++ b/monitoring/prober/infrastructure.py @@ -100,7 +100,7 @@ def wrapper_default_scope(*args, **kwargs): resource_type_code_descriptions: Dict[ResourceType, str] = {} -# Next code: 383 +# Next code: 381 def register_resource_type(code: int, description: str) -> ResourceType: """Register that the specified code refers to the described resource. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.md index 1662ae934b..a661a3a047 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.md @@ -205,6 +205,10 @@ it is in violation of **[astm.f3548.v21.DSS0210,A2-7-2,7](../../../../../require If the DSS does not allow the creation of an operational intent when valid credentials are presented, it is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../requirements/astm/f3548/v21.md)**. +#### [Create response format](../fragments/oir/crud/create_format.md) + +Check response format of a creation request. + #### 🛑 Get operational intent reference with missing credentials check If the DSS under test allows the fetching of an operational intent without any credentials being presented, @@ -255,6 +259,10 @@ it is in violation of **[astm.f3548.v21.DSS0210,A2-7-2,7](../../../../../require If the DSS does not allow the mutation of an operational intent when valid credentials are presented, it is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../requirements/astm/f3548/v21.md)**. +#### [Mutate response format](../fragments/oir/crud/update_format.md) + +Check response format of a mutation. + #### 🛑 Delete operational intent reference with missing credentials check If the DSS under test allows the deletion of an operational intent without any credentials being presented, diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.py index e89ad557bd..776b33e0e6 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.py @@ -41,7 +41,7 @@ class AuthenticationValidation(TestScenario): """ SUB_TYPE = register_resource_type( - 382, "Subscription, Operational Entity Id, Constraint" + 380, "Subscription, Operational Entity Id, Constraint" ) # Reuse the same ID for every type of entity. @@ -213,7 +213,7 @@ def _ensure_test_entities_dont_exist(self): query = qe.queries[0] check.record_failed( summary=f"Could not query OIR {self._test_id}", - details=f"When attempting to query OIR {self._test_id} from the DSS, received {query.response.status_code}: {qe.msg}", + details=f"When attempting to query OIR {self._test_id} from the DSS, received {query.response.status_code}: {qe}", query_timestamps=[query.request.timestamp], ) @@ -231,7 +231,6 @@ def _ensure_test_entities_dont_exist(self): details=f"When attempting to remove op intent reference {self._test_id} from the DSS, received {query.status_code}: {qe.msg}", query_timestamps=[query.request.timestamp], ) - self._dss.delete_op_intent(oir.id, oir.ovn) test_step_fragments.cleanup_sub(self, self._dss, self._test_id) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/oir_api_validator.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/oir_api_validator.py index d60ca62972..adf505e373 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/oir_api_validator.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/oir_api_validator.py @@ -136,7 +136,7 @@ def _verify_oir_creation(self): ) as check: if no_scope_q.status_code != 401: check.record_failed( - summary=f"Expected 403, got {no_scope_q.status_code}", + summary=f"Expected 401, got {no_scope_q.status_code}", query_timestamps=[no_scope_q.request.timestamp], ) self._sanity_check_oir_not_created(check, no_scope_q) @@ -171,9 +171,19 @@ def _verify_oir_creation(self): query_timestamps=[valid_q.request.timestamp], ) - oir_resp = ImplicitDict.parse( - valid_q.response.json, ChangeOperationalIntentReferenceResponse - ) + with self._scenario.check( + "Create operational intent reference response format conforms to spec" + ) as check: + try: + oir_resp = ImplicitDict.parse( + valid_q.response.json, ChangeOperationalIntentReferenceResponse + ) + except ValueError as e: + check.record_failed( + summary="Could not parse the response body", + details=f"Failed to parse the response body as a ChangeOperationalIntentReferenceResponse: {e}", + query_timestamps=[valid_q.request.timestamp], + ) # Save the current OIR self._current_oir = oir_resp.operational_intent_reference @@ -220,7 +230,7 @@ def _verify_oir_get(self): ) as check: if query_missing_scope.status_code != 401: check.record_failed( - summary=f"Expected 403, got {query_missing_scope.status_code}", + summary=f"Expected 401, got {query_missing_scope.status_code}", query_timestamps=[query_missing_scope.request.timestamp], ) @@ -332,13 +342,24 @@ def _verify_oir_mutation(self): if valid_q.status_code != 200: check.record_failed( summary=f"Expected 200, got {valid_q.status_code}", - details=f"Mutation is expected to have succeeded, but got status {valid_q.status_code} with body {valid_q.response.json} instead", + details=f"Mutation is expected to have succeeded, but got status {valid_q.status_code} with error {valid_q.error_message} instead", + query_timestamps=[valid_q.request.timestamp], + ) + + with self._scenario.check( + "Mutate operational intent reference response format conforms to spec" + ) as check: + try: + parsed_oir = ImplicitDict.parse( + valid_q.response.json, ChangeOperationalIntentReferenceResponse + ) + except ValueError as e: + check.record_failed( + summary="Could not parse the response body", + details=f"Failed to parse the response body as a ChangeOperationalIntentReferenceResponse: {e}", query_timestamps=[valid_q.request.timestamp], ) - parsed_oir = ImplicitDict.parse( - valid_q.response.json, ChangeOperationalIntentReferenceResponse - ) self._current_oir = parsed_oir.operational_intent_reference def _verify_oir_deletion(self): @@ -455,7 +476,7 @@ def _verify_oir_search(self): ) as check: if no_scope_q.status_code != 401: check.record_failed( - summary=f"Expected 403, got {no_scope_q.status_code}", + summary=f"Expected 401, got {no_scope_q.status_code}", query_timestamps=[no_scope_q.request.timestamp], ) @@ -493,11 +514,8 @@ def _sanity_check_oir_not_created( _, sanity_check = self._dss.get_op_intent_reference(self._test_id) self._scenario.record_query(sanity_check) except QueryError as qe: - for q in qe.queries: - self._scenario.record_query(q) - if qe.queries[0].response.status_code == 404: - pass # All is good - else: + self._scenario.record_queries(qe.queries) + if qe.queries[0].response.status_code != 404: check.record_failed( summary="OIR was created by an unauthorized request.", details="The Operational Intent Reference should not have been created, as the creation attempt was not authenticated.", @@ -523,17 +541,17 @@ def _sanity_check_oir_not_updated( ): check.record_failed( summary="OIR was updated by an unauthorized request.", - details="The Operational Intent Reference should not have been updated, as the update attempt was not authenticated.", + details=f"The Operational Intent Reference with id {self._test_id} should not have been updated, as the update attempt was not authenticated.", query_timestamps=[ creation_q.request.timestamp, sanity_check.request.timestamp, ], ) except QueryError as qe: - for q in qe.queries: - self._scenario.record_query(q) + self._scenario.record_queries(qe.queries) check.record_failed( summary="Could not fetch OIR to confirm it has not been mutated", + details=f"The Operational Intent Reference with id {self._test_id} could not be fetched to confirm it has not been mutated: {qe}", query_timestamps=[ creation_q.request.timestamp, qe.queries[0].request.timestamp, diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md index b6594e25d3..f0ae40f967 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md @@ -7,11 +7,9 @@ This test step fragment validates that operational intent references can be crea As per **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**, the DSS API must allow callers to create an operational intent reference with either one or both of the start and end time missing, provided all the required parameters are valid. -## 🛑 Create operational intent reference response format conforms to spec check +## [Response Format](./create_format.md) -The response to a successful operational intent reference creation query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. - -If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. +Check response format ## 🛑 Create operational intent reference response content is correct check diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create_format.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create_format.md new file mode 100644 index 0000000000..5c440de6bb --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create_format.md @@ -0,0 +1,9 @@ +# Update operational intent reference response format test step fragment + +This test step fragment validates that an operational intent references creation returns a body in the correct format. + +## 🛑 Create operational intent reference response format conforms to spec check + +The response to a successful operational intent reference creation query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. + +If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md index c65141c1e0..a9dd87f725 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md @@ -6,11 +6,9 @@ This test step fragment validates that operational intent references can be upda As per **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**, the DSS API must allow callers to mutate an operational intent reference. -## 🛑 Mutate operational intent reference response format conforms to spec check +## [Response Format](./update_format.md) -The response to a successful operational intent reference mutation query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. - -If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. +Check response format ## 🛑 Mutate operational intent reference response content is correct check diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update_format.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update_format.md new file mode 100644 index 0000000000..96b0f40cfe --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update_format.md @@ -0,0 +1,9 @@ +# Update operational intent reference response format test step fragment + +This test step fragment validates that updates to operational intent references return a body in the correct format. + +## 🛑 Mutate operational intent reference response format conforms to spec check + +The response to a successful operational intent reference mutation query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. + +If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py index 15a08799c7..08e609da68 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py @@ -65,7 +65,7 @@ def cleanup_sub( if deleted_sub.status_code != 200: check.record_failed( summary=f"Could not delete subscription {sub_id}", - details=f"When attempting to delete subscription {sub_id} from the DSS, received {deleted_sub.status_code} with body {deleted_sub.response.json}", + details=f"When attempting to delete subscription {sub_id} from the DSS, received {deleted_sub.status_code} with body {deleted_sub.error_message}", query_timestamps=[deleted_sub.request.timestamp], ) diff --git a/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md b/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md index 4f3808a99a..aa77032f43 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md @@ -23,7 +23,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005,1 Implemented ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
ASTM SCD DSS: Interfaces authentication
ASTM SCD DSS: Subscription Simple
ASTM SCD DSS: Subscription Synchronization
ASTM SCD DSS: Subscription Validation diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index 6e085e6431..1e9959bc3e 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -35,7 +35,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005,1 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
ASTM SCD DSS: Interfaces authentication
ASTM SCD DSS: Subscription Simple
ASTM SCD DSS: Subscription Synchronization
ASTM SCD DSS: Subscription Validation
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md index b8cadbc982..51ff8f2706 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md @@ -18,7 +18,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005,1 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
ASTM SCD DSS: Interfaces authentication
ASTM SCD DSS: Subscription Simple
ASTM SCD DSS: Subscription Synchronization
ASTM SCD DSS: Subscription Validation
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted diff --git a/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md b/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md index f4ed5d0281..f805a4a36e 100644 --- a/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md +++ b/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md @@ -408,7 +408,7 @@ ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation
ASTM NetRID DSS: Token Validation - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005,1 Implemented ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
ASTM SCD DSS: Interfaces authentication
ASTM SCD DSS: Subscription Simple
ASTM SCD DSS: Subscription Synchronization
ASTM SCD DSS: Subscription Validation diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md index 7811acce4b..1613c6fe67 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.md +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -19,7 +19,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005,1 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
ASTM SCD DSS: Interfaces authentication
ASTM SCD DSS: Subscription Simple
ASTM SCD DSS: Subscription Synchronization
ASTM SCD DSS: Subscription Validation
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index baa4184c53..60e1ff0ad4 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -454,7 +454,7 @@ ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation
ASTM NetRID DSS: Token Validation - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005,1 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
ASTM SCD DSS: Interfaces authentication
ASTM SCD DSS: Subscription Simple
ASTM SCD DSS: Subscription Synchronization
ASTM SCD DSS: Subscription Validation
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted