From 92c14860f213e517ba047c6360223ed56ffea6b2 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Wed, 20 Mar 2024 20:40:01 +0100 Subject: [PATCH] [uss_qualifier] adapt SCD subscription sync documentation to correct convention --- .../astm/utm/dss/fragments/sub/crud/create.md | 14 ++---- .../astm/utm/dss/fragments/sub/crud/delete.md | 8 +++- .../astm/utm/dss/fragments/sub/crud/read.md | 2 +- .../astm/utm/dss/fragments/sub/crud/update.md | 13 ++--- .../astm/utm/dss/fragments/sub/sync.md | 18 +++---- .../subscription_synchronization.md | 16 ------- .../subscription_synchronization.py | 48 ++++++++----------- .../astm/utm/dss/validators/__init__.py | 22 +++++++++ .../astm/utm/dss/validators/oir_validator.py | 31 +++--------- .../dss/validators/subscription_validator.py | 28 +++-------- 10 files changed, 86 insertions(+), 114 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/create.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/create.md index 6b20c141db..de0bc5cfab 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/create.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/create.md @@ -7,18 +7,14 @@ This test step fragment validates that subscriptions can be created. As per **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**, the DSS API must allow callers to create a subscription with either one or both of the start and end time missing, provided all the required parameters are valid. -## 🛑 Create subscription response is correct check - -A successful subscription creation query is expected to return a well-defined body, the content of which reflects the created subscription. -If the format and content of the response are not conforming, the DSS is failing to implement **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**. - -## ⚠️ Create subscription response format conforms to spec check +## 🛑 Create subscription response format conforms to spec check The response to a successful subscription creation query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Response to subscription creation contains a subscription check +## 🛑 Create subscription response content is correct check + +A successful subscription creation query is expected to return a well-defined body, the content of which reflects the created subscription. -As per **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**, upon creation of a subscription, -the newly created subscription must be part of its response. +If the content of the response does not correspond to the requested content, the DSS is failing to implement **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/delete.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/delete.md index c523ebb942..0aa55b66c3 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/delete.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/delete.md @@ -6,8 +6,14 @@ This test step fragment validates that subscriptions can be deleted. An attempt to delete a subscription when the correct version is provided should succeed, otherwise the DSS is in violation of **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**. -## ⚠️ Delete subscription response format conforms to spec check +## 🛑 Delete subscription response format conforms to spec check The response to a successful subscription deletion query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**. + +## 🛑 Delete subscription response content is correct check + +A successful subscription deletion query is expected to return a well-defined body, the content of which reflects the deleted subscription. + +If the content of the response does not correspond to the subscription at the time of deletion, the DSS is failing to implement **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/read.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/read.md index 507f2ed12d..66e268c0c7 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/read.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/read.md @@ -6,7 +6,7 @@ This test step fragment validates that subscriptions can be read. If a subscription cannot be queried using its ID, the DSS is failing to meet **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**. -## ⚠️ Get subscription response format conforms to spec check +## 🛑 Get subscription response format conforms to spec check The response to a successful get subscription query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/update.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/update.md index 8810035448..0221cf46d8 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/update.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/update.md @@ -6,13 +6,14 @@ This test step fragment validates that subscriptions can be updated. If a subscription cannot be modified with a valid set of parameters, the DSS is failing to meet **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Response to subscription mutation contains a subscription check - -As per **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**, upon mutation of a subscription, -the newly created subscription must be part of its response. - -## ⚠️ Mutate subscription response format conforms to spec check +## 🛑 Mutate subscription response format conforms to spec check The response to a successful subscription mutation query is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21. If it does not, the DSS is failing to implement **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**. + +## 🛑 Mutate subscription response content is correct check + +A successful subscription mutation query is expected to return a well-defined body, the content of which reflects the mutated subscription. + +If the content of the response does correspond to the requested mutation, the DSS is failing to implement **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/sync.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/sync.md index 8d3e921d09..f7dfac62ca 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/sync.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/sync.md @@ -12,7 +12,7 @@ is failing to implement one of the following requirements: As a result, the DSS pool under test is failing to meet **[astm.f3548.v21.DSS0020](../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Propagated subscription contains the correct USS base URL check +## ⚠️ Propagated subscription contains the correct USS base URL check If the subscription returned by a DSS to which the subscription was synchronized to does not contain the correct USS base URL, either one of the instances at which the subscription was created or the one that was queried, @@ -23,7 +23,7 @@ is failing to implement one of the following requirements: As a result, the DSS pool under test is failing to meet **[astm.f3548.v21.DSS0020](../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Propagated subscription contains the correct start time check +## ⚠️ Propagated subscription contains the correct start time check If the subscription returned by a DSS to which the subscription was synchronized to does not contain the correct start time, either one of the instances at which the subscription was created or the one that was queried, @@ -34,7 +34,7 @@ is failing to implement one of the following requirements: As a result, the DSS pool under test is failing to meet **[astm.f3548.v21.DSS0020](../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Propagated subscription contains the correct end time check +## ⚠️ Propagated subscription contains the correct end time check If the subscription returned by a DSS to which the subscription was synchronized to does not contain the correct end time, either one of the instances at which the subscription was created or the one that was queried, @@ -45,7 +45,7 @@ is failing to implement one of the following requirements: As a result, the DSS pool under test is failing to meet **[astm.f3548.v21.DSS0020](../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Propagated subscription contains the correct version check +## ⚠️ Propagated subscription contains the correct version check If the subscription returned by a DSS to which the subscription was synchronized to does not contain the correct version, either one of the instances at which the subscription was created or the one that was queried, @@ -56,7 +56,7 @@ is failing to implement one of the following requirements: As a result, the DSS pool under test is failing to meet **[astm.f3548.v21.DSS0020](../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Propagated subscription contains the correct notification flags check +## ⚠️ Propagated subscription contains the correct notification flags check If the subscription returned by a DSS to which the subscription was synchronized to does not contain the correct notification flags, either one of the instances at which the subscription was created or the one that was queried, @@ -67,7 +67,7 @@ is failing to implement one of the following requirements: As a result, the DSS pool under test is failing to meet **[astm.f3548.v21.DSS0020](../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Propagated subscription contains the correct implicit flag check +## ⚠️ Propagated subscription contains the correct implicit flag check If the subscription returned by a DSS to which the subscription was synchronized to does not contain the correct implicit flag, either one of the instances at which the subscription was created or the one that was queried, @@ -78,7 +78,7 @@ is failing to implement one of the following requirements: As a result, the DSS pool under test is failing to meet **[astm.f3548.v21.DSS0020](../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Propagated subscription contains expected notification count check +## ⚠️ Propagated subscription contains expected notification count check If the subscription returned by a DSS to which the subscription was synchronized to does not contain the expected notification count, either one of the instances at which the subscription was created or the one that was queried, @@ -89,7 +89,7 @@ is failing to implement one of the following requirements: As a result, the DSS pool under test is failing to meet **[astm.f3548.v21.DSS0020](../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Secondary DSS returns the subscription in searches for area that contains it check +## ⚠️ Secondary DSS returns the subscription in searches for area that contains it check The secondary DSS should be aware of the subscription's area: when a search query is issued for an area that encompasses the created subscription, the secondary DSS should return the subscription in its search results. @@ -101,7 +101,7 @@ Otherwise, it is in violation of one of the following requirements: As a result, the DSS pool under test is failing to meet **[astm.f3548.v21.DSS0020](../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Secondary DSS does not return the subscription in searches not encompassing the general area of the subscription check +## ⚠️ Secondary DSS does not return the subscription in searches not encompassing the general area of the subscription check The secondary DSS should be aware of the subscription's area: when a search query is issued for an area not in the vicinity of the created subscription, the secondary DSS should not return it in its search results. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.md index 8b981155e3..01c3dea057 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.md @@ -52,11 +52,6 @@ Verify that the subscription returned by the DSS under test is properly formatte Query the created subscription at every DSS provided in `dss_instances`. -#### 🛑 Subscription returned by a secondary DSS is valid and correct check - -When queried for a subscription that was created via another DSS, a DSS instance is expected to provide a valid subscription. - -If it does not, it might be in violation of **[astm.f3548.v21.DSS0005,5](../../../../../requirements/astm/f3548/v21.md)**. #### [Subscription is synchronized](../fragments/sub/sync.md) @@ -95,17 +90,6 @@ Verify that the version of the subscription returned by the DSS has been updated Query the updated subscription at every DSS provided in `dss_instances`. -#### 🛑 Subscription returned by a secondary DSS is valid and correct check - -When queried for a subscription that was mutated via another DSS, a DSS instance is expected to provide a valid subscription. - -If it does not, either one of the primary DSS or the DSS that returned the subscription is in violation of one of the following requirements: - -**[astm.f3548.v21.DSS0005,5](../../../../../requirements/astm/f3548/v21.md)**, if the API is not working as described by the OpenAPI specification; -**[astm.f3548.v21.DSS0215](../../../../../requirements/astm/f3548/v21.md)**, if the DSS through which the subscription was mutated is returning API calls to the client before having updated its underlying distributed storage. - -As a result, the DSS pool under test is failing to meet **[astm.f3548.v21.DSS0020](../../../../../requirements/astm/f3548/v21.md)**. - #### [Subscription is synchronized](../fragments/sub/sync.md) Confirm that the subscription that was just mutated is properly synchronized across all DSS instances. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.py index d7c0b9d9a1..224a28fec2 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.py @@ -5,10 +5,11 @@ from uas_standards.astm.f3548.v21.api import Subscription, SubscriptionID from uas_standards.astm.f3548.v21.constants import Scope -from monitoring.monitorlib import geo +from monitoring.monitorlib import geo, schema_validation from monitoring.monitorlib.geo import Volume3D from monitoring.monitorlib.geotemporal import Volume4D from monitoring.monitorlib.mutate.scd import MutatedSubscription +from monitoring.monitorlib.schema_validation import F3548_21 from monitoring.prober.infrastructure import register_resource_type from monitoring.uss_qualifier.resources.astm.f3548.v21 import PlanningAreaResource from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import ( @@ -21,6 +22,9 @@ ) from monitoring.uss_qualifier.resources.interuss.id_generator import IDGeneratorResource from monitoring.uss_qualifier.scenarios.astm.utm.dss import test_step_fragments +from monitoring.uss_qualifier.scenarios.astm.utm.dss.validators import ( + fail_with_schema_errors, +) from monitoring.uss_qualifier.scenarios.astm.utm.dss.validators.subscription_validator import ( SubscriptionValidator, ) @@ -252,7 +256,7 @@ def _create_sub_with_params( ) with self.check( - "Create subscription response is correct", [self._primary_pid] + "Create subscription response content is correct", [self._primary_pid] ) as check: SubscriptionValidator( check, @@ -488,22 +492,18 @@ def _validate_get_sub_from_secondary( query_timestamps=[fetched_sub.request.timestamp], ) + # Finally, validate the response schema with self.check( - "Subscription returned by a secondary DSS is valid and correct", - [secondary_dss.participant_id, self._primary_pid], + "Get subscription response format conforms to spec", + secondary_dss.participant_id, ) as check: - # Do a full validation of the subscription as a sanity check - SubscriptionValidator( - check, - self, - [secondary_dss.participant_id], - expected_sub_params, - ).validate_fetched_subscription( - expected_sub_id=expected_sub_params.sub_id, - fetched_sub=fetched_sub, - expected_version=self._current_subscription.version, - is_implicit=False, + errors = schema_validation.validate( + F3548_21.OpenAPIPath, + F3548_21.GetSubscriptionResponse, + fetched_sub.response.json, ) + if errors: + fail_with_schema_errors(check, errors, fetched_sub.request.timestamp) def _compare_upsert_resp_with_params( self, @@ -516,25 +516,19 @@ def _compare_upsert_resp_with_params( Verify that the response of the server is conforming to the spec and the parameters we used in the request. """ check_name = ( - "Response to subscription mutation contains a subscription" + "Mutate subscription response content is correct" if was_mutated - else "Response to subscription creation contains a subscription" + else "Create subscription response content is correct" ) - with self.check( - check_name, - [self._primary_pid], - ) as check: + + with self.check(check_name, [self._primary_pid]) as check: if not creation_resp_under_test.subscription: check.record_failed( "Response to subscription creation did not contain a subscription", - details="A subscription is expected to be returned in the response to a subscription creation request." + details=f"A subscription is expected to be returned in the response. " f"Parameters used: {creation_params}", query_timestamps=[creation_resp_under_test.request.timestamp], ) - - with self.check( - "Mutate subscription response format conforms to spec", [self._primary_pid] - ) as check: SubscriptionValidator( check, self, @@ -630,7 +624,7 @@ def _delete_sub_from_dss( return False with self.check( - "Delete subscription response format conforms to spec", + "Delete subscription response content is correct", [dss_instance.participant_id], ) as check: SubscriptionValidator( diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/__init__.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/__init__.py index e69de29bb2..9174334ae7 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/__init__.py @@ -0,0 +1,22 @@ +from datetime import datetime +from typing import List + +from monitoring.monitorlib import schema_validation +from monitoring.uss_qualifier.scenarios.scenario import PendingCheck + + +def fail_with_schema_errors( + check: PendingCheck, + errors: List[schema_validation.ValidationError], + t_dss: datetime, +) -> None: + """ + Fail the passed check with the passed schema validation errors. + """ + details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) + check.record_failed( + summary="Response format was invalid", + details="Found the following schema validation errors in the DSS response:\n" + + details, + query_timestamps=[t_dss], + ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/oir_validator.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/oir_validator.py index 2b073f2c70..07566e7fa7 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/oir_validator.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/oir_validator.py @@ -14,6 +14,9 @@ from monitoring.monitorlib import schema_validation, fetch from monitoring.monitorlib.schema_validation import F3548_21 +from monitoring.uss_qualifier.scenarios.astm.utm.dss.validators import ( + fail_with_schema_errors, +) from monitoring.uss_qualifier.scenarios.scenario import PendingCheck, TestScenario TIME_TOLERANCE_SEC = 1 @@ -288,7 +291,7 @@ def _validate_put_oir_response_schema( oir_query.response.json, ) if errors: - _fail_with_schema_errors(check, errors, t_dss) + fail_with_schema_errors(check, errors, t_dss) return False return True @@ -375,7 +378,7 @@ def validate_fetched_oir( fetched_oir.response.json, ) if errors: - _fail_with_schema_errors(check, errors, t_dss) + fail_with_schema_errors(check, errors, t_dss) parsed_resp = fetched_oir.parse_json_result( GetOperationalIntentReferenceResponse @@ -455,7 +458,7 @@ def validate_searched_oir_format( search_response.response.json, ) if errors: - _fail_with_schema_errors(check, errors, t_dss) + fail_with_schema_errors(check, errors, t_dss) def validate_deleted_oir( self, @@ -478,7 +481,7 @@ def validate_deleted_oir( deleted_oir.response.json, ) if errors: - _fail_with_schema_errors(check, errors, t_dss) + fail_with_schema_errors(check, errors, t_dss) oir_resp = deleted_oir.parse_json_result( ChangeOperationalIntentReferenceResponse @@ -494,23 +497,3 @@ def validate_deleted_oir( previous_version=None, expected_version=expected_version, ) - - -def _fail_with_schema_errors( - check: PendingCheck, - errors: List[schema_validation.ValidationError], - t_dss: datetime, -) -> None: - """ - Fail the passed check with the passed schema validation errors. - Note: - The main check IS NOT failed: - The main check pertains to the CONTENT of the response but not its FORMAT. - """ - details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) - check.record_failed( - summary="Response format was invalid", - details="Found the following schema validation errors in the DSS response:\n" - + details, - query_timestamps=[t_dss], - ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/subscription_validator.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/subscription_validator.py index 27295bcdbe..784851a28b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/subscription_validator.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/subscription_validator.py @@ -10,6 +10,9 @@ from monitoring.uss_qualifier.resources.astm.f3548.v21.subscription_params import ( SubscriptionParams, ) +from monitoring.uss_qualifier.scenarios.astm.utm.dss.validators import ( + fail_with_schema_errors, +) from monitoring.uss_qualifier.scenarios.scenario import PendingCheck, TestScenario TIME_TOLERANCE_SEC = 1 @@ -71,23 +74,6 @@ def _fail_sub_check( query_timestamps=[t_dss], ) - def _fail_with_schema_errors( - self, - check: PendingCheck, - errors: List[schema_validation.ValidationError], - t_dss: datetime, - ) -> None: - """Fail the passed check with the passed schema validation errors, and fail - the main check with the passed details.""" - details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) - self._fail_sub_check( - check, - summary="Response format was invalid", - details="Found the following schema validation errors in the DSS response:\n" - + details, - t_dss=t_dss, - ) - def _validate_sub( self, expected_sub_id: SubscriptionID, @@ -304,7 +290,7 @@ def _validate_put_sub_response_schema( new_sub.response.json, ) if errors: - self._fail_with_schema_errors(check, errors, t_dss) + fail_with_schema_errors(check, errors, t_dss) def validate_created_subscription( self, expected_sub_id: SubscriptionID, new_sub: MutatedSubscription @@ -407,7 +393,7 @@ def validate_fetched_subscription( fetched_sub.response.json, ) if errors: - self._fail_with_schema_errors(check, errors, t_dss) + fail_with_schema_errors(check, errors, t_dss) # Validate the subscription itself self._validate_sub( @@ -482,7 +468,7 @@ def validate_searched_subscriptions_format( searched_subscriptions.response.json, ) if errors: - self._fail_with_schema_errors(check, errors, t_dss) + fail_with_schema_errors(check, errors, t_dss) def validate_deleted_subscription( self, @@ -509,7 +495,7 @@ def validate_deleted_subscription( deleted_subscription.response.json, ) if errors: - self._fail_with_schema_errors(check, errors, t_dss) + fail_with_schema_errors(check, errors, t_dss) # Validate the subscription itself self._validate_sub(