From 5c7638e753bc2d13b4878805f704f6cbb76d78b3 Mon Sep 17 00:00:00 2001 From: Nikki <17799906+nikki-t@users.noreply.github.com> Date: Fri, 11 Oct 2024 16:54:45 -0400 Subject: [PATCH] Query to return granule files should query UAT when running in SIT or UAT environments --- hydrocron/db/track_ingest.py | 14 ++++--- tests/conftest.py | 2 +- tests/test_data/track_ingest_cnm_message.json | 2 +- tests/test_track_ingest.py | 4 +- tests/vcr_cassettes/publish_cnm.yaml | 42 +++++++++---------- 5 files changed, 34 insertions(+), 30 deletions(-) diff --git a/hydrocron/db/track_ingest.py b/hydrocron/db/track_ingest.py index 7668a19..434da01 100644 --- a/hydrocron/db/track_ingest.py +++ b/hydrocron/db/track_ingest.py @@ -109,7 +109,6 @@ def query_cmr(self, temporal): bearer_token = self._get_bearer_token() granules = query.short_name(self.SHORTNAME[self.collection_shortname]) \ .temporal(self.query_start, self.query_end) \ - .format("umm_json") \ .mode(CMR_UAT) \ .bearer_token(bearer_token) \ .get(query.hits()) @@ -117,7 +116,6 @@ def query_cmr(self, temporal): else: granules = query.short_name(self.collection_shortname) \ .temporal(self.query_start, self.query_end) \ - .format("umm_json") \ .get(query.hits()) else: logging.info("Querying CMR revision_date range: %s to %s.", self.query_start, self.query_end) @@ -125,7 +123,6 @@ def query_cmr(self, temporal): bearer_token = self._get_bearer_token() granules = query.short_name(self.SHORTNAME[self.collection_shortname]) \ .revision_date(self.query_start, self.query_end) \ - .format("umm_json") \ .mode(CMR_UAT) \ .bearer_token(bearer_token) \ .get(query.hits()) @@ -133,7 +130,6 @@ def query_cmr(self, temporal): else: granules = query.short_name(self.collection_shortname) \ .revision_date(self.query_start, self.query_end) \ - .format("umm_json") \ .get(query.hits()) cmr_granules = {} @@ -295,7 +291,15 @@ def _query_granule_files(self, granule_ur): """ query = GranuleQuery() - granules = query.short_name(self.collection_shortname).readable_granule_name(granule_ur).format("umm_json").get_all() + query = query.short_name(self.collection_shortname).readable_granule_name(granule_ur).format("umm_json") + + if self.ENV in ("sit", "uat"): + bearer_token = self._get_bearer_token() + query = query.bearer_token(bearer_token) \ + .mode(CMR_UAT) \ + .short_name(self.SHORTNAME[self.collection_shortname]) + + granules = query.get_all() cnm_files = [] for granule in granules: granule_json = json.loads(granule) diff --git a/tests/conftest.py b/tests/conftest.py index b01bee6..7c5f2e1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -228,7 +228,7 @@ def mock_sns(): os.environ["AWS_DEFAULT_REGION"] = "us-west-2" sns = boto3.client("sns") - sns.create_topic(Name="svc-hydrocron-sit-cnm-response") + sns.create_topic(Name="svc-hydrocron-test-cnm-response") yield sns diff --git a/tests/test_data/track_ingest_cnm_message.json b/tests/test_data/track_ingest_cnm_message.json index 3090f4a..5f4f7f6 100644 --- a/tests/test_data/track_ingest_cnm_message.json +++ b/tests/test_data/track_ingest_cnm_message.json @@ -15,7 +15,7 @@ "checksumType": "md5", "checksum": "6ce27e868bd90055252de186f554759f", "size": 745878, - "uri": "s3://podaac-swot-uat-cumulus-protected/SWOT_L2_HR_RiverSP_2.0/SWOT_L2_HR_RiverSP_Reach_020_457_NA_20240905T233134_20240905T233135_PIC0_01.zip" + "uri": "s3://podaac-swot-ops-cumulus-protected/SWOT_L2_HR_RiverSP_2.0/SWOT_L2_HR_RiverSP_Reach_020_457_NA_20240905T233134_20240905T233135_PIC0_01.zip" } ] } diff --git a/tests/test_track_ingest.py b/tests/test_track_ingest.py index 2410a09..4dc1c68 100644 --- a/tests/test_track_ingest.py +++ b/tests/test_track_ingest.py @@ -302,7 +302,7 @@ def test_track_ingest_publish_cnm(track_ingest_cnm_fixture): "actual_feature_count": 0, "status": "to_ingest" }] - track.ENV = "sit" + track.ENV = "test" vcr_cassette = pathlib.Path(os.path.dirname(os.path.realpath(__file__))) \ .joinpath('vcr_cassettes').joinpath('publish_cnm.yaml') @@ -310,7 +310,7 @@ def test_track_ingest_publish_cnm(track_ingest_cnm_fixture): track.publish_cnm_ingest(DEFAULT_ACCOUNT_ID) sns_backend = sns_backends[DEFAULT_ACCOUNT_ID]["us-west-2"] - actual = json.loads(sns_backend.topics[f"arn:aws:sns:us-west-2:{DEFAULT_ACCOUNT_ID}:svc-hydrocron-sit-cnm-response"].sent_notifications[0][1]) + actual = json.loads(sns_backend.topics[f"arn:aws:sns:us-west-2:{DEFAULT_ACCOUNT_ID}:svc-hydrocron-test-cnm-response"].sent_notifications[0][1]) expected_file = (pathlib.Path(os.path.dirname(os.path.realpath(__file__))) .joinpath('test_data').joinpath('track_ingest_cnm_message.json')) diff --git a/tests/vcr_cassettes/publish_cnm.yaml b/tests/vcr_cassettes/publish_cnm.yaml index 97360a1..37ecf5e 100644 --- a/tests/vcr_cassettes/publish_cnm.yaml +++ b/tests/vcr_cassettes/publish_cnm.yaml @@ -14,7 +14,7 @@ interactions: uri: https://cmr.earthdata.nasa.gov/search/granules.umm_json?short_name=SWOT_L2_HR_RiverSP_reach_2.0&readable_granule_name%5B%5D=SWOT_L2_HR_RiverSP_Reach_020_457_NA_20240905T233134_20240905T233135_PIC0_01.zip&options%5Breadable_granule_name%5D%5Bpattern%5D=true&page_size=0 response: body: - string: '{"hits":1,"took":105,"items":[]}' + string: '{"hits":1,"took":95,"items":[]}' headers: Access-Control-Allow-Origin: - '*' @@ -24,19 +24,19 @@ interactions: CMR-Hits: - '1' CMR-Request-Id: - - e3986cbe-e92f-4f66-9d3a-135846d23a3c + - 9f40c39a-12cc-4033-b67a-cb8fee448bc5 CMR-Took: - - '106' + - '95' Connection: - keep-alive Content-MD5: - - efb24cae14e6d340937153b1c3fcd8e7 + - 461384cf862923b25d050408730eacc0 Content-SHA1: - - 5f45a4e901063718301c91a36cdd34550fb0ca50 + - c6ee7948f5b33a1d02a79d1cd64cecafb8a78d11 Content-Type: - application/vnd.nasa.cmr.umm_results+json;version=1.6.6; charset=utf-8 Date: - - Thu, 10 Oct 2024 21:05:46 GMT + - Fri, 11 Oct 2024 20:42:19 GMT Server: - ServerTokens ProductOnly Strict-Transport-Security: @@ -46,11 +46,11 @@ interactions: Vary: - Accept-Encoding, User-Agent Via: - - 1.1 27b5e48b551f58a537d4366486948250.cloudfront.net (CloudFront) + - 1.1 c4e50e26fcbefa29d8d1be685a1f4242.cloudfront.net (CloudFront) X-Amz-Cf-Id: - - -dGCq9LYLoXF9oC5tqf6ZHugYJRfeqrk1uQw1IIpqRc8mTdbvzKVkA== + - OGrkgQzrwmJ_wu5tuaqCMCdg3rdW0DUZHyOoAZG4qphS18EuJmrEcA== X-Amz-Cf-Pop: - - LAX54-P3 + - LAX50-C1 X-Cache: - Miss from cloudfront X-Content-Type-Options: @@ -58,11 +58,11 @@ interactions: X-Frame-Options: - SAMEORIGIN X-Request-Id: - - -dGCq9LYLoXF9oC5tqf6ZHugYJRfeqrk1uQw1IIpqRc8mTdbvzKVkA== + - OGrkgQzrwmJ_wu5tuaqCMCdg3rdW0DUZHyOoAZG4qphS18EuJmrEcA== X-XSS-Protection: - 1; mode=block content-length: - - '32' + - '31' status: code: 200 message: OK @@ -81,7 +81,7 @@ interactions: uri: https://cmr.earthdata.nasa.gov/search/granules.umm_json?short_name=SWOT_L2_HR_RiverSP_reach_2.0&readable_granule_name%5B%5D=SWOT_L2_HR_RiverSP_Reach_020_457_NA_20240905T233134_20240905T233135_PIC0_01.zip&options%5Breadable_granule_name%5D%5Bpattern%5D=true&page_size=1 response: body: - string: '{"hits":1,"took":74,"items":[{"meta":{"concept-type":"granule","concept-id":"G3232820892-POCLOUD","revision-id":1,"native-id":"SWOT_L2_HR_RiverSP_Reach_020_457_NA_20240905T233134_20240905T233135_PIC0_01_swot","collection-concept-id":"C2799438303-POCLOUD","provider-id":"POCLOUD","format":"application/vnd.nasa.cmr.umm+json","revision-date":"2024-09-09T02:57:10.684Z"},"umm":{"TemporalExtent":{"RangeDateTime":{"EndingDateTime":"2024-09-05T23:31:35.589Z","BeginningDateTime":"2024-09-05T23:31:34.815Z"}},"GranuleUR":"SWOT_L2_HR_RiverSP_Reach_020_457_NA_20240905T233134_20240905T233135_PIC0_01_swot","AdditionalAttributes":[{"Values":["246L","247L","248L","249L","250L","251L","252L","253L","254L","255L","256L","257L","258L","246R","247R","248R","249R","250R","251R","252R","253R","254R","255R","256R","257R","258R"],"Name":"TILE"}],"MeasuredParameters":[{"QAStats":{"QAPercentMissingData":0,"QAPercentOutOfBoundsData":0},"ParameterName":"N/A"}],"SpatialExtent":{"HorizontalSpatialDomain":{"Geometry":{"BoundingRectangles":[{"WestBoundingCoordinate":-130.8307934819651,"SouthBoundingCoordinate":52.93360004865338,"EastBoundingCoordinate":-125.58639150138737,"NorthBoundingCoordinate":58.82204893062929}]},"Track":{"Cycle":20,"Passes":[{"Pass":457,"Tiles":["246L","247L","248L","249L","250L","251L","252L","253L","254L","255L","256L","257L","258L","246R","247R","248R","249R","250R","251R","252R","253R","254R","255R","256R","257R","258R"]}]}}},"ProviderDates":[{"Type":"Insert","Date":"2024-09-09T01:25:25.739Z"},{"Type":"Update","Date":"2024-09-09T01:25:25.739Z"}],"CollectionReference":{"Version":"2.0","ShortName":"SWOT_L2_HR_RiverSP_reach_2.0"},"PGEVersionClass":{"PGEName":"PGE_L2_HR_RiverSP","PGEVersion":"5.0.4"},"RelatedUrls":[{"URL":"https://archive.swot.podaac.earthdata.nasa.gov/podaac-swot-ops-cumulus-public/SWOT_L2_HR_RiverSP_2.0/SWOT_L2_HR_RiverSP_Reach_020_457_NA_20240905T233134_20240905T233135_PIC0_01.zip.iso.xml","Description":"Download + string: '{"hits":1,"took":92,"items":[{"meta":{"concept-type":"granule","concept-id":"G3232820892-POCLOUD","revision-id":1,"native-id":"SWOT_L2_HR_RiverSP_Reach_020_457_NA_20240905T233134_20240905T233135_PIC0_01_swot","collection-concept-id":"C2799438303-POCLOUD","provider-id":"POCLOUD","format":"application/vnd.nasa.cmr.umm+json","revision-date":"2024-09-09T02:57:10.684Z"},"umm":{"TemporalExtent":{"RangeDateTime":{"EndingDateTime":"2024-09-05T23:31:35.589Z","BeginningDateTime":"2024-09-05T23:31:34.815Z"}},"GranuleUR":"SWOT_L2_HR_RiverSP_Reach_020_457_NA_20240905T233134_20240905T233135_PIC0_01_swot","AdditionalAttributes":[{"Values":["246L","247L","248L","249L","250L","251L","252L","253L","254L","255L","256L","257L","258L","246R","247R","248R","249R","250R","251R","252R","253R","254R","255R","256R","257R","258R"],"Name":"TILE"}],"MeasuredParameters":[{"QAStats":{"QAPercentMissingData":0,"QAPercentOutOfBoundsData":0},"ParameterName":"N/A"}],"SpatialExtent":{"HorizontalSpatialDomain":{"Geometry":{"BoundingRectangles":[{"WestBoundingCoordinate":-130.8307934819651,"SouthBoundingCoordinate":52.93360004865338,"EastBoundingCoordinate":-125.58639150138737,"NorthBoundingCoordinate":58.82204893062929}]},"Track":{"Cycle":20,"Passes":[{"Pass":457,"Tiles":["246L","247L","248L","249L","250L","251L","252L","253L","254L","255L","256L","257L","258L","246R","247R","248R","249R","250R","251R","252R","253R","254R","255R","256R","257R","258R"]}]}}},"ProviderDates":[{"Type":"Insert","Date":"2024-09-09T01:25:25.739Z"},{"Type":"Update","Date":"2024-09-09T01:25:25.739Z"}],"CollectionReference":{"Version":"2.0","ShortName":"SWOT_L2_HR_RiverSP_reach_2.0"},"PGEVersionClass":{"PGEName":"PGE_L2_HR_RiverSP","PGEVersion":"5.0.4"},"RelatedUrls":[{"URL":"https://archive.swot.podaac.earthdata.nasa.gov/podaac-swot-ops-cumulus-public/SWOT_L2_HR_RiverSP_2.0/SWOT_L2_HR_RiverSP_Reach_020_457_NA_20240905T233134_20240905T233135_PIC0_01.zip.iso.xml","Description":"Download SWOT_L2_HR_RiverSP_Reach_020_457_NA_20240905T233134_20240905T233135_PIC0_01.zip.iso.xml","Type":"EXTENDED METADATA"},{"URL":"s3://podaac-swot-ops-cumulus-public/SWOT_L2_HR_RiverSP_2.0/SWOT_L2_HR_RiverSP_Reach_020_457_NA_20240905T233134_20240905T233135_PIC0_01.zip.iso.xml","Description":"This link provides direct download access via S3 to the granule","Type":"EXTENDED @@ -114,21 +114,21 @@ interactions: CMR-Hits: - '1' CMR-Request-Id: - - 21ec8088-e6d8-4bbe-b659-fd9a86915dc8 + - f3bd451b-250e-464f-aca3-261d20a36ae4 CMR-Search-After: - '["pocloud",1725579094815,3232820892]' CMR-Took: - - '75' + - '93' Connection: - keep-alive Content-MD5: - - 77046ee32e3289cf25ac89b8a8f2e431 + - 544062937074509d57e9507aa95f44a3 Content-SHA1: - - dab6e5694fc519744720a256a21ea1c522f2218f + - 4df473f06f2f0b28ee0e328160064de80869bf4f Content-Type: - application/vnd.nasa.cmr.umm_results+json;version=1.6.6; charset=utf-8 Date: - - Thu, 10 Oct 2024 21:05:46 GMT + - Fri, 11 Oct 2024 20:42:20 GMT Server: - ServerTokens ProductOnly Strict-Transport-Security: @@ -138,11 +138,11 @@ interactions: Vary: - Accept-Encoding, User-Agent Via: - - 1.1 e4cf17e75928ca088d52aecb26f1f1da.cloudfront.net (CloudFront) + - 1.1 92dd5512d5f290fe351674f3051d6d82.cloudfront.net (CloudFront) X-Amz-Cf-Id: - - YohmjCui14QWvPVi2hcsgadZt5eKbVh6WQscK946z7WnDuV74O2BRg== + - n2l1zw3GXkohQmQItnRDXs5Rh3HPLrSzLvWEeWDpaCjBvUH5Dm7_fw== X-Amz-Cf-Pop: - - LAX54-P3 + - LAX50-C1 X-Cache: - Miss from cloudfront X-Content-Type-Options: @@ -150,7 +150,7 @@ interactions: X-Frame-Options: - SAMEORIGIN X-Request-Id: - - YohmjCui14QWvPVi2hcsgadZt5eKbVh6WQscK946z7WnDuV74O2BRg== + - n2l1zw3GXkohQmQItnRDXs5Rh3HPLrSzLvWEeWDpaCjBvUH5Dm7_fw== X-XSS-Protection: - 1; mode=block content-length: