refactor: change s3 url patterns used with cogreader

nasaharvest · Dec 31, 2024 · 7a612f7 · 7a612f7
1 parent 4ce2e20
commit 7a612f7
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 27 deletions.
diff --git a/glam_api/glam/views/graphics.py b/glam_api/glam/views/graphics.py
@@ -11,6 +11,7 @@
 from drf_yasg.utils import swagger_auto_schema
 from drf_yasg import openapi
 
+from django.conf import settings
 from django.utils.decorators import method_decorator
 from django.views.decorators.cache import cache_page
 from django.core import serializers
@@ -316,7 +317,9 @@ def tick_formatter(x, pos):
 
         boundary_feature_geom = boundary_feature.geom.simplify(scale_factor)
 
-        with COGReader(product_ds.file_object.url) as image:
+        with COGReader(
+            f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{product_ds.file_object.name}"
+        ) as image:
             feat = image.feature(
                 json.loads(boundary_feature_geom.geojson), max_size=1024
             )
@@ -355,7 +358,9 @@ def tick_formatter(x, pos):
                     baseline_type=anom_type,
                 )
 
-            with COGReader(anomaly_ds.file_object.url) as anom_img:
+            with COGReader(
+                f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{anomaly_ds.file_object.name}"
+            ) as anom_img:
                 anom_feat = anom_img.feature(
                     json.loads(boundary_feature_geom.geojson), max_size=1024
                 )
@@ -369,7 +374,9 @@ def tick_formatter(x, pos):
                 mask_queryset, product__product_id=product_id, crop_mask=mask
             )
 
-            with COGReader(mask_ds.file_object.url) as mask_img:
+            with COGReader(
+                f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{mask_ds.file_object.name}"
+            ) as mask_img:
                 mask_feat = mask_img.feature(
                     json.loads(boundary_feature_geom.geojson), max_size=1024
                 )
@@ -664,7 +671,9 @@ def tick_formatter(x, pos):
                     buff = 1
                     admin_level = admin_1
 
-                with COGReader(product_ds.file_object.url) as image:
+                with COGReader(
+                    f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{product_ds.file_object.name}"
+                ) as image:
                     feat = image.feature(
                         json.loads(boundary_feature_geom.geojson), max_size=1024
                     )
@@ -706,7 +715,9 @@ def tick_formatter(x, pos):
                             baseline_type=anom_type,
                         )
 
-                    with COGReader(anomaly_ds.file_object.url) as anom_img:
+                    with COGReader(
+                        f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{anomaly_ds.file_object.name}"
+                    ) as anom_img:
                         anom_feat = anom_img.feature(geom, max_size=1024)
 
                     image = image - anom_feat.as_masked()
@@ -718,7 +729,9 @@ def tick_formatter(x, pos):
                         mask_queryset, product__product_id=product_id, crop_mask=mask
                     )
 
-                    with COGReader(mask_ds.file_object.url) as mask_img:
+                    with COGReader(
+                        f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{mask_ds.file_object.name}"
+                    ) as mask_img:
                         mask_feat = mask_img.feature(geom, max_size=1024)
 
                     image = image * mask_feat.as_masked()

diff --git a/glam_api/glam/views/histogram.py b/glam_api/glam/views/histogram.py
@@ -319,7 +319,7 @@ def custom_feature_histogram(self, request):
                 if not settings.USE_S3:
                     path = product_dataset.file_object.path
                 if settings.USE_S3:
-                    path = product_dataset.file_object.url
+                    path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{product_dataset.file_object.name}"
 
                 if geom["type"] == "Polygon" or geom["type"] == "MultiPolygon":
 
@@ -338,7 +338,7 @@ def custom_feature_histogram(self, request):
                         if not settings.USE_S3:
                             mask_path = mask_dataset.file_object.path
                         if settings.USE_S3:
-                            mask_path = mask_dataset.file_object.url
+                            mask_path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{mask_dataset.file_object.name}"
 
                         with COGReader(mask_path) as mask_src:
                             mask_feat = mask_src.feature(geom, max_size=1024)
@@ -382,7 +382,7 @@ def custom_feature_histogram(self, request):
                         if not settings.USE_S3:
                             baseline_path = anomaly_dataset.file_object.path
                         if settings.USE_S3:
-                            baseline_path = anomaly_dataset.file_object.url
+                            baseline_path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{anomaly_dataset.file_object.name}"
 
                         with COGReader(baseline_path) as baseline_src:
                             baseline_feat = baseline_src.feature(geom, max_size=1024)
@@ -507,7 +507,7 @@ def boundary_feature_histogram(
                 if not settings.USE_S3:
                     path = product_dataset.file_object.path
                 if settings.USE_S3:
-                    path = product_dataset.file_object.url
+                    path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{product_dataset.file_object.name}"
 
                 geom = json.loads(boundary_feature.geom.geojson)
 
@@ -526,7 +526,7 @@ def boundary_feature_histogram(
                     if not settings.USE_S3:
                         mask_path = mask_dataset.file_object.path
                     if settings.USE_S3:
-                        mask_path = mask_dataset.file_object.url
+                        mask_path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{mask_dataset.file_object.name}"
 
                     with COGReader(mask_path) as mask_src:
                         mask_feat = mask_src.feature(geom, max_size=1024)
@@ -570,7 +570,7 @@ def boundary_feature_histogram(
                     if not settings.USE_S3:
                         baseline_path = anomaly_dataset.file_object.path
                     if settings.USE_S3:
-                        baseline_path = anomaly_dataset.file_object.url
+                        baseline_path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{anomaly_dataset.file_object.name}"
 
                     with COGReader(baseline_path) as baseline_src:
                         baseline_feat = baseline_src.feature(geom, max_size=1024)

diff --git a/glam_api/glam/views/point.py b/glam_api/glam/views/point.py
@@ -173,7 +173,7 @@ def retrieve(
         if not settings.USE_S3:
             path = product_dataset.file_object.path
         if settings.USE_S3:
-            path = product_dataset.file_object.url
+            path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{product_dataset.file_object.name}"
 
         dataset_value = None
 
@@ -188,7 +188,7 @@ def retrieve(
             if not settings.USE_S3:
                 mask_path = mask_dataset.file_object.path
             if settings.USE_S3:
-                mask_path = mask_dataset.file_object.url
+                mask_path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{mask_dataset.file_object.name}"
 
             with COGReader(mask_path) as src:
                 mask_data = src.point(lon, lat)
@@ -240,7 +240,7 @@ def retrieve(
             if not settings.USE_S3:
                 baseline_path = anomaly_dataset.file_object.path
             if settings.USE_S3:
-                baseline_path = anomaly_dataset.file_object.url
+                baseline_path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{anomaly_dataset.file_object.name}"
 
             with COGReader(baseline_path) as baseline_img:
                 baseline_data = baseline_img.point(lon, lat)

diff --git a/glam_api/glam/views/query.py b/glam_api/glam/views/query.py
@@ -189,7 +189,7 @@ def query_custom_feature(self, request):
             if not settings.USE_S3:
                 path = product_dataset.file_object.path
             if settings.USE_S3:
-                path = product_dataset.file_object.url
+                path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{product_dataset.file_object.name}"
 
             if (
                 geom["geometry"]["type"] == "Polygon"
@@ -228,7 +228,7 @@ def query_custom_feature(self, request):
                         if not settings.USE_S3:
                             mask_path = mask_dataset.file_object.path
                         if settings.USE_S3:
-                            mask_path = mask_dataset.file_object.url
+                            mask_path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{mask_dataset.file_object.name}"
 
                         with COGReader(mask_path) as mask_src:
                             mask_feat = mask_src.feature(geom, max_size=1024)
@@ -289,7 +289,7 @@ def query_custom_feature(self, request):
                         if not settings.USE_S3:
                             baseline_path = anomaly_dataset.file_object.path
                         if settings.USE_S3:
-                            baseline_path = anomaly_dataset.file_object.url
+                            baseline_path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{anomaly_dataset.file_object.name}"
 
                         with COGReader(baseline_path) as baseline_src:
                             baseline_feat = baseline_src.feature(geom, max_size=1024)
@@ -383,7 +383,7 @@ def query_boundary_feature(
         if not settings.USE_S3:
             path = product_dataset.file_object.path
         if settings.USE_S3:
-            path = product_dataset.file_object.url
+            path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{product_dataset.file_object.name}"
 
         try:
             with COGReader(path) as product_src:
@@ -423,7 +423,7 @@ def query_boundary_feature(
                 if not settings.USE_S3:
                     mask_path = mask_dataset.file_object.path
                 if settings.USE_S3:
-                    mask_path = mask_dataset.file_object.url
+                    mask_path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{mask_dataset.file_object.name}"
 
                 with COGReader(mask_path) as mask_src:
                     mask_feat = mask_src.feature(
@@ -486,7 +486,7 @@ def query_boundary_feature(
                 if not settings.USE_S3:
                     baseline_path = anomaly_dataset.file_object.path
                 if settings.USE_S3:
-                    baseline_path = anomaly_dataset.file_object.url
+                    baseline_path = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{anomaly_dataset.file_object.name}"
 
                 with COGReader(baseline_path) as baseline_src:
                     baseline_feat = baseline_src.feature(

diff --git a/glam_api/glam/views/tiles.py b/glam_api/glam/views/tiles.py
@@ -269,10 +269,9 @@ def retrieve(
         if stretch_min is not None and stretch_max is not None:
             stretch_range = [stretch_min, stretch_max]
 
-        logging.debug(product_dataset.file_object.url)
         with rasterio.Env(CPL_DEBUG=True) as env:
             with COGReader(
-                product_dataset.file_object.url,
+                f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{product_dataset.file_object.name}"
             ) as cog:
                 img = cog.tile(x, y, z, tilesize=tile_size, reproject_method="bilinear")
 
@@ -315,7 +314,9 @@ def retrieve(
                     except:
                         stretch_range = [-100, 100]
 
-                with COGReader(anomaly_dataset.file_object.url) as cog:
+                with COGReader(
+                    f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{anomaly_dataset.file_object.name}"
+                ) as cog:
                     baseline = cog.tile(
                         x, y, z, tilesize=tile_size, reproject_method="bilinear"
                     )
@@ -328,7 +329,7 @@ def retrieve(
                 cropmask = CropMask.objects.get(cropmask_id=cropmask_id)
 
                 with COGReader(
-                    cropmask.map_raster.url,
+                    f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{cropmask.map_raster.name}"
                 ) as cog:
                     cm_img = cog.tile(
                         x, y, z, tilesize=tile_size, reproject_method="bilinear"
@@ -473,7 +474,9 @@ def preview(
         if stretch_min is not None and stretch_max is not None:
             stretch_range = [stretch_min, stretch_max]
 
-        with COGReader(product_dataset.file_object.url) as cog:
+        with COGReader(
+            f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{product_dataset.file_object.name}"
+        ) as cog:
             img = cog.tile(x, y, z, tilesize=tile_size)
             # todo if stretch_range = None get range from image
 
@@ -513,7 +516,9 @@ def preview(
                 except:
                     stretch_range = [-100, 100]
 
-            with COGReader(anomaly_dataset.file_object.url) as cog:
+            with COGReader(
+                f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{anomaly_dataset.file_object.name}"
+            ) as cog:
                 baseline = cog.tile(x, y, z, tilesize=tile_size)
 
             anom = img.as_masked().data - baseline.as_masked().data