From d91946b930e17c198572c1301d40bccb83fe55f6 Mon Sep 17 00:00:00 2001 From: Theo Satabin Date: Tue, 13 Feb 2024 14:57:58 +0100 Subject: [PATCH 1/4] Correction du test d'existence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `Storage`: la réponse à un HEAD (test existence en S3) donne un code 404 et non NoSuchKey (confusion avec la lecture d'objet) --- CHANGELOG.md | 6 ++++++ src/rok4/storage.py | 10 +++++----- tests/test_storage.py | 4 +++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d7b91f..4556cb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.1.4 + +### [Fixed] + +* Storage : la réponse à un HEAD (test existence en S3) donne un code 404 et non NoSuchKey (confusion avec la lecture d'objet) + ## 2.1.3 ### [Fixed] diff --git a/src/rok4/storage.py b/src/rok4/storage.py index 075baa0..232d86e 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -116,7 +116,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s StorageError: S3 client configuration issue Returns: - Tuple[Dict[str, Union['boto3.client',str]], str, str]: the S3 informations (client, host, key, secret) and the simple bucket name + Tuple[Dict[str, Union['boto3.client',str]], str]: the S3 informations (client, host, key, secret) and the simple bucket name """ global __S3_CLIENTS, __S3_DEFAULT_CLIENT @@ -156,6 +156,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s "secret_key": secret_keys[i], "url": urls[i], "host": h, + "secure": urls[i].startswith("https://") } if i == 0: @@ -443,9 +444,6 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: Returns: str: Data binary content """ - print("########################################") - print(f"get_data_binary {path}") - print("########################################") return __get_cached_data_binary(path, __get_ttl_hash(), range) @@ -576,7 +574,7 @@ def exists(path: str) -> bool: s3_client["client"].head_object(Bucket=bucket_name, Key=base_name) return True except botocore.exceptions.ClientError as e: - if e.response["Error"]["Code"] == "NoSuchKey": + if e.response["Error"]["Code"] == "404": return False else: raise StorageError("S3", e) @@ -1076,6 +1074,8 @@ def get_osgeo_path(path: str) -> str: gdal.SetConfigOption("AWS_ACCESS_KEY_ID", s3_client["key"]) gdal.SetConfigOption("AWS_S3_ENDPOINT", s3_client["host"]) gdal.SetConfigOption("AWS_VIRTUAL_HOSTING", "FALSE") + if not s3_client["secure"]: + gdal.SetConfigOption("AWS_HTTPS", "NO") return f"/vsis3/{bucket_name}/{base_name}" diff --git a/tests/test_storage.py b/tests/test_storage.py index 4e5e543..3c8104a 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -27,6 +27,7 @@ ) + @mock.patch.dict(os.environ, {}, clear=True) @patch("builtins.open", new_callable=mock_open, read_data=b"data") def test_hash_file_ok(mock_file): @@ -70,6 +71,7 @@ def test_get_path_from_infos(): @mock.patch.dict(os.environ, {}, clear=True) def test_s3_missing_env(): + disconnect_s3_clients() with pytest.raises(MissingEnvironmentError): get_data_str("s3://bucket/path/to/object") @@ -809,7 +811,7 @@ def test_exists_s3_ok(mocked_s3_client): assert False, f"S3 exists raises an exception: {exc}" s3_instance.head_object.side_effect = botocore.exceptions.ClientError( - operation_name="InvalidKeyPair.Duplicate", error_response={"Error": {"Code": "NoSuchKey"}} + operation_name="InvalidKeyPair.Duplicate", error_response={"Error": {"Code": "404"}} ) try: assert not exists("s3://bucket/object.ext") From 1c1fc5488b7cb2ae022d1c330a2f0d6c8b22a055 Mon Sep 17 00:00:00 2001 From: Theo Satabin Date: Tue, 13 Feb 2024 16:43:23 +0100 Subject: [PATCH 2/4] Correction du chargement d'un Rasterset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### [Fixed] - `RasterSet`: le chargement d'un raster set à partir d'un fichier ou d'un descripteur utilise la librairie Storage et non la librairie GDAL --- src/rok4/pyramid.py | 17 ++-- src/rok4/raster.py | 177 ++++++++++++++++++++---------------------- src/rok4/storage.py | 2 +- src/rok4/vector.py | 3 +- tests/test_pyramid.py | 2 +- tests/test_raster.py | 48 +++++------- 6 files changed, 116 insertions(+), 133 deletions(-) diff --git a/src/rok4/pyramid.py b/src/rok4/pyramid.py index 09ea0e4..679a69a 100644 --- a/src/rok4/pyramid.py +++ b/src/rok4/pyramid.py @@ -382,7 +382,7 @@ class Pyramid: __tms (rok4.tile_matrix_set.TileMatrixSet): Used grid __levels (Dict[str, Level]): Pyramid's levels __format (str): Data format - __storage (Dict[str, Union[rok4.storage.StorageType,str,int]]): Pyramid's storage informations (type, root and depth if FILE storage) + __storage (Dict[str, Union[rok4.enums.StorageType,str,int]]): Pyramid's storage informations (type, root and depth if FILE storage) __raster_specifications (Dict): If raster pyramid, raster specifications __content (Dict): Loading status (loaded) and list content (cache). @@ -600,12 +600,15 @@ def raster_specifications(self) -> Dict: """Get raster specifications for a RASTER pyramid Example: - { - "channels": 3, - "nodata": "255,0,0", - "photometric": "rgb", - "interpolation": "bicubic" - } + + RGB pyramid with red nodata + + { + "channels": 3, + "nodata": "255,0,0", + "photometric": "rgb", + "interpolation": "bicubic" + } Returns: Dict: Raster specifications, None if VECTOR pyramid diff --git a/src/rok4/raster.py b/src/rok4/raster.py index d52617e..5f7d954 100644 --- a/src/rok4/raster.py +++ b/src/rok4/raster.py @@ -2,16 +2,18 @@ The module contains the following class : - - Raster - Structure describing raster data. - - RasterSet - Structure describing a set of raster data. +- `Raster` - Structure describing raster data. +- `RasterSet` - Structure describing a set of raster data. """ # -- IMPORTS -- # standard library -import copy +from copy import deepcopy import json import re +from json.decoder import JSONDecodeError +import tempfile from typing import Dict, Tuple # 3rd party @@ -19,7 +21,7 @@ # package from rok4.enums import ColorFormat -from rok4.storage import exists, get_osgeo_path, put_data_str +from rok4.storage import exists, get_osgeo_path, put_data_str, copy, remove, get_data_str from rok4.utils import compute_bbox, compute_format # -- GLOBALS -- @@ -32,18 +34,13 @@ class Raster: """A structure describing raster data - Attributes : - path (str): path to the file/object (ex: - file:///path/to/image.tif or s3://bucket/path/to/image.tif) - bbox (Tuple[float, float, float, float]): bounding rectange - in the data projection - bands (int): number of color bands (or channels) - format (ColorFormat): numeric variable format for color values. - Bit depth, as bits per channel, can be derived from it. - mask (str): path to the associated mask file or object, if any, - or None (same path as the image, but with a ".msk" extension - and TIFF format. ex: - file:///path/to/image.msk or s3://bucket/path/to/image.msk) + Attributes: + path (str): path to the file/object (ex: file:///path/to/image.tif or s3://bucket/path/to/image.tif) + bbox (Tuple[float, float, float, float]): bounding rectange in the data projection + bands (int): number of color bands (or channels) format (ColorFormat). Numeric variable format for color values. Bit depth, as bits per channel, + can be derived from it. + mask (str): path to the associated mask file or object, if any, or None (same path as the image, but with a ".msk" extension and TIFF format. + Ex: file:///path/to/image.msk or s3://bucket/path/to/image.msk) dimensions (Tuple[int, int]): image width and height, in pixels """ @@ -77,14 +74,16 @@ def from_file(cls, path: str) -> "Raster": print(f"Cannot load information from image : {e}") Raises: + FormatError: MASK file is not a TIFF RuntimeError: raised by OGR/GDAL if anything goes wrong NotImplementedError: Storage type not handled + FileNotFoundError: File or object does not exists Returns: Raster: a Raster instance """ if not exists(path): - raise Exception(f"No file or object found at path '{path}'.") + raise FileNotFoundError(f"No file or object found at path '{path}'.") self = cls() @@ -100,11 +99,8 @@ def from_file(cls, path: str) -> "Raster": work_mask_path = get_osgeo_path(mask_path) mask_driver = gdal.IdentifyDriver(work_mask_path).ShortName if "GTiff" != mask_driver: - message = ( - f"Mask file '{mask_path}' is not a TIFF image." - + f" (GDAL driver : '{mask_driver}'" - ) - raise Exception(message) + message = f"Mask file '{mask_path}' use GDAL driver : '{mask_driver}'" + raise FormatError("TIFF", mask_path, message) self.mask = mask_path else: self.mask = None @@ -129,19 +125,13 @@ def from_parameters( """Creates a Raster object from parameters Args: - path (str): path to the file/object (ex: - file:///path/to/image.tif or s3://bucket/image.tif) + path (str): path to the file/object (ex: file:///path/to/image.tif or s3://bucket/image.tif) bands (int): number of color bands (or channels) - bbox (Tuple[float, float, float, float]): bounding rectange - in the data projection - dimensions (Tuple[int, int]): image width and height - expressed in pixels - format (ColorFormat): numeric format for color values. - Bit depth, as bits per channel, can be derived from it. - mask (str, optionnal): path to the associated mask, if any, - or None (same path as the image, but with a - ".msk" extension and TIFF format. ex: - file:///path/to/image.msk or s3://bucket/image.msk) + bbox (Tuple[float, float, float, float]): bounding rectange in the data projection + dimensions (Tuple[int, int]): image width and height expressed in pixels + format (ColorFormat): numeric format for color values. Bit depth, as bits per channel, can be derived from it. + mask (str, optionnal): path to the associated mask, if any, or None (same path as the image, but with a ".msk" + extension and TIFF format. ex: file:///path/to/image.msk or s3://bucket/image.msk) Examples: @@ -152,13 +142,12 @@ def from_parameters( try: raster = Raster.from_parameters( - path="file:///data/SC1000/_0040_6150_L93.tif", - mask="file:///data/SC1000/0040_6150_L93.msk", - bands=3, - format=ColorFormat.UINT8, - dimensions=(2000, 2000), - bbox=(40000.000, 5950000.000, - 240000.000, 6150000.000) + path="file:///data/SC1000/_0040_6150_L93.tif", + mask="file:///data/SC1000/0040_6150_L93.msk", + bands=3, + format=ColorFormat.UINT8, + dimensions=(2000, 2000), + bbox=(40000.000, 5950000.000, 240000.000, 6150000.000) ) except Exception as e: @@ -186,24 +175,16 @@ def from_parameters( class RasterSet: """A structure describing a set of raster data - Attributes : + Attributes: raster_list (List[Raster]): List of Raster instances in the set - colors (List[Dict]): List of color properties for each raster - instance. Contains only one element if - the set is homogenous. - Element properties: - bands (int): number of color bands (or channels) - format (ColorFormat): numeric variable format for - color values. Bit depth, as bits per channel, - can be derived from it. + colors (Set[Tuple[int, ColorFormat]]): Set (distinct values) of color properties (bands and format) found in the raster set. srs (str): Name of the set's spatial reference system - bbox (Tuple[float, float, float, float]): bounding rectange - in the data projection, enclosing the whole set + bbox (Tuple[float, float, float, float]): bounding rectange in the data projection, enclosing the whole set """ def __init__(self) -> None: self.bbox = (None, None, None, None) - self.colors = [] + self.colors = set() self.raster_list = [] self.srs = None @@ -212,9 +193,8 @@ def from_list(cls, path: str, srs: str) -> "RasterSet": """Instanciate a RasterSet from an images list path and a srs Args: - path (str): path to the images list file or object - (each line in this list contains the path to - an image file or object in the set) + path (str): path to the images list file or object (each line in this list contains the path to an image file or object in the set) + srs (str): images' coordinates system Examples: @@ -224,13 +204,13 @@ def from_list(cls, path: str, srs: str) -> "RasterSet": try: raster_set = RasterSet.from_list( - path="file:///data/SC1000.list", - srs="EPSG:3857" + path="file:///data/SC1000.list", + srs="EPSG:3857" ) except Exception as e: print( - f"Cannot load information from list file : {e}" + f"Cannot load information from list file : {e}" ) Raises: @@ -243,33 +223,42 @@ def from_list(cls, path: str, srs: str) -> "RasterSet": self = cls() self.srs = srs - local_list_path = get_osgeo_path(path) + # Chargement de la liste des images (la liste peut être un fichier ou un objet) + list_obj = tempfile.NamedTemporaryFile(mode="r", delete=False) + list_file = list_obj.name + copy(path, f"file://{list_file}") + list_obj.close() image_list = [] - with open(file=local_list_path) as list_file: - for line in list_file: + with open(list_file) as listin: + for line in listin: image_path = line.strip(" \t\n\r") image_list.append(image_path) - temp_bbox = [None, None, None, None] + remove(f"file://{list_file}") + + bbox = [None, None, None, None] for image_path in image_list: raster = Raster.from_file(image_path) self.raster_list.append(raster) - if temp_bbox == [None, None, None, None]: - for i in range(0, 4, 1): - temp_bbox[i] = raster.bbox[i] + + # Mise à jour de la bbox globale + if bbox == [None, None, None, None]: + bbox = list(raster.bbox) else: - if temp_bbox[0] > raster.bbox[0]: - temp_bbox[0] = raster.bbox[0] - if temp_bbox[1] > raster.bbox[1]: - temp_bbox[1] = raster.bbox[1] - if temp_bbox[2] < raster.bbox[2]: - temp_bbox[2] = raster.bbox[2] - if temp_bbox[3] < raster.bbox[3]: - temp_bbox[3] = raster.bbox[3] - color_dict = {"bands": raster.bands, "format": raster.format} - if color_dict not in self.colors: - self.colors.append(color_dict) - self.bbox = tuple(temp_bbox) + if bbox[0] > raster.bbox[0]: + bbox[0] = raster.bbox[0] + if bbox[1] > raster.bbox[1]: + bbox[1] = raster.bbox[1] + if bbox[2] < raster.bbox[2]: + bbox[2] = raster.bbox[2] + if bbox[3] < raster.bbox[3]: + bbox[3] = raster.bbox[3] + + # Inventaire des colors distinctes + self.colors.add((raster.bands, raster.format)) + + self.bbox = tuple(bbox) + return self @classmethod @@ -287,12 +276,11 @@ def from_descriptor(cls, path: str) -> "RasterSet": try: raster_set = RasterSet.from_descriptor( - "file:///data/images/descriptor.json" + "file:///data/images/descriptor.json" ) except Exception as e: - message = ("Cannot load information from " - + f"descriptor file : {e}") + message = ("Cannot load information from descriptor file : {e}") print(message) Raises: @@ -303,24 +291,26 @@ def from_descriptor(cls, path: str) -> "RasterSet": RasterSet: a RasterSet instance """ self = cls() - descriptor_path = get_osgeo_path(path) - with open(file=descriptor_path) as file_handle: - raw_content = file_handle.read() - serialization = json.loads(raw_content) + + try: + serialization = json.loads(get_data_str(path)) + + except JSONDecodeError as e: + raise FormatError("JSON", path, e) + self.srs = serialization["srs"] self.raster_list = [] for raster_dict in serialization["raster_list"]: - parameters = copy.deepcopy(raster_dict) + parameters = deepcopy(raster_dict) parameters["bbox"] = tuple(raster_dict["bbox"]) parameters["dimensions"] = tuple(raster_dict["dimensions"]) parameters["format"] = ColorFormat[raster_dict["format"]] self.raster_list.append(Raster.from_parameters(**parameters)) + self.bbox = tuple(serialization["bbox"]) - self.colors = [] for color_dict in serialization["colors"]: - color_item = copy.deepcopy(color_dict) - color_item["format"] = ColorFormat[color_dict["format"]] - self.colors.append(color_item) + self.colors.add((color_dict["bands"], ColorFormat[color_dict["format"]])) + return self @property @@ -332,7 +322,7 @@ def serializable(self) -> Dict: """ serialization = {"bbox": list(self.bbox), "srs": self.srs, "colors": [], "raster_list": []} for color in self.colors: - color_serial = {"bands": color["bands"], "format": color["format"].name} + color_serial = {"bands": color[0], "format": color[1].name} serialization["colors"].append(color_serial) for raster in self.raster_list: raster_dict = { @@ -345,15 +335,14 @@ def serializable(self) -> Dict: if raster.mask is not None: raster_dict["mask"] = raster.mask serialization["raster_list"].append(raster_dict) + return serialization def write_descriptor(self, path: str = None) -> None: """Print raster set's descriptor as JSON Args: - path (str, optional): Complete path (file or object) - where to print the raster set's JSON. Defaults to None, - JSON is printed to standard output. + path (str, optional): Complete path (file or object) where to print the raster set's JSON. Defaults to None, JSON is printed to standard output. """ content = json.dumps(self.serializable, sort_keys=True) if path is None: diff --git a/src/rok4/storage.py b/src/rok4/storage.py index 232d86e..071c16e 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -27,7 +27,7 @@ To use several S3 clusters, each environment variable have to contain a list (comma-separated), with the same number of elements -Example: work with 2 S3 clusters: +Example, work with 2 S3 clusters: - ROK4_S3_KEY=KEY1,KEY2 - ROK4_S3_SECRETKEY=SKEY1,SKEY2 diff --git a/src/rok4/vector.py b/src/rok4/vector.py index 268d181..a092469 100644 --- a/src/rok4/vector.py +++ b/src/rok4/vector.py @@ -2,8 +2,7 @@ The module contains the following class : - - 'Vector' - Data Vector - +- `Vector` - Data Vector """ # -- IMPORTS -- diff --git a/tests/test_pyramid.py b/tests/test_pyramid.py index 007ef45..f2e3fcd 100644 --- a/tests/test_pyramid.py +++ b/tests/test_pyramid.py @@ -265,7 +265,7 @@ def test_list_read(mocked_tms_class): try: pyramid = Pyramid.from_descriptor("file://tests/fixtures/TIFF_PBF_MVT.json") pyramid.load_list() - pyramid.load_list() # on passe par la détection d'une liste déjà chrargée ainsi + pyramid.load_list() # on passe par la détection d'une liste déjà chargée ainsi for (slab_type, level, column, row), infos in pyramid.list_generator(): assert slab_type == SlabType.DATA assert level == "4" diff --git a/tests/test_raster.py b/tests/test_raster.py index 3222bc2..2f61e82 100644 --- a/tests/test_raster.py +++ b/tests/test_raster.py @@ -262,7 +262,7 @@ def test_default(self): and len(rasterset.bbox) == 4 and all(coordinate is None for coordinate in rasterset.bbox) ) - assert isinstance(rasterset.colors, list) and not rasterset.colors + assert isinstance(rasterset.colors, set) and not rasterset.colors assert isinstance(rasterset.raster_list, list) and not rasterset.raster_list assert rasterset.srs is None @@ -270,9 +270,9 @@ def test_default(self): class TestRasterSetFromList(TestCase): """rok4.raster.RasterSet.from_list(path, srs) class constructor.""" - @mock.patch("rok4.raster.get_osgeo_path") + @mock.patch("rok4.raster.copy") @mock.patch("rok4.raster.Raster.from_file") - def test_ok_at_least_3_files(self, m_from_file, m_get_osgeo_path): + def test_ok_at_least_3_files(self, m_from_file, m_copy): """List of 3 or more valid image files""" file_number = random.randint(3, 100) file_list = [] @@ -282,9 +282,8 @@ def test_ok_at_least_3_files(self, m_from_file, m_get_osgeo_path): m_open = mock_open(read_data=file_list_string) list_path = "s3://test_bucket/raster_set.list" list_local_path = "/tmp/raster_set.list" - m_get_osgeo_path.return_value = list_local_path raster_list = [] - colors = [] + colors = set() serial_in = {"raster_list": [], "colors": []} for n in range(0, file_number, 1): raster = MagicMock(Raster) @@ -305,9 +304,8 @@ def test_ok_at_least_3_files(self, m_from_file, m_get_osgeo_path): else: raster.mask = None color_dict = {"bands": raster.bands, "format": raster.format} - if color_dict not in colors: - colors.append(color_dict) - serial_in["colors"].append({"bands": raster.bands, "format": raster.format.name}) + if (raster.bands, raster.format) not in colors: + colors.add((raster.bands, raster.format)) raster.dimensions = (5000, 5000) raster_list.append(raster) raster_serial = { @@ -320,6 +318,8 @@ def test_ok_at_least_3_files(self, m_from_file, m_get_osgeo_path): if raster.mask: raster_serial["mask"] = raster.mask serial_in["raster_list"].append(raster_serial) + for c in colors: + serial_in["colors"].append({"bands": c[0], "format": c[1].name}) m_from_file.side_effect = raster_list srs = "EPSG:4326" serial_in["srs"] = srs @@ -331,8 +331,8 @@ def test_ok_at_least_3_files(self, m_from_file, m_get_osgeo_path): serial_out = rasterset.serializable assert rasterset.srs == srs - m_get_osgeo_path.assert_called_once_with(list_path) - m_open.assert_called_once_with(file=list_local_path) + m_copy.assert_called_once() + m_open.assert_called_once() assert rasterset.raster_list == raster_list assert isinstance(serial_out["bbox"], list) for i in range(0, 4, 1): @@ -349,9 +349,9 @@ def test_ok_at_least_3_files(self, m_from_file, m_get_osgeo_path): class TestRasterSetFromDescriptor(TestCase): """rok4.raster.RasterSet.from_descriptor(path) class constructor.""" - @mock.patch("rok4.raster.get_osgeo_path") + @mock.patch("rok4.raster.get_data_str") @mock.patch("rok4.raster.Raster.from_parameters") - def test_simple_ok(self, m_from_parameters, m_get_osgeo_path): + def test_simple_ok(self, m_from_parameters,m_get_data_str): serial_in = { "bbox": [550000.000, 6210000.000, 570000.000, 6230000.000], "colors": [{"bands": 3, "format": "UINT8"}], @@ -398,14 +398,10 @@ def test_simple_ok(self, m_from_parameters, m_get_osgeo_path): raster_list.append(raster) raster_args_list.append(raster_properties) m_from_parameters.side_effect = raster_list - m_get_osgeo_path.return_value = local_path - m_open = mock_open(read_data=desc_content) + m_get_data_str.return_value = desc_content - with mock.patch("rok4.raster.open", m_open): - rasterset = RasterSet.from_descriptor(desc_path) + rasterset = RasterSet.from_descriptor(desc_path) - m_get_osgeo_path.assert_called_once_with(desc_path) - m_open.assert_called_once_with(file=local_path) assert rasterset.srs == serial_in["srs"] m_from_parameters.assert_called() assert m_from_parameters.call_count == 3 @@ -413,11 +409,11 @@ def test_simple_ok(self, m_from_parameters, m_get_osgeo_path): assert m_from_parameters.call_args_list[i] == call(**raster_args_list[i]) assert rasterset.raster_list == raster_list assert isinstance(rasterset.bbox, tuple) and len(rasterset.bbox) == 4 - assert isinstance(rasterset.colors, list) and rasterset.colors + assert isinstance(rasterset.colors, set) and rasterset.colors for i in range(0, len(serial_in["colors"]), 1): expected_color = copy.deepcopy(serial_in["colors"][i]) expected_color["format"] = ColorFormat[serial_in["colors"][i]["format"]] - assert rasterset.colors[i] == expected_color + assert (expected_color["bands"], expected_color["format"]) in rasterset.colors serial_out = rasterset.serializable assert isinstance(serial_out["bbox"], list) and len(serial_out["bbox"]) == 4 for i in range(0, 4, 1): @@ -469,11 +465,9 @@ def test_ok_with_output_path(self, m_put_data_str): rasterset = RasterSet() rasterset.bbox = tuple(serial_in["bbox"]) rasterset.srs = serial_in["srs"] - rasterset.colors = [] + rasterset.colors = set() for color_dict in serial_in["colors"]: - rasterset.colors.append( - {"bands": color_dict["bands"], "format": ColorFormat[color_dict["format"]]} - ) + rasterset.colors.add((color_dict["bands"], ColorFormat[color_dict["format"]])) rasterset.raster_list = [] for raster_dict in serial_in["raster_list"]: raster_args = copy.deepcopy(raster_dict) @@ -526,11 +520,9 @@ def test_ok_no_output_path(self, m_print): rasterset = RasterSet() rasterset.bbox = tuple(serial_in["bbox"]) rasterset.srs = serial_in["srs"] - rasterset.colors = [] + rasterset.colors = set() for color_dict in serial_in["colors"]: - rasterset.colors.append( - {"bands": color_dict["bands"], "format": ColorFormat[color_dict["format"]]} - ) + rasterset.colors.add((color_dict["bands"], ColorFormat[color_dict["format"]])) rasterset.raster_list = [] for raster_dict in serial_in["raster_list"]: raster_args = copy.deepcopy(raster_dict) From d699d0ceadd6be660baf0419deeb1a5671246304 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 13 Feb 2024 15:45:32 +0000 Subject: [PATCH 3/4] =?UTF-8?q?[pre-commit.ci]=20Corrections=20automatique?= =?UTF-8?q?s=20appliqu=C3=A9es=20par=20les=20git=20hooks.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/rok4/pyramid.py | 2 +- src/rok4/raster.py | 24 ++++++++++++++++-------- src/rok4/storage.py | 2 +- tests/test_raster.py | 2 +- tests/test_storage.py | 1 - 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/rok4/pyramid.py b/src/rok4/pyramid.py index 679a69a..0a80f9f 100644 --- a/src/rok4/pyramid.py +++ b/src/rok4/pyramid.py @@ -602,7 +602,7 @@ def raster_specifications(self) -> Dict: Example: RGB pyramid with red nodata - + { "channels": 3, "nodata": "255,0,0", diff --git a/src/rok4/raster.py b/src/rok4/raster.py index 5f7d954..f6f71fe 100644 --- a/src/rok4/raster.py +++ b/src/rok4/raster.py @@ -8,12 +8,13 @@ # -- IMPORTS -- -# standard library -from copy import deepcopy import json import re -from json.decoder import JSONDecodeError import tempfile + +# standard library +from copy import deepcopy +from json.decoder import JSONDecodeError from typing import Dict, Tuple # 3rd party @@ -21,7 +22,14 @@ # package from rok4.enums import ColorFormat -from rok4.storage import exists, get_osgeo_path, put_data_str, copy, remove, get_data_str +from rok4.storage import ( + copy, + exists, + get_data_str, + get_osgeo_path, + put_data_str, + remove, +) from rok4.utils import compute_bbox, compute_format # -- GLOBALS -- @@ -37,9 +45,9 @@ class Raster: Attributes: path (str): path to the file/object (ex: file:///path/to/image.tif or s3://bucket/path/to/image.tif) bbox (Tuple[float, float, float, float]): bounding rectange in the data projection - bands (int): number of color bands (or channels) format (ColorFormat). Numeric variable format for color values. Bit depth, as bits per channel, + bands (int): number of color bands (or channels) format (ColorFormat). Numeric variable format for color values. Bit depth, as bits per channel, can be derived from it. - mask (str): path to the associated mask file or object, if any, or None (same path as the image, but with a ".msk" extension and TIFF format. + mask (str): path to the associated mask file or object, if any, or None (same path as the image, but with a ".msk" extension and TIFF format. Ex: file:///path/to/image.msk or s3://bucket/path/to/image.msk) dimensions (Tuple[int, int]): image width and height, in pixels """ @@ -130,7 +138,7 @@ def from_parameters( bbox (Tuple[float, float, float, float]): bounding rectange in the data projection dimensions (Tuple[int, int]): image width and height expressed in pixels format (ColorFormat): numeric format for color values. Bit depth, as bits per channel, can be derived from it. - mask (str, optionnal): path to the associated mask, if any, or None (same path as the image, but with a ".msk" + mask (str, optionnal): path to the associated mask, if any, or None (same path as the image, but with a ".msk" extension and TIFF format. ex: file:///path/to/image.msk or s3://bucket/image.msk) Examples: @@ -310,7 +318,7 @@ def from_descriptor(cls, path: str) -> "RasterSet": self.bbox = tuple(serialization["bbox"]) for color_dict in serialization["colors"]: self.colors.add((color_dict["bands"], ColorFormat[color_dict["format"]])) - + return self @property diff --git a/src/rok4/storage.py b/src/rok4/storage.py index 071c16e..25eb9b4 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -156,7 +156,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s "secret_key": secret_keys[i], "url": urls[i], "host": h, - "secure": urls[i].startswith("https://") + "secure": urls[i].startswith("https://"), } if i == 0: diff --git a/tests/test_raster.py b/tests/test_raster.py index 2f61e82..af1543e 100644 --- a/tests/test_raster.py +++ b/tests/test_raster.py @@ -351,7 +351,7 @@ class TestRasterSetFromDescriptor(TestCase): @mock.patch("rok4.raster.get_data_str") @mock.patch("rok4.raster.Raster.from_parameters") - def test_simple_ok(self, m_from_parameters,m_get_data_str): + def test_simple_ok(self, m_from_parameters, m_get_data_str): serial_in = { "bbox": [550000.000, 6210000.000, 570000.000, 6230000.000], "colors": [{"bands": 3, "format": "UINT8"}], diff --git a/tests/test_storage.py b/tests/test_storage.py index 3c8104a..20049b6 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -27,7 +27,6 @@ ) - @mock.patch.dict(os.environ, {}, clear=True) @patch("builtins.open", new_callable=mock_open, read_data=b"data") def test_hash_file_ok(mock_file): From 358a5516e20c06a684f5ff27b699b8634783a8d6 Mon Sep 17 00:00:00 2001 From: Theo Satabin Date: Tue, 13 Feb 2024 16:52:28 +0100 Subject: [PATCH 4/4] =?UTF-8?q?Compl=C3=A9tion=20du=20changelog=20pour=20l?= =?UTF-8?q?a=202.1.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4556cb2..971c852 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### [Fixed] * Storage : la réponse à un HEAD (test existence en S3) donne un code 404 et non NoSuchKey (confusion avec la lecture d'objet) +* RasterSet: le chargement d'un raster set à partir d'un fichier ou d'un descripteur utilise la librairie Storage et non la librairie GDAL ## 2.1.3