diff --git a/src/rok4/layer.py b/src/rok4/layer.py index ba2f211..631a870 100644 --- a/src/rok4/layer.py +++ b/src/rok4/layer.py @@ -12,7 +12,7 @@ import re from rok4.exceptions import * -from rok4.pyramid import Pyramid, PyramidType +from rok4.pyramid import Pyramid from rok4.tile_matrix_set import TileMatrixSet from rok4.storage import * from rok4.utils import * diff --git a/src/rok4/pyramid.py b/src/rok4/pyramid.py index 8aa51aa..ebb63b0 100644 --- a/src/rok4/pyramid.py +++ b/src/rok4/pyramid.py @@ -21,11 +21,8 @@ from rok4.tile_matrix_set import TileMatrixSet, TileMatrix from rok4.storage import * from rok4.utils import * - - from rok4.enums import PyramidType, SlabType, StorageType - ROK4_IMAGE_HEADER_SIZE = 2048 """Slab's header size, 2048 bytes""" @@ -609,7 +606,7 @@ def storage_root(self) -> str: Returns: str: Pyramid's storage root """ - + return self.__storage["root"].split("@", 1)[ 0 ] # Suppression de l'éventuel hôte de spécification du cluster S3 @@ -735,7 +732,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict] S3 stored descriptor from rok4.pyramid import Pyramid - + try: pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") @@ -759,6 +756,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict] 'slab': 'DATA_18_5424_7526' } ) + Raises: StorageError: Unhandled pyramid storage to copy list MissingEnvironmentError: Missing object storage informations @@ -826,7 +824,7 @@ def get_level(self, level_id: str) -> "Level": Returns: The corresponding pyramid's level, None if not present """ - + return self.__levels.get(level_id, None) def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: @@ -1010,7 +1008,6 @@ def get_slab_path_from_infos( else: return slab_path - def get_tile_data_binary(self, level: str, column: int, row: int) -> str: """Get a pyramid's tile as binary string @@ -1256,6 +1253,7 @@ def get_tile_data_vector(self, level: str, column: int, row: int) -> Dict: S3 stored vector pyramid, to print a tile as GeoJSON from rok4.pyramid import Pyramid + import json try: @@ -1370,6 +1368,10 @@ def size(self) -> int: Returns: int: size of the pyramid """ - if not hasattr(self,"_Pyramid__size") : - self.__size = size_path(get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name)) + + if not hasattr(self, "_Pyramid__size"): + self.__size = size_path( + get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name) + ) + return self.__size diff --git a/src/rok4/raster.py b/src/rok4/raster.py index 1c1ac36..95ccf68 100644 --- a/src/rok4/raster.py +++ b/src/rok4/raster.py @@ -9,7 +9,7 @@ import copy import json import re -from enum import Enum + from typing import Dict, Tuple from osgeo import gdal, ogr diff --git a/src/rok4/storage.py b/src/rok4/storage.py index 54e2837..6e2eca8 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -39,7 +39,6 @@ import hashlib import requests from typing import Dict, List, Tuple, Union -from enum import Enum from shutil import copyfile from osgeo import gdal @@ -68,7 +67,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s Returns: Tuple[Dict[str, Union['boto3.client',str]], str, str]: the S3 informations (client, host, key, secret) and the simple bucket name """ - + global __S3_CLIENTS, __S3_DEFAULT_CLIENT if not __S3_CLIENTS: @@ -127,7 +126,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s def disconnect_s3_clients() -> None: """Clean S3 clients""" - + global __S3_CLIENTS, __S3_DEFAULT_CLIENT __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None @@ -838,10 +837,12 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: response = requests.get(from_type.value + from_path, stream = True) with open(to_path, "wb") as f: for chunk in response.iter_content(chunk_size=65536) : + if chunk: f.write(chunk) except Exception as e: + raise StorageError(f"HTTP(S) and FILE", f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}") elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.CEPH : @@ -858,6 +859,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: offset += size except Exception as e: + raise StorageError(f"HTTP(S) and CEPH", f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}") elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.S3 : @@ -869,6 +871,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: with tempfile.NamedTemporaryFile("w+b",delete=False) as f: name_fich = f.name for chunk in response.iter_content(chunk_size=65536) : + if chunk: f.write(chunk) @@ -879,6 +882,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: except Exception as e: raise StorageError(f"HTTP(S) and S3", f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}") + else: raise StorageError( f"{from_type.name} and {to_type.name}", @@ -1022,6 +1026,7 @@ def size_path(path: str) -> int : elif storage_type == StorageType.S3: s3_client, bucket_name = __get_s3_client(tray_name) + try : paginator = s3_client["client"].get_paginator('list_objects_v2') pages = paginator.paginate( @@ -1039,7 +1044,6 @@ def size_path(path: str) -> int : except Exception as e: raise StorageError("S3", e) - elif storage_type == StorageType.CEPH: raise NotImplementedError else: diff --git a/src/rok4/tile_matrix_set.py b/src/rok4/tile_matrix_set.py index 06d4466..b6685bd 100644 --- a/src/rok4/tile_matrix_set.py +++ b/src/rok4/tile_matrix_set.py @@ -18,7 +18,6 @@ from rok4.storage import get_data_str from rok4.utils import * - class TileMatrix: """A tile matrix is a tile matrix set's level. diff --git a/src/rok4/utils.py b/src/rok4/utils.py index d7d3ef7..4db30d4 100644 --- a/src/rok4/utils.py +++ b/src/rok4/utils.py @@ -4,7 +4,6 @@ import os import re -from enum import Enum from typing import Dict, List, Tuple, Union from osgeo import gdal, ogr, osr diff --git a/src/rok4/vector.py b/src/rok4/vector.py index 5478056..c3c045b 100644 --- a/src/rok4/vector.py +++ b/src/rok4/vector.py @@ -16,7 +16,6 @@ from rok4.storage import get_osgeo_path, copy from rok4.exceptions import * - class Vector: """A data vector diff --git a/tests/test_layer.py b/tests/test_layer.py index 8d515da..a9a79a2 100644 --- a/tests/test_layer.py +++ b/tests/test_layer.py @@ -8,7 +8,6 @@ from rok4.enums import PyramidType from rok4.exceptions import * - @mock.patch.dict(os.environ, {}, clear=True) @mock.patch( "rok4.layer.get_data_str", diff --git a/tests/test_storage.py b/tests/test_storage.py index a3f8799..d60e8c0 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -103,6 +103,7 @@ def test_file_read_ok(mock_file): except Exception as exc: assert False, f"FILE read raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -179,6 +180,7 @@ def test_http_read_range_error(): @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("requests.get") def test_http_read_ok(mock_http): + try : requests_instance = MagicMock() requests_instance.content = b'data' @@ -186,7 +188,7 @@ def test_http_read_ok(mock_http): data = get_data_str("http://path/to/file.ext") mock_http.assert_called_with("http://path/to/file.ext", stream=True) - assert data == 'data' + assert data == "data" except Exception as exc: assert False, f"HTTP read raises an exception: {exc}" @@ -373,6 +375,7 @@ def test_copy_s3_s3_intercluster_nok(mocked_s3_client): with pytest.raises(StorageError): copy("s3://bucket@a/source.ext", "s3://bucket@c/destination.ext", "toto") + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -423,6 +426,7 @@ def test_copy_file_ceph_ok(mock_file, mocked_rados_client): except Exception as exc: assert False, f"FILE -> CEPH copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -513,7 +517,6 @@ def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): http_instance.iter_content.return_value = ["data","data2"] mock_requests.return_value = http_instance - disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.write.return_value = None @@ -546,7 +549,7 @@ def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_cl copy("http://path/to/source.ext", "s3://bucket/destination.ext") mock_requests.assert_called_once_with("http://path/to/source.ext", stream=True) - mock_tempfile.assert_called_once_with("w+b",delete=False) + mock_tempfile.assert_called_once_with("w+b", delete=False) except Exception as exc: assert False, f"HTTP -> CEPH copy raises an exception: {exc}" @@ -563,6 +566,7 @@ def test_link_hard_nok(): with pytest.raises(StorageError): link("ceph://pool1/source.ext", "ceph://pool2/destination.ext", True) + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.symlink", return_value=None) def test_link_file_ok(mock_link): @@ -619,6 +623,7 @@ def test_link_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 link raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -792,6 +797,7 @@ def test_exists_http_ok(mock_requests): ############ remove + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.remove") def test_remove_file_ok(mock_remove): @@ -807,6 +813,7 @@ def test_remove_file_ok(mock_remove): except Exception as exc: assert False, f"FILE deletion (not found) raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -889,6 +896,7 @@ def test_size_path_file_ok(): except Exception as exc: assert False, f"FILE size of the path raises an exception: {exc}" + def test_size_file_nok(): with pytest.raises(StorageError) : size = size_path("file://tests/fixtures/TIFF_PBF_M") diff --git a/tests/test_tile_matrix_set.py b/tests/test_tile_matrix_set.py index b4e58a9..98bccd6 100644 --- a/tests/test_tile_matrix_set.py +++ b/tests/test_tile_matrix_set.py @@ -1,3 +1,4 @@ + import pytest import os from unittest.mock import * diff --git a/tests/test_utils.py b/tests/test_utils.py index 37941aa..bfcb636 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -75,7 +75,6 @@ def test_reproject_point_ok(): # Tests for the rok4.utils.compute_bbox function. - def test_compute_bbox_epsg_3857_ok(): try: mocked_datasource = MagicMock(gdal.Dataset) @@ -216,7 +215,6 @@ def test_compute_bbox_no_srs_ok(): # Tests for the rok4.utils.compute_format function. - @mock.patch("rok4.utils.gdal.Info") @mock.patch("rok4.utils.gdal.GetColorInterpretationName", return_value="Palette") @mock.patch("rok4.utils.gdal.GetDataTypeSize", return_value=8)