diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 31a1cee..3bbe193 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,3 +25,18 @@ repos: rev: 23.3.0 hooks: - id: black + args: ["--target-version=py38"] + + + - repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + args: ["--profile", "black", "--filter-files"] + + - repo: https://github.com/asottile/pyupgrade + rev: v3.3.1 + hooks: + - id: pyupgrade + args: + - "--py38-plus" diff --git a/CHANGELOG.md b/CHANGELOG.md index be082c3..24eaed0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ * Documentation interne * Tests unitaires pour la classe RasterSet * Classe Raster : constructeur à partir des paramètres - + * Pyramid * Fonction de calcul de la taille d'une pyramide * Générateur de lecture de la liste du contenu @@ -75,7 +75,7 @@ Lecture par système de fichier virtuel avec GDAL * Utils * Meilleure gestion de reprojection par `reproject_bbox` : on détecte des systèmes identiques en entrée ou quand seul l'ordre des axes changent, pour éviter le calcul * Ajout de la fonction de reprojection d'un point `reproject_point` : on détecte des systèmes identiques en entrée ou quand seul l'ordre des axes changent, pour éviter le calcul - + ### [Changed] * Utils : @@ -98,8 +98,8 @@ Ajout de fonctionnalités de lecture de donnée d'une pyramide et suivi des reco * Storage : * Fonction de lecture binaire, complète ou partielle, d'un fichier ou objet S3 ou CEPH * Exceptions : NotImplementedError permet de préciser qu'une fonctionnalité n'a pas été implémentée pour tous les cas. Ici, on ne gère pas la décompression des données raster pour les compressions packbit et LZW - -* Ajout de la publication PyPI dans la CI GitHub + +* Ajout de la publication PyPI dans la CI GitHub ### [Changed] @@ -181,4 +181,4 @@ Initialisation des librairies Python utilisées par les outils python à venir d * Librairie de gestion d'un descripteur de pyramide * chargement depuis un descripteur ou par clone (avec changement de stockage) * écriture du descripteur -* Tests unitaires couvrant ces librairies \ No newline at end of file +* Tests unitaires couvrant ces librairies diff --git a/pyproject.toml b/pyproject.toml index 4e1599b..f172c36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ doc = [ dev = [ "black", + "isort >= 5.12.0", "pre-commit >3,<4" ] diff --git a/src/rok4/Layer.py b/src/rok4/Layer.py index e57e3ab..b13e55a 100644 --- a/src/rok4/Layer.py +++ b/src/rok4/Layer.py @@ -5,11 +5,11 @@ - `Layer` - Descriptor to broadcast pyramids' data """ -from typing import Dict, List, Tuple, Union import json -from json.decoder import JSONDecodeError import os import re +from json.decoder import JSONDecodeError +from typing import Dict, List, Tuple, Union from rok4.Exceptions import * from rok4.Pyramid import Pyramid diff --git a/src/rok4/Pyramid.py b/src/rok4/Pyramid.py index 0d7e382..f6c670f 100644 --- a/src/rok4/Pyramid.py +++ b/src/rok4/Pyramid.py @@ -6,20 +6,21 @@ - `Level` - Level of a pyramid """ -from typing import Dict, List, Tuple, Union, Iterator +import io import json -from json.decoder import JSONDecodeError import os import re -import numpy import zlib -import io +from json.decoder import JSONDecodeError +from typing import Dict, Iterator, List, Tuple, Union + import mapbox_vector_tile +import numpy from PIL import Image from rok4.Exceptions import * -from rok4.TileMatrixSet import TileMatrixSet, TileMatrix from rok4.Storage import * +from rok4.TileMatrixSet import TileMatrix, TileMatrixSet from rok4.Utils import * from rok4.enums import PyramidType, SlabType, StorageType @@ -768,7 +769,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict] roots = {} s3_cluster = self.storage_s3_cluster - with open(list_file, "r") as listin: + with open(list_file) as listin: # Lecture des racines for line in listin: line = line.rstrip() diff --git a/src/rok4/Raster.py b/src/rok4/Raster.py index 640f4bf..271fc6a 100644 --- a/src/rok4/Raster.py +++ b/src/rok4/Raster.py @@ -10,9 +10,9 @@ import json import re from enum import Enum -from typing import Tuple, Dict +from typing import Dict, Tuple -from osgeo import ogr, gdal +from osgeo import gdal, ogr from rok4.Storage import exists, get_osgeo_path, put_data_str from rok4.Utils import ColorFormat, compute_bbox, compute_format diff --git a/src/rok4/Storage.py b/src/rok4/Storage.py index 406e431..efc3696 100644 --- a/src/rok4/Storage.py +++ b/src/rok4/Storage.py @@ -30,17 +30,18 @@ To precise the cluster to use, bucket name should be bucket_name@s3.storage.fr or bucket_name@s4.storage.fr. If no host is defined (no @) in the bucket name, first S3 cluster is used """ +import hashlib +import os +import re +import tempfile +from enum import Enum +from shutil import copyfile +from typing import Dict, List, Tuple, Union + import boto3 import botocore.exceptions -import tempfile -import re -import os import rados -import hashlib import requests -from typing import Dict, List, Tuple, Union -from enum import Enum -from shutil import copyfile from osgeo import gdal gdal.UseExceptions() @@ -933,11 +934,9 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: try: target_s3_client["client"].put_object( - Body=f"{__OBJECT_SYMLINK_SIGNATURE}{target_bucket}/{target_base_name}".encode( - "utf-8" - ), - Bucket=link_bucket, - Key=link_base_name, + Body = f"{__OBJECT_SYMLINK_SIGNATURE}{target_bucket}/{target_base_name}".encode(), + Bucket = link_bucket, + Key = link_base_name ) except Exception as e: raise StorageError("S3", e) @@ -946,9 +945,7 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: ioctx = __get_ceph_ioctx(link_tray) try: - ioctx.write_full( - link_base_name, f"{__OBJECT_SYMLINK_SIGNATURE}{target_path}".encode("utf-8") - ) + ioctx.write_full(link_base_name, f"{__OBJECT_SYMLINK_SIGNATURE}{target_path}".encode()) except Exception as e: raise StorageError("CEPH", e) diff --git a/src/rok4/TileMatrixSet.py b/src/rok4/TileMatrixSet.py index 472810c..3000492 100644 --- a/src/rok4/TileMatrixSet.py +++ b/src/rok4/TileMatrixSet.py @@ -9,15 +9,15 @@ - ROK4_TMS_DIRECTORY """ +import json +import os +from json.decoder import JSONDecodeError +from typing import Dict, List, Tuple + from rok4.Exceptions import * from rok4.Storage import get_data_str from rok4.Utils import * -from typing import Dict, List, Tuple -from json.decoder import JSONDecodeError -import json -import os - class TileMatrix: """A tile matrix is a tile matrix set's level. diff --git a/src/rok4/Utils.py b/src/rok4/Utils.py index 627686c..53450d7 100644 --- a/src/rok4/Utils.py +++ b/src/rok4/Utils.py @@ -4,8 +4,10 @@ import os import re +from enum import Enum from typing import Dict, List, Tuple, Union -from osgeo import ogr, osr, gdal + +from osgeo import gdal, ogr, osr ogr.UseExceptions() osr.UseExceptions() diff --git a/src/rok4/Vector.py b/src/rok4/Vector.py index 94e6def..46656c2 100644 --- a/src/rok4/Vector.py +++ b/src/rok4/Vector.py @@ -6,12 +6,14 @@ """ -from osgeo import ogr -from rok4.Storage import get_osgeo_path, copy -from rok4.Exceptions import * import os import tempfile +from osgeo import ogr + +from rok4.Exceptions import * +from rok4.Storage import copy, get_osgeo_path + # Enable GDAL/OGR exceptions ogr.UseExceptions() diff --git a/tests/test_Layer.py b/tests/test_Layer.py index 7a3788e..bbf71eb 100644 --- a/tests/test_Layer.py +++ b/tests/test_Layer.py @@ -1,12 +1,12 @@ -from rok4.Layer import Layer -from rok4.enums import PyramidType -from rok4.Exceptions import * +import os +from unittest import mock +from unittest.mock import * import pytest -import os -from unittest.mock import * -from unittest import mock +from rok4.Exceptions import * +from rok4.Layer import Layer +from rok4.enums import PyramidType @mock.patch.dict(os.environ, {}, clear=True) diff --git a/tests/test_Pyramid.py b/tests/test_Pyramid.py index 60576f3..3b75be5 100644 --- a/tests/test_Pyramid.py +++ b/tests/test_Pyramid.py @@ -1,13 +1,15 @@ +import os +from unittest import mock +from unittest.mock import * + +import pytest + +from rok4.Exceptions import * from rok4.Pyramid import * from rok4.TileMatrixSet import TileMatrixSet from rok4.enums import SlabType, StorageType -from rok4.Utils import * -from rok4.Exceptions import * -import pytest -import os -from unittest.mock import * -from unittest import mock +from rok4.Utils import * @mock.patch("rok4.Pyramid.get_data_str", side_effect=StorageError("FILE", "Not found")) diff --git a/tests/test_Raster.py b/tests/test_Raster.py index de35de1..cc79846 100644 --- a/tests/test_Raster.py +++ b/tests/test_Raster.py @@ -6,17 +6,17 @@ """ import copy -import math import json +import math import random +from unittest import TestCase, mock +from unittest.mock import MagicMock, Mock, call, mock_open, patch + import pytest -from unittest import mock, TestCase -from unittest.mock import call, MagicMock, Mock, mock_open, patch from rok4.Raster import Raster, RasterSet from rok4.Utils import ColorFormat - # rok4.Raster.Raster class tests diff --git a/tests/test_Storage.py b/tests/test_Storage.py index 540632c..8426ccf 100644 --- a/tests/test_Storage.py +++ b/tests/test_Storage.py @@ -1,15 +1,14 @@ -from rok4.Storage import * -from rok4.Exceptions import * -from rok4.enums import StorageType - -import pytest import os +from unittest import mock +from unittest.mock import * import botocore.exceptions +import pytest from rados import ObjectNotFound -from unittest import mock -from unittest.mock import * +from rok4.Exceptions import * +from rok4.Storage import * +from rok4.enums import StorageType @mock.patch.dict(os.environ, {}, clear=True) diff --git a/tests/test_TileMatrixSet.py b/tests/test_TileMatrixSet.py index 4750f50..5cf6062 100644 --- a/tests/test_TileMatrixSet.py +++ b/tests/test_TileMatrixSet.py @@ -1,10 +1,11 @@ -from rok4.TileMatrixSet import TileMatrixSet -from rok4.Exceptions import * - -import pytest import os -from unittest.mock import * from unittest import mock +from unittest.mock import * + +import pytest + +from rok4.Exceptions import * +from rok4.TileMatrixSet import TileMatrixSet @mock.patch.dict(os.environ, {}, clear=True) diff --git a/tests/test_Utils.py b/tests/test_Utils.py index cdba571..0f5d67d 100644 --- a/tests/test_Utils.py +++ b/tests/test_Utils.py @@ -1,14 +1,14 @@ -from rok4.Utils import * -from rok4.Exceptions import * +import math +import os +import random +from unittest import mock +from unittest.mock import * import pytest -import os from osgeo import gdal, osr -import math -import random -from unittest.mock import * -from unittest import mock +from rok4.Exceptions import * +from rok4.Utils import * def test_srs_to_spatialreference_ignf_ok(): diff --git a/tests/test_Vector.py b/tests/test_Vector.py index 221f5e4..3fe0a1c 100644 --- a/tests/test_Vector.py +++ b/tests/test_Vector.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -from rok4.Vector import * -from rok4.Exceptions import * -from rok4.Storage import disconnect_ceph_clients - -import pytest import os -from unittest.mock import * from unittest import mock +from unittest.mock import * + +import pytest + +from rok4.Exceptions import * +from rok4.Storage import disconnect_ceph_clients +from rok4.Vector import * @mock.patch.dict(os.environ, {}, clear=True)