From 27918896016b40354317bcef04ac460c23fff320 Mon Sep 17 00:00:00 2001
From: CorentinPeutin <corentin.peutin@ensg.eu>
Date: Thu, 16 Mar 2023 16:12:07 +0100
Subject: [PATCH 1/7] Ajout copie HTTP -> FILE/S3/CEPH

---
 src/rok4/Storage.py   | 135 +++++++++++++++++++++++------
 tests/test_Storage.py | 192 ++++++++++++++++++++++++++++++++++--------
 2 files changed, 268 insertions(+), 59 deletions(-)

diff --git a/src/rok4/Storage.py b/src/rok4/Storage.py
index db4ec6c..e8a8a7f 100644
--- a/src/rok4/Storage.py
+++ b/src/rok4/Storage.py
@@ -4,6 +4,8 @@
 - S3 (path are preffixed with `s3://`)
 - CEPH (path are preffixed with `ceph://`)
 - FILE (path are preffixed with `file://`, but it is the default paths' interpretation)
+- HTTP (path are preffixed with `https://`)
+- HTTPS (path are preffixed with `https://`)
 
 According to functions, all storage types are not necessarily available.
 
@@ -35,6 +37,7 @@
 import os
 import rados
 import hashlib
+import requests
 from typing import Dict, List, Tuple
 from enum import Enum
 from shutil import copyfile
@@ -45,6 +48,8 @@ class StorageType(Enum):
     FILE = "file://"
     S3 = "s3://"
     CEPH = "ceph://"
+    HTTP = "http://"
+    HTTPS = "https://"
 
 __S3_CLIENTS = dict()
 __S3_DEFAULT_CLIENT = None
@@ -62,7 +67,7 @@ def __get_s3_client(bucket_name: str) -> Tuple['boto3.client', str, str]:
 
     Returns:
         Tuple['boto3.client', str, str]: the S3 client, the cluster host and the simple bucket name
-    """    
+    """
     global __S3_CLIENTS, __S3_DEFAULT_CLIENT
 
     if not __S3_CLIENTS:
@@ -113,7 +118,7 @@ def __get_s3_client(bucket_name: str) -> Tuple['boto3.client', str, str]:
 
 def disconnect_s3_clients() -> None:
     """Clean S3 clients
-    """    
+    """
     global __S3_CLIENTS, __S3_DEFAULT_CLIENT
     __S3_CLIENTS = dict()
     __S3_DEFAULT_CLIENT = None
@@ -134,7 +139,7 @@ def __get_ceph_ioctx(pool: str) -> 'rados.Ioctx':
 
     Returns:
         rados.Ioctx: IO ceph context
-    """    
+    """
     global __CEPH_CLIENT, __CEPH_IOCTXS
 
     if __CEPH_CLIENT is None:
@@ -146,7 +151,7 @@ def __get_ceph_ioctx(pool: str) -> 'rados.Ioctx':
             )
 
             __CEPH_CLIENT.connect()
-            
+
         except KeyError as e:
             raise MissingEnvironmentError(e)
         except Exception as e:
@@ -157,12 +162,12 @@ def __get_ceph_ioctx(pool: str) -> 'rados.Ioctx':
             __CEPH_IOCTXS[pool] = __CEPH_CLIENT.open_ioctx(pool)
         except Exception as e:
             raise StorageError("CEPH", e)
-    
+
     return __CEPH_IOCTXS[pool]
 
 def disconnect_ceph_clients() -> None:
     """Clean CEPH clients
-    """    
+    """
     global __CEPH_CLIENT, __CEPH_IOCTXS
     __CEPH_CLIENT = None
     __CEPH_IOCTXS = dict()
@@ -173,8 +178,8 @@ def get_infos_from_path(path: str) -> Tuple[StorageType, str, str, str]:
     """Extract storage type, the unprefixed path, the container and the basename from path (Default: FILE storage)
 
     For a FILE storage, the tray is the directory and the basename is the file name.
-    
-    For an object storage (CEPH or S3), the tray is the bucket or the pool and the basename is the object name. 
+
+    For an object storage (CEPH or S3), the tray is the bucket or the pool and the basename is the object name.
     For a S3 bucket, format can be <bucket name>@<cluster name> to use several clusters. Cluster name is the host (without protocol)
 
     Args:
@@ -192,6 +197,10 @@ def get_infos_from_path(path: str) -> Tuple[StorageType, str, str, str]:
         return StorageType.CEPH, path[7:], pool_name, object_name
     elif path.startswith("file://"):
         return StorageType.FILE, path[7:], os.path.dirname(path[7:]), os.path.basename(path[7:])
+    elif path.startswith("http://"):
+        return StorageType.HTTP, path, "NULL", "NULL"
+    elif path.startswith("https://"):
+        return StorageType.HTTPS, path, "NULL", "NULL"
     else:
         return StorageType.FILE, path, os.path.dirname(path), os.path.basename(path)
 
@@ -206,7 +215,7 @@ def get_path_from_infos(storage_type: StorageType, *args) -> str:
 
     Returns:
         str: Full path
-    """    
+    """
     return f"{storage_type.value}{os.path.join(*args)}"
 
 
@@ -247,7 +256,7 @@ def get_data_str(path: str) -> str:
     return get_data_binary(path).decode('utf-8')
 
 
-def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: 
+def get_data_binary(path: str, range: Tuple[int, int] = None) -> str:
     """Load data into a binary string
 
     Args:
@@ -262,10 +271,10 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str:
         str: Data binary content
     """
 
-    storage_type, path, tray_name, base_name  = get_infos_from_path(path)        
+    storage_type, path, tray_name, base_name  = get_infos_from_path(path)
 
     if storage_type == StorageType.S3:
-        
+
         s3_client, host, tray_name = __get_s3_client(tray_name)
 
         try:
@@ -285,7 +294,7 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str:
             raise StorageError("S3", e)
 
     elif storage_type == StorageType.CEPH:
-        
+
         ioctx = __get_ceph_ioctx(tray_name)
 
         try:
@@ -307,11 +316,22 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str:
             else:
                 f.seek(range[0])
                 data = f.read(range[1])
-            
+
             f.close()
         except Exception as e:
             raise StorageError("FILE", e)
 
+    elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS:
+
+        try:
+            reponse = requests.get(path)
+            data = reponse.content
+            print(reponse.content)
+            if reponse.status_code == 404 :
+                raise StorageError("HTTP", "Requested file does not exist")
+        except Exception as e:
+            raise StorageError("HTTP", e)
+
     else:
         raise StorageError("UNKNOWN", "Unhandled storage type to read binary data")
 
@@ -334,7 +354,7 @@ def put_data_str(data: str, path: str) -> None:
     storage_type, path, tray_name, base_name  = get_infos_from_path(path)
 
     if storage_type == StorageType.S3:
-        
+
         s3_client, host, tray_name = __get_s3_client(tray_name)
 
         try:
@@ -347,7 +367,7 @@ def put_data_str(data: str, path: str) -> None:
             raise StorageError("S3", e)
 
     elif storage_type == StorageType.CEPH:
-        
+
         ioctx = __get_ceph_ioctx(tray_name)
 
         try:
@@ -412,6 +432,14 @@ def get_size(path: str) -> int:
         except Exception as e:
             raise StorageError("FILE", e)
 
+    elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS:
+
+        try:
+            reponse = requests.get(path)
+            return reponse.content.__sizeof__()
+        except Exception as e:
+            raise StorageError("HTTP", e)
+
     else:
         raise StorageError("UNKNOWN", "Unhandled storage type to get size")
 
@@ -461,6 +489,17 @@ def exists(path: str) -> bool:
 
         return os.path.exists(path)
 
+    elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS:
+
+        try:
+            reponse = requests.get(path)
+            if reponse.status_code == 200 :
+                return True
+            else :
+                return False
+        except Exception as e:
+            raise StorageError("HTTP", e)
+
     else:
         raise StorageError("UNKNOWN", "Unhandled storage type to test if exists")
 
@@ -529,7 +568,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
 
     # Réalisation de la copie, selon les types de stockage
     if from_type == StorageType.FILE and to_type == StorageType.FILE :
-        
+
         try:
             if to_tray != "":
                 os.makedirs(to_tray, exist_ok=True)
@@ -545,13 +584,13 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
             raise StorageError(f"FILE", f"Cannot copy file {from_path} to {to_path} : {e}")
 
     elif from_type == StorageType.S3 and to_type == StorageType.FILE :
-        
+
         s3_client, host, from_tray = __get_s3_client(from_tray)
 
         try:
             if to_tray != "":
                 os.makedirs(to_tray, exist_ok=True)
-            
+
             s3_client.download_file(from_tray, from_base_name, to_path)
 
             if from_md5 is not None :
@@ -565,7 +604,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
     elif from_type == StorageType.FILE and to_type == StorageType.S3 :
 
         s3_client, host, to_tray = __get_s3_client(to_tray)
-        
+
         try:
             s3_client.upload_file(from_path, to_tray, to_base_name)
 
@@ -587,7 +626,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
                     {
                         'Bucket': from_tray,
                         'Key': from_base_name
-                    }, 
+                    },
                     to_tray, to_base_name
                 )
             else:
@@ -602,7 +641,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
 
         except Exception as e:
             raise StorageError(f"S3", f"Cannot copy S3 object {from_path} to {to_path} : {e}")
-        
+
 
     elif from_type == StorageType.CEPH and to_type == StorageType.FILE :
 
@@ -644,7 +683,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
     elif from_type == StorageType.FILE and to_type == StorageType.CEPH :
 
         ioctx = __get_ceph_ioctx(to_tray)
-        
+
         if from_md5 is not None:
             checker = hashlib.md5()
 
@@ -706,7 +745,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
 
         except Exception as e:
             raise StorageError(f"CEPH", f"Cannot copy CEPH object {from_path} to {to_path} : {e}")
-            
+
     elif from_type == StorageType.CEPH and to_type == StorageType.S3 :
 
         from_ioctx = __get_ceph_ioctx(from_tray)
@@ -749,6 +788,52 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
         except Exception as e:
             raise StorageError(f"CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}")
 
+    elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.FILE :
+
+        try:
+            reponse = requests.get(from_path, stream = True)
+            with open(to_path, "wb") as f:
+                for chunk in reponse.iter_content(chunk_size=65536) :
+                    if chunk:
+                        f.write(chunk)
+
+        except Exception as e:
+            raise StorageError(f"HTTP(S) and FILE", f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}")
+
+    elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.CEPH :
+
+        to_ioctx = __get_ceph_ioctx(to_tray)
+
+        try:
+            reponse = requests.get(from_path, stream = True)
+            offset = 0
+            for chunk in reponse.iter_content(chunk_size=65536) :
+                if chunk:
+                    to_ioctx.write(to_base_name, chunk, offset)
+                    offset += 65536
+
+        except Exception as e:
+            raise StorageError(f"HTTP(S) and CEPH", f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}")
+
+    elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.S3 :
+
+        to_s3_client, to_host, to_tray = __get_s3_client(to_tray)
+
+        try:
+            reponse = requests.get(from_path, stream = True)
+            with tempfile.NamedTemporaryFile("w+b",delete=False) as f:
+                name_fich = f.name
+                for chunk in reponse.iter_content(chunk_size=65536) :
+                    if chunk:
+                        f.write(chunk)
+
+            to_s3_client.upload_file(name_fich, to_tray, to_base_name)
+
+            os.remove(name_fich)
+
+        except Exception as e:
+            raise StorageError(f"HTTP(S) and S3", f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}")
+
     else:
         raise StorageError(f"{from_type.name} and {to_type.name}", f"Cannot copy from {from_type.name} to {to_type.name}")
 
@@ -795,7 +880,7 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None:
             raise StorageError("S3", e)
 
     elif target_type == StorageType.CEPH:
-        
+
         ioctx = __get_ceph_ioctx(link_tray)
 
         try:
diff --git a/tests/test_Storage.py b/tests/test_Storage.py
index 175568c..985a1ea 100644
--- a/tests/test_Storage.py
+++ b/tests/test_Storage.py
@@ -20,7 +20,7 @@ def test_hash_file_ok(mock_file):
     except Exception as exc:
         assert False, f"FILE md5 sum raises an exception: {exc}"
 
-@mock.patch.dict(os.environ, {}, clear=True)  
+@mock.patch.dict(os.environ, {}, clear=True)
 def test_get_infos_from_path():
     assert (StorageType.S3, "toto/titi", "toto", "titi") == get_infos_from_path("s3://toto/titi")
     assert (StorageType.FILE, "/toto/titi/tutu.json", "/toto/titi", "tutu.json") == get_infos_from_path("file:///toto/titi/tutu.json")
@@ -28,7 +28,7 @@ def test_get_infos_from_path():
     assert (StorageType.FILE, "wrong://toto/titi", "wrong://toto", "titi") == get_infos_from_path("wrong://toto/titi")
 
 
-@mock.patch.dict(os.environ, {}, clear=True)  
+@mock.patch.dict(os.environ, {}, clear=True)
 def test_get_path_from_infos():
     assert get_path_from_infos(StorageType.S3, "toto", "toto/titi") == "s3://toto/toto/titi"
     assert get_path_from_infos(StorageType.FILE, "/toto/titi", "tutu.json") == "file:///toto/titi/tutu.json"
@@ -66,7 +66,7 @@ def test_s3_invalid_endpoint(mocked_s3_client):
 def test_file_read_error(mock_file):
     with pytest.raises(StorageError):
         data = get_data_str("file:///path/to/file.ext")
-    
+
     mock_file.assert_called_with("/path/to/file.ext", "rb")
 
 
@@ -80,7 +80,7 @@ def test_file_read_ok(mock_file):
     except Exception as exc:
         assert False, f"FILE read raises an exception: {exc}"
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 def test_s3_read_nok(mocked_s3_client):
     disconnect_s3_clients()
@@ -90,7 +90,7 @@ def test_s3_read_nok(mocked_s3_client):
     with pytest.raises(StorageError):
         data = get_data_str("s3://bucket/path/to/object")
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 def test_s3_read_ok(mocked_s3_client):
 
@@ -110,7 +110,7 @@ def test_s3_read_ok(mocked_s3_client):
         assert False, f"S3 read raises an exception: {exc}"
 
 
-@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)
 @mock.patch('rok4.Storage.rados.Rados')
 def test_ceph_read_ok(mocked_rados_client):
 
@@ -128,10 +128,36 @@ def test_ceph_read_ok(mocked_rados_client):
     except Exception as exc:
         assert False, f"CEPH read raises an exception: {exc}"
 
+@mock.patch.dict(os.environ, {}, clear=True)
+@mock.patch("requests.get", side_effect={"status_code":404})
+def test_http_read_error(mock_http):
+    with pytest.raises(StorageError):
+        requests_instance = MagicMock()
+        requests_instance.content = "NULL"
+        requests_instance.status_code = 404
+        mock_http.return_value = requests_instance
+        data = get_data_str("http:///path/to/file.ext")
+
+    mock_http.assert_called_with("http:///path/to/file.ext")
+
+@mock.patch.dict(os.environ, {}, clear=True)
+@mock.patch("requests.get")
+def test_http_read_ok(mock_http):
+    try :
+        requests_instance = MagicMock()
+        requests_instance.content = b'data'
+        mock_http.return_value = requests_instance
+
+        data = get_data_str("http:///path/to/file.ext")
+        mock_http.assert_called_with("http:///path/to/file.ext")
+        assert data == 'data'
+    except Exception as exc:
+        assert False, f"HTTP read raises an exception: {exc}"
+
 
 ############ put_data_str
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 def test_s3_write_nok(mocked_s3_client):
 
@@ -144,7 +170,7 @@ def test_s3_write_nok(mocked_s3_client):
         put_data_str("data", "s3://bucket/path/to/object")
 
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 def test_s3_write_ok(mocked_s3_client):
 
@@ -158,7 +184,7 @@ def test_s3_write_ok(mocked_s3_client):
         assert False, f"S3 write raises an exception: {exc}"
 
 
-@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)
 @mock.patch('rok4.Storage.rados.Rados')
 def test_ceph_write_ok(mocked_rados_client):
 
@@ -190,7 +216,7 @@ def test_copy_file_file_ok(mock_hash_file, mock_copyfile, mock_makedirs):
         assert False, f"FILE -> FILE copy raises an exception: {exc}"
 
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 @mock.patch('os.makedirs', return_value=None)
 @mock.patch('rok4.Storage.hash_file', return_value="toto")
@@ -210,7 +236,7 @@ def test_copy_s3_file_ok(mock_hash_file, mock_makedirs, mocked_s3_client):
         assert False, f"S3 -> FILE copy raises an exception: {exc}"
 
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 @mock.patch('os.makedirs', return_value=None)
 @mock.patch('rok4.Storage.hash_file', return_value="toto")
@@ -226,7 +252,7 @@ def test_copy_s3_file_nok(mock_hash_file, mock_makedirs, mocked_s3_client):
         mock_makedirs.assert_called_once_with("/path/to", exist_ok=True)
 
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 def test_copy_file_s3_ok(mocked_s3_client):
 
@@ -242,7 +268,7 @@ def test_copy_file_s3_ok(mocked_s3_client):
         assert False, f"FILE -> S3 copy raises an exception: {exc}"
 
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 def test_copy_s3_s3_ok(mocked_s3_client):
 
@@ -258,7 +284,7 @@ def test_copy_s3_s3_ok(mocked_s3_client):
         assert False, f"S3 -> S3 copy raises an exception: {exc}"
 
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 def test_copy_s3_s3_intercluster_ok(mocked_s3_client):
 
@@ -274,7 +300,7 @@ def test_copy_s3_s3_intercluster_ok(mocked_s3_client):
         assert False, f"S3 -> S3 inter cluster copy raises an exception: {exc}"
 
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 def test_copy_s3_s3_intercluster_nok(mocked_s3_client):
 
@@ -287,7 +313,7 @@ def test_copy_s3_s3_intercluster_nok(mocked_s3_client):
     with pytest.raises(StorageError):
         copy("s3://bucket@a/source.ext", "s3://bucket@c/destination.ext", "toto")
 
-@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)
 @mock.patch('rok4.Storage.rados.Rados')
 @mock.patch('os.makedirs', return_value=None)
 @patch("builtins.open", new_callable=mock_open)
@@ -306,7 +332,7 @@ def test_copy_ceph_file_ok(mock_file, mock_makedirs, mocked_rados_client):
     except Exception as exc:
         assert False, f"CEPH -> FILE copy raises an exception: {exc}"
 
-@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)
 @mock.patch('rok4.Storage.rados.Rados')
 @patch("builtins.open", new_callable=mock_open, read_data=b"data")
 def test_copy_file_ceph_ok(mock_file, mocked_rados_client):
@@ -323,7 +349,7 @@ def test_copy_file_ceph_ok(mock_file, mocked_rados_client):
     except Exception as exc:
         assert False, f"FILE -> CEPH copy raises an exception: {exc}"
 
-@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)
 @mock.patch('rok4.Storage.rados.Rados')
 @patch("builtins.open", new_callable=mock_open, read_data=b"data")
 def test_copy_ceph_ceph_ok(mock_file, mocked_rados_client):
@@ -342,7 +368,7 @@ def test_copy_ceph_ceph_ok(mock_file, mocked_rados_client):
         assert False, f"CEPH -> CEPH copy raises an exception: {exc}"
 
 
-@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c", "ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c", "ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.rados.Rados')
 @mock.patch('rok4.Storage.boto3.client')
 @patch("builtins.open", new_callable=mock_open, read_data=b"data")
@@ -367,6 +393,69 @@ def test_copy_ceph_s3_ok(mock_file, mocked_s3_client, mocked_rados_client):
     except Exception as exc:
         assert False, f"CEPH -> S3 copy raises an exception: {exc}"
 
+@mock.patch.dict(os.environ, {}, clear=True)
+@mock.patch('requests.get')
+@patch('builtins.open', new_callable=mock_open)
+def test_copy_http_file_ok(mock_open, mock_requests):
+    try:
+
+        http_instance = MagicMock()
+        http_instance.iter_content.return_value = ["data","data2"]
+        mock_requests.return_value = http_instance
+
+        copy("http:///path/to/source.ext", "file:///path/to/destination.ext")
+        mock_requests.assert_called_once_with("http:///path/to/source.ext", stream=True)
+        mock_open.assert_called_once_with("/path/to/destination.ext", "wb")
+    except Exception as exc:
+        assert False, f"HTTP -> FILE copy raises an exception: {exc}"
+
+@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)
+@mock.patch('rok4.Storage.rados.Rados')
+@mock.patch('requests.get')
+def test_copy_http_ceph_ok(mock_requests, mocked_rados_client):
+    try:
+
+        http_instance = MagicMock()
+        http_instance.iter_content.return_value = ["data","data2"]
+        mock_requests.return_value = http_instance
+
+
+        disconnect_ceph_clients()
+        ioctx_instance = MagicMock()
+        ioctx_instance.write.return_value = None
+        ceph_instance = MagicMock()
+        ceph_instance.open_ioctx.return_value = ioctx_instance
+        mocked_rados_client.return_value = ceph_instance
+
+        copy("http:///path/to/source.ext", "ceph://pool1/source.ext")
+        mock_requests.assert_called_once_with("http:///path/to/source.ext", stream=True)
+    except Exception as exc:
+        assert False, f"HTTP -> CEPH copy raises an exception: {exc}"
+
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
+@mock.patch('rok4.Storage.boto3.client')
+@mock.patch('requests.get')
+@patch('tempfile.NamedTemporaryFile', new_callable=mock_open)
+@mock.patch('os.remove')
+def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_client):
+    try:
+
+        http_instance = MagicMock()
+        http_instance.iter_content.return_value = ["data","data2"]
+        mock_requests.return_value = http_instance
+
+        disconnect_s3_clients()
+        s3_instance = MagicMock()
+        s3_instance.upload_file.return_value = None
+        s3_instance.head_object.return_value = {"ETag": "8d777f385d3dfec8815d20f7496026dc"}
+        mocked_s3_client.return_value = s3_instance
+
+        copy("http:///path/to/source.ext", "s3://bucket/destination.ext")
+        mock_requests.assert_called_once_with("http:///path/to/source.ext", stream=True)
+        mock_tempfile.assert_called_once_with("w+b",delete=False)
+    except Exception as exc:
+        assert False, f"HTTP -> CEPH copy raises an exception: {exc}"
+
 
 
 ############ link
@@ -379,7 +468,7 @@ def test_link_hard_nok():
     with pytest.raises(StorageError):
         link("ceph://pool1/source.ext", "ceph://pool2/destination.ext", True)
 
-@mock.patch.dict(os.environ, {}, clear=True)  
+@mock.patch.dict(os.environ, {}, clear=True)
 @mock.patch('os.symlink', return_value=None)
 def test_link_file_ok(mock_link):
     try:
@@ -389,7 +478,7 @@ def test_link_file_ok(mock_link):
         assert False, f"FILE link raises an exception: {exc}"
 
 
-@mock.patch.dict(os.environ, {}, clear=True)  
+@mock.patch.dict(os.environ, {}, clear=True)
 @mock.patch('os.link', return_value=None)
 def test_hlink_file_ok(mock_link):
     try:
@@ -398,7 +487,7 @@ def test_hlink_file_ok(mock_link):
     except Exception as exc:
         assert False, f"FILE hard link raises an exception: {exc}"
 
-@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)
 @mock.patch('rok4.Storage.rados.Rados')
 def test_link_ceph_ok(mocked_rados_client):
 
@@ -415,7 +504,7 @@ def test_link_ceph_ok(mocked_rados_client):
         assert False, f"CEPH link raises an exception: {exc}"
 
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 def test_link_s3_ok(mocked_s3_client):
 
@@ -430,7 +519,7 @@ def test_link_s3_ok(mocked_s3_client):
         assert False, f"S3 link raises an exception: {exc}"
 
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 def test_link_s3_nok(mocked_s3_client):
 
@@ -444,7 +533,7 @@ def test_link_s3_nok(mocked_s3_client):
 
 ############ get_size
 
-@mock.patch.dict(os.environ, {}, clear=True)  
+@mock.patch.dict(os.environ, {}, clear=True)
 @mock.patch('os.stat')
 def test_size_file_ok(mock_stat):
     mock_stat.return_value.st_size = 12
@@ -454,7 +543,7 @@ def test_size_file_ok(mock_stat):
     except Exception as exc:
         assert False, f"FILE size raises an exception: {exc}"
 
-@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)
 @mock.patch('rok4.Storage.rados.Rados')
 def test_size_ceph_ok(mocked_rados_client):
 
@@ -472,7 +561,7 @@ def test_size_ceph_ok(mocked_rados_client):
         assert False, f"CEPH size raises an exception: {exc}"
 
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 def test_size_s3_ok(mocked_s3_client):
 
@@ -487,10 +576,24 @@ def test_size_s3_ok(mocked_s3_client):
     except Exception as exc:
         assert False, f"S3 size raises an exception: {exc}"
 
+@mock.patch.dict(os.environ, {}, clear=True)
+@mock.patch('requests.get')
+def test_size_http_ok(mock_requests):
+
+    http_instance = MagicMock()
+    http_instance.content.__sizeof__.return_value = 12
+    mock_requests.return_value = http_instance
+
+    try:
+        size = get_size("http:///path/to/file.ext")
+        assert size == 12
+    except Exception as exc:
+        assert False, f"HTTP size raises an exception: {exc}"
+
 
 ############ exists
 
-@mock.patch.dict(os.environ, {}, clear=True)  
+@mock.patch.dict(os.environ, {}, clear=True)
 @mock.patch('os.path.exists', return_value=True)
 def test_exists_file_ok(mock_exists):
     try:
@@ -504,7 +607,7 @@ def test_exists_file_ok(mock_exists):
     except Exception as exc:
         assert False, f"FILE not exists raises an exception: {exc}"
 
-@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)
 @mock.patch('rok4.Storage.rados.Rados')
 def test_exists_ceph_ok(mocked_rados_client):
 
@@ -527,7 +630,7 @@ def test_exists_ceph_ok(mocked_rados_client):
         assert False, f"CEPH not exists raises an exception: {exc}"
 
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 def test_exists_s3_ok(mocked_s3_client):
 
@@ -547,10 +650,31 @@ def test_exists_s3_ok(mocked_s3_client):
     except Exception as exc:
         assert False, f"CEPH not exists raises an exception: {exc}"
 
+@mock.patch.dict(os.environ, {}, clear=True)
+@mock.patch('requests.get')
+def test_exists_http_ok(mock_requests):
+
+    http_instance = MagicMock()
+    http_instance.status_code = 200
+    mock_requests.return_value = http_instance
+
+    try:
+        assert exists("http:///path/to/file.ext")
+    except Exception as exc:
+        assert False, f"HTTP exists raises an exception: {exc}"
+
+    http_instance.status_code = 404
+    mock_requests.return_value = http_instance
+
+    try:
+        assert not exists("http:///path/to/file.ext")
+    except Exception as exc:
+        assert False, f"HTTP exists raises an exception: {exc}"
+
 
 ############ remove
 
-@mock.patch.dict(os.environ, {}, clear=True)  
+@mock.patch.dict(os.environ, {}, clear=True)
 @mock.patch('os.remove')
 def test_remove_file_ok(mock_remove):
     mock_remove.return_value = None
@@ -565,7 +689,7 @@ def test_remove_file_ok(mock_remove):
     except Exception as exc:
         assert False, f"FILE deletion (not found) raises an exception: {exc}"
 
-@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)
 @mock.patch('rok4.Storage.rados.Rados')
 def test_remove_ceph_ok(mocked_rados_client):
 
@@ -588,7 +712,7 @@ def test_remove_ceph_ok(mocked_rados_client):
         assert False, f"CEPH deletion (not found) raises an exception: {exc}"
 
 
-@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)  
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
 @mock.patch('rok4.Storage.boto3.client')
 def test_remove_s3_ok(mocked_s3_client):
 
@@ -600,4 +724,4 @@ def test_remove_s3_ok(mocked_s3_client):
     try:
         remove("s3://bucket/object.ext")
     except Exception as exc:
-        assert False, f"S3 deletion raises an exception: {exc}"
\ No newline at end of file
+        assert False, f"S3 deletion raises an exception: {exc}"

From 298f0c58ee400538006d3a2f4e842e35ae65eb79 Mon Sep 17 00:00:00 2001
From: CorentinPeutin <corentin.peutin@ensg.eu>
Date: Thu, 16 Mar 2023 16:30:23 +0100
Subject: [PATCH 2/7] =?UTF-8?q?Mise=20=C3=A0=20jour=20du=20CHANGELOG.md?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 CHANGELOG.md | 21 +++------------------
 1 file changed, 3 insertions(+), 18 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5a62581..256187b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,29 +1,14 @@
 ## Summary
 
-Ajout de fonctionnalités de lecture de donnée d'une pyramide et suivi des recommandations PyPA pour la gestion du projet.
+Ajout du type de stockage HTTP.
 
 ## Changelog
 
 ### [Added]
 
-* TileMatrix :
-    * Fonction de calcul des indices de tuile et de pixel dans la tuile à partir d'un point dans le système de coordonnées du TMS
-* Pyramid :
-    * Fonction de calcul des indices de tuile et de pixel dans la tuile à partir d'un point dans le système de coordonnées du TMS et éventuellement un niveau
-    * Fonctions de lecture d'une tuile : au format binaire source ou au format tableau à 3 dimensions pour les tuiles raster
 * Storage :
-    * Fonction de lecture binaire, complète ou partielle, d'un fichier ou objet S3 ou CEPH
-* Exceptions : NotImplementedError permet de préciser qu'une fonctionnalité n'a pas été implémentée pour tous les cas. Ici, on ne gère pas la décompression des données raster pour les compressions packbit et LZW
-  
-* Ajout de la publication PyPI dans la CI GitHub 
-
-### [Changed]
-
-* Storage :
-    * La lecture sous forme de chaîne s'appuie sur la lecture complète binaire. Aucun changement à l'usage.
-* TileMatrixSet : quelque soit le système de coordonnées, on ne gère que un ordre des axes X,Y ou Lon,Lat. Cependant, les fonctions de calcul de ou à partir de bbox respectent l'ordre du système dans ces dernières.
-
-* Passage de la configuration du projet dans le fichier `pyproject.toml`
+    * Ajout de la copie de HTTP vers FILE/S3/CEPH
+    * Ajout de la fonction de lecture d'un fichier HTTP, de l'existence d'un fichier HTTP et du calcul de taille d'un fichier HTTP
 
 <!--
 ### [Added]

From ce0f152a4d9f38dc103c22570622436ce05f53d4 Mon Sep 17 00:00:00 2001
From: CorentinPeutin <corentin.peutin@ensg.eu>
Date: Fri, 17 Mar 2023 14:36:53 +0100
Subject: [PATCH 3/7] Gestion des fichiers lourds

---
 src/rok4/Storage.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/rok4/Storage.py b/src/rok4/Storage.py
index e8a8a7f..a46e13f 100644
--- a/src/rok4/Storage.py
+++ b/src/rok4/Storage.py
@@ -324,7 +324,7 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str:
     elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS:
 
         try:
-            reponse = requests.get(path)
+            reponse = requests.get(path, stream=True)
             data = reponse.content
             print(reponse.content)
             if reponse.status_code == 404 :
@@ -435,7 +435,7 @@ def get_size(path: str) -> int:
     elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS:
 
         try:
-            reponse = requests.get(path)
+            reponse = requests.get(path, stream=True)
             return reponse.content.__sizeof__()
         except Exception as e:
             raise StorageError("HTTP", e)
@@ -492,7 +492,7 @@ def exists(path: str) -> bool:
     elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS:
 
         try:
-            reponse = requests.get(path)
+            reponse = requests.get(path, stream=True)
             if reponse.status_code == 200 :
                 return True
             else :
@@ -809,8 +809,9 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
             offset = 0
             for chunk in reponse.iter_content(chunk_size=65536) :
                 if chunk:
+                    size = len(chunk)
                     to_ioctx.write(to_base_name, chunk, offset)
-                    offset += 65536
+                    offset += size
 
         except Exception as e:
             raise StorageError(f"HTTP(S) and CEPH", f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}")

From 569b785e7344ba56bb4b1a98fef7d3d163f47b82 Mon Sep 17 00:00:00 2001
From: CorentinPeutin <corentin.peutin@ensg.eu>
Date: Thu, 30 Mar 2023 14:52:33 +0200
Subject: [PATCH 4/7] Merge branch 'develop' of
 https://github.com/rok4/core-python into feature/stockage_HTTP

---
 .github/workflows/build-and-release.yaml |  11 +-
 CHANGELOG.md                             |  18 ++-
 README.md                                |   3 +
 src/rok4/Exceptions.py                   |   8 --
 src/rok4/Storage.py                      | 161 ++++++++++++++---------
 tests/test_Storage.py                    |  32 ++++-
 tests/test_Utils.py                      |   6 +-
 7 files changed, 162 insertions(+), 77 deletions(-)

diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml
index 2802b78..40fb13e 100644
--- a/.github/workflows/build-and-release.yaml
+++ b/.github/workflows/build-and-release.yaml
@@ -73,7 +73,7 @@ jobs:
           echo "/usr/lib/python3/dist-packages/" >.venv/lib/python${{ matrix.python-version }}/site-packages/system.pth
 
       - name: Run unit tests
-        if: "(${{ matrix.os }}=='ubuntu-20.04')&&(${{ matrix.python-version }}=='3.8')"
+        if: "matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'"
         run: |
           source .venv/bin/activate
           pip install -e .[test]
@@ -81,20 +81,20 @@ jobs:
           coverage report -m
 
       - name: Build unit tests report
-        if: "(${{ matrix.os }}=='ubuntu-20.04')&&(${{ matrix.python-version }}=='3.8')"
+        if: "matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'"
         run: |
           source .venv/bin/activate
           coverage html -d dist/tests/
           rm dist/tests/.gitignore
 
       - name: Build package
-        if: "(${{ matrix.os }}=='ubuntu-20.04')&&(${{ matrix.python-version }}=='3.8')"
+        if: "matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'"
         run: |
           source .venv/bin/activate
           python3 -m build
 
       - name: Build documentation
-        if: "(${{ matrix.os }}=='ubuntu-20.04')&&(${{ matrix.python-version }}=='3.8')"
+        if: "matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'"
         run: |
           source .venv/bin/activate
           pip install -e .[doc]
@@ -102,7 +102,7 @@ jobs:
           cp README.md CHANGELOG.md dist/
 
       - name: Upload packages, tests results and documentation
-        if: "(${{ matrix.os }}=='ubuntu-20.04')&&(${{ matrix.python-version }}=='3.8')"
+        if: "matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8'"
         uses: actions/upload-artifact@v3
         with:
           name: dist-py3
@@ -189,6 +189,7 @@ jobs:
           cat artifact/CHANGELOG.md >>docs/versions/${{ github.ref_name }}/CHANGELOG.md
           sed "s#__version__#${{ github.ref_name }}#" templates/README.template.md >docs/versions/${{ github.ref_name }}/README.md
           cat artifact/README.md >>docs/versions/${{ github.ref_name }}/README.md
+          sed -i "s#x.y.z#${{ github.ref_name }}#g" docs/versions/${{ github.ref_name }}/README.md
           sed "s#__version__#${{ github.ref_name }}#" templates/index.template.md >docs/index.md
           echo "# Versions" >docs/versions.md
           echo "" >>docs/versions.md
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 256187b..6df50cf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,15 +1,31 @@
 ## Summary
 
 Ajout du type de stockage HTTP.
+Lecture par système de fichier virtuel avec GDAL
 
 ## Changelog
 
 ### [Added]
 
-* Storage :
+* Storage
+    * Fonction `get_osgeo_path` permettant de configurer le bon sytème de fichier virtuel en fonction du chemin fourni, et retourne celui à utiliser dans le Open de gdal ou ogr
+
+### [Changed]
+
+* Storage
+    * la récupération d'un client S3 (`__get_s3_client`) permet de récupérer le client, l'hôte, les clés d'accès et secrète, ainsi que le nom du bucket sans l'éventuel hôte du cluster
     * Ajout de la copie de HTTP vers FILE/S3/CEPH
     * Ajout de la fonction de lecture d'un fichier HTTP, de l'existence d'un fichier HTTP et du calcul de taille d'un fichier HTTP
 
+### [Fixed]
+
+* Storage
+    * Lecture binaire S3 : mauvaise configuration du nom du bucket et de l'objet et mauvaise lecture partielle
+
+### [Removed]
+
+* Exceptions
+    * `NotImplementedError` est une exceptions native
 <!--
 ### [Added]
 
diff --git a/README.md b/README.md
index 400645d..191feab 100644
--- a/README.md
+++ b/README.md
@@ -16,8 +16,11 @@ Installations système requises :
 * debian : `apt install python3-rados python3-gdal`
 
 Depuis [PyPI](https://pypi.org/project/rok4/) : `pip install rok4`
+
 Depuis [GitHub](https://github.com/rok4/core-python/releases/) : `pip install https://github.com/rok4/core-python/releases/download/x.y.z/rok4-x.y.z-py3-none-any.whl`
 
+L'environnement d'exécution doit avoir accès aux librairies système. Dans le cas d'une utilisation au sein d'un environnement python, précisez bien à la création `python3 -m venv --system-site-packages .venv`.
+
 ## Utiliser la librairie
 
 ```python
diff --git a/src/rok4/Exceptions.py b/src/rok4/Exceptions.py
index 3f0ff4f..92c75e3 100644
--- a/src/rok4/Exceptions.py
+++ b/src/rok4/Exceptions.py
@@ -37,11 +37,3 @@ def __init__(self, expected_format, content, issue):
         self.content = content
         self.issue = issue
         super().__init__(f"Expected format {expected_format} to read '{content}' : {issue}")
-
-class NotImplementedError(Exception):
-    """
-    Exception raised when a feature could be provided but is not implemented yet
-    """
-
-    def __init__(self, issue):
-        super().__init__(issue)
\ No newline at end of file
diff --git a/src/rok4/Storage.py b/src/rok4/Storage.py
index 1db35e9..af95c85 100644
--- a/src/rok4/Storage.py
+++ b/src/rok4/Storage.py
@@ -38,9 +38,12 @@
 import rados
 import hashlib
 import requests
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Union
 from enum import Enum
 from shutil import copyfile
+from osgeo import gdal
+
+gdal.UseExceptions()
 
 from rok4.Exceptions import *
 
@@ -53,7 +56,7 @@ class StorageType(Enum):
 
 __S3_CLIENTS = dict()
 __S3_DEFAULT_CLIENT = None
-def __get_s3_client(bucket_name: str) -> Tuple['boto3.client', str, str]:
+def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union['boto3.client',str]], str, str]:
     """Get the S3 client
 
     Create it if not already done
@@ -66,7 +69,7 @@ def __get_s3_client(bucket_name: str) -> Tuple['boto3.client', str, str]:
         StorageError: S3 client configuration issue
 
     Returns:
-        Tuple['boto3.client', str, str]: the S3 client, the cluster host and the simple bucket name
+        Tuple[Dict[str, Union['boto3.client',str]], str, str]: the S3 informations (client, host, key, secret) and the simple bucket name
     """
     global __S3_CLIENTS, __S3_DEFAULT_CLIENT
 
@@ -84,15 +87,21 @@ def __get_s3_client(bucket_name: str) -> Tuple['boto3.client', str, str]:
 
                 h = re.sub("https?://", "", urls[i])
 
-                if urls[i] in __S3_CLIENTS:
+                if h in __S3_CLIENTS:
                     raise StorageError("S3", "A S3 cluster is defined twice (based on URL)")
 
-                __S3_CLIENTS[h] = boto3.client(
-                    's3',
-                    aws_access_key_id = keys[i],
-                    aws_secret_access_key = secret_keys[i],
-                    endpoint_url = urls[i]
-                )
+                __S3_CLIENTS[h] = {
+                    "client": boto3.client(
+                        's3',
+                        aws_access_key_id = keys[i],
+                        aws_secret_access_key = secret_keys[i],
+                        endpoint_url = urls[i]
+                    ),
+                    "key": keys[i],
+                    "secret_key": secret_keys[i],
+                    "url": urls[i],
+                    "host": h
+                }
 
                 if i == 0:
                     # Le premier cluster est celui par défaut
@@ -114,7 +123,7 @@ def __get_s3_client(bucket_name: str) -> Tuple['boto3.client', str, str]:
     if host not in __S3_CLIENTS:
         raise StorageError("S3", f"Unknown S3 cluster, according to host '{host}'")
 
-    return __S3_CLIENTS[host], host, bucket_name
+    return __S3_CLIENTS[host], bucket_name
 
 def disconnect_s3_clients() -> None:
     """Clean S3 clients
@@ -277,19 +286,19 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str:
 
     if storage_type == StorageType.S3:
 
-        s3_client, host, tray_name = __get_s3_client(tray_name)
+        s3_client, bucket_name = __get_s3_client(tray_name)
 
         try:
             if range is None:
-                data = s3_client.get_object(
-                    Bucket='tray_name',
-                    Key='base_name',
+                data = s3_client["client"].get_object(
+                    Bucket=bucket_name,
+                    Key=base_name,
                 )['Body'].read()
             else:
-                data = s3_client.get_object(
-                    Bucket='tray_name',
-                    Key='base_name',
-                    Range=f"bytes=${range[0]}-${range[1] - range[0] - 1}"
+                data = s3_client["client"].get_object(
+                    Bucket=bucket_name,
+                    Key=base_name,
+                    Range=f"bytes={range[0]}-{range[0] + range[1] - 1}"
                 )['Body'].read()
 
         except botocore.exceptions.ClientError as e:
@@ -370,12 +379,12 @@ def put_data_str(data: str, path: str) -> None:
 
     if storage_type == StorageType.S3:
 
-        s3_client, host, tray_name = __get_s3_client(tray_name)
+        s3_client, bucket_name = __get_s3_client(tray_name)
 
         try:
-            s3_client.put_object(
+            s3_client["client"].put_object(
                 Body = data.encode('utf-8'),
-                Bucket = tray_name,
+                Bucket = bucket_name,
                 Key = base_name
             )
         except Exception as e:
@@ -421,10 +430,10 @@ def get_size(path: str) -> int:
 
     if storage_type == StorageType.S3:
 
-        s3_client, host, tray_name = __get_s3_client(tray_name)
+        s3_client, bucket_name = __get_s3_client(tray_name)
 
         try:
-            size = s3_client.head_object(Bucket=tray_name, Key=base_name)["ContentLength"].strip('"')
+            size = s3_client["client"].head_object(Bucket=bucket_name, Key=base_name)["ContentLength"].strip('"')
             return int(size)
         except Exception as e:
             raise StorageError("S3", e)
@@ -477,10 +486,10 @@ def exists(path: str) -> bool:
 
     if storage_type == StorageType.S3:
 
-        s3_client, host, tray_name = __get_s3_client(tray_name)
+        s3_client, bucket_name = __get_s3_client(tray_name)
 
         try:
-            s3_client.head_object(Bucket=tray_name, Key=base_name)
+            s3_client["client"].head_object(Bucket=bucket_name, Key=base_name)
             return True
         except botocore.exceptions.ClientError as e:
             if e.response['Error']['Code'] == "404":
@@ -532,11 +541,11 @@ def remove(path: str) -> None:
 
     if storage_type == StorageType.S3:
 
-        s3_client, host, tray_name = __get_s3_client(tray_name)
+        s3_client, bucket_name = __get_s3_client(tray_name)
 
         try:
-            s3_client.delete_object(
-                Bucket=tray_name,
+            s3_client["client"].delete_object(
+                Bucket=bucket_name,
                 Key=base_name
             )
         except Exception as e:
@@ -600,13 +609,13 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
 
     elif from_type == StorageType.S3 and to_type == StorageType.FILE :
 
-        s3_client, host, from_tray = __get_s3_client(from_tray)
+        s3_client, from_bucket = __get_s3_client(from_tray)
 
         try:
             if to_tray != "":
                 os.makedirs(to_tray, exist_ok=True)
 
-            s3_client.download_file(from_tray, from_base_name, to_path)
+            s3_client["client"].download_file(from_bucket, from_base_name, to_path)
 
             if from_md5 is not None :
                 to_md5 = hash_file(to_path)
@@ -618,13 +627,13 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
 
     elif from_type == StorageType.FILE and to_type == StorageType.S3 :
 
-        s3_client, host, to_tray = __get_s3_client(to_tray)
+        s3_client, to_bucket = __get_s3_client(to_tray)
 
         try:
-            s3_client.upload_file(from_path, to_tray, to_base_name)
+            s3_client["client"].upload_file(from_path, to_bucket, to_base_name)
 
             if from_md5 is not None :
-                to_md5 = s3_client.head_object(Bucket=to_tray, Key=to_base_name)["ETag"].strip('"')
+                to_md5 = s3_client["client"].head_object(Bucket=to_bucket, Key=to_base_name)["ETag"].strip('"')
                 if to_md5 != from_md5:
                     raise StorageError(f"FILE and S3", f"Invalid MD5 sum control for copy file {from_path} to S3 object {to_path} : {from_md5} != {to_md5}")
         except Exception as e:
@@ -632,25 +641,25 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
 
     elif from_type == StorageType.S3 and to_type == StorageType.S3 :
 
-        from_s3_client, from_host, from_tray = __get_s3_client(from_tray)
-        to_s3_client, to_host, to_tray = __get_s3_client(to_tray)
+        from_s3_client, from_bucket = __get_s3_client(from_tray)
+        to_s3_client, to_bucket = __get_s3_client(to_tray)
 
         try:
-            if to_host == from_host:
-                to_s3_client.copy(
+            if to_s3_client["host"] == from_s3_client["host"]:
+                to_s3_client["client"].copy(
                     {
-                        'Bucket': from_tray,
+                        'Bucket': from_bucket,
                         'Key': from_base_name
                     },
-                    to_tray, to_base_name
+                    to_bucket, to_base_name
                 )
             else:
                 with tempfile.NamedTemporaryFile("w+b") as f:
-                    from_s3_client.download_fileobj(from_tray, from_base_name, f)
-                    to_s3_client.upload_file(f.name, to_tray, to_base_name)
+                    from_s3_client["client"].download_fileobj(from_bucket, from_base_name, f)
+                    to_s3_client["client"].upload_file(f.name, to_bucket, to_base_name)
 
             if from_md5 is not None :
-                to_md5 = to_s3_client.head_object(Bucket=to_tray, Key=to_base_name)["ETag"].strip('"')
+                to_md5 = to_s3_client["client"].head_object(Bucket=to_bucket, Key=to_base_name)["ETag"].strip('"')
                 if to_md5 != from_md5:
                     raise StorageError(f"S3", f"Invalid MD5 sum control for copy S3 object {from_path} to {to_path} : {from_md5} != {to_md5}")
 
@@ -765,7 +774,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
 
         from_ioctx = __get_ceph_ioctx(from_tray)
 
-        to_s3_client, to_host, to_tray = __get_s3_client(to_tray)
+        s3_client, to_bucket = __get_s3_client(to_tray)
 
         if from_md5 is not None:
             checker = hashlib.md5()
@@ -775,7 +784,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
             size = 0
 
             with tempfile.NamedTemporaryFile("w+b",delete=False) as f:
-                name_fich = f.name
+                name_tmp = f.name
                 while True:
                     chunk = from_ioctx.read(from_base_name, 65536, offset)
                     size = len(chunk)
@@ -788,18 +797,13 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
                     if size < 65536:
                         break
 
-            to_s3_client.upload_file(name_fich, to_tray, to_base_name)
+            s3_client["client"].upload_file(name_tmp, to_bucket, to_base_name)
 
-            os.remove(name_fich)
+            os.remove(name_tmp)
 
             if from_md5 is not None and from_md5 != checker.hexdigest():
                 raise StorageError(f"CEPH and S3", f"Invalid MD5 sum control for copy CEPH object {from_path} to S3 object {to_path} : {from_md5} != {checker.hexdigest()}")
 
-            if from_md5 is not None :
-                to_md5 = to_s3_client.head_object(Bucket=to_tray, Key=to_base_name)["ETag"].strip('"')
-                if to_md5 != from_md5 and "-" not in to_md5:
-                    raise StorageError(f"CEPH and S3", f"Invalid MD5 sum control for copy CEPH object {from_path} to S3 object {to_path} : {from_md5} != {to_md5}")
-
         except Exception as e:
             raise StorageError(f"CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}")
 
@@ -833,7 +837,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
 
     elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.S3 :
 
-        to_s3_client, to_host, to_tray = __get_s3_client(to_tray)
+        to_s3_client, to_bucket = __get_s3_client(to_tray)
 
         try:
             reponse = requests.get(from_path, stream = True)
@@ -843,7 +847,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
                     if chunk:
                         f.write(chunk)
 
-            to_s3_client.upload_file(name_fich, to_tray, to_base_name)
+            to_s3_client["client"].upload_file(name_fich, to_tray, to_base_name)
 
             os.remove(name_fich)
 
@@ -880,16 +884,16 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None:
     # Réalisation du lien, selon les types de stockage
     if target_type == StorageType.S3:
 
-        target_s3_client, target_host, target_tray = __get_s3_client(target_tray)
-        link_s3_client, link_host, link_tray = __get_s3_client(link_tray)
+        target_s3_client, target_bucket = __get_s3_client(target_tray)
+        link_s3_client, link_bucket = __get_s3_client(link_tray)
 
-        if link_host != target_host:
+        if target_s3_client["host"] != link_s3_client["host"]:
             raise StorageError(f"S3", f"Cannot make link {link_path} -> {target_path} : link works only on the same S3 cluster")
 
         try:
-            target_s3_client.put_object(
-                Body = f"{__OBJECT_SYMLINK_SIGNATURE}{target_tray}/{target_base_name}".encode('utf-8'),
-                Bucket = link_tray,
+            target_s3_client["client"].put_object(
+                Body = f"{__OBJECT_SYMLINK_SIGNATURE}{target_bucket}/{target_base_name}".encode('utf-8'),
+                Bucket = link_bucket,
                 Key = link_base_name
             )
         except Exception as e:
@@ -916,3 +920,40 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None:
 
     else:
         raise StorageError("UNKNOWN", "Unhandled storage type to make link")
+
+def get_osgeo_path(path: str) -> str:
+    """Return GDAL/OGR Open compliant path and configure storage access
+
+    For a S3 input path, endpoint, access and secret keys are set and path is built with "/vsis3" root.
+
+    For a FILE input path, only storage prefix is removed
+
+    Args:
+        path (str): Source path
+
+    Raises:
+        NotImplementedError: Storage type not handled
+
+    Returns:
+        str: GDAL/OGR Open compliant path
+    """
+
+    storage_type, unprefixed_path, tray_name, base_name  = get_infos_from_path(path)
+
+
+    if storage_type == StorageType.S3:
+
+        s3_client, bucket_name = __get_s3_client(tray_name)
+
+        gdal.SetConfigOption('AWS_SECRET_ACCESS_KEY', s3_client["secret_key"])
+        gdal.SetConfigOption('AWS_ACCESS_KEY_ID', s3_client["key"])
+        gdal.SetConfigOption('AWS_S3_ENDPOINT', s3_client["host"])
+        gdal.SetConfigOption('AWS_VIRTUAL_HOSTING', 'FALSE')
+
+        return f"/vsis3/{bucket_name}/{base_name}"
+
+    elif storage_type == StorageType.FILE:
+        return unprefixed_path
+
+    else:
+        raise NotImplementedError(f"Cannot get a GDAL/OGR compliant path from {path}")
diff --git a/tests/test_Storage.py b/tests/test_Storage.py
index 9f73ae4..862b5a0 100644
--- a/tests/test_Storage.py
+++ b/tests/test_Storage.py
@@ -138,7 +138,7 @@ def test_http_read_error(mock_http):
         mock_http.return_value = requests_instance
         data = get_data_str("http:///path/to/file.ext")
 
-    mock_http.assert_called_with("http:///path/to/file.ext")
+    mock_http.assert_called_with("http:///path/to/file.ext", stream=True)
 
 @mock.patch.dict(os.environ, {}, clear=True)
 @mock.patch("requests.get")
@@ -149,7 +149,7 @@ def test_http_read_ok(mock_http):
         mock_http.return_value = requests_instance
 
         data = get_data_str("http:///path/to/file.ext")
-        mock_http.assert_called_with("http:///path/to/file.ext")
+        mock_http.assert_called_with("http:///path/to/file.ext", stream=True)
         assert data == 'data'
     except Exception as exc:
         assert False, f"HTTP read raises an exception: {exc}"
@@ -725,3 +725,31 @@ def test_remove_s3_ok(mocked_s3_client):
         remove("s3://bucket/object.ext")
     except Exception as exc:
         assert False, f"S3 deletion raises an exception: {exc}"
+
+
+############ get_osgeo_path
+
+@mock.patch.dict(os.environ, {}, clear=True)
+def test_get_osgeo_path_file_ok():
+    try:
+        path = get_osgeo_path("file:///path/to/file.ext")
+        assert path == "/path/to/file.ext"
+    except Exception as exc:
+        assert False, f"FILE osgeo path raises an exception: {exc}"
+
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
+def test_get_osgeo_path_s3_ok():
+
+    disconnect_s3_clients()
+
+    try:
+        path = get_osgeo_path("s3://bucket@b/to/object.ext")
+        assert path == "/vsis3/bucket/to/object.ext"
+    except Exception as exc:
+        assert False, f"S3 osgeo path raises an exception: {exc}"
+
+
+@mock.patch.dict(os.environ, {}, clear=True)
+def test_get_osgeo_path_nok():
+    with pytest.raises(NotImplementedError):
+        get_osgeo_path("ceph://pool/data.ext")
diff --git a/tests/test_Utils.py b/tests/test_Utils.py
index f5dfe6a..075e7d9 100644
--- a/tests/test_Utils.py
+++ b/tests/test_Utils.py
@@ -3,6 +3,7 @@
 
 import pytest
 import os
+import math
 
 from unittest.mock import *
 from unittest import mock
@@ -52,7 +53,10 @@ def test_reproject_point_ok():
         sr_3857 = srs_to_spatialreference("EPSG:3857")
         sr_ignf = srs_to_spatialreference("IGNF:WGS84G")
         x,y = reproject_point((43, 3), sr_4326, sr_3857)
-        assert (x,y) == (333958.4723798207, 5311971.846945471)
+
+        assert math.isclose(x, 333958.4723798207, rel_tol=1e-5)
+        assert math.isclose(y, 5311971.846945471, rel_tol=1e-5)
+
         x,y = reproject_point((43, 3), sr_4326, sr_ignf)
         assert (x,y) == (3, 43)
 

From 9336da29280cdd50b7a1f04de33994a3409ac0d3 Mon Sep 17 00:00:00 2001
From: CorentinPeutin <corentin.peutin@ensg.eu>
Date: Thu, 13 Apr 2023 14:17:49 +0200
Subject: [PATCH 5/7] Modifcation du get_infos_from path pour HTTP +
 modifications mineurs pour le HTTP

---
 src/rok4/Storage.py   | 37 ++++++++++++++++++++-----------------
 tests/test_Storage.py |  7 ++++++-
 2 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/src/rok4/Storage.py b/src/rok4/Storage.py
index 0f12a62..d19d437 100644
--- a/src/rok4/Storage.py
+++ b/src/rok4/Storage.py
@@ -207,9 +207,9 @@ def get_infos_from_path(path: str) -> Tuple[StorageType, str, str, str]:
     elif path.startswith("file://"):
         return StorageType.FILE, path[7:], os.path.dirname(path[7:]), os.path.basename(path[7:])
     elif path.startswith("http://"):
-        return StorageType.HTTP, path, "NULL", "NULL"
+        return StorageType.HTTP, path[7:], os.path.dirname(path[7:]), os.path.basename(path[7:])
     elif path.startswith("https://"):
-        return StorageType.HTTPS, path, "NULL", "NULL"
+        return StorageType.HTTPS, path[8:], os.path.dirname(path[8:]), os.path.basename(path[8:])
     else:
         return StorageType.FILE, path, os.path.dirname(path), os.path.basename(path)
 
@@ -347,14 +347,16 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str:
 
     elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS:
 
-        try:
-            reponse = requests.get(path, stream=True)
-            data = reponse.content
-            print(reponse.content)
-            if reponse.status_code == 404 :
-                raise StorageError("HTTP", "Requested file does not exist")
-        except Exception as e:
-            raise StorageError("HTTP", e)
+        if range is None :
+            try:
+                reponse = requests.get(storage_type.value + path, stream=True)
+                data = reponse.content
+                if reponse.status_code == 404 :
+                    raise StorageError(storage_type.name, "Requested file does not exist")
+            except Exception as e:
+                raise StorageError(storage_type.name, e)
+        else :
+            raise NotImplementedError
 
     else:
         raise StorageError("UNKNOWN", "Unhandled storage type to read binary data")
@@ -459,10 +461,11 @@ def get_size(path: str) -> int:
     elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS:
 
         try:
-            reponse = requests.get(path, stream=True)
-            return reponse.content.__sizeof__()
+            # Le stream=True permet de ne télécharger que le header initialement
+            reponse = requests.get(storage_type.value + path, stream=True).headers["content-length"]
+            return reponse
         except Exception as e:
-            raise StorageError("HTTP", e)
+            raise StorageError(storage_type.name, e)
 
     else:
         raise StorageError("UNKNOWN", "Unhandled storage type to get size")
@@ -516,7 +519,7 @@ def exists(path: str) -> bool:
     elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS:
 
         try:
-            reponse = requests.get(path, stream=True)
+            reponse = requests.get(storage_type.value + path, stream=True)
             if reponse.status_code == 200 :
                 return True
             else :
@@ -810,7 +813,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
     elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.FILE :
 
         try:
-            reponse = requests.get(from_path, stream = True)
+            reponse = requests.get(from_type.value + from_path, stream = True)
             with open(to_path, "wb") as f:
                 for chunk in reponse.iter_content(chunk_size=65536) :
                     if chunk:
@@ -824,7 +827,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
         to_ioctx = __get_ceph_ioctx(to_tray)
 
         try:
-            reponse = requests.get(from_path, stream = True)
+            reponse = requests.get(from_type.value + from_path, stream = True)
             offset = 0
             for chunk in reponse.iter_content(chunk_size=65536) :
                 if chunk:
@@ -840,7 +843,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
         to_s3_client, to_bucket = __get_s3_client(to_tray)
 
         try:
-            reponse = requests.get(from_path, stream = True)
+            reponse = requests.get(from_type.value + from_path, stream = True)
             with tempfile.NamedTemporaryFile("w+b",delete=False) as f:
                 name_fich = f.name
                 for chunk in reponse.iter_content(chunk_size=65536) :
diff --git a/tests/test_Storage.py b/tests/test_Storage.py
index d1de8b5..07006e6 100644
--- a/tests/test_Storage.py
+++ b/tests/test_Storage.py
@@ -140,6 +140,11 @@ def test_http_read_error(mock_http):
 
     mock_http.assert_called_with("http:///path/to/file.ext", stream=True)
 
+@mock.patch.dict(os.environ, {}, clear=True)
+def test_http_read_range_error():
+    with pytest.raises(NotImplementedError):
+        data = get_data_binary("http:///path/to/file.ext", (0,100))
+
 @mock.patch.dict(os.environ, {}, clear=True)
 @mock.patch("requests.get")
 def test_http_read_ok(mock_http):
@@ -581,7 +586,7 @@ def test_size_s3_ok(mocked_s3_client):
 def test_size_http_ok(mock_requests):
 
     http_instance = MagicMock()
-    http_instance.content.__sizeof__.return_value = 12
+    http_instance.headers = {"content-length":12}
     mock_requests.return_value = http_instance
 
     try:

From d043916019738d8de3022e5b0ff382af0c0e5ba6 Mon Sep 17 00:00:00 2001
From: CorentinPeutin <corentin.peutin@ensg.eu>
Date: Mon, 17 Apr 2023 16:31:43 +0200
Subject: [PATCH 6/7] Correction de coquilles

---
 src/rok4/Storage.py   | 10 +++++-----
 tests/test_Storage.py | 28 ++++++++++++++--------------
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/rok4/Storage.py b/src/rok4/Storage.py
index d19d437..b938670 100644
--- a/src/rok4/Storage.py
+++ b/src/rok4/Storage.py
@@ -2,10 +2,10 @@
 
 Available storage types are :
 - S3 (path are preffixed with `s3://`)
-- CEPH (path are preffixed with `ceph://`)
-- FILE (path are preffixed with `file://`, but it is the default paths' interpretation)
-- HTTP (path are preffixed with `https://`)
-- HTTPS (path are preffixed with `https://`)
+- CEPH (path are prefixed with `ceph://`)
+- FILE (path are prefixed with `file://`, but it is the default paths' interpretation)
+- HTTP (path are prefixed with `http://`)
+- HTTPS (path are prefixed with `https://`)
 
 According to functions, all storage types are not necessarily available.
 
@@ -525,7 +525,7 @@ def exists(path: str) -> bool:
             else :
                 return False
         except Exception as e:
-            raise StorageError("HTTP", e)
+            raise StorageError(storage_type.name, e)
 
     else:
         raise StorageError("UNKNOWN", "Unhandled storage type to test if exists")
diff --git a/tests/test_Storage.py b/tests/test_Storage.py
index 07006e6..d0ca499 100644
--- a/tests/test_Storage.py
+++ b/tests/test_Storage.py
@@ -136,14 +136,14 @@ def test_http_read_error(mock_http):
         requests_instance.content = "NULL"
         requests_instance.status_code = 404
         mock_http.return_value = requests_instance
-        data = get_data_str("http:///path/to/file.ext")
+        data = get_data_str("http://path/to/file.ext")
 
-    mock_http.assert_called_with("http:///path/to/file.ext", stream=True)
+    mock_http.assert_called_with("http://path/to/file.ext", stream=True)
 
 @mock.patch.dict(os.environ, {}, clear=True)
 def test_http_read_range_error():
     with pytest.raises(NotImplementedError):
-        data = get_data_binary("http:///path/to/file.ext", (0,100))
+        data = get_data_binary("http://path/to/file.ext", (0,100))
 
 @mock.patch.dict(os.environ, {}, clear=True)
 @mock.patch("requests.get")
@@ -153,8 +153,8 @@ def test_http_read_ok(mock_http):
         requests_instance.content = b'data'
         mock_http.return_value = requests_instance
 
-        data = get_data_str("http:///path/to/file.ext")
-        mock_http.assert_called_with("http:///path/to/file.ext", stream=True)
+        data = get_data_str("http://path/to/file.ext")
+        mock_http.assert_called_with("http://path/to/file.ext", stream=True)
         assert data == 'data'
     except Exception as exc:
         assert False, f"HTTP read raises an exception: {exc}"
@@ -408,8 +408,8 @@ def test_copy_http_file_ok(mock_open, mock_requests):
         http_instance.iter_content.return_value = ["data","data2"]
         mock_requests.return_value = http_instance
 
-        copy("http:///path/to/source.ext", "file:///path/to/destination.ext")
-        mock_requests.assert_called_once_with("http:///path/to/source.ext", stream=True)
+        copy("http://path/to/source.ext", "file:///path/to/destination.ext")
+        mock_requests.assert_called_once_with("http://path/to/source.ext", stream=True)
         mock_open.assert_called_once_with("/path/to/destination.ext", "wb")
     except Exception as exc:
         assert False, f"HTTP -> FILE copy raises an exception: {exc}"
@@ -432,8 +432,8 @@ def test_copy_http_ceph_ok(mock_requests, mocked_rados_client):
         ceph_instance.open_ioctx.return_value = ioctx_instance
         mocked_rados_client.return_value = ceph_instance
 
-        copy("http:///path/to/source.ext", "ceph://pool1/source.ext")
-        mock_requests.assert_called_once_with("http:///path/to/source.ext", stream=True)
+        copy("http://path/to/source.ext", "ceph://pool1/source.ext")
+        mock_requests.assert_called_once_with("http://path/to/source.ext", stream=True)
     except Exception as exc:
         assert False, f"HTTP -> CEPH copy raises an exception: {exc}"
 
@@ -455,8 +455,8 @@ def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_cl
         s3_instance.head_object.return_value = {"ETag": "8d777f385d3dfec8815d20f7496026dc"}
         mocked_s3_client.return_value = s3_instance
 
-        copy("http:///path/to/source.ext", "s3://bucket/destination.ext")
-        mock_requests.assert_called_once_with("http:///path/to/source.ext", stream=True)
+        copy("http://path/to/source.ext", "s3://bucket/destination.ext")
+        mock_requests.assert_called_once_with("http://path/to/source.ext", stream=True)
         mock_tempfile.assert_called_once_with("w+b",delete=False)
     except Exception as exc:
         assert False, f"HTTP -> CEPH copy raises an exception: {exc}"
@@ -590,7 +590,7 @@ def test_size_http_ok(mock_requests):
     mock_requests.return_value = http_instance
 
     try:
-        size = get_size("http:///path/to/file.ext")
+        size = get_size("http://path/to/file.ext")
         assert size == 12
     except Exception as exc:
         assert False, f"HTTP size raises an exception: {exc}"
@@ -664,7 +664,7 @@ def test_exists_http_ok(mock_requests):
     mock_requests.return_value = http_instance
 
     try:
-        assert exists("http:///path/to/file.ext")
+        assert exists("http://path/to/file.ext")
     except Exception as exc:
         assert False, f"HTTP exists raises an exception: {exc}"
 
@@ -672,7 +672,7 @@ def test_exists_http_ok(mock_requests):
     mock_requests.return_value = http_instance
 
     try:
-        assert not exists("http:///path/to/file.ext")
+        assert not exists("http://path/to/file.ext")
     except Exception as exc:
         assert False, f"HTTP exists raises an exception: {exc}"
 

From 6efe37ae3b344a8263d9eec3d92e455c89b1732a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9o=20Satabin?= <theo.satabin@ign.fr>
Date: Thu, 4 May 2023 13:41:37 +0200
Subject: [PATCH 7/7] =?UTF-8?q?Homog=C3=A9n=C3=A9isation=20des=20erreurs?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/rok4/Storage.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/rok4/Storage.py b/src/rok4/Storage.py
index b938670..6d172cd 100644
--- a/src/rok4/Storage.py
+++ b/src/rok4/Storage.py
@@ -349,10 +349,10 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str:
 
         if range is None :
             try:
-                reponse = requests.get(storage_type.value + path, stream=True)
+                reponse = requests.get(f"{storage_type.value}{path}", stream=True)
                 data = reponse.content
                 if reponse.status_code == 404 :
-                    raise StorageError(storage_type.name, "Requested file does not exist")
+                    raise FileNotFoundError(f"{storage_type.value}{path}")
             except Exception as e:
                 raise StorageError(storage_type.name, e)
         else :
@@ -519,8 +519,8 @@ def exists(path: str) -> bool:
     elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS:
 
         try:
-            reponse = requests.get(storage_type.value + path, stream=True)
-            if reponse.status_code == 200 :
+            response = requests.get(storage_type.value + path, stream=True)
+            if response.status_code == 200 :
                 return True
             else :
                 return False
@@ -813,9 +813,9 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
     elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.FILE :
 
         try:
-            reponse = requests.get(from_type.value + from_path, stream = True)
+            response = requests.get(from_type.value + from_path, stream = True)
             with open(to_path, "wb") as f:
-                for chunk in reponse.iter_content(chunk_size=65536) :
+                for chunk in response.iter_content(chunk_size=65536) :
                     if chunk:
                         f.write(chunk)
 
@@ -827,9 +827,9 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
         to_ioctx = __get_ceph_ioctx(to_tray)
 
         try:
-            reponse = requests.get(from_type.value + from_path, stream = True)
+            response = requests.get(from_type.value + from_path, stream = True)
             offset = 0
-            for chunk in reponse.iter_content(chunk_size=65536) :
+            for chunk in response.iter_content(chunk_size=65536) :
                 if chunk:
                     size = len(chunk)
                     to_ioctx.write(to_base_name, chunk, offset)
@@ -843,10 +843,10 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
         to_s3_client, to_bucket = __get_s3_client(to_tray)
 
         try:
-            reponse = requests.get(from_type.value + from_path, stream = True)
+            response = requests.get(from_type.value + from_path, stream = True)
             with tempfile.NamedTemporaryFile("w+b",delete=False) as f:
                 name_fich = f.name
-                for chunk in reponse.iter_content(chunk_size=65536) :
+                for chunk in response.iter_content(chunk_size=65536) :
                     if chunk:
                         f.write(chunk)