Merge pull request #606 from dcs4cop/toniof-xxx-fix_for_s3_configs

Corrected retrieval of store params
xcube-dev · Feb 8, 2022 · 018d86e · 018d86e
2 parents 421c6b3 + 2788571
commit 018d86e
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 6 deletions.
diff --git a/test/webapi/test_context.py b/test/webapi/test_context.py
@@ -197,6 +197,27 @@ def test_get_global_place_group(self):
             ctx.get_global_place_group("bibo", "http://localhost:9090")
         self.assertEqual('HTTP 404: Place group "bibo" not found', f"{cm.exception}")
 
+    def test_get_other_store_params_than_root(self):
+        ctx = new_test_service_context()
+        dataset_config = {'Identifier': 'two',
+                          'Title': 'Test 2',
+                          'FileSystem': 's3',
+                          'Anonymous': False,
+                          'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
+                          'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
+                          'Region': 'eu-central-1'}
+        store_params = ctx._get_other_store_params_than_root(dataset_config)
+        expected_dict = \
+            {'storage_options':
+                 {'anon': False,
+                  'client_kwargs':
+                      {'endpoint_url': 'https://s3.eu-central-1.amazonaws.com',
+                       'region_name': 'eu-central-1'}
+                  }
+             }
+        self.assertIsNotNone(store_params)
+        self.assertEqual(expected_dict, store_params)
+
 
 class NormalizePrefixTest(unittest.TestCase):
     def test_empty(self):

diff --git a/xcube/webapi/context.py b/xcube/webapi/context.py
@@ -308,7 +308,7 @@ def _maybe_assign_store_instance_ids(self):
             # Determine common prefixes of paths (and call them roots)
             prefixes = _get_common_prefixes(paths)
             if len(prefixes) < 1:
-                roots = prefixes
+                roots = ['']
             else:
                 # perform further step to merge prefixes with same start
                 prefixes = list(set(prefixes))
@@ -329,6 +329,8 @@ def _maybe_assign_store_instance_ids(self):
                 while not root.endswith("/") and not root.endswith("\\") and \
                         len(root) > 0:
                     root = root[:-1]
+                if root.endswith("/") or root.endswith("\\"):
+                    root = root[:-1]
                 abs_root = root
                 # For local file systems: Determine absolute root from base dir
                 fs_protocol = FS_TYPE_TO_PROTOCOL.get(file_system, file_system)
@@ -352,28 +354,32 @@ def _maybe_assign_store_instance_ids(self):
                     store_instance_id = f'{fs_protocol}_{counter}'
                     data_store_pool.add_store_config(store_instance_id,
                                                      data_store_config)
-
                 for config in config_list:
                     if config['Path'].startswith(root):
                         config['StoreInstanceId'] = store_instance_id
-                        config['Path'] = config['Path'][len(root):]
+                        new_path = config['Path'][len(root):]
+                        while new_path.startswith("/") or \
+                                new_path.startswith("\\"):
+                            new_path = new_path[1:]
+                        config['Path'] = new_path
 
     def _get_other_store_params_than_root(self,
                                           dataset_config: DatasetConfigDict) \
             -> Dict:
         if FS_TYPE_TO_PROTOCOL.get(dataset_config.get('FileSystem',
                                                       'file')) != 's3':
             return {}
-        store_params = dict()
+        storage_options=dict()
         if 'Anonymous' in dataset_config:
-            store_params['anon'] = dataset_config['Anonymous']
+            storage_options['anon'] = dataset_config['Anonymous']
         client_kwargs = dict(
         )
         if 'Endpoint' in dataset_config:
             client_kwargs['endpoint_url'] = dataset_config['Endpoint']
         if 'Region' in dataset_config:
             client_kwargs['region_name'] = dataset_config['Region']
-        store_params['client_kwargs'] = client_kwargs
+        storage_options['client_kwargs'] = client_kwargs
+        store_params = dict(storage_options=storage_options)
         return store_params
 
     def get_dataset_configs_from_stores(self) \