Merge pull request #1049 from CCI-Tools/forman-xxx-configure_data_sto…

…res_by_given_root

Configure data store "local" by given root

CHANGES.md

@@ -9,8 +9,8 @@
   For this reason, the Cate App UI is now bundled with the Cate server
   and served via the endpoint "/app".
 
-* The "local" data store is now configured to use the current 
-  working directory as root.
+* The "local" data store is now configured to use a given
+  server root path.
 
 * Removed the `cate upd` CLI command.
 

cate/ds/__init__.py

@@ -1,75 +1,26 @@
 # The MIT License (MIT)
-# Copyright (c) 2016, 2017 by the ESA CCI Toolbox development team and contributors
+# Copyright (c) 2016-2023 by the ESA CCI Toolbox team and contributors
 #
-# Permission is hereby granted, free of charge, to any person obtaining a copy of
-# this software and associated documentation files (the "Software"), to deal in
-# the Software without restriction, including without limitation the rights to
-# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-# of the Software, and to permit persons to whom the Software is furnished to do
-# so, subject to the following conditions:
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
 #
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-import logging
-
-from xcube.util.assertions import assert_given
-
-_LOG = logging.getLogger('cate')
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
 
 
 def cate_init():
-    # Plugin initializer.
-    import yaml
-    import os
-    from xcube.core.store import DataStoreConfig
-    from xcube.core.store import get_data_store_params_schema
-
-    from cate.conf import get_data_stores_path
-    from cate.conf.defaults import STORES_CONF_FILE
-    from cate.core.common import default_user_agent
-    from cate.core.ds import DATA_STORE_POOL
-
-    dir_path = os.path.dirname(os.path.abspath(__file__))
-    default_stores_file = os.path.join(dir_path, 'data/stores.yml')
-
-    if os.path.exists(STORES_CONF_FILE):
-        with open(STORES_CONF_FILE, 'r') as fp:
-            store_configs = yaml.safe_load(fp)
-    else:
-        with open(default_stores_file, 'r') as fp:
-            store_configs = yaml.safe_load(fp)
-
-    for store_name, store_config in store_configs.items():
-        store_id = store_config.get('store_id')
-        assert_given(store_id, name='store_id', exception_type=RuntimeError)
-
-        if store_id == 'file' \
-                and 'store_params' in store_config:
-            root = store_config.get('store_params', {}).get('root')
-            if root is None:
-                root = os.environ.get('CATE_LOCAL_DATA_STORE_PATH',
-                                      os.path.join(get_data_stores_path(),
-                                                   store_name))
-            else:
-                root = os.path.abspath(os.path.expanduser(root))
-            # Note: even if the root directory doesn't exist yet,
-            # the xcube "file" data store will create it for us.
-            store_config['store_params']['root'] = root
-
-        store_params_schema = get_data_store_params_schema(store_id)
-        if 'user_agent' in store_params_schema.properties:
-            if 'store_params' not in store_config:
-                store_config['store_params'] = {}
-            store_config['store_params']['user_agent'] = default_user_agent()
-
-        store_config = DataStoreConfig.from_dict(store_config)
-        DATA_STORE_POOL.add_store_config(store_name, store_config)
+    """Cate plugin initializer."""
+    from .stores import configure_data_stores
+    configure_data_stores()

cate/ds/data/stores.yml

@@ -23,8 +23,11 @@ cci-zarr-store:
 local:
   store_id: "file"
   store_params:
-    root: "~"
+    root: null
     max_depth: 4
+    excludes:
+      - ".*"
+      - "*/.*"
   title: "File Data Sources"
   description: >
     This data store represents

cate/ds/stores.py

@@ -0,0 +1,78 @@
+# The MIT License (MIT)
+# Copyright (c) 2016-2023 by the ESA CCI Toolbox team and contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import os
+from typing import Optional
+
+import yaml
+
+from cate.conf import get_data_stores_path
+from cate.conf.defaults import STORES_CONF_FILE
+from cate.core.common import default_user_agent
+from cate.core.ds import DATA_STORE_POOL
+from xcube.core.store import DataStoreConfig
+from xcube.core.store import get_data_store_params_schema
+from xcube.util.assertions import assert_given
+
+
+def configure_data_stores(local_root_path: Optional[str] = None):
+    """Configure Cate's data stores.
+
+    :param local_root_path: Optional root path
+        for the data store named "local".
+    """
+
+    if os.path.exists(STORES_CONF_FILE):
+        with open(STORES_CONF_FILE, 'r') as fp:
+            store_configs = yaml.safe_load(fp)
+    else:
+        dir_path = os.path.dirname(os.path.abspath(__file__))
+        default_stores_file = os.path.join(dir_path, 'data/stores.yml')
+
+        with open(default_stores_file, 'r') as fp:
+            store_configs = yaml.safe_load(fp)
+
+    for store_name, store_config in store_configs.items():
+        store_id = store_config.get('store_id')
+        assert_given(store_id, name='store_id', exception_type=RuntimeError)
+
+        if store_name == 'local':
+            if local_root_path is not None:
+                root = local_root_path
+            else:
+                root = store_config.get('store_params', {}).get('root')
+                if root is None:
+                    root = os.environ.get('CATE_LOCAL_DATA_STORE_PATH',
+                                          os.path.join(get_data_stores_path(),
+                                                       store_name))
+            root = os.path.abspath(os.path.expanduser(root))
+            # Note: even if the root directory doesn't exist yet,
+            # the xcube "file" data store will create it for us.
+            store_config['store_params']['root'] = root
+
+        store_params_schema = get_data_store_params_schema(store_id)
+        if 'user_agent' in store_params_schema.properties:
+            if 'store_params' not in store_config:
+                store_config['store_params'] = {}
+            store_config['store_params']['user_agent'] = default_user_agent()
+
+        store_config = DataStoreConfig.from_dict(store_config)
+        DATA_STORE_POOL.add_store_config(store_name, store_config)

cate/webapi/start.py

@@ -56,6 +56,7 @@
 from cate.conf.defaults import WEBAPI_PROGRESS_DEFER_PERIOD
 from cate.core.types import ValidationError
 from cate.core.wsmanag import FSWorkspaceManager
+from cate.ds.stores import configure_data_stores
 from cate.util.misc import get_dependencies
 from cate.util.web import JsonRpcWebSocketHandler
 from cate.util.web.webapi import (run_start,
@@ -94,8 +95,9 @@
 class WebAPIInfoHandler(WebAPIRequestHandler):
     def get(self):
         # noinspection PyUnresolvedReferences
-        user_root_mode = isinstance(self.application.workspace_manager, FSWorkspaceManager) \
-                         and self.application.workspace_manager.root_path is not None
+        workspace_manager = self.application.workspace_manager
+        user_root_mode = isinstance(workspace_manager, FSWorkspaceManager) \
+                         and workspace_manager.root_path is not None
 
         self.write_status_ok(content={
             'name': SERVICE_NAME,
@@ -116,7 +118,8 @@ def service_factory(application):
     )
 
 
-# All JSON REST responses should have same structure, namely a dictionary as follows:
+# All JSON REST responses should have same structure,
+# namely a dictionary as follows:
 #
 # {
 #    "status": "ok" | "error",
@@ -190,6 +193,8 @@ def create_application(user_root_path: str = None):
         _LOG.warning(f"user root path given by environment variable"
                      f" CATE_USER_ROOT superseded by {user_root_path}")
 
+    if user_root_path is not None:
+        configure_data_stores(local_root_path=user_root_path)
     application.workspace_manager = FSWorkspaceManager(user_root_path)
 
     return application