feat: initialise ckan extension

.github/workflows/test.yml

@@ -0,0 +1,48 @@
+name: Tests
+on: [push, pull_request]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    container:
+      # The CKAN version tag of the Solr and Postgres containers should match
+      # the one of the container the tests run on.
+      # You can switch this base image with a custom image tailored to your project
+      image: ckan/ckan-base:2.10.4
+    services:
+      solr:
+        image: ckan/ckan-solr:2.10-solr8
+      postgres:
+        image: ckan/ckan-postgres-dev:2.10
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+          image: redis:3
+
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install requirements
+      # Install any extra requirements your extension has here (dev requirements, other extensions etc)
+      run: |
+        pip install -r requirements.txt
+        pip install -r dev-requirements.txt
+        pip install -e .
+    - name: Setup extension
+      # Extra initialization steps
+      run: |
+        # Replace default path to CKAN core config file with the one on the container
+        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
+
+        ckan -c test.ini db init
+    - name: Run tests
+      run: pytest --ckan-ini=test.ini --cov=ckanext.zarr --disable-warnings ckanext/zarr
+

MANIFEST.in

@@ -0,0 +1,5 @@
+include README.rst
+include LICENSE
+include requirements.txt
+recursive-include ckanext/zarr *.html *.json *.js *.less *.css *.mo *.yml
+recursive-include ckanext/zarr/migration *.ini *.py *.mako

ckanext/__init__.py

@@ -0,0 +1,9 @@
+# encoding: utf-8
+
+# this is a namespace package
+try:
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+    __path__ = pkgutil.extend_path(__path__, __name__)

ckanext/zarr/actions.py

@@ -0,0 +1,183 @@
+import logging
+import ckan.plugins.toolkit as toolkit
+from ckan.plugins.toolkit import ValidationError, _
+
+
+log = logging.getLogger(__name__)
+
+
+@toolkit.side_effect_free
+def user_show_me(context, resource_dict):
+    """
+    Returns the current user object.  Raises NotAuthorized error if no user
+    object found.
+
+    No input params.
+
+    :rtype dictionary
+    :returns The user object as a dictionary, which takes the following structure:
+        ```
+            {
+                "id": "7f88caf3-e68b-4c96-883e-b49f3d547d84",
+                "name": "fjelltopp_editor",
+                "fullname": "Fjelltopp Editor",
+                "email": "[email protected]",
+                "created": "2021-10-29 12:51:56.277305",
+                "reset_key": null,
+                "about": null,
+                "activity_streams_email_notifications": false,
+                "sysadmin": false,
+                "state": "active",
+                "image_url": null,
+                "plugin_extras": null
+            }
+        ```
+
+    """
+    model = context['model']
+    auth_user_obj = context.get('auth_user_obj', model.user.AnonymousUser())
+    if isinstance(auth_user_obj, model.user.AnonymousUser):
+        raise toolkit.NotAuthorized
+    else:
+        return auth_user_obj.as_dict()
+
+
+def dataset_duplicate(context, data_dict):
+    """
+    Quick way to duplicate a dataset and record the relationship between parent and child.
+    """
+    dataset_id_or_name = toolkit.get_or_bust(data_dict, 'id')
+    dataset = toolkit.get_action('package_show')(context, {'id': dataset_id_or_name})
+    dataset_id = dataset['id']
+
+    dataset.pop('id', None)
+    dataset.pop('name', None)
+    data_dict.pop('id', None)
+    context.pop('package', None)
+
+    dataset = {**dataset, **data_dict}
+
+    for resource in dataset.get('resources', []):
+        del resource['id']
+        del resource['package_id']
+
+    duplicate_dataset = toolkit.get_action('package_create')(context, dataset)
+    _record_dataset_duplication(dataset_id, duplicate_dataset['id'], context)
+
+    return toolkit.get_action('package_show')(context, {'id': duplicate_dataset['id']})
+
+
+@toolkit.chained_action
+def package_create(next_action, context, data_dict):
+    """
+    Validates the type parameter when creating a dataset.
+    """
+    dataset_type = data_dict.get('type', '')
+
+    valid_types = toolkit.get_action("scheming_dataset_schema_list")(context, {})
+    if 'dataset' not in valid_types:
+        valid_types.append("dataset")
+
+    if dataset_type:
+        if dataset_type not in valid_types:
+            raise toolkit.ValidationError(f"Type '{dataset_type}' is invalid, valid types are: '{', '.join(valid_types)}'")
+
+    return next_action(context, data_dict)
+
+
+@toolkit.chained_action
+def user_list(next_action, context, data_dict):
+    """
+    Allows you to search for users by id as well as by name.
+    """
+    try:
+        user_from_id = toolkit.get_action('user_show')(
+            context,
+            {'id': data_dict.get('q', '')}
+        )
+        data_dict['q'] = user_from_id.get('name')
+    except toolkit.ObjectNotFound:
+        pass
+    return next_action(context, data_dict)
+
+
+def dataset_tag_replace(context, data_dict):
+    """
+    Allows you to bulk replace one tag with another tag.
+    """
+    if 'tags' not in data_dict or not isinstance(data_dict['tags'], dict):
+        raise toolkit.ValidationError(toolkit._(
+            "Must specify 'tags' dict of tags for update in form "
+            "{'old_tag_name1': 'new_tag_name1', 'old_tag_name2': 'new_tag_name2'}"))
+
+    tags = data_dict.pop("tags")
+    package_search_params = _restrict_datasets_to_those_with_tags(data_dict, tags)
+
+    datasets = toolkit.get_action('package_search')(context, package_search_params).get('results', [])
+
+    _check_user_access_to_all_datasets(context, datasets)
+    _update_tags(context, datasets, tags)
+
+    return {'datasets_modified': len(datasets)}
+
+
+def _check_user_access_to_all_datasets(context, datasets):
+    for ds in datasets:
+        toolkit.check_access('package_patch', context, {"id": ds['id']})
+
+
+def _restrict_datasets_to_those_with_tags(package_search_params, tags):
+    fq_tag_restriction = " OR ".join([f"tags:{key}" for key in tags])
+
+    if 'fq' in package_search_params:
+        original_fq = package_search_params['fq']
+        package_search_params['fq'] = f"({original_fq}) AND ({fq_tag_restriction})"
+    else:
+        package_search_params['fq'] = f"({fq_tag_restriction})"
+
+    return package_search_params
+
+
+def _update_tags(context, datasets, tags_to_be_replaced):
+    dataset_patch_action = toolkit.get_action('package_patch')
+
+    for ds in datasets:
+        final_tags = _prepare_final_tag_list(ds['tags'], tags_to_be_replaced)
+        dataset_patch_action(context, {'id': ds['id'], 'tags': final_tags})
+
+
+def _prepare_final_tag_list(original_tags, tags_to_be_replaced):
+    final_tags = []
+    for tag in original_tags:
+        tag_name = tag['name']
+        final_tags.append({"name": tags_to_be_replaced[tag_name] if tag_name in tags_to_be_replaced else tag_name})
+
+    return final_tags
+
+
+def _record_dataset_duplication(dataset_id, new_dataset_id, context):
+    # We should probably use activities to record duplication in CKAN 2.10
+
+    relationship = {
+        'subject': new_dataset_id,
+        'object': dataset_id,
+        'type': 'child_of'
+    }
+
+    try:
+        current_activity_id = toolkit.get_action('package_activity_list')(
+            context,
+            {'id': dataset_id}
+        )[0]['id']
+        relationship['comment'] = f"Duplicated from activity {current_activity_id}"
+    except Exception as e:
+        log.error(f"Failed to get current activity for package {dataset_id} ...")
+        log.exception(e)
+
+    try:
+        toolkit.get_action('package_relationship_create')(context, relationship)
+    except Exception as e:
+        log.error(f"Failed to record duplication of {dataset_id} to {new_dataset_id} ...")
+        log.exception(e)
+
+

ckanext/zarr/assets/css/FileInputComponent.css

@@ -0,0 +1,18 @@
+.dropzone{
+    text-align: center;
+    padding: 20px;
+    border-width: 3px;
+    border-radius: 2px;
+    border-color: #ebebeb;
+    border-style: dashed;
+    background-color: #fafafa;
+    margin-bottom: 20px;
+}
+
+#FileInputComponent h3{
+    margin-bottom: 20px;
+}
+
+#FileInputComponent .field-url-input-group{
+    margin-bottom: 30px;
+}

ckanext/zarr/assets/css/source-sans-3VF.css

@@ -0,0 +1,17 @@
+@font-face{
+    font-family: 'Source Sans 3 VF';
+    font-weight: 200 900;
+    font-style: normal;
+    font-stretch: normal;
+    src: url('/fonts/SourceSans3VF-Upright.otf.woff2') format('woff2'),
+         url('/fonts/SourceSans3VF-Upright.ttf.woff2') format('woff'),
+}
+
+@font-face{
+    font-family: 'Source Sans 3 VF';
+    font-weight: 200 900;
+    font-style: italic;
+    font-stretch: normal;
+    src: url('/fonts/SourceSans3VF-Italic.otf.woff2') format('woff2'),
+         url('/fonts/SourceSans3VF-Italic.ttf.woff2') format('woff'),
+}

ckanext/zarr/assets/css/zarr.css

@@ -0,0 +1,2 @@
+/* This file overwrites css from the base ckan theme */
+

ckanext/zarr/assets/webassets.yml

@@ -0,0 +1,17 @@
+zarr-theme:
+  filter: cssrewrite
+  output: zarr/%(version)s-zarr.css
+  contents:
+    - css/source-sans-3VF.css
+    - css/zarr.css
+
+FileInputComponentStyles:
+    contents:
+        - css/FileInputComponent.css
+    output: zarr/%(version)s_FileInputComponent.css
+
+FileInputComponentScripts:
+    contents:
+        - frictionless-js.js
+        - build/FileInputComponent.js
+    output: zarr/%(version)s_FileInputComponent.js