Merge branch 'dev' into ckan-2.11

.github/workflows/main.yml

@@ -10,6 +10,15 @@ env:
   PROJECT_NAME: wri-odp
   BRANCH_NAME: dev
   ENABLE_TRIVY_SCANNING: true
+  ENABLE_BUILD_CKAN: true
+  ENABLE_BUILD_FRONTEND: true
+  ENABLE_BUILD_DATAPUSHER: true
+  ENABLE_BUILD_MIGRATION: true
+  ENABLE_INT_TESTS: true
+  ENABLE_UNIT_TESTS: true
+  ENABLE_FRONTEND_TESTS: true
+  ENABLE_DEBUG_LOGS: false
+  ENABLE_FRONTEND_DOCKER: true
 permissions:
   id-token: write
   contents: read
@@ -38,6 +47,7 @@ jobs:
           sudo echo "127.0.0.1 frontend" | sudo tee -a /etc/hosts
           sudo echo "127.0.0.1 prefect" | sudo tee -a /etc/hosts
       - name: Build and push CKAN image to ECR
+        if: ${{ env.ENABLE_BUILD_CKAN == 'true' }}
         env:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           CKAN_REPO: ${{ secrets.ECR_CKAN_REPO}}
@@ -47,6 +57,7 @@ jobs:
           docker build -t $REGISTRY/$CKAN_REPO:$IMAGE_TAG deployment/ckan
           docker push $REGISTRY/$CKAN_REPO:$IMAGE_TAG
       - name: Build and push Frontend image to ECR
+        if: ${{ env.ENABLE_BUILD_FRONTEND == 'true' }}
         env:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           FRONTEND_REPO: ${{ secrets.ECR_FRONTEND_REPO}}
@@ -66,6 +77,7 @@ jobs:
           deployment/frontend
           docker push $REGISTRY/$FRONTEND_REPO:$IMAGE_TAG
       - name: Build and push Datapusher image to ECR
+        if: ${{ env.ENABLE_BUILD_DATAPUSHER == 'true' }}
         env:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           DATAPUSHER_REPO: ${{ secrets.ECR_DATAPUSHER_REPO}}
@@ -75,6 +87,7 @@ jobs:
           datapusher
           docker push $REGISTRY/$DATAPUSHER_REPO:$IMAGE_TAG
       - name: Build and push Migration image to ECR
+        if: ${{ env.ENABLE_BUILD_MIGRATION == 'true' }}
         env:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           DATAPUSHER_REPO: ${{ secrets.ECR_DATAPUSHER_REPO}}
@@ -93,6 +106,7 @@ jobs:
       - name: Initialize the Extensions
         run: bash ./ckan-backend-dev/ckan/scripts/init-extensions.sh
       - name: Cypress Install and CKAN setup
+        if: ${{ env.ENABLE_INT_TESTS == 'true' }}
         uses: cypress-io/github-action@v6
         with:
           wait-on: 'http://localhost:5000/private-admin/en'
@@ -103,18 +117,21 @@ jobs:
       - name: Create sysadmin API for Authorization
         run: bash ./ckan-backend-dev/ckan/scripts/cypress_setup.sh
       - name: Set up Frontend Docker Container
+        if: ${{ env.ENABLE_FRONTEND_DOCKER == 'true' }}
         env:
           CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}'
         run: docker compose -f docker-compose.test.yml --env-file .env.example up --build -d frontend
         working-directory: ./ckan-backend-dev
       - name: Initialize the Extensions
         run: bash ./ckan-backend-dev/ckan/scripts/init-extensions.sh
       - name: Print Logs
+        if: ${{ env.ENABLE_DEBUG_LOGS == 'true' }}
         env:
           CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}'
         run: docker compose -f docker-compose.test.yml --env-file .env.example logs
         working-directory: ./ckan-backend-dev
       - name: Cypress Install and Frontend setup
+        if: ${{ env.ENABLE_INT_TESTS == 'true' }}
         uses: cypress-io/github-action@v6
         with:
           wait-on: 'http://localhost:3000'
@@ -123,6 +140,7 @@ jobs:
           runTests: false
           working-directory: ./e2e-tests
       - name: Run Integration tests 🧪
+        if: ${{ env.ENABLE_INT_TESTS == 'true' }}
         uses: cypress-io/github-action@v6
         with:
           command: node test.js
@@ -137,6 +155,7 @@ jobs:
           # Fail this action after this many failed attempts
           max-attempts: 3 # Optional, defaults to 1
       - name: Run frontend tests 🧪
+        if: ${{ env.ENABLE_FRONTEND_TESTS == 'true' }}
         uses: cypress-io/github-action@v6
         with:
           command: npm run test
@@ -172,6 +191,13 @@ jobs:
           CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:${{ github.sha }}'
         run: docker compose -f docker-compose.test.yml --env-file .env.example exec -T ckan-dev /bin/bash -c "/srv/app/run_unit_tests.sh"
         working-directory: ./ckan-backend-dev
+        timeout-minutes: 10
+      - name: Print Error Logs After Unit Tests
+        if: failure()
+        env:
+          CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:dev-${{ github.sha }}'
+        run: docker compose -f docker-compose.test.yml --env-file .env.example logs
+        working-directory: ./ckan-backend-dev
       - name: Run Trivy Vulnerability Scanner for CKAN Container 🧪
         if: ${{ env.ENABLE_TRIVY_SCANNING == 'true' }}
         uses: aquasecurity/trivy-action@master

ckan-backend-dev/ckan/Dockerfile.dev

@@ -136,10 +136,11 @@ RUN ckan config-tool ${CKAN_INI} -s app:main "ckan.datapusher.api_token = xxxx-x
 
 COPY setup/who.ini ${APP_DIR}/src/ckan/who.ini
 
-COPY setup/supervisord.conf /etc/supervisord.d/supervisord.conf
+COPY setup/supervisord.conf /etc/supervisord.conf
 COPY setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
 COPY setup/supervisor.harvest.conf /etc/supervisord.d/harvest.conf
 
 RUN chown -R ckan:ckan /etc/supervisord.d
+RUN chown -R ckan:ckan /etc/supervisord.conf
 
 CMD ["sh", "-c", "${APP_DIR}/start_ckan_development.sh"]

ckan-backend-dev/ckan/scripts/run_unit_tests.sh

@@ -16,7 +16,7 @@ echo "Test Summary" > "$ROOT_DIR/test_summary.txt"
 if [ -d "src_extensions/ckanext-wri" ]; then
   cd src_extensions/ckanext-wri
 
-  pytest --ckan-ini=test.ini ckanext/wri/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
+  pytest --ckan-ini=test.ini -v --tb=long ckanext/wri/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
   PYTEST_EXIT_CODE=${PIPESTATUS[0]}
 
   if [ $PYTEST_EXIT_CODE -eq 0 ]; then
@@ -30,7 +30,7 @@ fi
 if [ -d "$ROOT_DIR/src/ckanext-wri" ]; then
   cd $ROOT_DIR/src/ckanext-wri
 
-  pytest --ckan-ini=test.ini ckanext/wri/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
+  pytest --ckan-ini=test.ini -v --tb=long ckanext/wri/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
   PYTEST_EXIT_CODE=${PIPESTATUS[0]}
 
   if [ $PYTEST_EXIT_CODE -eq 0 ]; then
@@ -52,7 +52,7 @@ for dir in ckanext-*; do
 
     cd $dir
 
-    pytest --ckan-ini=test.ini ckanext/${dir#ckanext-}/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
+    pytest --ckan-ini=test.ini -v --tb=long ckanext/${dir#ckanext-}/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
     PYTEST_EXIT_CODE=${PIPESTATUS[0]}
 
     if [ $PYTEST_EXIT_CODE -eq 0 ]; then

ckan-backend-dev/ckan/setup/start_ckan.sh.override

@@ -51,6 +51,11 @@ echo "From GitHub Actions: $GITHUB_ACTIONS"
 # Check if we are running in GitHub Actions
 if [ "$GITHUB_ACTIONS" = "true" ]; then
   UWSGI_OPTS="$UWSGI_OPTS --single-interpreter --manage-script-name --mount /private-admin=/srv/app/wsgi.py"
+  SUPERVISOR_USER='root'
+  SUPERVISOR_HOME='/tmp'
+else
+  SUPERVISOR_USER='ckan'
+  SUPERVISOR_HOME='/srv/app'
 fi
 echo "Starting CKAN with the following uwsgi options: $UWSGI_OPTS"
 
@@ -59,6 +64,17 @@ export SCRIPT_NAME="/private-admin"
 if [ $? -eq 0 ]
 then
     # Start supervisord
+    echo "==================================="
+    echo "SUPERVISOR_USER: $SUPERVISOR_USER"
+    echo "SUPERVISOR_HOME: $SUPERVISOR_HOME"
+    echo "==================================="
+
+    sed -i "s|SUPERVISOR_USER|$SUPERVISOR_USER|g" /etc/supervisord.d/harvest.conf
+    sed -i "s|SUPERVISOR_HOME|$SUPERVISOR_HOME|g" /etc/supervisord.d/harvest.conf
+    sed -i "s|SUPERVISOR_USER|$SUPERVISOR_USER|g" /etc/supervisord.d/worker.conf
+    sed -i "s|SUPERVISOR_HOME|$SUPERVISOR_HOME|g" /etc/supervisord.d/worker.conf
+    sed -i "s|SUPERVISOR_HOME|$SUPERVISOR_HOME|g" /etc/supervisord.conf
+
     supervisord --configuration /etc/supervisord.conf &
     # Start uwsgi
     sudo -u ckan -EH uwsgi $UWSGI_OPTS

ckan-backend-dev/ckan/setup/start_ckan_development.sh.override

@@ -107,8 +107,28 @@ then
     done
 fi
 
+# Check if we are running in GitHub Actions
+if [ "$GITHUB_ACTIONS" = "true" ]; then
+  SUPERVISOR_USER='root'
+  SUPERVISOR_HOME='/tmp'
+else
+  SUPERVISOR_USER='ckan'
+  SUPERVISOR_HOME='/srv/app'
+fi
+
 # Start supervisord
-supervisord --configuration /etc/supervisord.d/supervisord.conf &
+echo "==================================="
+echo "SUPERVISOR_USER: $SUPERVISOR_USER"
+echo "SUPERVISOR_HOME: $SUPERVISOR_HOME"
+echo "==================================="
+
+sed -i "s|SUPERVISOR_USER|$SUPERVISOR_USER|g" /etc/supervisord.d/harvest.conf
+sed -i "s|SUPERVISOR_HOME|$SUPERVISOR_HOME|g" /etc/supervisord.d/harvest.conf
+sed -i "s|SUPERVISOR_USER|$SUPERVISOR_USER|g" /etc/supervisord.d/worker.conf
+sed -i "s|SUPERVISOR_HOME|$SUPERVISOR_HOME|g" /etc/supervisord.d/worker.conf
+sed -i "s|SUPERVISOR_HOME|$SUPERVISOR_HOME|g" /etc/supervisord.conf
+
+supervisord --configuration /etc/supervisord.conf &
 
 # Start the development server with automatic reload
 ckan -c $CKAN_INI run -H 0.0.0.0 --prefix /private-admin

ckan-backend-dev/ckan/setup/supervisor.harvest.conf

@@ -8,8 +8,8 @@ stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stdout
 stderr_logfile_maxbytes=0
-user=ckan
-environment=HOME="/srv/app",USER="ckan"
+user=SUPERVISOR_USER
+environment=HOME="SUPERVISOR_HOME",USER="SUPERVISOR_USER"
 
 
 [program:ckan-gather]
@@ -22,5 +22,5 @@ stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stdout
 stderr_logfile_maxbytes=0
-user=ckan
-environment=HOME="/srv/app",USER="ckan"
+user=SUPERVISOR_USER
+environment=HOME="SUPERVISOR_HOME",USER="SUPERVISOR_USER"

ckan-backend-dev/ckan/setup/supervisor.worker.conf

@@ -8,5 +8,5 @@ stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stdout
 stderr_logfile_maxbytes=0
-user=ckan
-environment=HOME="/srv/app",USER="ckan"
+user=SUPERVISOR_USER
+environment=HOME="SUPERVISOR_HOME",USER="SUPERVISOR_USER"

ckan-backend-dev/ckan/setup/supervisord.conf

@@ -1,20 +1,20 @@
 [unix_http_server]
-file = /tmp/supervisor.sock
+file = SUPERVISOR_HOME/supervisor.sock
 chmod = 0777
 chown = ckan:ckan
 
 [supervisord]
-logfile = /tmp/supervisord.log
+logfile = SUPERVISOR_HOME/supervisord.log
 logfile_maxbytes = 50MB
 logfile_backups=10
 loglevel = info
-pidfile = /tmp/supervisord.pid
+pidfile = SUPERVISOR_HOME/supervisord.pid
 nodaemon = true
 umask = 022
 identifier = supervisor
 
 [supervisorctl]
-serverurl = unix:///tmp/supervisor.sock
+serverurl = unix://SUPERVISOR_HOME/supervisor.sock
 
 [rpcinterface:supervisor]
 supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

ckan-backend-dev/docker-compose.dev.yml

@@ -5,7 +5,6 @@ volumes:
   minio_data:
 
 services:
-
   ckan-dev:
     container_name: ${CKAN_CONTAINER_NAME}
     build:
@@ -29,9 +28,6 @@ services:
       - ./src:/srv/app/src_extensions
     restart: unless-stopped
     image: ${CKAN_IMAGE}
-    #healthcheck:
-    #  test: ["CMD-SHELL", "wget -qO /dev/null http://localhost:5000 || (echo 'Health check failed' > /tmp/ckan-healthcheck.log && wget -O - http://localhost:5000 2>> /tmp/ckan-healthcheck.log && exit 1)"]
-
 
   frontend:
     container_name: wri-frontend
@@ -148,7 +144,8 @@ services:
       - pg_data:/var/lib/postgresql/data
     restart: unless-stopped
     healthcheck:
-      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
+      test:
+        ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
 
   solr:
     container_name: ${SOLR_CONTAINER_NAME}

ckan-backend-dev/src/ckanext-wri/ckanext/wri/logic/action/create.py

@@ -17,6 +17,7 @@
     send_group_notification,
 )
 import ckan.logic as logic
+from ckanext.wri.logic.action.get import validate_visibility
 
 from ckan.common import _, config
 import ckan.plugins.toolkit as tk
@@ -34,6 +35,10 @@
     _before_dataset_create_or_update,
 )
 import uuid
+from ckan.logic.action.create import (
+    organization_create as old_organization_create)
+
+import ckan.authz as authz
 
 NotificationGetUserViewedActivity: TypeAlias = None
 log = logging.getLogger(__name__)
@@ -425,6 +430,8 @@ def migration_status(context: Context, data_dict: DataDict):
 
 
 def package_create(context: Context, data_dict: DataDict):
+
+    validate_visibility(context, data_dict)
     if data_dict.get("type") == "harvest":
         return old_package_create(context, data_dict)
 
@@ -858,8 +865,35 @@ def download_event_create(context: Context, data_dict: DataDict):
             raise tk.ValidationError("Missing required field " + item[0])
 
     events = []
+    download_id = str(uuid.uuid4())  # Generate a unique UUID for this group of downloads
     for resource_id in resources:
-        event = DownloadEvent.create(email, first_name, last_name, affiliation, organization, job_title, country, interests, package_id, resource_id)
+        event = DownloadEvent.create(email, first_name, last_name, affiliation, organization, job_title, country, interests, package_id, resource_id, download_id)
         events.append(event)
 
     return download_event_list_dictize(events, context)
+
+
+@logic.side_effect_free
+def organization_create(context, data_dict):
+    visibility = data_dict.get('visibility', "public")
+
+    if visibility == "public":
+        parent_org = data_dict.get("parent")
+        parent_org = parent_org.get("value") if parent_org else None
+        if parent_org:
+            parent_org = logic.get_action("organization_show")(context, {"id": parent_org})
+            users = parent_org.get("users", [])
+            username = context.get("user")
+            if users:
+                user_capacity = [user.get("capacity") for user in users if user.get("name") == username]
+                if "admin" not in user_capacity:
+                    raise ValidationError({"message": _("User does not have admin access to create a sub team")})
+            if parent_org.get("visibility", "public") == "private":
+                raise ValidationError({"message": _("Parent Organization has private visibility and cannot create public teams")})
+
+        else:
+            if not authz.is_sysadmin(context.get("user")):
+                raise ValidationError({"message": _("Only sysadmins can create public teams without a parent")})
+
+    result = old_organization_create(context, data_dict)
+    return result