Merge pull request #21 from fractal-analytics-platform/18-integrate-s…

…ome-part-of-fractal-demos

18 integrate some part of fractal demos

.github/workflows/test.yml

@@ -1,15 +1,20 @@
 name: Test
 
-on: workflow_dispatch
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+
 
 jobs:
   tests:
-    timeout-minutes: 20
+    timeout-minutes: 15
     runs-on: ubuntu-latest
 
     steps:
     - name: Checkout
       uses: actions/checkout@v4
 
-    - name: Start containers
-      run: docker compose --file docker-compose-demos.yml up --build --force-recreate
+    - name: Run demos
+      run: make run-demos-github

.gitignore

@@ -5,3 +5,5 @@ fractal-db-data/
 advanced/fractal-share/
 advanced/fractal-db-data/
 advanced/fractal-slurm-data/
+
+*.swp

docker-compose.yml → Old/docker-compose.yml

@@ -48,14 +48,14 @@ services:
       timeout: 2s
       retries: 5
 
-  client:
-    container_name: fractal-client
+  web:
+    container_name: fractal-web
     depends_on:
       server:
         condition: service_healthy
-    image: fractal-client
+    image: fractal-web
     build:
       context: .
-      dockerfile: client/Dockerfile.client
+      dockerfile: web/Dockerfile.web
     ports:
       - 5173:5173

README.md

@@ -1,5 +1,11 @@
 # fractal-containers
 
+> Note: this repository is under active development, and the README is most
+> likely not up-to-date.
+
+---
+
+
 `fractal-containers` is a containerized implementation of the
 [Fractal](https://fractal-analytics-platform.github.io) analytics platform for
 image analysis. The goal of this project is to allow rapid local deployment of

config.env

@@ -0,0 +1,17 @@
+# fractal-server
+# FRACTAL_SERVER_RELEASE=1.3.10
+# FRACTAL_SERVER_GIT=...
+
+# fractal-client
+# FRACTAL_CLIENT_RELEASE=1.3.6
+# FRACTAL_CLIENT_GIT=...
+
+# fractal-tasks-core
+# FRACTAL_TASKS_CORE_RELEASE=0.12.0
+# FRACTAL_TASKS_CORE_GIT
+
+# fractal-web
+# FRACTAL_WEB_GIT=0.6.0
+
+# fractal-demos
+# FRACTAL_DEMOS_GIT=...

demos/Dockerfile.demos.step1

@@ -0,0 +1,11 @@
+# syntax=docker/dockerfile:1
+
+FROM python:3.11-slim
+
+# Install system libraries
+RUN apt-get update
+RUN apt-get -y install git  # required for installing client/tasks from git
+
+# Install requirements for output-validation Python script
+ADD demos/requirements-validation.txt ./requirements-validation.txt
+RUN python3 -m pip install -r requirements-validation.txt

demos/Dockerfile.demos.step2

@@ -0,0 +1,16 @@
+# syntax=docker/dockerfile:1
+
+FROM fractal/demos-base:1
+
+ADD config.env .
+
+COPY demos/get_fractal_client.sh .
+RUN bash get_fractal_client.sh
+
+COPY demos/get_fractal_demos.sh .
+RUN bash get_fractal_demos.sh
+
+ADD config.env ./fractal-demos/examples/config.env
+COPY demos/get_fractal_tasks_core.sh ./fractal-demos/examples/get_fractal_tasks_core.sh
+ADD demos/fractal.env ./fractal-demos/examples/.fractal.env
+ADD demos/run_demos.sh ./fractal-demos/examples/run_demos.sh

demos/get_fractal_client.sh

@@ -0,0 +1,20 @@
+source config.env
+echo "FRACTAL_CLIENT_RELEASE=$FRACTAL_CLIENT_RELEASE"
+echo "FRACTAL_CLIENT_GIT=$FRACTAL_CLIENT_GIT"
+if [ -z "${FRACTAL_CLIENT_RELEASE}" ]; then
+    if [ -z "${FRACTAL_CLIENT_GIT}" ]; then
+        # Case 1: no release, no git
+        python3 -m pip install fractal-client
+    else
+        # Case 2: only git set  # TODO: add git support
+        echo "FRACTAL_CLIENT_GIT not supported" && exit 1
+    fi
+else
+    if [ -z "${FRACTAL_CLIENT_GIT}" ]; then
+        # Case 3: only release set
+        python3 -m pip install fractal-client==$FRACTAL_CLIENT_RELEASE
+    else
+        # Case 4: both release and git set
+        echo "Error: cannot set both FRACTAL_CLIENT_RELEASE and FRACTAL_CLIENT_GIT." && exit 1
+    fi
+fi

demos/get_fractal_demos.sh

@@ -0,0 +1,10 @@
+git clone https://github.com/fractal-analytics-platform/fractal-demos.git
+
+source config.env
+
+echo "FRACTAL_DEMOS_GIT=$FRACTAL_DEMOS_GIT"
+if [ ! -z "${FRACTAL_DEMOS_GIT}" ]; then
+    cd fractal-demos
+    git checkout $FRACTAL_DEMOS_GIT
+    cd ..
+fi

demos/get_fractal_tasks_core.sh

@@ -0,0 +1,22 @@
+source config.env
+echo "FRACTAL_TASKS_CORE_RELEASE=$FRACTAL_TASKS_CORE_RELEASE"
+echo "FRACTAL_TASKS_CORE_GIT=$FRACTAL_TASKS_CORE_GIT"
+if [ -z "${FRACTAL_TASKS_CORE_RELEASE}" ]; then
+    if [ -z "${FRACTAL_TASKS_CORE_GIT}" ]; then
+        # Case 1: no release, no git
+        fractal task collect fractal-tasks-core --package-extras fractal-tasks
+    else
+        # Case 2: only git set
+        # TODO: git clone, git checkout, install poetry, poetry build, collect from local wheel file
+        echo "Error: FRACTAL_TASKS_CORE_GIT not supported" && exit 1
+    fi
+else
+    if [ -z "${FRACTAL_TASKS_CORE_GIT}" ]; then
+        # Case 3: only release set
+        fractal task collect fractal-tasks-core --package-extras fractal-tasks --package-version $FRACTAL_TASKS_CORE_RELEASE
+    else
+        # Case 4: both release and git set
+        echo "Error: cannot set both FRACTAL_TASKS_CORE_RELEASE and FRACTAL_TASKS_CORE_GIT."
+        exit 1
+    fi
+fi

demos/requirements-validation.txt

@@ -0,0 +1 @@
+zarr

demos/run_demos.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+echo "START run_all_demos.sh"
+
+# Due to how fractal-demos scripts are written, we should have a .fractal.env
+# file also in 00_user_setup
+cp .fractal.env 00_user_setup
+cp .fractal.env 01_cardio_tiny_dataset
+
+# Copy images from Resources folder
+mkdir images
+cp -r /home/fractal_share/Resources/images/10.5281_zenodo.8287221 images/
+
+# Trigger task collection
+bash get_fractal_tasks_core.sh
+
+# Wait for task collection to be complete
+while [ "$(fractal task list)" == "[]" ]; do
+    echo "No task available, wait 10 seconds.";
+    sleep 10;
+done
+
+# Enter 01_cardio_tiny_dataset folder
+cd 01_cardio_tiny_dataset
+
+# Run example 01 and capture exit code
+echo "START examples/01 API calls"
+TMPFILE="tmp_01_api.txt"
+./run_example.sh >> $TMPFILE 2>&1
+API_EXITCODE=$?
+cat $TMPFILE
+
+# Check exit code
+if [ $API_EXITCODE -ne 0 ]; then
+    echo "Error: API_EXITCODE=$API_EXITCODE"
+    exit 1
+fi
+
+# Parse temporary file to extract PROJECT_ID and JOB_ID
+PROJECT_ID=$(cat $TMPFILE | grep "PROJECT_ID" | cut -d '=' -f 2)
+JOB_ID=$(cat $TMPFILE | grep "JOB_ID" | cut -d '=' -f 2)
+echo "PROJECT_ID=$PROJECT_ID"
+echo "JOB_ID=$JOB_ID"
+
+# Wait for job to be done or failed
+while true; do
+    STATUS_LINE=$(fractal job show $PROJECT_ID $JOB_ID | grep "status")
+    echo $STATUS_LINE
+    if [[ "$STATUS_LINE" == *done* || "$STATUS_LINE" == *failed* ]]; then
+        break
+    fi
+    sleep 1
+done
+
+# Check job status, once again
+fractal job show $PROJECT_ID $JOB_ID
+echo "END examples/01 API calls"
+
+# Start output validation
+echo "START examples/01 output validation"
+TMPFILE="tmp_01_validation.txt"
+python validate_results.py >> $TMPFILE 2>&1
+VALIDATION_EXIT_CODE=$?
+cat $TMPFILE
+
+# Check exit code
+if [ $VALIDATION_EXIT_CODE -ne 0 ]; then
+    echo "Error: VALIDATION_EXIT_CODE=$VALIDATION_EXIT_CODE"
+    exit 1
+fi
+
+echo "END examples/01 output validation"
+echo
+
+echo "API_EXITCODE=$API_EXITCODE"
+echo "VALIDATION_EXIT_CODE=$VALIDATION_EXIT_CODE"
+echo "END run_all_demos.sh"