diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ecce996..bccd493 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,20 +10,6 @@ on: jobs: - tests-v1: - timeout-minutes: 25 - runs-on: ubuntu-22.04 - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set FRACTAL_RUNNER_BACKEND to 'slurm' - run: sed -i 's/local_experimental$/local/' ./testing/server/server.env - - - name: Run demos - run: cd testing && make run-demos-v1-github - tests-v2: timeout-minutes: 25 runs-on: ubuntu-22.04 diff --git a/testing/config.env b/testing/config.env index 664a8b3..e8743f4 100644 --- a/testing/config.env +++ b/testing/config.env @@ -15,12 +15,3 @@ FRACTAL_WEB_GIT=main # fractal-demos FRACTAL_DEMOS_V2_GIT=reduce-memory-requirements-examples-01-02 - -### LEGACY - -# fractal-client (Fractal V1) -FRACTAL_CLIENT_V1_RELEASE=1.4.4 -# fractal-tasks-core (Fractal V1) -FRACTAL_TASKS_CORE_V1_RELEASE=0.14.3 -# fractal-demos (Fractal V1) -FRACTAL_DEMOS_V1_GIT=v1 diff --git a/testing/demos-v1/Dockerfile b/testing/demos-v1/Dockerfile deleted file mode 100644 index 2502332..0000000 --- a/testing/demos-v1/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM python:3.11-slim - -# Install system libraries -RUN apt update && apt install -y git curl - -# Install requirements for output-validation Python script -ADD demos-v2/requirements-validation.txt ./requirements-validation.txt -RUN python3 -m pip install -r requirements-validation.txt - -ADD config.env . - -COPY demos-v1/get_fractal_client.sh . -RUN bash get_fractal_client.sh - -COPY demos-v1/get_fractal_demos.sh . -RUN bash get_fractal_demos.sh - -ADD config.env ./fractal-demos-v1/examples/config.env -COPY demos-v1/get_fractal_tasks_core.sh ./fractal-demos-v1/examples/get_fractal_tasks_core.sh -ADD demos-v1/fractal.env ./fractal-demos-v1/examples/.fractal.env -ADD demos-v1/run_demos.sh ./fractal-demos-v1/examples/run_demos.sh diff --git a/testing/demos-v1/fractal.env b/testing/demos-v1/fractal.env deleted file mode 100644 index 4e2e9d3..0000000 --- a/testing/demos-v1/fractal.env +++ /dev/null @@ -1,3 +0,0 @@ -FRACTAL_SERVER=http://fractal-server:8000 -FRACTAL_USER=admin@fractal.xy -FRACTAL_PASSWORD=1234 diff --git a/testing/demos-v1/get_fractal_client.sh b/testing/demos-v1/get_fractal_client.sh deleted file mode 100644 index 33cf562..0000000 --- a/testing/demos-v1/get_fractal_client.sh +++ /dev/null @@ -1,26 +0,0 @@ -source config.env -echo "FRACTAL_CLIENT_V1_RELEASE=$FRACTAL_CLIENT_V1_RELEASE" -echo "FRACTAL_CLIENT_V1_GIT=$FRACTAL_CLIENT_V1_GIT" -if [ -z "${FRACTAL_CLIENT_V1_RELEASE}" ]; then - if [ -z "${FRACTAL_CLIENT_V1_GIT}" ]; then - # Case 1: no release, no git - python3 -m pip install fractal-client - else - # Case 2: only git set - python3 -m pip install git+https://github.com/fractal-analytics-platform/fractal-client.git@$FRACTAL_CLIENT_V1_GIT - # Check exit code - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "Error: Installing fractal-client@$FRACTAL_CLIENT_V1_GIT (EXIT_CODE=$EXIT_CODE)" - exit 1 - fi - fi -else - if [ -z "${FRACTAL_CLIENT_V1_GIT}" ]; then - # Case 3: only release set - python3 -m pip install "fractal-client==$FRACTAL_CLIENT_V1_RELEASE" - else - # Case 4: both release and git set - echo "Error: cannot set both FRACTAL_CLIENT_V1_RELEASE and FRACTAL_CLIENT_V1_GIT." && exit 1 - fi -fi diff --git a/testing/demos-v1/get_fractal_demos.sh b/testing/demos-v1/get_fractal_demos.sh deleted file mode 100755 index 16eb78e..0000000 --- a/testing/demos-v1/get_fractal_demos.sh +++ /dev/null @@ -1,14 +0,0 @@ -source config.env - -if [ -z "${FRACTAL_DEMOS_V1_GIT}" ]; then - FRACTAL_DEMOS_V1_GIT=main -fi - -git clone --single-branch --branch "$FRACTAL_DEMOS_V1_GIT" https://github.com/fractal-analytics-platform/fractal-demos.git -mv fractal-demos fractal-demos-v1 - -EXIT_CODE=$? -if [ $EXIT_CODE -ne 0 ]; then - check_exit_code "Cloning fractal-demos@$FRACTAL_DEMOS_V1_GIT (EXIT_CODE=$EXIT_CODE)" - exit 1 -fi diff --git a/testing/demos-v1/get_fractal_tasks_core.sh b/testing/demos-v1/get_fractal_tasks_core.sh deleted file mode 100644 index d8014ec..0000000 --- a/testing/demos-v1/get_fractal_tasks_core.sh +++ /dev/null @@ -1,45 +0,0 @@ -check_exit_code() { - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "Error: $1 (EXIT_CODE=$EXIT_CODE)" - exit 1 - fi -} - -source config.env -echo "FRACTAL_TASKS_CORE_V1_RELEASE=$FRACTAL_TASKS_CORE_V1_RELEASE" -echo "FRACTAL_TASKS_CORE_V1_GIT=$FRACTAL_TASKS_CORE_V1_GIT" -if [ -z "${FRACTAL_TASKS_CORE_V1_RELEASE}" ]; then - if [ -z "${FRACTAL_TASKS_CORE_V1_GIT}" ]; then - # Case 1: no release, no git - fractal task collect fractal-tasks-core --package-extras fractal-tasks - else - # Case 2: only git set - git clone https://github.com/fractal-analytics-platform/fractal-tasks-core.git - cd fractal-tasks-core - git checkout $FRACTAL_TASKS_CORE_V1_GIT - check_exit_code "Checking out to fractal-tasks-core@$FRACTAL_TASKS_CORE_V1_GIT" - - curl -sSL https://install.python-poetry.org | python3 - - check_exit_code "Poetry installation" - - /root/.local/bin/poetry build - - WHL=$(ls dist/*.whl) - check_exit_code "File .whl not found" - ABS_WHL="$(pwd)/$WHL" - chmod 777 "$ABS_WHL" - - cd .. - fractal task collect $ABS_WHL --package-extras fractal-tasks - fi -else - if [ -z "${FRACTAL_TASKS_CORE_V1_GIT}" ]; then - # Case 3: only release set - fractal task collect fractal-tasks-core --package-extras fractal-tasks --package-version $FRACTAL_TASKS_CORE_V1_RELEASE - else - # Case 4: both release and git set - echo "Error: cannot set both FRACTAL_TASKS_CORE_V1_RELEASE and FRACTAL_TASKS_CORE_V1_GIT." - exit 1 - fi -fi diff --git a/testing/demos-v1/requirements-validation.txt b/testing/demos-v1/requirements-validation.txt deleted file mode 100644 index c90e524..0000000 --- a/testing/demos-v1/requirements-validation.txt +++ /dev/null @@ -1 +0,0 @@ -zarr diff --git a/testing/demos-v1/run_demos.sh b/testing/demos-v1/run_demos.sh deleted file mode 100644 index 44de046..0000000 --- a/testing/demos-v1/run_demos.sh +++ /dev/null @@ -1,164 +0,0 @@ -#!/bin/bash - -set -e - -VERSION="[v1]" - -wait_for_job() { - THIS_PROJECT_ID=$1 - THIS_JOB_ID=$2 - DESCRIPTION=$3 - while true; do - STATUS_LINE=$(fractal job show "$THIS_PROJECT_ID" "$THIS_JOB_ID" | grep "status") - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "$VERSION Error: $DESCRIPTION (EXIT_CODE=$EXIT_CODE)" - exit 1 - fi - echo "$VERSION $DESCRIPTION $STATUS_LINE" - if [[ "$STATUS_LINE" == *done* || "$STATUS_LINE" == *failed* ]]; then - break - fi - sleep 4 - done -} - - -echo "$VERSION START run_all_demos.sh" - -# Due to how fractal-demos scripts are written, we should have a .fractal.env -# file also in 00_user_setup -cp .fractal.env 00_user_setup -cp .fractal.env 01_cardio_tiny_dataset -cp .fractal.env 02_cardio_small - -# Copy images from Resources folder -mkdir images -cp -r /home/fractal_share/Resources/images/10.5281_zenodo.8287221 images/ -cp -r /home/fractal_share/Resources/images/10.5281_zenodo.7057076 images/ - -# Pre-populate `zarr_dir` folders for both examples, so that we can make -# them broadly accessible -ZARRDIR01=./01_cardio_tiny_dataset/output_cardiac-tiny -ZARRDIR02=./02_cardio_small/output-cardio-2x2-zenodo-subset -PARAMETERSDIR01=./01_cardio_tiny_dataset/Parameters -PARAMETERSDIR02=./02_cardio_small/Parameters - -for THISDIR in "$ZARRDIR01" "$ZARRDIR02" "$PARAMETERSDIR01" "$PARAMETERSDIR02"; do - mkdir -p "$THISDIR" - chmod -R 777 "$THISDIR" -done - -# Whoami -fractal user whoami -FRACTAL_USER_ID=$(fractal --batch user whoami ) - -fractal user edit "$FRACTAL_USER_ID" --new-cache-dir /home/test01 --new-slurm-user test01 -# Trigger task collection -bash get_fractal_tasks_core.sh - -# Wait for task collection to be complete -while [ "$(fractal task list)" == "[]" ]; do - echo "$VERSION No task available, wait 10 seconds."; - sleep 10; -done - -# Enter 01_cardio_tiny_dataset folder -cd 01_cardio_tiny_dataset - -# Run example 01 and capture exit code -echo "$VERSION START examples/01 API calls" -TMPFILE="tmp_01_api.txt" -./run_example.sh >> $TMPFILE 2>&1 -API_EXITCODE=$? -cat $TMPFILE - -# Check exit code -if [ $API_EXITCODE -ne 0 ]; then - echo "$VERSION Error: examples/01 API_EXITCODE=$API_EXITCODE" - exit 1 -fi - -# Parse temporary file to extract PROJECT_ID and JOB_ID -PROJECT_ID=$(cat $TMPFILE | grep "PROJECT_ID" | cut -d '=' -f 2) -JOB_ID=$(cat $TMPFILE | grep "JOB_ID" | cut -d '=' -f 2) -echo "$VERSION PROJECT_ID=$PROJECT_ID" -echo "$VERSION JOB_ID=$JOB_ID" - -# Wait for job to be done or failed -wait_for_job "$PROJECT_ID" "$JOB_ID" "examples/01" - -# Check job status, once again -fractal job show "$PROJECT_ID" "$JOB_ID" -echo "$VERSION END examples/01 API calls" - -# Start output validation -echo "$VERSION START examples/01 output validation" -TMPFILE="tmp_01_validation.txt" -python validate_results.py >> $TMPFILE 2>&1 -VALIDATION_EXIT_CODE=$? -cat $TMPFILE - -# Check exit code -if [ $VALIDATION_EXIT_CODE -ne 0 ]; then - echo "$VERSION Error: examples/01 VALIDATION_EXIT_CODE=$VALIDATION_EXIT_CODE" - exit 1 -fi - -echo "$VERSION examples/01 API_EXITCODE=$API_EXITCODE" -echo "$VERSION examples/01 VALIDATION_EXIT_CODE=$VALIDATION_EXIT_CODE" -echo "$VERSION END examples/01 output validation" -echo "$VERSION" - -cd .. - -# Enter 02_cardio_small -cd 02_cardio_small - -# Run example 02 and capture exit code - -echo "$VERSION START examples/02 API calls" -TMPFILE="tmp_02_api.txt" - -./run_example_on_image_subset.sh >> $TMPFILE 2>&1 -API_EXITCODE=$? -cat $TMPFILE - -# Check exit code -if [ $API_EXITCODE -ne 0 ]; then - echo "$VERSION Error: examples/02 API_EXITCODE=$API_EXITCODE" - exit 1 -fi - -# Parse temporary file to extract PROJECT_ID and JOB_ID -PROJECT_ID=$(cat $TMPFILE | grep "PROJECT_ID" | cut -d '=' -f 2) -JOB_ID=$(cat $TMPFILE | grep "JOB_ID" | cut -d '=' -f 2) -echo "$VERSION PROJECT_ID=$PROJECT_ID" -echo "$VERSION JOB_ID=$JOB_ID" - -# Wait for job to be done or failed -wait_for_job "$PROJECT_ID" "$JOB_ID" "examples/02" - -# Check job status, once again -fractal job show "$PROJECT_ID" "$JOB_ID" -echo "$VERSION END examples/02 API calls" - -# Start output validation -echo "$VERSION START examples/02 output validation" -TMPFILE="tmp_02_validation.txt" -python validate_results.py >> $TMPFILE 2>&1 -VALIDATION_EXIT_CODE=$? -cat $TMPFILE - -# Check exit code -if [ $VALIDATION_EXIT_CODE -ne 0 ]; then - echo "$VERSION Error: examples/02 VALIDATION_EXIT_CODE=$VALIDATION_EXIT_CODE" - exit 1 -fi - -echo "$VERSION examples/02 API_EXITCODE=$API_EXITCODE" -echo "$VERSION examples/02 VALIDATION_EXIT_CODE=$VALIDATION_EXIT_CODE" -echo "$VERSION END examples/02 output validation" -echo "$VERSION" - -echo "$VERSION END run_all_demos.sh" diff --git a/testing/docker-compose.yml b/testing/docker-compose.yml index 93c4471..4ded82d 100644 --- a/testing/docker-compose.yml +++ b/testing/docker-compose.yml @@ -71,28 +71,6 @@ services: networks: - fractal - fractal-demos-v1: - container_name: fractal-demos-v1 - depends_on: - fractal-server: - condition: service_healthy - hostname: slurm - image: fractal-demos-v1 - build: - context: . - dockerfile: demos-v1/Dockerfile - ports: - - 6817:6818 - volumes: - - fractal_share:/home/fractal_share/ - - ./Resources:/home/fractal_share/Resources - command: bash -c " - cp -r ./fractal-demos-v1 /home/fractal_share/ - && cd /home/fractal_share/fractal-demos-v1/examples - && . ./run_demos.sh" - networks: - - fractal - fractal-demos-v2: container_name: fractal-demos-v2 depends_on: