integration-test.yml

name: integration test
concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: false

on:
  workflow_dispatch:
    inputs:
      push_containers:
        type: boolean
        required: false
        default: true
        description: Build docker images and push them to the registry
  schedule:
    - cron: '33 2 * * *'

jobs:
  # Build and create containers
  build:
    uses: ./.github/workflows/main-build.yml
    with:
      push_containers: ${{ github.event_name == 'schedule' || inputs.push_containers }}
      version: 'integration_build'
      target_tag: 'integration_build'
      deploy_branch: 'integration_build'
    secrets: inherit

  # Synchronize S3 Bucket and create cache for per-component runs
  sync_s3:
    env:
      s3_bucket: s3://openpipelines-data/
    runs-on: ubuntu-latest

    outputs:
      cache_key: ${{ steps.cache.outputs.cache_key }}
      dest_paths: ${{ steps.cache.outputs.dest_paths }}

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - uses: viash-io/viash-actions/project/sync-and-cache@v6
      id: cache

  # phase 3
  integration_test:
    runs-on: ubuntu-latest
    needs: [ build, sync_s3 ]
    if: "${{ needs.build.outputs.workflow_matrix != '[]' }}"

    strategy:
      fail-fast: false
      matrix:
        component: ${{ fromJson(needs.build.outputs.workflow_matrix) }}

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
    - uses: data-intuitive/reclaim-the-bytes@v2
    
    - name: Keep symlinks as-is
      run: |
        git config --global core.symlinks true

    - uses: actions/checkout@v4
      with:
        ref: 'integration_build'

    - uses: viash-io/viash-actions/setup@v6

    - uses: nf-core/setup-nextflow@v2.0.0

    # use cache
    - name: Cache resources data
      if: ${{ needs.sync_s3.outputs.cache_key != '' }}
      uses: actions/cache/restore@v4
      timeout-minutes: 10
      with:
        path: ${{ needs.sync_s3.outputs.dest_paths }}
        key: ${{ needs.sync_s3.outputs.cache_key }}
        fail-on-cache-miss: true

    - name: Remove unused test resources to save space
      shell: bash
      run: |
        readarray -t resources < <(viash config view --format json "${{ matrix.component.config }}" -c 'del(.dependencies)' | jq -r -c '(.build_info.config | capture("^(?<dir>.*\/)").dir) as $dir | .test_resources | map(select(.type == "file")) | map($dir + .path) | unique | .[]')
        to_not_remove=()
        for resource in "${resources[@]}"; do
          if [[ $resource == *"resources_test"* ]]; then
            relative_path=${resource#*resources_test/}
            relative_path_trailing_slash_removed=${relative_path%/}
            to_not_remove+=("-path" "./resources_test/$relative_path_trailing_slash_removed" "-prune" "-o")
          fi
        done
        # Remove last prune and -o
        if (( ${#errors[@]} )); then
          unset 'to_not_remove[${#to_not_remove[@]}-1]'
          unset 'to_not_remove[${#to_not_remove[@]}-1]'
          to_not_remove+=( "(" "${to_not_remove[@]}" ")" "-prune" "-o")
        fi
        echo "Not removing ${to_not_remove[@]}"
        find ./resources_test/ "${to_not_remove[@]}" -type f -exec rm {} +
        tree ./resources_test/

    - name: Build test dependencies
      uses: viash-io/viash-actions/ns-build@v6
      with:
        config_mod: |
          .version := 'integration_build'
          .engines[.type == 'docker'].target_tag := 'integration_build'
        src: ${{ matrix.component.dir }}
        query: "^test_workflows"
        setup: build

    - name: Run integration test
      timeout-minutes: 60
      run: |
        # todo: replace with viash test command
        export NXF_VER=22.04.5
        nextflow run . \
          -main-script "${{ matrix.component.main_script }}" \
          -entry "${{ matrix.component.entry }}" \
          -profile docker,mount_temp,no_publish \
          -c src/workflows/utils/labels_ci.config \
          -c src/workflows/utils/integration_tests.config