Fix integration tests #7467
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: viash test | |
| on: | |
| pull_request: | |
| push: | |
| branches: [ main ] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ !contains(github.ref, 'main')}} | |
| jobs: | |
| # phase 1 | |
| list: | |
| env: | |
| s3_bucket: s3://openpipelines-data/ | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.detect_changed_components.outputs.output_matrix }} | |
| cache_key: ${{ steps.cache.outputs.cache_key }} | |
| dest_paths: ${{ steps.cache.outputs.dest_paths }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' | |
| - uses: data-intuitive/reclaim-the-bytes@v2 | |
| - uses: viash-io/viash-actions/setup@v6 | |
| - name: Check if all config can be parsed if there is no unicode support | |
| run: | | |
| LANG=C viash ns list > /dev/null | |
| JAVA_TOOL_OPTIONS=-Dfile.encoding=ascii viash ns build --parallel > /dev/null | |
| - uses: viash-io/viash-actions/project/sync-and-cache@v6 | |
| id: cache | |
| - id: ns_list | |
| uses: viash-io/viash-actions/ns-list@v6 | |
| with: | |
| engine: docker | |
| runner: executable | |
| format: json | |
| query_namespace: ^(?!workflows|test_workflows) | |
| - id: detect_changed_components | |
| uses: viash-io/viash-actions/project/detect-changed-components@v6 | |
| with: | |
| input_file: "${{ steps.ns_list.outputs.output_file }}" | |
| # phase 2 | |
| viash_test: | |
| needs: list | |
| if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| component: ${{ fromJson(needs.list.outputs.matrix) }} | |
| steps: | |
| # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' | |
| - uses: data-intuitive/reclaim-the-bytes@v2 | |
| - uses: actions/checkout@v4 | |
| - uses: viash-io/viash-actions/setup@v6 | |
| - uses: viash-io/viash-actions/project/update-docker-engine@v6 | |
| # use cache | |
| - name: Restore test resources cache | |
| if: ${{ needs.list.outputs.cache_key != '' }} | |
| uses: actions/cache/restore@v4 | |
| timeout-minutes: 10 | |
| with: | |
| path: ${{ needs.list.outputs.dest_paths }} | |
| key: ${{ needs.list.outputs.cache_key }} | |
| fail-on-cache-miss: true | |
| - name: Remove unused test resources to save space (only when restoring from cache) | |
| shell: bash | |
| run: | | |
| readarray -t resources < <(viash config view --format json "${{ matrix.component.config }}" | jq -r -c '(.build_info.config | capture("^(?<dir>.*\/)").dir) as $dir | .test_resources | map(select(.type == "file")) | map($dir + .path) | unique | .[]') | |
| to_not_remove=() | |
| for resource in "${resources[@]}"; do | |
| if [[ $resource == *"resources_test"* ]]; then | |
| relative_path=${resource#*resources_test/} | |
| relative_path_trailing_slash_removed=${relative_path%/} | |
| to_not_remove+=("-path" "./resources_test/$relative_path_trailing_slash_removed" "-prune" "-o") | |
| fi | |
| done | |
| # Remove last prune and -o | |
| if (( ${#errors[@]} )); then | |
| unset 'to_not_remove[${#to_not_remove[@]}-1]' | |
| unset 'to_not_remove[${#to_not_remove[@]}-1]' | |
| to_not_remove+=( "(" "${to_not_remove[@]}" ")" "-prune" "-o") | |
| fi | |
| find ./resources_test/ "${to_not_remove[@]}" -type f -exec rm {} + | |
| - name: Login to the nvidia container registry | |
| uses: docker/login-action@v3 | |
| env: | |
| NVIDIA_PASSWORD: ${{ secrets.NVIDIA_PASSWORD }} | |
| if: ${{ env.NVIDIA_PASSWORD != '' }} | |
| with: | |
| registry: nvcr.io | |
| username: $oauthtoken | |
| password: ${{ env.NVIDIA_PASSWORD }} | |
| - name: Run test | |
| timeout-minutes: 40 | |
| run: | | |
| viash test \ | |
| "${{ matrix.component.config }}" \ | |
| --cpus 4 \ | |
| --memory "14gb" \ | |
| --engine docker \ | |
| --runner executable |