Add component to subset obsp (#888) #1094
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ inputs.deploy_branch && inputs.deploy_branch || format('{0}_build', github.ref_name) }} | |
| cancel-in-progress: true | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| push_containers: | |
| type: boolean | |
| required: false | |
| default: false | |
| description: Build docker images and push them to the registry | |
| version: | |
| type: string | |
| required: false | |
| description: | | |
| Version to tag the build components with (e.i functionality.version). | |
| Defaults to name of the branch that triggered the workflow, suffixed by "_build". | |
| target_tag: | |
| type: string | |
| required: false | |
| description: | | |
| Version tag of containers to use. Defaults to name of the branch that triggered the workflow, | |
| suffixed by "_build". Can be used in combination with 'push_containers' (by unchecking it) | |
| to re-use existing docker images or set the tag for new builds. | |
| # when used as a subworkflow | |
| workflow_call: | |
| inputs: | |
| push_containers: | |
| type: boolean | |
| required: false | |
| default: false | |
| description: push the containers to the registry | |
| version: | |
| type: string | |
| required: false | |
| description: | | |
| Version to tag the build components with (e.i functionality.version). | |
| Defaults to name of the branch that triggered the workflow, suffixed by "_build". | |
| target_tag: | |
| type: string | |
| required: false | |
| description: | | |
| Version tag of containers to use. Defaults to name of the branch that triggered the workflow, | |
| suffixed by "_build". Can be used in combination with 'push_containers' (by setting it to 'false') | |
| to re-use existing docker images or set the tag for new builds. | |
| deploy_branch: | |
| type: string | |
| required: false | |
| description: | | |
| Branch to deploy the build to. Defaults to name of the branch | |
| that triggered the workflow, suffixed by "_build". | |
| is_workflow_call: | |
| description: | | |
| Flag to destinguish between a 'workflow_call' and 'workflow_dispatch'. | |
| github.event_name can not be used for this because it refers to the event of | |
| the top-level workflow. See https://github.com/actions/runner/issues/3146 | |
| type: boolean | |
| required: false | |
| default: true | |
| outputs: | |
| component_matrix: | |
| description: "A JSON object that can be used to populate a github actions matrix for component jobs." | |
| value: ${{ jobs.build_and_deploy_target_folder.outputs.component_matrix }} | |
| workflow_matrix: | |
| description: "A JSON object that can be used to populate a github actions matrix for workflow jobs." | |
| value: ${{ jobs.build_and_deploy_target_folder.outputs.workflow_matrix}} | |
| secrets: | |
| GTHB_USER: | |
| required: true | |
| GTHB_PAT: | |
| required: true | |
| push: | |
| branches: [ 'main' ] | |
| jobs: | |
| # phase 1 | |
| build_and_deploy_target_folder: | |
| name: "Build and push target folder" | |
| runs-on: ubuntu-latest | |
| outputs: | |
| component_matrix: ${{ steps.set_matrix.outputs.components }} | |
| workflow_matrix: ${{ steps.set_matrix.outputs.workflows }} | |
| env: | |
| DEPLOY_BRANCH: ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch }} | |
| steps: | |
| - name: Do not allow manual trigger if the branch is main | |
| # github.event_name is the event of the top-level workflow, so use dummy 'is_workflow_call' input | |
| if: ${{ github.event_name != 'push' && (!inputs.is_workflow_call) && github.ref == 'refs/heads/main' }} | |
| run: | | |
| echo "This workflow should not be triggered with workflow_dispatch from the main branch" | |
| exit 1 | |
| - name: Check input arguments | |
| run: | | |
| input_version="${{ inputs.version || format('{0}_build', github.ref_name) }}" | |
| target_tag="${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}" | |
| should_push="${{inputs.push_containers }}" | |
| if [ "$input_version" != "$target_tag" ] && [ "$should_push" == "true" ]; then | |
| echo "When trying to push new docker images, the tag for the components must be equal to the target_tag for the docker images." | |
| exit 1 | |
| fi | |
| - name: Keep symlinks as-is | |
| run: | | |
| git config --global core.symlinks true | |
| - name: Branch to checkout (use existing target branch if it exists) | |
| id: get_checkout_branch | |
| run: | | |
| if ! git ls-remote --heads --exit-code https://github.com/openpipelines-bio/openpipeline.git "$DEPLOY_BRANCH" > /dev/null; then | |
| echo "Remote branch does not exist, fetching current branch and building on top of it" | |
| echo "checkout_branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Remote branch exists, checking out existing branch" | |
| echo "checkout_branch=$DEPLOY_BRANCH" >> "$GITHUB_OUTPUT" | |
| fi | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.get_checkout_branch.outputs.checkout_branch }} | |
| fetch-depth: 0 | |
| - name: Fetch changes from ${{github.ref_name}} | |
| run: | | |
| git fetch origin ${{github.ref_name}} | |
| git checkout -f --no-overlay origin/${{github.ref_name}} -- '.' | |
| - uses: viash-io/viash-actions/setup@v6 | |
| - name: Remove target folder from .gitignore | |
| run: | | |
| # allow publishing the target folder | |
| sed -i '/^\/target\/$/d' .gitignore | |
| - uses: viash-io/viash-actions/ns-build@v6 | |
| with: | |
| config_mod: | | |
| .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" | |
| .engines[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}' | |
| parallel: true | |
| query: ^(?!workflows|test_workflows) | |
| - uses: viash-io/viash-actions/ns-build@v6 | |
| with: | |
| config_mod: .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" | |
| parallel: true | |
| query: ^workflows | |
| - name: Build nextflow schemas | |
| uses: viash-io/viash-actions/pro/build-nextflow-schemas@v6 | |
| with: | |
| target_dir: target/nextflow | |
| viash_pro_token: ${{ secrets.GTHB_PAT }} | |
| tools_version: 'main_build' | |
| enable_dataset_input: true | |
| - name: Build parameter files | |
| uses: viash-io/viash-actions/pro/build-nextflow-params@v6 | |
| with: | |
| target_dir: target/nextflow | |
| viash_pro_token: ${{ secrets.GTHB_PAT }} | |
| tools_version: 'main_build' | |
| - name: Deploy to target branch | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| create_branch: true | |
| commit_message: "deploy: ${{github.sha}}" | |
| skip_dirty_check: true | |
| branch: ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch }} | |
| - name: "List components" | |
| id: ns_list | |
| uses: viash-io/viash-actions/ns-list@v6 | |
| with: | |
| engine: docker | |
| src: src | |
| format: json | |
| query_namespace: ^(?!workflows)|(workflows/test_workflows) | |
| - name: "List workflows" | |
| id: ns_list_workflows | |
| uses: viash-io/viash-actions/ns-list@v6 | |
| with: | |
| src: src | |
| format: json | |
| query_namespace: ^workflows/(?!test_workflows) | |
| - name: "Parse JSON output from 'viash ns list' as input for matrix." | |
| id: set_matrix | |
| run: | | |
| echo "components=$(jq -c '[ .[] | | |
| { | |
| "name": .name, | |
| "namespace_separator": .engines | map(select(.type == "docker"))[0].namespace_separator, | |
| "namespace": .namespace, | |
| "config": .build_info.config, | |
| "dir": .build_info.config | capture("^(?<dir>.*\/)").dir | |
| } | |
| ] | unique' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT | |
| echo "workflows=$(jq -c '[ .[] | . as $config | (.test_resources // [])[] | select(.type == "nextflow_script", .entrypoint) | | |
| { | |
| "name": $config.name, | |
| "namespace": $config.namespace, | |
| "main_script": (($config.build_info.config | capture("^(?<dir>.*\/)").dir) + "/" + .path), | |
| "entry": .entrypoint, | |
| "config": $config.build_info.config | |
| } | |
| ] | unique' ${{ steps.ns_list_workflows.outputs.output_file }} )" >> $GITHUB_OUTPUT | |
| # phase 2 | |
| build_and_deploy_docker_containers: | |
| name: "Build and Deploy Docker Images" | |
| needs: build_and_deploy_target_folder | |
| if: ${{github.event_name == 'push' || inputs.push_containers }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| component: ${{ fromJson(needs.build_and_deploy_target_folder.outputs.component_matrix) }} | |
| steps: | |
| # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' | |
| - uses: data-intuitive/reclaim-the-bytes@v2 | |
| - uses: actions/checkout@v4 | |
| - uses: viash-io/viash-actions/setup@v6 | |
| - name: Build container | |
| uses: viash-io/viash-actions/ns-build@v6 | |
| with: | |
| config_mod: | | |
| .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" | |
| .engines[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || inputs.target_tag }}' | |
| engine: docker | |
| src: ${{ matrix.component.dir }} | |
| setup: build | |
| - name: Login to container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.GTHB_USER }} | |
| password: ${{ secrets.GTHB_PAT }} | |
| - name: Push container | |
| uses: viash-io/viash-actions/ns-build@v6 | |
| with: | |
| config_mod: .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" | |
| engine: docker | |
| src: ${{ matrix.component.dir }} | |
| setup: push |