Integrate the vineyard kedro with argo workflow. #877
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2020-2023 Alibaba Group Holding Limited. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: Vineyard Operator | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - 'v*' | |
| paths: | |
| - '.github/workflows/vineyard-operator.yaml' | |
| - 'k8s/**' | |
| - 'src/server/util/kubectl*' | |
| - 'charts/vineyard-operator/**' | |
| pull_request: | |
| branches: | |
| - main | |
| - dev/kubernetes | |
| paths: | |
| - '.github/workflows/vineyard-operator.yaml' | |
| - 'k8s/**' | |
| - 'src/server/util/kubectl*' | |
| - 'charts/vineyard-operator/**' | |
| concurrency: | |
| group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} | |
| cancel-in-progress: true | |
| env: | |
| REGISTRY: ghcr.io/v6d-io/v6d | |
| IMG: vineyardcloudnative/vineyard-operator:latest | |
| NIGHTLY_IMG: ghcr.io/v6d-io/v6d/vineyard-operator:nightly | |
| LATEST_IMG: ghcr.io/v6d-io/v6d/vineyard-operator:latest | |
| KUBECONFIG: /tmp/e2e-k8s.config | |
| jobs: | |
| ci: | |
| runs-on: ubuntu-20.04 | |
| if: ${{ github.repository == 'v6d-io/v6d' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| job: | |
| - release | |
| - e2e-tests-assembly-local | |
| - e2e-tests-assembly-distributed | |
| - e2e-tests-autogenerated-helm-chart | |
| - e2e-tests-failover | |
| - e2e-tests-repartition-dask | |
| - e2e-tests-scheduler-outside-cluster | |
| - e2e-tests-serialize | |
| - e2e-tests-sidecar | |
| - e2e-tests-spill | |
| - e2e-tests-workflow | |
| - e2e-tests-mars-examples | |
| - e2e-tests-vineyardctl | |
| - e2e-tests-deploy-raw-backup-and-recover | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: 1.19 | |
| - name: Cache for golang | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/go | |
| key: vineyard-kubernetes-go-${{ hashFiles('**/go.sum') }} | |
| restore-keys: | | |
| vineyard-kubernetes-go- | |
| - name: golangci-lint | |
| uses: golangci/golangci-lint-action@v3 | |
| with: | |
| version: v1.49.0 | |
| working-directory: k8s | |
| skip-pkg-cache: true | |
| - name: Golang format | |
| run: | | |
| function prepend() { while read line; do echo "${1}${line}"; done; } | |
| make -C k8s fmt | |
| GIT_DIFF=$(git diff --ignore-submodules) | |
| if [[ -n $GIT_DIFF ]]; then | |
| echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" | |
| echo "| gofmt failures found!" | |
| echo "|" | |
| echo "$GIT_DIFF" | prepend "| " | |
| echo "|" | |
| echo "| Run: " | |
| echo "|" | |
| echo "| make -C k8s fmt" | |
| echo "|" | |
| echo "| to fix this error." | |
| echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" | |
| exit -1 | |
| fi | |
| - name: Log in to GitHub Docker Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build docker image | |
| run: | | |
| make -C k8s docker-build | |
| - name: Extract tag name | |
| id: tag | |
| if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} | |
| run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT | |
| - name: Set up qemu | |
| if: ${{ matrix.job == 'release' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }} | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Set up Docker Buildx | |
| if: ${{ matrix.job == 'release' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }} | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Upload latest docker image | |
| if: ${{ matrix.job == 'release' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }} | |
| run: | | |
| # pre-publish to ghcr.io to avoid the "no basic auth credentials" error | |
| export IMG=ghcr.io/v6d-io/v6d/vineyard-operator:${{ steps.tag.outputs.TAG }} | |
| make -C k8s docker-build-push-multi-arch REGISTRY=${{ env.REGISTRY }} VERSION=${{ steps.tag.outputs.TAG }} | |
| export IMG=ghcr.io/v6d-io/v6d/vineyard-operator:latest | |
| make -C k8s docker-build-push-multi-arch REGISTRY=${{ env.REGISTRY }} VERSION=latest | |
| - name: Generate the python and vineyardd image for tests | |
| if: ${{ matrix.job != 'release' }} | |
| run: | | |
| make -C k8s/test/e2e build-base-images | |
| - name: Prepare directories for logs | |
| if: ${{ matrix.job != 'release' }} | |
| run: | | |
| # create log directories for all tasks to prevent the "Path to upload does not exist" warnings | |
| for job in e2e-tests-assembly-local \ | |
| e2e-tests-assembly-distributed \ | |
| e2e-tests-autogenerated-helm-chart \ | |
| e2e-tests-failover \ | |
| e2e-tests-repartition-dask \ | |
| e2e-tests-scheduler-outside-cluster \ | |
| e2e-tests-serialize \ | |
| e2e-tests-sidecar \ | |
| e2e-tests-spill \ | |
| e2e-tests-workflow \ | |
| e2e-tests-mars-examples \ | |
| e2e-tests-vineyardctl \ | |
| e2e-tests-deploy-raw-backup-and-recover; do | |
| mkdir -p $job-logs | |
| chmod a+rwx $job-logs | |
| done | |
| - name: e2e-tests-assembly-local | |
| if: ${{ matrix.job == 'e2e-tests-assembly-local' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-assembly-local | |
| - name: e2e-tests-assembly-distributed | |
| if: ${{ matrix.job == 'e2e-tests-assembly-distributed' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-assembly-distributed | |
| - name: e2e-tests-autogenerated-helm-chart | |
| if: ${{ matrix.job == 'e2e-tests-autogenerated-helm-chart' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-autogenerated-helm-chart | |
| - name: e2e-tests-failover | |
| if: ${{ matrix.job == 'e2e-tests-failover' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-failover | |
| - name: e2e-tests-repartition-dask | |
| if: ${{ matrix.job == 'e2e-tests-repartition-dask' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-repartition-dask | |
| - name: e2e-tests-scheduler-outside-cluster | |
| if: ${{ matrix.job == 'e2e-tests-scheduler-outside-cluster' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-scheduler-outside-cluster | |
| - name: e2e-tests-serialize | |
| if: ${{ matrix.job == 'e2e-tests-serialize' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-serialize | |
| - name: e2e-tests-sidecar | |
| if: ${{ matrix.job == 'e2e-tests-sidecar' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-sidecar | |
| - name: e2e-tests-spill | |
| if: ${{ matrix.job == 'e2e-tests-spill' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-spill | |
| - name: e2e-tests-workflow | |
| if: ${{ matrix.job == 'e2e-tests-workflow' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-workflow | |
| - name: e2e-tests-mars-examples | |
| if: false # temporarily disabled, we need some changes in mars | |
| # if: ${{ matrix.job == 'e2e-tests-mars-examples' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-mars-examples | |
| - name: e2e-tests-vineyardctl | |
| if: ${{ matrix.job == 'e2e-tests-vineyardctl' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-vineyardctl | |
| - name: e2e-tests-deploy-raw-backup-and-recover | |
| if: ${{ matrix.job == 'e2e-tests-deploy-raw-backup-and-recover' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-deploy-raw-backup-and-recover | |
| - name: Export kubernetes logs | |
| uses: dashanji/kubernetes-log-export-action@v3 | |
| if: ${{ matrix.job != 'release' && failure() }} | |
| env: | |
| SHOW_TIMESTAMPS: 'true' | |
| OUTPUT_DIR: ${{ github.workspace }}/${{ matrix.job }}-logs | |
| NAMESPACES: vineyard* | |
| - uses: stoat-dev/stoat-action@v0 | |
| if: ${{ matrix.job != 'release' && failure() }} | |
| - name: Leave a message for logs URL | |
| if: ${{ matrix.job != 'release' && failure() }} | |
| run: | | |
| short_sha=$(git rev-parse --short ${{ github.event.pull_request.head.sha }}) | |
| case ${{ matrix.job }} in | |
| e2e-tests-spill) | |
| signed_url_key=0707 | |
| ;; | |
| e2e-tests-sidecar) | |
| signed_url_key=eb62 | |
| ;; | |
| e2e-tests-failover) | |
| signed_url_key=4302 | |
| ;; | |
| e2e-tests-workflow) | |
| signed_url_key=2355 | |
| ;; | |
| e2e-tests-serialize) | |
| signed_url_key=ab53 | |
| ;; | |
| e2e-tests-vineyardctl) | |
| signed_url_key=cb83 | |
| ;; | |
| e2e-tests-assembly-local) | |
| signed_url_key=2cdf | |
| ;; | |
| e2e-tests-repartition-dask) | |
| signed_url_key=fb84 | |
| ;; | |
| e2e-tests-assembly-distributed) | |
| signed_url_key=2573 | |
| ;; | |
| e2e-tests-autogenerated-helm-chart) | |
| signed_url_key=7a77 | |
| ;; | |
| e2e-tests-scheduler-outside-cluster) | |
| signed_url_key=0c6a | |
| ;; | |
| e2e-tests-deploy-raw-backup-and-recover) | |
| signed_url_key=6552 | |
| ;; | |
| esac | |
| echo "::notice:: ${{ matrix.job }}: https://www.stoat.dev/file-viewer?root=https://v6d-io--v6d--$short_sha--e2e-tests-$signed_url_key.stoat.page" |