Add the example code of how to accelerate the data sharing with vineyard on fluid platform. #1279
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2020-2023 Alibaba Group Holding Limited. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: Vineyard Operator | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - 'v*' | |
| paths: | |
| - '.github/workflows/vineyard-operator.yaml' | |
| - 'charts/vineyard-operator/**' | |
| - 'docker/Dockerfile.vineyardd*' | |
| - 'k8s/**' | |
| - 'src/server/util/kubectl*' | |
| pull_request: | |
| branches: | |
| - main | |
| - dev/kubernetes | |
| paths: | |
| - '.github/workflows/vineyard-operator.yaml' | |
| - 'charts/vineyard-operator/**' | |
| - 'docker/Dockerfile.vineyardd*' | |
| - 'k8s/**' | |
| - 'src/server/util/kubectl*' | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} | |
| cancel-in-progress: true | |
| env: | |
| REGISTRY: ghcr.io/v6d-io/v6d | |
| IMG: vineyardcloudnative/vineyard-operator:latest | |
| NIGHTLY_IMG: ghcr.io/v6d-io/v6d/vineyard-operator:nightly | |
| LATEST_IMG: ghcr.io/v6d-io/v6d/vineyard-operator:latest | |
| KUBECONFIG: /tmp/e2e-k8s.config | |
| jobs: | |
| ci: | |
| runs-on: ubuntu-20.04 | |
| if: ${{ github.repository == 'v6d-io/v6d' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| job: | |
| - release | |
| - unit-tests | |
| - e2e-tests-airflow-integration | |
| - e2e-tests-assembly-local | |
| - e2e-tests-assembly-distributed | |
| - e2e-tests-autogenerated-helm-chart | |
| - e2e-tests-failover | |
| - e2e-tests-repartition-dask | |
| - e2e-tests-schedule-workflow | |
| - e2e-tests-schedule-workflow-without-crd | |
| - e2e-tests-schedule-workload | |
| - e2e-tests-serialize | |
| - e2e-tests-sidecar | |
| - e2e-tests-spill | |
| - e2e-tests-workflow | |
| - e2e-tests-mars-examples | |
| - e2e-tests-vineyardctl | |
| - e2e-tests-deploy-raw-backup-and-recover | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Free disk space | |
| run: | | |
| bash .github/free-disk-space.sh | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: 1.19 | |
| - name: Cache for golang | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/go | |
| key: vineyard-kubernetes-go-${{ hashFiles('**/go.sum') }} | |
| restore-keys: | | |
| vineyard-kubernetes-go- | |
| - name: golangci-lint | |
| uses: golangci/golangci-lint-action@v3 | |
| with: | |
| version: v1.50.0 | |
| working-directory: k8s | |
| skip-pkg-cache: true | |
| - name: Golang format | |
| run: | | |
| function prepend() { while read line; do echo "${1}${line}"; done; } | |
| make -C k8s fmt | |
| GIT_DIFF=$(git diff --ignore-submodules) | |
| if [[ -n $GIT_DIFF ]]; then | |
| echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" | |
| echo "| gofmt failures found!" | |
| echo "|" | |
| echo "$GIT_DIFF" | prepend "| " | |
| echo "|" | |
| echo "| Run: " | |
| echo "|" | |
| echo "| make -C k8s fmt" | |
| echo "|" | |
| echo "| to fix this error." | |
| echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" | |
| exit -1 | |
| fi | |
| - name: Log in to GitHub Docker Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build docker image | |
| run: | | |
| make -C k8s docker-build | |
| - name: Extract tag name | |
| id: tag | |
| if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} | |
| run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT | |
| - name: Set up qemu | |
| if: ${{ matrix.job == 'release' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }} | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Set up Docker Buildx | |
| if: ${{ matrix.job == 'release' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }} | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Upload latest docker image of vineyard-operator | |
| if: ${{ matrix.job == 'release' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }} | |
| run: | | |
| # pre-publish to ghcr.io to avoid the "no basic auth credentials" error | |
| export IMG=ghcr.io/v6d-io/v6d/vineyard-operator:${{ steps.tag.outputs.TAG }} | |
| make -C k8s docker-build-push-multi-arch REGISTRY=${{ env.REGISTRY }} VERSION=${{ steps.tag.outputs.TAG }} | |
| export IMG=ghcr.io/v6d-io/v6d/vineyard-operator:latest | |
| make -C k8s docker-build-push-multi-arch REGISTRY=${{ env.REGISTRY }} VERSION=latest | |
| - name: Upload latest docker image of vineyard-fluid-fuse | |
| if: ${{ matrix.job == 'release' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }} | |
| run: | | |
| make -C docker vineyard-fluid-fuse | |
| docker tag ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:latest \ | |
| ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:${{ steps.tag.outputs.TAG }} | |
| docker push ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:${{ steps.tag.outputs.TAG }} | |
| docker push ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:latest | |
| - name: Generate the python and vineyardd image for tests | |
| run: | | |
| make -C k8s/test/e2e build-base-images | |
| - name: unit-tests | |
| if: ${{ matrix.job == 'unit-tests' }} | |
| run: | | |
| make -C k8s/test/e2e unittest | |
| - name: Prepare directories for logs | |
| run: | | |
| # create log directories for all tasks to prevent the "Path to upload does not exist" warnings | |
| for job in e2e-tests-airflow-integration \ | |
| e2e-tests-assembly-local \ | |
| e2e-tests-assembly-distributed \ | |
| e2e-tests-autogenerated-helm-chart \ | |
| e2e-tests-failover \ | |
| e2e-tests-repartition-dask \ | |
| e2e-tests-schedule-workflow \ | |
| e2e-tests-schedule-workflow-without-crd \ | |
| e2e-tests-schedule-workload \ | |
| e2e-tests-serialize \ | |
| e2e-tests-sidecar \ | |
| e2e-tests-spill \ | |
| e2e-tests-workflow \ | |
| e2e-tests-mars-examples \ | |
| e2e-tests-vineyardctl \ | |
| e2e-tests-deploy-raw-backup-and-recover \ | |
| unit-tests; do | |
| mkdir -p $job-logs | |
| chmod a+rwx $job-logs | |
| done | |
| - name: Start to export kubernetes logs | |
| uses: dashanji/kubernetes-log-export-action@v6 | |
| env: | |
| SHOW_TIMESTAMPS: 'true' | |
| OUTPUT_DIR: ${{ github.workspace }}/${{ matrix.job }}-logs | |
| NAMESPACES: vineyard* | |
| RESOURCES: configmap | |
| MODE: start | |
| - name: e2e-tests-airflow-integration | |
| if: ${{ matrix.job == 'e2e-tests-airflow-integration' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-airflow-integration | |
| - name: e2e-tests-assembly-local | |
| if: ${{ matrix.job == 'e2e-tests-assembly-local' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-assembly-local | |
| - name: e2e-tests-assembly-distributed | |
| if: ${{ matrix.job == 'e2e-tests-assembly-distributed' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-assembly-distributed | |
| - name: e2e-tests-autogenerated-helm-chart | |
| if: ${{ matrix.job == 'e2e-tests-autogenerated-helm-chart' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-autogenerated-helm-chart | |
| - name: e2e-tests-failover | |
| if: ${{ matrix.job == 'e2e-tests-failover' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-failover | |
| - name: e2e-tests-repartition-dask | |
| if: ${{ matrix.job == 'e2e-tests-repartition-dask' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-repartition-dask | |
| - name: e2e-tests-schedule-workflow | |
| if: ${{ matrix.job == 'e2e-tests-schedule-workflow' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-schedule-workflow | |
| - name: e2e-tests-schedule-workflow-without-crd | |
| if: ${{ matrix.job == 'e2e-tests-schedule-workflow-without-crd' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-schedule-workflow-without-crd | |
| - name: e2e-tests-schedule-workload | |
| if: ${{ matrix.job == 'e2e-tests-schedule-workload' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-schedule-workload | |
| - name: e2e-tests-serialize | |
| if: ${{ matrix.job == 'e2e-tests-serialize' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-serialize | |
| - name: e2e-tests-sidecar | |
| if: ${{ matrix.job == 'e2e-tests-sidecar' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-sidecar | |
| - name: e2e-tests-spill | |
| if: ${{ matrix.job == 'e2e-tests-spill' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-spill | |
| - name: e2e-tests-workflow | |
| if: ${{ matrix.job == 'e2e-tests-workflow' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-workflow | |
| - name: e2e-tests-mars-examples | |
| if: false # temporarily disabled, we need some changes in mars | |
| # if: ${{ matrix.job == 'e2e-tests-mars-examples' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-mars-examples | |
| - name: e2e-tests-vineyardctl | |
| if: ${{ matrix.job == 'e2e-tests-vineyardctl' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-vineyardctl | |
| - name: e2e-tests-deploy-raw-backup-and-recover | |
| if: ${{ matrix.job == 'e2e-tests-deploy-raw-backup-and-recover' }} | |
| run: | | |
| make -C k8s/test/e2e e2e-tests-deploy-raw-backup-and-recover | |
| - name: Stop to export kubernetes logs | |
| uses: dashanji/kubernetes-log-export-action@v6 | |
| if: ${{ failure() }} | |
| env: | |
| MODE: stop | |
| OUTPUT_DIR: ${{ github.workspace }}/${{ matrix.job }}-logs | |
| WITH_STOAT: true | |
| - name: Setup tmate session | |
| if: false | |
| uses: mxschmitt/action-tmate@v3 | |
| - uses: stoat-dev/stoat-action@v0 | |
| if: ${{ failure() }} | |
| - name: Leave a message for logs URL | |
| if: ${{ failure() }} | |
| run: | | |
| sha=${{ github.event.pull_request.head.sha || github.sha }} | |
| short_sha=$(git rev-parse --short $sha) | |
| case ${{ matrix.job }} in | |
| e2e-tests-spill) | |
| signed_url_key=0707 | |
| ;; | |
| e2e-tests-sidecar) | |
| signed_url_key=eb62 | |
| ;; | |
| e2e-tests-failover) | |
| signed_url_key=4302 | |
| ;; | |
| e2e-tests-workflow) | |
| signed_url_key=2355 | |
| ;; | |
| e2e-tests-serialize) | |
| signed_url_key=ab53 | |
| ;; | |
| e2e-tests-vineyardctl) | |
| signed_url_key=cb83 | |
| ;; | |
| e2e-tests-assembly-local) | |
| signed_url_key=2cdf | |
| ;; | |
| e2e-tests-repartition-dask) | |
| signed_url_key=fb84 | |
| ;; | |
| e2e-tests-assembly-distributed) | |
| signed_url_key=2573 | |
| ;; | |
| e2e-tests-autogenerated-helm-chart) | |
| signed_url_key=7a77 | |
| ;; | |
| e2e-tests-schedule-workload) | |
| signed_url_key=3d07 | |
| ;; | |
| e2e-tests-schedule-workflow) | |
| signed_url_key=44c1 | |
| ;; | |
| e2e-tests-schedule-workflow-without-crd) | |
| signed_url_key=9a01 | |
| ;; | |
| e2e-tests-deploy-raw-backup-and-recover) | |
| signed_url_key=6552 | |
| ;; | |
| e2e-tests-airflow-integration) | |
| signed_url_key=2e12 | |
| ;; | |
| esac | |
| echo "::notice:: ${{ matrix.job }}: https://www.stoat.dev/file-viewer?root=https://v6d-io--v6d--$short_sha--e2e-tests-$signed_url_key.stoat.page" |