Merge remote-tracking branch 'origin/feature/diff-based-ci' into w/12… #3076
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Pre merge | |
| concurrency: | |
| # This stops any build on a branch if a new commit is pushed on that branch | |
| # the second check is here to run each build when pushing on the branch `q/*` | |
| # in order for bert-e to merge any branch that passes the queue before waiting | |
| # for the entire queue to passe the build. | |
| # if the branch is 'q/*' then we append the commit sha to the concurrency group to make it unique. | |
| # this works because the '||' or condition will only evaluate the right side if the left side is false ;-) | |
| group: "tests-${{ github.ref_name }}-${{ ( !startsWith(github.ref_name, 'q/') || github.sha) }}" | |
| cancel-in-progress: true | |
| on: | |
| push: | |
| branches: | |
| - "release/**" | |
| - "feature/**" | |
| - "improvement/**" | |
| - "hotfix/**" | |
| - "bugfix/**" | |
| - "documentation/**" | |
| - "user/**" | |
| - "dependabot/**" | |
| - "w/**" | |
| - "q/*/**" | |
| jobs: | |
| changed-files: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| docs: ${{ steps.diff.outputs.docs_any_modified == 'true' }} | |
| shell-ui: ${{ steps.diff.outputs.shell-ui_any_modified == 'true' }} | |
| unit-tests-ui: ${{ steps.diff.outputs.unit-tests-ui_any_modified == 'true' }} | |
| unit-tests-shell-ui: ${{ steps.diff.outputs.unit-tests-shell-ui_any_modified == 'true' }} | |
| unit-tests-crd-client-generator: ${{ steps.diff.outputs.unit-tests-crd-client-generator_any_modified == 'true' }} | |
| unit-tests-metalk8s-operator: ${{ steps.diff.outputs.unit-tests-metalk8s-operator_any_modified == 'true' }} | |
| unit-tests-storage-operator: ${{ steps.diff.outputs.unit-tests-storage-operator_any_modified == 'true' }} | |
| unit-tests-salt: ${{ steps.diff.outputs.unit-tests-salt_any_modified == 'true' }} | |
| unit-tests-lib-alert-tree: ${{ steps.diff.outputs.unit-tests-lib-alert-tree_any_modified == 'true' }} | |
| integration-tests-ui: ${{ steps.diff.outputs.integration-tests-ui_any_modified == 'true' }} | |
| # If the workflow is triggered on a development branch let's always build | |
| build: ${{ startsWith(github.ref_name, 'development/') || steps.diff.outputs.build_any_modified == 'true' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get base sha | |
| run: | | |
| source VERSION | |
| branch=development/${VERSION_MAJOR}.${VERSION_MINOR} | |
| if [[ ! $(git ls-remote --heads origin $branch) ]]; then | |
| branch=development/${VERSION_MAJOR} | |
| fi | |
| echo "BASE_SHA=$(git merge-base HEAD origin/$branch)" >> $GITHUB_ENV | |
| - name: Get changed files | |
| id: diff | |
| uses: tj-actions/changed-files@v45 | |
| with: | |
| base_sha: ${{ env.BASE_SHA }} | |
| files_yaml_from_source_file: .github/changed-files.yaml | |
| - name: List all changed files | |
| env: | |
| ALL_DOCS_CHANGED_FILES: ${{ steps.diff.outputs.docs_all_changed_files }} | |
| ALL_SHELL_UI_CHANGED_FILES: ${{ steps.diff.outputs.shell-ui_all_changed_files }} | |
| ALL_UNIT_TESTS_UI_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-ui_all_changed_files }} | |
| ALL_UNIT_TESTS_SHELL_UI_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-shell-ui_all_changed_files }} | |
| ALL_UNIT_TESTS_CRD_CLIENT_GENERATOR_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-crd-client-generator_all_changed_files }} | |
| ALL_UNIT_TESTS_METALK8S_OPERATOR_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-metalk8s-operator_all_changed_files }} | |
| ALL_UNIT_TESTS_STORAGE_OPERATOR_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-storage-operator_all_changed_files }} | |
| ALL_UNIT_TESTS_SALT_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-salt_all_changed_files }} | |
| ALL_UNIT_TESTS_LIB_ALERT_TREE_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-lib-alert-tree_all_changed_files }} | |
| ALL_INTEGRATION_TESTS_UI_CHANGED_FILES: ${{ steps.diff.outputs.integration-tests-ui_all_changed_files }} | |
| ALL_BUILD_CHANGED_FILES: ${{ steps.diff.outputs.build_all_changed_files }} | |
| run: | | |
| for type in docs shell-ui unit-tests-ui unit-tests-shell-ui unit-tests-crd-client-generator unit-tests-metalk8s-operator unit-tests-storage-operator unit-tests-salt unit-tests-lib-alert-tree integration-tests-ui build; do | |
| echo "All $type changed files:" | |
| changed_files_var=$(echo "ALL_${type^^}_CHANGED_FILES" | tr '-' '_') | |
| echo "${!changed_files_var}" | |
| done | |
| build: | |
| needs: changed-files | |
| if: needs.changed-files.outputs.build == 'true' | |
| uses: ./.github/workflows/build.yaml | |
| secrets: inherit | |
| build-docs: | |
| needs: changed-files | |
| # Build of docs is triggered by "build" so do not re-trigger it | |
| if: needs.changed-files.outputs.docs == 'true' && needs.changed-files.outputs.build != 'true' | |
| uses: ./.github/workflows/build-docs.yaml | |
| secrets: inherit | |
| build-shell-ui: | |
| needs: changed-files | |
| # Build of shell-ui is triggered by "build" so do not re-trigger it | |
| if: needs.changed-files.outputs.shell-ui == 'true' && needs.changed-files.outputs.build != 'true' | |
| uses: ./.github/workflows/build-shell-ui.yaml | |
| secrets: inherit | |
| e2e-tests: | |
| needs: | |
| - build | |
| - changed-files | |
| if: needs.changed-files.outputs.build == 'true' | |
| uses: ./.github/workflows/e2e-tests.yaml | |
| secrets: inherit | |
| with: | |
| artifacts-url: ${{ needs.build.outputs.artifact-link }} | |
| lint: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Python 3 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| # NOTE: Today python3.6 is needed by some lint steps | |
| - name: Install Python 3.6 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.6" | |
| - name: Install Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.20.14" | |
| - name: Install deps | |
| run: | | |
| export DEBIAN_FRONTEND=noninteractive | |
| sudo apt-get update | |
| sudo apt-get install --no-install-recommends -y shellcheck | |
| python3.10 -m pip install tox~=4.0.11 | |
| - name: Install Helm | |
| env: | |
| HELM_VERSION: "3.10.0" | |
| run: | | |
| sudo curl -O https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz \ | |
| && sudo tar -zxvf helm-v${HELM_VERSION}-linux-amd64.tar.gz \ | |
| && sudo mv linux-amd64/helm /usr/local/bin/helm | |
| - name: Run all linting targets | |
| run: ./doit.sh lint | |
| unit_tests_ui: | |
| needs: changed-files | |
| if: needs.changed-files.outputs.unit-tests-ui == 'true' | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - name: Cache node modules | |
| id: cache-npm | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| # npm cache files are stored in `~/.npm` on Linux/macOS | |
| path: ~/.npm | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: Install UI dependencies | |
| run: | | |
| cd ui | |
| npm ci --prefer-offline --no-audit --legacy-peer-deps --unsafe-perm --no-optional | |
| - name: Run all UI unit tests | |
| run: | | |
| cd ui | |
| npm run test:nowatch --no-update-notifier | |
| - name: Prepare artifacts | |
| run: mkdir -p "artifacts/ui" && mv ui/junit artifacts/ui/ | |
| - name: Upload artifacts | |
| uses: scality/action-artifacts@v4 | |
| with: | |
| method: upload | |
| url: https://artifacts.scality.net | |
| user: ${{ secrets.ARTIFACTS_USER }} | |
| password: ${{ secrets.ARTIFACTS_PASSWORD }} | |
| source: artifacts | |
| unit_tests_shell_ui: | |
| needs: changed-files | |
| if: needs.changed-files.outputs.unit-tests-shell-ui == 'true' | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - name: Cache node modules | |
| id: cache-npm | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| # npm cache files are stored in `~/.npm` on Linux/macOS | |
| path: ~/.npm | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: Install UI dependencies | |
| run: | | |
| cd shell-ui | |
| npm ci --prefer-offline --no-audit --legacy-peer-deps -d | |
| - name: Run all Shell UI unit tests | |
| run: | | |
| cd shell-ui | |
| npm run test --no-update-notifier | |
| unit_tests_crd_client_generator: | |
| needs: changed-files | |
| if: needs.changed-files.outputs.unit-tests-crd-client-generator == 'true' | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| working-directory: "tools/crd-client-generator-js" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - name: Cache node modules | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| # npm cache files are stored in `~/.npm` on Linux/macOS | |
| path: ~/.npm | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('package-lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: Install CRD client generator dependencies | |
| run: npm ci --prefer-offline --no-audit | |
| - name: Run all CRD client generator unit tests | |
| run: npm run test --no-update-notifier | |
| unit_tests_metalk8s_operator: | |
| needs: changed-files | |
| if: needs.changed-files.outputs.unit-tests-metalk8s-operator == 'true' | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| working-directory: "operator" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.20.14" | |
| - name: Cache Go modules | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-go-modules | |
| with: | |
| path: | | |
| ~/.cache/go-build | |
| ~/go/pkg/mod | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('go.sum') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: Run all MetalK8s operator unit and integration tests | |
| run: make test | |
| unit_tests_storage_operator: | |
| needs: changed-files | |
| if: needs.changed-files.outputs.unit-tests-storage-operator == 'true' | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| working-directory: "storage-operator" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.20.14" | |
| - name: Cache Go modules | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-go-modules | |
| with: | |
| path: | | |
| ~/.cache/go-build | |
| ~/go/pkg/mod | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('go.sum') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: Run all storage-operator unit and integration tests | |
| run: make test | |
| unit_tests_salt: | |
| needs: changed-files | |
| if: needs.changed-files.outputs.unit-tests-salt == 'true' | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Python 3.6 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.6" | |
| - name: Install deps | |
| run: | | |
| pip install tox | |
| - name: Run all salt unit tests | |
| run: tox -e unit-tests | |
| unit_tests_lib_alert_tree: | |
| needs: changed-files | |
| if: needs.changed-files.outputs.unit-tests-lib-alert-tree == 'true' | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| working-directory: "tools/lib-alert-tree" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Python 3 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.6" | |
| - name: Install deps | |
| run: python3.6 -m pip install poetry~=1.1.9 | |
| - name: Install lib_alert_tree | |
| run: poetry install -E cli | |
| - name: Run all lib_alert_tree unit tests | |
| run: poetry run pytest -vv | |
| build_integration_container_nginx: | |
| needs: changed-files | |
| if: needs.changed-files.outputs.integration-tests-ui == 'true' | |
| runs-on: ubuntu-20.04 | |
| permissions: write-all | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Install node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - name: Cache node modules | |
| id: cache-npm | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| # npm cache files are stored in `~/.npm` on Linux/macOS | |
| path: ~/.npm | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: Install Shell UI dependencies | |
| run: | | |
| cd shell-ui | |
| npm ci --legacy-peer-deps | |
| - name: Build Shell UI | |
| run: | | |
| cd shell-ui | |
| npm run rsbuild | |
| - name: Install UI dependencies | |
| run: | | |
| cd ui | |
| npm ci --legacy-peer-deps | |
| - name: Build UI | |
| run: | | |
| cd ui | |
| npm run build | |
| - name: Prepare Services Volumes | |
| run: | | |
| mkdir -p service | |
| mkdir -p service/shell | |
| cp -r ui/build/* service/ | |
| cp -r ui/build/.well-known service/.well-known | |
| cp -r shell-ui/build/* service/shell/ | |
| - name: Build container with nginx configuration for integration tests | |
| run: | | |
| cat > Dockerfile <<EOF | |
| FROM nginx:alpine | |
| COPY ui/standalone-nginx.conf /etc/nginx/conf.d/default.conf | |
| ADD service /usr/share/nginx/html | |
| EOF | |
| - name: Login to Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: docker/build-push-action@v6 | |
| with: | |
| push: true | |
| file: Dockerfile | |
| context: . | |
| tags: ghcr.io/${{ github.repository_owner }}/metalk8s-nginx-integration-tests:${{ github.sha }} | |
| integration_tests_ui: | |
| runs-on: ubuntu-20.04 | |
| needs: | |
| - build_integration_container_nginx | |
| - changed-files | |
| if: needs.changed-files.outputs.integration-tests-ui == 'true' | |
| services: | |
| app: | |
| image: ghcr.io/${{ github.repository_owner }}/metalk8s-nginx-integration-tests:${{ github.sha }} | |
| credentials: | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| ports: | |
| - 80:80 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - name: Cache node modules | |
| id: cache-npm | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| # npm cache files are stored in `~/.npm` on Linux/macOS | |
| path: ~/.npm | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: Install Cypress and its dependencies | |
| run: | | |
| cd ui | |
| PKGS="har-validator cypress cypress-cucumber-preprocessor cypress-wait-until @testing-library/cypress" | |
| for pkg in $PKGS; do | |
| npm install --no-save --no-package-lock --legacy-peer-deps $pkg@$(node -p \ | |
| -e "require('./package-lock.json').dependencies['$pkg'].version" \ | |
| ) || exit 1 | |
| done | |
| - name: Wait for application to be available | |
| run: | | |
| bash -c ' | |
| attempts=0 | |
| until curl -Isfo /dev/null http://localhost/; do | |
| (( attempts++ )) | |
| if [ $attempts -gt 100 ]; then | |
| >&2 echo "Failed to reach application after 5 minutes" | |
| exit 1 | |
| fi | |
| sleep 3 | |
| done | |
| ' | |
| - name: Run all UI integration tests | |
| env: | |
| CYPRESS_BASE_URL: http://localhost | |
| run: | | |
| cd ui | |
| rm -rf babel.config.js | |
| npm run test:integration --no-update-notifier | |
| - name: Prepare upload folder | |
| if: always() | |
| run: | | |
| mkdir -p upload/ui/cypress | |
| mv ui/cypress/screenshots upload/ui/cypress/screenshots || true | |
| mv ui/cypress/videos upload/ui/cypress/videos || true | |
| mv junit upload/ui || true | |
| - name: Upload artifacts | |
| if: always() | |
| uses: scality/action-artifacts@v4 | |
| with: | |
| method: upload | |
| url: https://artifacts.scality.net | |
| user: ${{ secrets.ARTIFACTS_USER }} | |
| password: ${{ secrets.ARTIFACTS_PASSWORD }} | |
| source: upload | |
| write-final-status: | |
| runs-on: ubuntu-20.04 | |
| needs: | |
| - changed-files | |
| - build-docs | |
| - build-shell-ui | |
| - e2e-tests | |
| - lint | |
| - unit_tests_ui | |
| - unit_tests_shell_ui | |
| - unit_tests_crd_client_generator | |
| - unit_tests_metalk8s_operator | |
| - unit_tests_storage_operator | |
| - unit_tests_salt | |
| - unit_tests_lib_alert_tree | |
| - integration_tests_ui | |
| if: always() | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Upload final status | |
| if: always() | |
| uses: scality/actions/[email protected] | |
| with: | |
| ARTIFACTS_USER: ${{ secrets.ARTIFACTS_USER }} | |
| ARTIFACTS_PASSWORD: ${{ secrets.ARTIFACTS_PASSWORD }} | |
| JOBS_RESULTS: ${{ join(needs.*.result) }} |