E2E Cross-version Tests #13
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Cross-version Tests | |
| on: | |
| schedule: | |
| - cron: '0 9 * * 0' | |
| jobs: | |
| cross-version-testing: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| strategy: | |
| matrix: | |
| version: [ | |
| # Major OSS upgrade | |
| { source: v0.42.2, target: v0.48.7 }, | |
| # OSS upgrade | |
| { source: v0.41.3.1, target: v0.42.2 }, | |
| # EE upgrade | |
| { source: v1.43.4, target: v1.44.0 }, | |
| # EE downgrade | |
| { source: v1.42.0, target: v1.41.3.1 }, | |
| # Cross-edition upgrade | |
| { source: v0.41.3.1, target: v1.42.2 } | |
| ] | |
| fail-fast: false | |
| env: | |
| CROSS_VERSION_SOURCE: ${{ matrix.version.source }} | |
| CROSS_VERSION_TARGET: ${{ matrix.version.target }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set source Docker image to OSS | |
| run: | | |
| echo "DOCKER_SOURCE_IMAGE=metabase/metabase:${{ matrix.version.source }}" >> $GITHUB_ENV | |
| if: startsWith(matrix.version.source, 'v0') | |
| - name: Set source Docker image to EE | |
| run: | | |
| echo "DOCKER_SOURCE_IMAGE=metabase/metabase-enterprise:${{ matrix.version.source }}" >> $GITHUB_ENV | |
| if: startsWith(matrix.version.source, 'v1') | |
| - name: Prepare front-end environment | |
| uses: ./.github/actions/prepare-frontend | |
| - name: Start Metabase ${{ matrix.version.source }} | |
| run: | | |
| docker run -d \ | |
| -v $PWD/my-metabase:/metabase.db \ | |
| -p 3000:3000 \ | |
| -e CYPRESS_ALL_FEATURES_TOKEN=${{ secrets.ENTERPRISE_TOKEN }} \ | |
| -e CYPRESS_NO_FEATURES_TOKEN=${{ secrets.E2E_STARTER_TOKEN }} \ | |
| --name metabase-${{ matrix.version.source }} \ | |
| ${{ env.DOCKER_SOURCE_IMAGE }} | |
| - name: Wait for Metabase to start on port 3000 | |
| run: while ! curl -s 'http://localhost:3000/api/health' | grep '{"status":"ok"}'; do sleep 1; done | |
| - name: Prepare Cypress environment | |
| uses: ./.github/actions/prepare-cypress | |
| - name: Run Cypress on the source | |
| run: | | |
| yarn cypress run \ | |
| --browser chrome \ | |
| --config-file e2e/test/scenarios/cross-version/source/shared/cross-version-source.config.js \ | |
| --spec e2e/test/scenarios/cross-version/source/**/*.cy.spec.js | |
| - name: Stop Metabase ${{ matrix.version.source }} | |
| run: docker stop metabase-${{ matrix.version.source }} | |
| - name: Set target Docker image to OSS | |
| run: | | |
| echo "DOCKER_TARGET_IMAGE=metabase/metabase:${{ matrix.version.source }}" >> $GITHUB_ENV | |
| if: startsWith(matrix.version.source, 'v0') | |
| - name: Set target Docker image to EE | |
| run: | | |
| echo "DOCKER_TARGET_IMAGE=metabase/metabase-enterprise:${{ matrix.version.source }}" >> $GITHUB_ENV | |
| if: startsWith(matrix.version.source, 'v1') | |
| - name: Start Metabase ${{ matrix.version.target }} | |
| run: | | |
| docker run -d \ | |
| -v $PWD/my-metabase:/metabase.db \ | |
| -p 3001:3000 \ | |
| -e CYPRESS_ALL_FEATURES_TOKEN=${{ secrets.ENTERPRISE_TOKEN }} \ | |
| -e CYPRESS_NO_FEATURES_TOKEN=${{ secrets.E2E_STARTER_TOKEN }} \ | |
| --name metabase-${{ matrix.version.target }} \ | |
| ${{ env.DOCKER_TARGET_IMAGE }} | |
| - name: Wait for Metabase to start on port 3001 | |
| run: while ! curl -s 'http://localhost:3001/api/health' | grep '{"status":"ok"}'; do sleep 1; done | |
| - name: Run Cypress on the target | |
| run: | | |
| yarn cypress run \ | |
| --browser chrome \ | |
| --config-file e2e/test/scenarios/cross-version/target/shared/cross-version-target.config.js \ | |
| --spec e2e/test/scenarios/cross-version/target/**/*.cy.spec.js | |
| - name: Upload Cypress Artifacts upon failure | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: cypress-artifacts-${{ matrix.version.source}}-to-${{ matrix.version.target }} | |
| path: | | |
| ./cypress | |
| if-no-files-found: ignore | |
| notify-on-failure: | |
| needs: cross-version-testing | |
| if: ${{ always() && contains(needs.*.result, 'failure') }} | |
| name: Cross-version test failure | |
| runs-on: ubuntu-22.04 | |
| env: | |
| BRANCH_NAME: ${{ github.event.workflow_run.head_branch }} | |
| AUTHOR_NAME: ${{ github.event.workflow_run.head_commit.author.name }} | |
| steps: | |
| - name: Generate job summary | |
| run: | | |
| echo "# ${{ github.event.workflow_run.name }} workflow failed! :x:" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "View the failed run attempt (#${{ github.event.workflow_run.run_attempt }}) using the following link:" >> $GITHUB_STEP_SUMMARY | |
| echo "${{ github.event.workflow_run.html_url }}" >> $GITHUB_STEP_SUMMARY | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: lts/Hydrogen # 18.x.x | |
| - run: npm install @slack/web-api | |
| - name: notify slack of failure | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | # js | |
| const FAILED_RUN_URL = "${{ github.event.workflow_run.html_url }}"; | |
| const FAILED_RUN_NAME = "${{ github.event.workflow_run.name }}"; | |
| const BREAKING_COMMIT = "${{ github.event.workflow_run.head_sha }}"; | |
| const AUTHOR = process.env.AUTHOR_NAME; | |
| const BRANCH = process.env.BRANCH_NAME; | |
| // notify slack of failure | |
| const { WebClient } = require('@slack/web-api'); | |
| const slack = new WebClient('${{ secrets.SLACK_BOT_TOKEN }}'); | |
| await slack.chat.postMessage({ | |
| channel: 'engineering-ci', | |
| text: 'Cross-version Tests Have Failed', | |
| blocks: [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": `:broken_heart: Cross-version tests are failing on ${BRANCH}`, | |
| "emoji": true, | |
| } | |
| }, | |
| ], | |
| attachments: [{ | |
| color: "#BF40BF", | |
| blocks: [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": `Commit <https://github.com/metabase/metabase/commit/${BREAKING_COMMIT}|${BREAKING_COMMIT.slice(0,7)}> has failing <${FAILED_RUN_URL}|${FAILED_RUN_NAME}> tests on the <https://github.com/metabase/metabase/commits/${BRANCH}|\`${BRANCH}\`> branch.` | |
| } | |
| }, | |
| ] | |
| }] | |
| }); |