From be1e759d9fd0bd3a0bf64941ae21f6bd12b4bcd0 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 14 Feb 2024 14:31:42 -0800 Subject: [PATCH] Stop running Mac BATS tests on all PRs, and run nightly instead --- .github/workflows/ci-bats-macos.yaml | 142 +++++++++++++++++++++++++++ .github/workflows/ci-bats-unix.yaml | 2 +- 2 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci-bats-macos.yaml diff --git a/.github/workflows/ci-bats-macos.yaml b/.github/workflows/ci-bats-macos.yaml new file mode 100644 index 00000000000..8557e5a2ca4 --- /dev/null +++ b/.github/workflows/ci-bats-macos.yaml @@ -0,0 +1,142 @@ +name: Test Bats MacOS + +on: + workflow_dispatch: + schedule: + - cron: '0 1 * * *' + +concurrency: + group: ci-bats-mac-nightly + cancel-in-progress: true + +jobs: + test: + name: Bats tests + defaults: + run: + shell: bash + runs-on: macos-latest + strategy: + fail-fast: true + env: + use_credentials: ${{ secrets.AWS_SECRET_ACCESS_KEY != '' && secrets.AWS_ACCESS_KEY_ID != '' }} + steps: + - name: Conditionally Set ENV VARS for AWS tests + run: | + if [[ $use_credentials == true ]]; then + echo "AWS_SDK_LOAD_CONFIG=1" >> $GITHUB_ENV + echo "AWS_REGION=us-west-2" >> $GITHUB_ENV + echo "DOLT_BATS_AWS_TABLE=dolt-ci-bats-manifests-us-west-2" >> $GITHUB_ENV + echo "DOLT_BATS_AWS_BUCKET=dolt-ci-bats-chunks-us-west-2" >> $GITHUB_ENV + echo "DOLT_BATS_AWS_EXISTING_REPO=aws_remote_bats_tests__dolt__" >> $GITHUB_ENV + fi + - name: Configure AWS Credentials + if: ${{ env.use_credentials == 'true' }} + uses: aws-actions/configure-aws-credentials@v2.2.0 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + role-duration-seconds: 10800 # 3 hours D: + - name: Setup Go 1.x + uses: actions/setup-go@v3 + with: + go-version: ^1.21 + id: go + - name: Setup Python 3.x + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ^16 + - name: Create CI Bin + run: | + mkdir -p ./.ci_bin + echo "$(pwd)/.ci_bin" >> $GITHUB_PATH + - name: Install Bats + run: | + npm i bats + echo "$(pwd)/node_modules/.bin" >> $GITHUB_PATH + working-directory: ./.ci_bin + - name: Install Python Deps + run: | + pip install mysql-connector-python + pip install pandas + pip install pyarrow + - name: Install Dolt + working-directory: ./go + run: | + go build -mod=readonly -o ../.ci_bin/dolt ./cmd/dolt/. + go build -mod=readonly -o ../.ci_bin/remotesrv ./utils/remotesrv/. + go build -mod=readonly -o ../.ci_bin/noms ./store/cmd/noms/. + - name: Setup Dolt Config + run: | + dolt config --global --add user.name 'Dolthub Actions' + dolt config --global --add user.email 'actions@dolthub.com' + - name: Install Maven + working-directory: ./.ci_bin + run: | + curl -LO https://dlcdn.apache.org/maven/maven-3/3.9.4/binaries/apache-maven-3.9.4-bin.tar.gz + tar -xf apache-maven-3.9.4-bin.tar.gz + echo "$(pwd)/apache-maven-3.9.4/bin" >> $GITHUB_PATH + - name: Install Hadoop + working-directory: ./.ci_bin + run: | + curl -LO https://downloads.apache.org/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz + tar xvf hadoop-3.3.6.tar.gz + echo "$(pwd)/hadoop-3.3.6/bin" >> $GITHUB_PATH + - name: Install parquet-cli + id: parquet_cli + working-directory: ./.ci_bin + run: | + curl -OL https://github.com/apache/parquet-mr/archive/refs/tags/apache-parquet-1.12.3.tar.gz + tar zxvf apache-parquet-1.12.3.tar.gz + cd parquet-mr-apache-parquet-1.12.3/parquet-cli + mvn clean install -DskipTests + runtime_jar="$(pwd)"/target/parquet-cli-1.12.3-runtime.jar + echo "runtime_jar=$runtime_jar" >> $GITHUB_OUTPUT + - name: Check expect + run: expect -v + - name: Test all Mac + env: + SQL_ENGINE: "local-engine" + PARQUET_RUNTIME_JAR: ${{ steps.parquet_cli.outputs.runtime_jar }} + BATS_TEST_RETRIES: "3" + run: | + bats --tap $BATS_FILTER . + working-directory: ./integration-tests/bats + + report-bats-failure: + name: Report Bats MacOS Failure via Email + needs: test + runs-on: ubuntu-22.04 + if: always() && (needs.test.result == 'failure') + steps: + - uses: actions/checkout@v3 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2.2.0 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Send Email + if: ${{ github.event_name == 'workflow_dispatch' }} + uses: ./.github/actions/ses-email-action + with: + region: us-west-2 + version: ${{ github.ref }} + template: 'BatsMacFailureTemplate' + toAddresses: '["${{ github.event.inputs.email }}"]' + workflowURL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + - name: Send Email + if: ${{ github.event_name == 'schedule' }} + uses: ./.github/actions/ses-email-action + with: + region: us-west-2 + version: ${{ github.event.client_payload.ref }} + template: 'BatsMacFailureTemplate' + toAddresses: '["dolts@dolthub.com", "tim@dolthub.com"]' + workflowURL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/ci-bats-unix.yaml b/.github/workflows/ci-bats-unix.yaml index d20f9904877..e2df9bb1eb4 100644 --- a/.github/workflows/ci-bats-unix.yaml +++ b/.github/workflows/ci-bats-unix.yaml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: true matrix: - os: [ ubuntu-22.04, macos-latest ] + os: [ ubuntu-22.04 ] env: use_credentials: ${{ secrets.AWS_SECRET_ACCESS_KEY != '' && secrets.AWS_ACCESS_KEY_ID != '' }} steps: