Skip to content

Commit

Permalink
split file to reuse steps
Browse files Browse the repository at this point in the history
  • Loading branch information
mchades committed Sep 11, 2024
1 parent 76109fb commit 2e58d3c
Show file tree
Hide file tree
Showing 6 changed files with 252 additions and 309 deletions.
77 changes: 77 additions & 0 deletions .github/workflows/backend-integration-test-action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
name: Backend Integration Test Action

# run backend integration test
on:
workflow_call:
inputs:
architecture:
required: true
description: 'Architecture of the platform'
type: string
java-version:
required: true
description: 'Java version'
type: string
backend:
required: true
description: 'Backend storage for Gravitino'
type: string
test-mode:
required: true
description: 'run on embedded or deploy mode'
type: string

jobs:
start-runner:
name: JDK${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }}
runs-on: ubuntu-latest
timeout-minutes: 90
env:
PLATFORM: ${{ inputs.architecture }}
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v4
with:
java-version: ${{ inputs.java-version }}
distribution: 'temurin'
cache: 'gradle'

- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Check required command
run: |
dev/ci/check_commands.sh
- name: Package Gravitino
if: ${{ inputs.test-mode == 'deploy' }}
run: |
./gradlew compileDistribution -x test -PjdkVersion=${{ inputs.java-version }}
- name: Free up disk space
run: |
dev/ci/util_free_space.sh
- name: Backend Integration Test (JDK${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }})
id: integrationTest
run: >
./gradlew test -PskipTests -PtestMode=${{ inputs.test-mode }} -PjdkVersion=${{ inputs.java-version }} -PjdbcBackend=${{ inputs.backend }} -PskipDockerTests=false
-x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test
-x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test
-x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
-x :authorizations:authorization-ranger:test -x :web:integration-test:test -x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
with:
name: integrate-test-reports-${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }}
path: |
build/reports
iceberg/iceberg-rest-server/build/*.log
distribution/package/logs/*.out
distribution/package/logs/*.log
catalogs/**/*.log
catalogs/**/*.tar
distribution/**/*.log
124 changes: 13 additions & 111 deletions .github/workflows/backend-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ name: Backend Integration Test

# Controls when the workflow will run
on:
# Triggers the workflow on push or pull request events but only for the "main" branch
push:
branches: [ "main", "branch-*" ]
pull_request:
Expand Down Expand Up @@ -50,11 +49,8 @@ jobs:
BackendIT-on-push:
needs: changes
if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true')
runs-on: ubuntu-latest
timeout-minutes: 90
strategy:
matrix:
# Integration test for AMD64 architecture
architecture: [linux/amd64]
java-version: [ 8, 11, 17 ]
backend: [ h2, mysql, postgresql ]
Expand All @@ -66,126 +62,32 @@ jobs:
backend: 'postgresql'
- test-mode: 'deploy'
backend: 'h2'

env:
PLATFORM: ${{ matrix.architecture }}
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: 'temurin'
cache: 'gradle'

- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Check required command
run: |
dev/ci/check_commands.sh
- name: Package Gravitino
if : ${{ matrix.test-mode == 'deploy' }}
run: |
./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
- name: Free up disk space
run: |
dev/ci/util_free_space.sh
- name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
id: integrationTest
run: >
./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipDockerTests=false
-x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test
-x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test
-x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
-x :authorizations:authorization-ranger:test -x :web:integration-test:test -x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
with:
name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}
path: |
build/reports
iceberg/iceberg-rest-server/build/*.log
integration-test/build/*.log
integration-test/build/*.tar
integration-test/build/trino-ci-container-log
distribution/package/logs/*.out
distribution/package/logs/*.log
catalogs/**/*.log
catalogs/**/*.tar
distribution/**/*.log
uses: ./.github/workflows/backend-integration-test-action.yml
with:
architecture: ${{ matrix.architecture }}
java-version: ${{ matrix.java-version }}
backend: ${{ matrix.backend }}
test-mode: ${{ matrix.test-mode }}

BackendIT-on-pr:
needs: changes
if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true')
runs-on: ubuntu-latest
timeout-minutes: 90
strategy:
matrix:
# Integration test for AMD64 architecture
architecture: [ linux/amd64 ]
java-version: [ 17 ]
test-mode: [ embedded, deploy ]
backend: [ h2, mysql, postgresql ]
test-mode: [ embedded, deploy ]
exclude:
- test-mode: 'embedded'
backend: 'mysql'
- test-mode: 'embedded'
backend: 'postgresql'
- test-mode: 'deploy'
backend: 'h2'

env:
PLATFORM: ${{ matrix.architecture }}
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: 'temurin'
cache: 'gradle'

- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Check required command
run: |
dev/ci/check_commands.sh
- name: Package Gravitino
if: ${{ matrix.test-mode == 'deploy' }}
run: |
./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
- name: Free up disk space
run: |
dev/ci/util_free_space.sh
- name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
id: integrationTest
run: >
./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipDockerTests=false
-x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test
-x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test
-x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
-x :authorizations:authorization-ranger:test -x :web:integration-test:test -x :trino-connector:integration-test:test -x :trino-connector:test
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
with:
name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}
path: |
build/reports
iceberg/iceberg-rest-server/build/*.log
distribution/package/logs/*.out
distribution/package/logs/*.log
catalogs/**/*.log
catalogs/**/*.tar
distribution/**/*.log
uses: ./.github/workflows/backend-integration-test-action.yml
with:
architecture: ${{ matrix.architecture }}
java-version: ${{ matrix.java-version }}
backend: ${{ matrix.backend }}
test-mode: ${{ matrix.test-mode }}
66 changes: 66 additions & 0 deletions .github/workflows/flink-integration-test-action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
name: Flink Integration Test Action

# run flink integration test
on:
workflow_call:
inputs:
architecture:
required: true
description: 'Architecture of the platform'
type: string
java-version:
required: true
description: 'Java version'
type: string

jobs:
start-runner:
name: JDK${{ inputs.java-version }}
runs-on: ubuntu-latest
timeout-minutes: 30
env:
PLATFORM: ${{ inputs.architecture }}
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v4
with:
java-version: ${{ inputs.java-version }}
distribution: 'temurin'
cache: 'gradle'

- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Check required command
run: |
dev/ci/check_commands.sh
- name: Package Gravitino
run: |
./gradlew compileDistribution -x test -PjdkVersion=${{ inputs.java-version }}
- name: Free up disk space
run: |
dev/ci/util_free_space.sh
- name: Flink Integration Test
id: integrationTest
# run embedded mode and deploy mode integration tests
run: |
./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ inputs.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ inputs.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
with:
name: flink-connector-integrate-test-reports-${{ inputs.java-version }}
path: |
build/reports
flink-connector/flink/build/*.log
flink-connector/flink/build/*.tar
distribution/package/logs/gravitino-server.out
distribution/package/logs/gravitino-server.log
catalogs/**/*.log
catalogs/**/*.tar
Loading

0 comments on commit 2e58d3c

Please sign in to comment.