Skip to content

Commit

Permalink
Merge pull request #5 from linkedin/main
Browse files Browse the repository at this point in the history
Update fork
  • Loading branch information
pthirun committed Jan 23, 2025
2 parents 3cba91b + a8173c8 commit dd6680f
Show file tree
Hide file tree
Showing 229 changed files with 12,501 additions and 1,925 deletions.
63 changes: 2 additions & 61 deletions .github/workflows/VeniceCI-CompatibilityTests.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# GitHub Actions workflow for running compatibility tests: Avro, Alpini unit, Alpini functional tests, and Pulsar Venice integration tests
# GitHub Actions workflow for running compatibility tests: Avro and Pulsar Venice integration tests

name: TestsDeCompatibilite

Expand Down Expand Up @@ -50,65 +50,6 @@ jobs:
path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz
retention-days: 30

AlpiniUnitTests:
strategy:
fail-fast: false
matrix:
jdk: [8, 11, 17]
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
- uses: actions/checkout@v4
with:
# Checkout as many commits as needed for the diff
fetch-depth: 2
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up JDK
uses: actions/setup-java@v4
with:
java-version: ${{ matrix.jdk }}
distribution: 'temurin'
cache: 'gradle'
# - name: Allow Deprecated TLS versions for Alpini tests
# run: |
# echo "java.security file before modifications: "
# cat "$JAVA_HOME/conf/security/java.security"

# # This is possibly flaky but
# sed -i 's/TLSv1, //g' "$JAVA_HOME/conf/security/java.security" # Allow TLSv1
# sed -i 's/TLSv1.1, //g' "$JAVA_HOME/conf/security/java.security" # Allow TLSv1.1

# echo "java.security file after modifications: "
# cat "$JAVA_HOME/conf/security/java.security"
- shell: bash
run: |
git remote set-head origin --auto
git remote add upstream https://github.com/linkedin/venice
git fetch upstream
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v4
with:
add-job-summary: never
- name: Run alpini unit tests
run: ./gradlew --continue --no-daemon -DmaxParallelForks=1 alpiniUnitTest
- name: Package Build Artifacts
if: (success() || failure())
shell: bash
run: |
mkdir ${{ github.job }}-artifacts
find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts
tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts
- name: Upload Build Artifacts
if: (success() || failure())
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}-jdk${{ matrix.jdk }}
path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz
retention-days: 30

PulsarVeniceIntegrationTests:
strategy:
fail-fast: false
Expand Down Expand Up @@ -178,7 +119,7 @@ jobs:
strategy:
fail-fast: false
runs-on: ubuntu-latest
needs: [AvroCompatibilityTests, AlpiniUnitTests, PulsarVeniceIntegrationTests]
needs: [AvroCompatibilityTests, PulsarVeniceIntegrationTests]
timeout-minutes: 120
steps:
- name: AllIsWell
Expand Down
97 changes: 96 additions & 1 deletion .github/workflows/VeniceCI-E2ETests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,101 @@ jobs:
key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}

IntegrationTests_1001:
name: IntegrationTests_1001
strategy:
fail-fast: false
matrix:
jdk: [17]
runs-on: ubuntu-latest
permissions:
id-token: write
contents: read
checks: write
pull-requests: write
issues: write
timeout-minutes: 120
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk${{ matrix.jdk }}-IntegrationTests_1001
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up JDK
uses: actions/setup-java@v4
with:
java-version: ${{ matrix.jdk }}
distribution: 'temurin'
cache: 'gradle'
- shell: bash
run: |
git remote set-head origin --auto
git remote add upstream https://github.com/linkedin/venice
git fetch upstream
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v4
with:
add-job-summary: never
- name: Run Integration Tests
run: ./gradlew --continue --no-daemon -DforkEvery=1 -DmaxParallelForks=1 integrationTests_1001
- name: Package Build Artifacts
if: success() || failure()
shell: bash
run: |
mkdir ${{ github.job }}-artifacts
echo "Repository owner: ${{ github.repository_owner }}"
echo "Repository name: ${{ github.repository }}"
echo "event name: ${{ github.event_name }}"
find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts
tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts
- name: Generate Fork Repo Test Reports
if: ${{ (github.repository_owner != 'linkedin') && (success() || failure()) }}
uses: dorny/[email protected]
env:
NODE_OPTIONS: --max-old-space-size=9182
with:
token: ${{ secrets.GITHUB_TOKEN }}
name: ${{ github.job }} Test Reports # Name where it report the test results
path: '**/TEST-*.xml'
fail-on-error: 'false'
max-annotations: '10'
list-tests: 'all'
list-suites: 'all'
reporter: java-junit
- name: Publish Test Report
continue-on-error: true
env:
NODE_OPTIONS: "--max_old_space_size=8192"
uses: mikepenz/action-junit-report@v5
if: always()
with:
check_name: ${{ github.job }}-jdk${{ matrix.jdk }} Report
comment: false
annotate_only: true
flaky_summary: true
commit: ${{github.event.workflow_run.head_sha}}
detailed_summary: true
report_paths: '**/build/test-results/test/TEST-*.xml'
- name: Upload Build Artifacts
if: success() || failure()
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}
path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz
retention-days: 30
- name: Upload test results to BuildPulse for flaky test detection
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && !cancelled()
uses: buildpulse/buildpulse-action@main
with:
account: 100582612927
repository: 100441445875
path: |
**/TEST-*.xml
key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}

IntegrationTests_1010:
name: IntegrationTests_1010
strategy:
Expand Down Expand Up @@ -2964,7 +3059,7 @@ jobs:
matrix:
jdk: [17]
runs-on: ubuntu-latest
needs: [IntegrationTests_1000, IntegrationTests_1010, IntegrationTests_1020, IntegrationTests_1030, IntegrationTests_1040, IntegrationTests_1050, IntegrationTests_1060, IntegrationTests_1070, IntegrationTests_1080, IntegrationTests_1090, IntegrationTests_1100, IntegrationTests_1110, IntegrationTests_1120, IntegrationTests_1130, IntegrationTests_1200, IntegrationTests_1210, IntegrationTests_1220, IntegrationTests_1230, IntegrationTests_1240, IntegrationTests_1250, IntegrationTests_1260, IntegrationTests_1270, IntegrationTests_1280, IntegrationTests_1400, IntegrationTests_1410, IntegrationTests_1420, IntegrationTests_1430, IntegrationTests_1440, IntegrationTests_1500, IntegrationTests_1550, IntegrationTests_9999]
needs: [IntegrationTests_1000, IntegrationTests_1001, IntegrationTests_1010, IntegrationTests_1020, IntegrationTests_1030, IntegrationTests_1040, IntegrationTests_1050, IntegrationTests_1060, IntegrationTests_1070, IntegrationTests_1080, IntegrationTests_1090, IntegrationTests_1100, IntegrationTests_1110, IntegrationTests_1120, IntegrationTests_1130, IntegrationTests_1200, IntegrationTests_1210, IntegrationTests_1220, IntegrationTests_1230, IntegrationTests_1240, IntegrationTests_1250, IntegrationTests_1260, IntegrationTests_1270, IntegrationTests_1280, IntegrationTests_1400, IntegrationTests_1410, IntegrationTests_1420, IntegrationTests_1430, IntegrationTests_1440, IntegrationTests_1500, IntegrationTests_1550, IntegrationTests_9999]
timeout-minutes: 20
steps:
- name: NoOp
Expand Down
58 changes: 41 additions & 17 deletions .github/workflows/VeniceCI-StaticAnalysisAndUnitTests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,46 +67,66 @@ jobs:
uses: ./.github/workflows/UnitTests-core.yml
with:
artifact_suffix: clients
arg: :clients:venice-admin-tool:jacocoTestCoverageVerification :clients:venice-admin-tool:diffCoverage
:clients:venice-producer:jacocoTestCoverageVerification :clients:venice-producer:diffCoverage
:integrations:venice-pulsar:jacocoTestCoverageVerification :integrations:venice-pulsar:diffCoverage
:clients:venice-client:jacocoTestCoverageVerification :clients:venice-client:diffCoverage
:clients:venice-push-job:jacocoTestCoverageVerification :clients:venice-push-job:diffCoverage
:integrations:venice-samza:jacocoTestCoverageVerification :integrations:venice-samza:diffCoverage
:clients:venice-thin-client:jacocoTestCoverageVerification :clients:venice-thin-client:diffCoverage --continue
arg:
# Cannot use :clients:recursiveDiffCoverage because that would include DVC, and we want this one to run in server...
:clients:venice-admin-tool:diffCoverage
:clients:venice-producer:diffCoverage
:clients:venice-client:diffCoverage
:clients:venice-push-job:diffCoverage
:clients:venice-thin-client:diffCoverage
--continue

Integrations:
uses: ./.github/workflows/UnitTests-core.yml
with:
artifact_suffix: integrations
arg:
:integrations:recursiveDiffCoverage
--continue

Internal:
uses: ./.github/workflows/UnitTests-core.yml
with:
artifact_suffix: internal
arg: :internal:venice-client-common:jacocoTestCoverageVerification :internal:venice-client-common:diffCoverage
:internal:venice-common:jacocoTestCoverageVerification :internal:venice-common:diffCoverage
:internal:venice-jdk-compatibility-test:jacocoTestCoverageVerification :internal:venice-jdk-compatibility-test:diffCoverage
:internal:venice-test-common:jacocoTestCoverageVerification :internal:venice-test-common:diffCoverage --continue

arg:
# Cannot use :internal:recursiveDiffCoverage because that would include the avro compat test, and we want this one to run in the Compatibility group (TODO: move it out of internal?)...
:internal:venice-client-common:diffCoverage
:internal:venice-common:diffCoverage
:internal:venice-jdk-compatibility-test:diffCoverage
:internal:venice-test-common:diffCoverage
--continue

Controller:
uses: ./.github/workflows/UnitTests-core.yml
with:
artifact_suffix: controller
arg: :services:venice-controller:jacocoTestCoverageVerification :services:venice-controller:diffCoverage --continue
arg:
:services:venice-controller:diffCoverage
--continue

Server:
uses: ./.github/workflows/UnitTests-core.yml
with:
artifact_suffix: server
arg: :clients:da-vinci-client:jacocoTestCoverageVerification :clients:da-vinci-client:diffCoverage
:services:venice-server:jacocoTestCoverageVerification :services:venice-server:diffCoverage --continue
arg:
:clients:da-vinci-client:diffCoverage
:services:venice-server:diffCoverage
--continue

Router:
uses: ./.github/workflows/UnitTests-core.yml
with:
artifact_suffix: router
arg: :services:venice-router:jacocoTestCoverageVerification :services:venice-router:diffCoverage --continue
arg:
:services:venice-router:diffCoverage
alpiniUnitTest
--continue

StaticAnalysisAndUnitTestsCompletionCheck:
strategy:
fail-fast: false
runs-on: ubuntu-latest
needs: [ValidateGradleWrapper, StaticAnalysis, Clients, Internal, Controller, Server, Router]
needs: [ValidateGradleWrapper, StaticAnalysis, Clients, Integrations, Internal, Controller, Server, Router]
timeout-minutes: 120
if: success() || failure() # Always run this job, regardless of previous job status
steps:
Expand All @@ -128,6 +148,10 @@ jobs:
echo "Internal module unit tests failed."
exit 1
fi
if [ "${{ needs.Integrations.result }}" != "success" ]; then
echo "Integrations module unit tests failed."
exit 1
fi
if [ "${{ needs.Controller.result }}" != "success" ]; then
echo "Controller module unit tests failed."
exit 1
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,4 @@ Gemfile.lock
.bundles_cache
docs/vendor/
clients/da-vinci-client/classHash*.txt
integrations/venice-duckdb/classHash*.txt
Loading

0 comments on commit dd6680f

Please sign in to comment.