From 899ec1e76c6bc303f613631958bee7fbf968ac53 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 30 Oct 2023 21:01:53 -0400 Subject: [PATCH] Add more Signed-off-by: Peter Zhu --- .github/workflows/add-untriaged.yml | 19 +++ .github/workflows/auto-release.yml | 29 ++++ .github/workflows/backport.yml | 28 ++++ .github/workflows/build.yml | 125 +++++++++--------- .github/workflows/delete_backport_branch.yml | 15 +++ .github/workflows/maven-publish.yml | 34 +++++ .../workflows/multi-node-test-workflow.yml | 56 ++++++++ .github/workflows/release-workflow.yml | 123 +++++++++++++++++ 8 files changed, 366 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/add-untriaged.yml create mode 100644 .github/workflows/auto-release.yml create mode 100644 .github/workflows/backport.yml create mode 100644 .github/workflows/delete_backport_branch.yml create mode 100644 .github/workflows/maven-publish.yml create mode 100644 .github/workflows/multi-node-test-workflow.yml create mode 100644 .github/workflows/release-workflow.yml diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml new file mode 100644 index 00000000..9dcc7020 --- /dev/null +++ b/.github/workflows/add-untriaged.yml @@ -0,0 +1,19 @@ +name: Apply 'untriaged' label during issue lifecycle + +on: + issues: + types: [opened, reopened, transferred] + +jobs: + apply-label: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v6 + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['untriaged'] + }) diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml new file mode 100644 index 00000000..a598da2c --- /dev/null +++ b/.github/workflows/auto-release.yml @@ -0,0 +1,29 @@ +name: Releases + +on: + push: + tags: + - '*' + +jobs: + + build: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.5.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + - name: Get tag + id: tag + uses: dawidd6/action-get-tag@v1 + - uses: actions/checkout@v2 + - uses: ncipollo/release-action@v1 + with: + github_token: ${{ steps.github_app_token.outputs.token }} + bodyFile: release-notes/opensearch-asynchronous-search.release-notes-${{steps.tag.outputs.tag}}.md \ No newline at end of file diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 00000000..e47d8d88 --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,28 @@ +name: Backport +on: + pull_request_target: + types: + - closed + - labeled + +jobs: + backport: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + name: Backport + steps: + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.5.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + + - name: Backport + uses: VachaShah/backport@v1.1.4 + with: + github_token: ${{ steps.github_app_token.outputs.token }} + branch_name: backport/backport-${{ github.event.number }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index faca8283..9c26cda3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,16 +47,15 @@ jobs: id: step-build-test-linux run: | chown -R 1000:1000 `pwd` - su `id -un 1000` -c "./gradlew assemble" + su `id -un 1000` -c "./gradlew build" plugin=`basename $(ls build/distributions/*.zip)` echo plugin $plugin mv -v build/distributions/$plugin ./ echo "build-test-linux=$plugin" >> $GITHUB_OUTPUT - #run: ./gradlew build -# - name: Uploads coverage -# uses: codecov/codecov-action@v1.2.1 -# with: -# token: ${{ secrets.CODECOV_TOKEN }} + - name: Uploads coverage + uses: codecov/codecov-action@v1.2.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} - uses: actions/upload-artifact@v3 with: name: async-plugin-linux-${{ matrix.java }} @@ -137,60 +136,60 @@ jobs: name: logs path: build/testclusters/integTest-*/logs/* -# windows-build: -# # Job name -# name: Build Asynchronous Search -# # This job runs on Windows. -# runs-on: windows-latest -# steps: -# # This step uses the setup-java Github action: https://github.com/actions/setup-java -# - name: Set Up JDK 11 -# uses: actions/setup-java@v1 -# with: -# java-version: 11 -# # This step uses the checkout Github action: https://github.com/actions/checkout -# - name: Checkout Branch -# uses: actions/checkout@v2 -# - name: Build with Gradle -# run: ./gradlew.bat build -x integTest -x jacocoTestReport -# env: -# _JAVA_OPTIONS: -Xmx4096M -# - name: Create Artifact Path -# run: | -# mkdir -p asynchronous-search-artifacts -# cp ./build/distributions/*.zip asynchronous-search-artifacts -# # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact -# - name: Upload Artifacts -# uses: actions/upload-artifact@v1 -# with: -# name: asynchronous-search-plugin-windows -# path: asynchronous-search-artifacts -# -# mac-os-build: -# # Job name -# name: Build Asynchronous Search -# # This job runs on Mac OS. -# runs-on: macos-latest -# steps: -# # This step uses the setup-java Github action: https://github.com/actions/setup-java -# - name: Set Up JDK 11 -# uses: actions/setup-java@v1 -# with: -# java-version: 11 -# # This step uses the checkout Github action: https://github.com/actions/checkout -# - name: Checkout Branch -# uses: actions/checkout@v2 -# - name: Build with Gradle -# run: ./gradlew build -x integTest -x jacocoTestReport -# env: -# _JAVA_OPTIONS: -Xmx4096M -# - name: Create Artifact Path -# run: | -# mkdir -p asynchronous-search-artifacts -# cp ./build/distributions/*.zip asynchronous-search-artifacts -# # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact -# - name: Upload Artifacts -# uses: actions/upload-artifact@v1 -# with: -# name: asynchronous-search-plugin-mac -# path: asynchronous-search-artifacts + windows-build: + # Job name + name: Build Asynchronous Search + # This job runs on Windows. + runs-on: windows-latest + steps: + # This step uses the setup-java Github action: https://github.com/actions/setup-java + - name: Set Up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + # This step uses the checkout Github action: https://github.com/actions/checkout + - name: Checkout Branch + uses: actions/checkout@v2 + - name: Build with Gradle + run: ./gradlew.bat build -x integTest -x jacocoTestReport + env: + _JAVA_OPTIONS: -Xmx4096M + - name: Create Artifact Path + run: | + mkdir -p asynchronous-search-artifacts + cp ./build/distributions/*.zip asynchronous-search-artifacts + # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact + - name: Upload Artifacts + uses: actions/upload-artifact@v1 + with: + name: asynchronous-search-plugin-windows + path: asynchronous-search-artifacts + + mac-os-build: + # Job name + name: Build Asynchronous Search + # This job runs on Mac OS. + runs-on: macos-latest + steps: + # This step uses the setup-java Github action: https://github.com/actions/setup-java + - name: Set Up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + # This step uses the checkout Github action: https://github.com/actions/checkout + - name: Checkout Branch + uses: actions/checkout@v2 + - name: Build with Gradle + run: ./gradlew build -x integTest -x jacocoTestReport + env: + _JAVA_OPTIONS: -Xmx4096M + - name: Create Artifact Path + run: | + mkdir -p asynchronous-search-artifacts + cp ./build/distributions/*.zip asynchronous-search-artifacts + # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact + - name: Upload Artifacts + uses: actions/upload-artifact@v1 + with: + name: asynchronous-search-plugin-mac + path: asynchronous-search-artifacts diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml new file mode 100644 index 00000000..f24f022b --- /dev/null +++ b/.github/workflows/delete_backport_branch.yml @@ -0,0 +1,15 @@ +name: Delete merged branch of the backport PRs +on: + pull_request: + types: + - closed + +jobs: + delete-branch: + runs-on: ubuntu-latest + if: startsWith(github.event.pull_request.head.ref,'backport/') + steps: + - name: Delete merged branch + uses: SvanBoxel/delete-merged-branch@main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml new file mode 100644 index 00000000..3e924caf --- /dev/null +++ b/.github/workflows/maven-publish.yml @@ -0,0 +1,34 @@ +name: Publish snapshots to maven + +on: + workflow_dispatch: + push: + branches: + - 'main' + - '2.*' + +jobs: + build-and-publish-snapshots: + runs-on: ubuntu-latest + + permissions: + id-token: write + contents: write + + steps: + - uses: actions/setup-java@v3 + with: + distribution: temurin # Temurin is a distribution of adoptium + java-version: 11 + - uses: actions/checkout@v3 + - uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} + aws-region: us-east-1 + - name: publish snapshots to maven + run: | + export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text) + export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text) + echo "::add-mask::$SONATYPE_USERNAME" + echo "::add-mask::$SONATYPE_PASSWORD" + ./gradlew publishPluginZipPublicationToSnapshotsRepository \ No newline at end of file diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml new file mode 100644 index 00000000..6d07db88 --- /dev/null +++ b/.github/workflows/multi-node-test-workflow.yml @@ -0,0 +1,56 @@ +name: Multi node test workflow + +env: + java_version: 11 +# This workflow is triggered on pull requests to master +on: + pull_request: + branches: + - "*" + push: + branches: + - "*" + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + build: + # Job name + needs: Get-CI-Image-Tag + name: Build Asynchronous Search + # This job runs on Linux + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + steps: + # This step uses the setup-java Github action: https://github.com/actions/setup-java + - name: Set Up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: ${{ env.java_version }} + # This step uses the checkout Github action: https://github.com/actions/checkout + - name: Checkout Branch + uses: actions/checkout@v2 + - name: Run integration tests with multi node config + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew integTest -PnumNodes=5" + - name: Run Backwards Compatibility Tests + run: | + echo "Running backwards compatibility tests ..." + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew bwcTestSuite -Dtests.security.manager=false" + - name: Upload failed logs + uses: actions/upload-artifact@v2 + if: failure() + with: + name: logs + path: build/testclusters/integTest-*/logs/* diff --git a/.github/workflows/release-workflow.yml b/.github/workflows/release-workflow.yml new file mode 100644 index 00000000..848f345f --- /dev/null +++ b/.github/workflows/release-workflow.yml @@ -0,0 +1,123 @@ +name: Release workflow + +env: + aws_staging_region: us-west-2 + +# This workflow is triggered on creating tags to master or a opendistro release branch +on: + push: + tags: + - 'v*' + +jobs: + build: + strategy: + matrix: + java: [11,17] + # Job name + name: Build Asynchronous Search with JDK ${{ matrix.java }} + # This job runs on Linux + runs-on: ubuntu-latest + steps: + # This step uses the checkout Github action: https://github.com/actions/checkout + - name: Checkout Branch + uses: actions/checkout@v2 + # This step uses the setup-java Github action: https://github.com/actions/setup-java + - name: Set Up JDK ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + # Building zip, deb and rpm files + - name: Build with Gradle + run: ./gradlew build buildDeb buildRpm --no-daemon -Dbuild.snapshot=false + + # Creating artifact path as well as individual folders for rpm, zip and deb assets + - name: Create Artifact Path + run: | + artifacts_dir="asynchronous-search-artifacts" + build_dir="./build/distributions" + mkdir -p ${artifacts_dir} + cp ${build_dir}/*.deb ${artifacts_dir} + cp ${build_dir}/*.deb ${artifacts_dir}_deb + cp ${build_dir}/*.rpm ${artifacts_dir} + cp ${build_dir}/*.rpm ${artifacts_dir}_rpm + cp ${build_dir}/*.zip ${artifacts_dir} + cp ${build_dir}/*.zip ${artifacts_dir}_zip + echo "TAG_VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV + # AWS authentication + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} + aws-region: ${{ env.aws_staging_region }} + + # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact + - name: Upload Artifacts to S3 + run: | + zip=`ls asynchronous-search-artifacts/*.zip` + rpm=`ls asynchronous-search-artifacts/*.rpm` + deb=`ls asynchronous-search-artifacts/*.deb` + # Inject the build number before the suffix + zip_outfile=`basename ${zip%.zip}-build-${GITHUB_RUN_NUMBER}.zip` + rpm_outfile=`basename ${rpm%.rpm}-build-${GITHUB_RUN_NUMBER}.rpm` + deb_outfile=`basename ${deb%.deb}-build-${GITHUB_RUN_NUMBER}.deb` + s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-plugins/asynchronous-search/" + echo "Copying ${zip} to ${s3_prefix}${zip_outfile}" + aws s3 cp --quiet $zip ${s3_prefix}${zip_outfile} + echo "Copying ${rpm} to ${s3_prefix}${rpm_outfile}" + aws s3 cp --quiet $rpm ${s3_prefix}${rpm_outfile} + echo "Copying ${deb} to ${s3_prefix}${deb_outfile}" + aws s3 cp --quiet $deb ${s3_prefix}${deb_outfile} + + - name: Create Github Draft Release + id: create_release + uses: actions/create-release@v1.0.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ env.TAG_VERSION }} + draft: true + prerelease: false + + # Upload the release with .zip as asset + - name: Upload Release Asset + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_name: asynchronous-search.zip + asset_path: asynchronous-search-artifacts_zip + asset_content_type: application/zip + + # Upload the release with .rpm as asset + - name: Upload Release Asset + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_name: asynchronous-search.rpm + asset_path: asynchronous-search-artifacts_rpm + asset_content_type: application/zip + + # Upload the release with .deb as asset + - name: Upload Release Asset + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_name: asynchronous-search.deb + asset_path: asynchronous-search-artifacts_deb + asset_content_type: application/zip + + - name: Upload Workflow Artifacts + uses: actions/upload-artifact@v1 + with: + name: asynchronous-search-plugin + path: asynchronous-search-artifacts +