Onboard jenkins prod docker images to github actions (#507) (#508)

* Onboard jenkins prod docker images to github actions Signed-off-by: Peter Zhu <[email protected]> * Add more Signed-off-by: Peter Zhu <[email protected]> * Add more Signed-off-by: Peter Zhu <[email protected]> * Add more Signed-off-by: Peter Zhu <[email protected]> --------- Signed-off-by: Peter Zhu <[email protected]> (cherry picked from commit e70aa78) Co-authored-by: Peter Zhu <[email protected]>
opensearch-project · Oct 31, 2023 · 3fd781e · 3fd781e
1 parent e291c1c
commit 3fd781e
Show file tree

Hide file tree

Showing 3 changed files with 88 additions and 52 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -9,7 +9,13 @@ on:
       - "*"
 
 jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
   linux-build:
+    needs: Get-CI-Image-Tag
     strategy:
       matrix:
         java:
@@ -18,7 +24,16 @@ jobs:
     # Job name
     name: Build Asynchronous Search
     # This job runs on Linux.
+    outputs:
+      build-test-linux: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK ${{ matrix.java }}
@@ -29,33 +44,68 @@ jobs:
       - name: Checkout Branch
         uses: actions/checkout@v2
       - name: Build with Gradle
-        run: ./gradlew build
+        id: step-build-test-linux
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew build"
+          plugin=`basename $(ls build/distributions/*.zip)`
+          echo plugin $plugin
+          mv -v build/distributions/$plugin ./
+          echo "build-test-linux=$plugin" >> $GITHUB_OUTPUT
+      - name: Uploads coverage
+        uses: codecov/[email protected]
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+      - uses: actions/upload-artifact@v3
+        with:
+          name: async-plugin-linux-${{ matrix.java }}
+          path: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
+          if-no-files-found: error
+
+  linux-test-docker:
+    needs: linux-build
+    strategy:
+      matrix:
+        java:
+          - 11
+          - 17
+    # Job name
+    name: Test Asynchronous Search with opensearchstaging docker
+    # This job runs on Linux.
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      - uses: actions/download-artifact@v3
+        with:
+          name: async-plugin-linux-${{ matrix.java }}
       - name: Pull and Run Docker for security tests
         run: |
-          version=`./gradlew properties -q | grep "opensearch_version:" | awk '{print $2}'`
-          IFS='-' read -r -a version_array <<< "$version"
-          plugin_version="${version_array[0]}.0"; for entry in ${version_array[@]:1}; do plugin_version+="-$entry"; done
-          docker_version="${version_array[0]}-${version_array[1]}"
+          plugin=${{ needs.linux-build.outputs.build-test-linux }}
+          version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3`
+          plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4`
+          qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1`
 
-          echo version $version
-          echo plugin_version $plugin_version
-          echo docker_version $docker_version
+          if [ -n "$qualifier" ] && [ "$qualifier" != "SNAPSHOT" ]; then
+              qualifier=-${qualifier}
+          else
+              qualifier=""
+          fi
 
-          pwd=`pwd`
-          echo $pwd
-          cd ..
-          pwd1=`pwd`
-          echo $pwd1
-          list_of_all_files=`ls asynchronous-search/build/distributions/`
-          echo "listing distributions"
-          echo $list_of_all_files
+          docker_version=$version$qualifier
+
+
+          echo plugin version plugin_version qualifier docker_version
+          echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)"
+
+          pwd && ls -l ./$plugin
 
           if docker pull opensearchstaging/opensearch:$docker_version
           then
             echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
             echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-asynchronous-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-asynchronous-search; fi" >> Dockerfile
-            echo "ADD asynchronous-search/build/distributions/opensearch-asynchronous-search-$plugin_version.zip /tmp/" >> Dockerfile
-            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/opensearch-asynchronous-search-$plugin_version.zip" >> Dockerfile
+            echo "ADD $plugin /tmp/" >> Dockerfile
+            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
             docker build -t opensearch-asynchronous-search:test .
             echo "imagePresent=true" >> $GITHUB_ENV
           else
@@ -85,20 +135,7 @@ jobs:
         with:
           name: logs
           path: build/testclusters/integTest-*/logs/*
-      - name: Create Artifact Path
-        run: |
-          mkdir -p asynchronous-search-artifacts
-          cp ./build/distributions/*.zip asynchronous-search-artifacts     
-      - name: Uploads coverage
-        uses: codecov/[email protected]
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-      # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v1
-        with:
-          name: asynchronous-search-plugin-ubuntu
-          path: asynchronous-search-artifacts
+
   windows-build:
     # Job name
     name: Build Asynchronous Search
@@ -127,6 +164,7 @@ jobs:
         with:
           name: asynchronous-search-plugin-windows
           path: asynchronous-search-artifacts
+
   mac-os-build:
     # Job name
     name: Build Asynchronous Search

diff --git a/.github/workflows/dco-check.yml b/.github/workflows/dco-check.yml
diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml
@@ -12,11 +12,24 @@ on:
       - "*"
 
 jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
   build:
     # Job name
+    needs: Get-CI-Image-Tag
     name: Build Asynchronous Search
     # This job runs on Linux
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK 11
@@ -27,11 +40,14 @@ jobs:
       - name: Checkout Branch
         uses: actions/checkout@v2
       - name: Run integration tests with multi node config
-        run: ./gradlew integTest -PnumNodes=5
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew integTest -PnumNodes=5"
       - name: Run Backwards Compatibility Tests
         run: |
           echo "Running backwards compatibility tests ..."
-          ./gradlew bwcTestSuite -Dtests.security.manager=false
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew bwcTestSuite -Dtests.security.manager=false"
       - name: Upload failed logs
         uses: actions/upload-artifact@v2
         if: failure()