MAINT: GitHub action for e2e secrets (opensearch-project#3704)

* ADD: initial AWS testing resources CDK Signed-off-by: George Chen <[email protected]> * MNT: renaming Signed-off-by: George Chen <[email protected]> * E2E: basicLogWithAwsSecretsEndToEndTest Signed-off-by: George Chen <[email protected]> * MAINT: refactoring on build.gradle Signed-off-by: George Chen <[email protected]> * MAINT: use AWS_PROFILE Signed-off-by: George Chen <[email protected]> * MAINT: check aws credentials Signed-off-by: George Chen <[email protected]> * maint: check absolute path of .aws Signed-off-by: George Chen <[email protected]> * MAINT: check output Signed-off-by: George Chen <[email protected]> * MAINT: check output Signed-off-by: George Chen <[email protected]> * MAINT: check step output Signed-off-by: George Chen <[email protected]> * MAINT: aws configure Signed-off-by: George Chen <[email protected]> * MAINT: set files Signed-off-by: George Chen <[email protected]> * MNT: format Signed-off-by: George Chen <[email protected]> * MAINT: check files Signed-off-by: George Chen <[email protected]> * testing Signed-off-by: George Chen <[email protected]> * MAINT: run actual tests Signed-off-by: George Chen <[email protected]> * TST: assume new role Signed-off-by: George Chen <[email protected]> * MAINT: add test Signed-off-by: George Chen <[email protected]> * MAINT: title Signed-off-by: George Chen <[email protected]> * MAINT: back test on branch Signed-off-by: George Chen <[email protected]> * MAINT: revert Signed-off-by: George Chen <[email protected]> * MAINT: job name Signed-off-by: George Chen <[email protected]> * MAINT: file name Signed-off-by: George Chen <[email protected]> * MAINT: us-east-2 Signed-off-by: George Chen <[email protected]> * MAINT: manual workflow trigger Signed-off-by: George Chen <[email protected]> * MAINT: add back variable Signed-off-by: George Chen <[email protected]> * MAINT: use repo variables Signed-off-by: George Chen <[email protected]> * MAINT: target change Signed-off-by: George Chen <[email protected]> * TST: any push Signed-off-by: George Chen <[email protected]> * MAINT: revert change Signed-off-by: George Chen <[email protected]> --------- Signed-off-by: George Chen <[email protected]>
kkondaka · Nov 28, 2023 · 9449f24 · 9449f24
1 parent 2cb1724
commit 9449f24
Show file tree

Hide file tree

Showing 2 changed files with 54 additions and 0 deletions.
diff --git a/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml b/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml
@@ -0,0 +1,53 @@
+# This workflow will build a Java project with Gradle
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle
+
+name: Data Prepper AWS secrets End-to-end test with Gradle
+on:
+  push:
+    branches: [ main ]
+  pull_request_target:
+    types: [ opened, synchronize, reopened ]
+  workflow_dispatch:
+# permission can be added at job level or workflow level
+permissions:
+  id-token: write   # This is required for requesting the JWT
+  contents: read    # This is required for actions/checkout
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        java: [ 11, 17, 21, docker ]
+        test: [ 'basicLogWithAwsSecretsEndToEndTest' ]
+      fail-fast: false
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Git clone the repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: configure aws credentials
+        id: creds
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.TEST_IAM_ROLE_ARN }}
+          aws-region: ${{ secrets.TEST_REGION }}
+          output-credentials: true
+      - name: get caller identity 1
+        run: |
+          aws sts get-caller-identity
+      - name: Configure AWS Credentials file
+        run: |
+          aws configure set default.region ${{ secrets.TEST_REGION }}
+          aws configure set default.aws_access_key_id ${{ steps.creds.outputs.aws-access-key-id }}
+          aws configure set default.aws_secret_access_key ${{ steps.creds.outputs.aws-secret-access-key }}
+          aws configure set default.aws_session_token ${{ steps.creds.outputs.aws-session-token }}
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Checkout Data Prepper
+        uses: actions/checkout@v2
+      - name: Run basic grok with AWS secrets end-to-end tests with Gradle
+        run: AWS_PROFILE=default ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:log:${{ matrix.test }}
diff --git a/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml b/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml
@@ -3,6 +3,7 @@ pipeline_configurations:
     secrets:
       opensearch-sink:
         secret_id: "opensearch-sink-basic-credentials"
+        region: "us-east-2"
 grok-pipeline:
   source:
     http: