[Backport main] [Infra] Add CI groups for OSD core test cases to avoi…

…d flaky test (#1368) * [Infra] Add CI groups for OSD core test cases to avoid flaky test (#1352) * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: run all Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: add ci-groups Signed-off-by: SuZhou-Joe <[email protected]> * feat: revert the deletion of windows flow Signed-off-by: SuZhou-Joe <[email protected]> * fix: integration test failure Signed-off-by: SuZhou-Joe <[email protected]> * fix: integration test failure Signed-off-by: SuZhou-Joe <[email protected]> --------- Signed-off-by: SuZhou-Joe <[email protected]> Co-authored-by: Yulong Ruan <[email protected]> (cherry picked from commit bd5e0fb) * feat: merge Signed-off-by: SuZhou-Joe <[email protected]> --------- Signed-off-by: SuZhou-Joe <[email protected]> Co-authored-by: SuZhou-Joe <[email protected]>
opensearch-project · Jun 12, 2024 · 24b5659 · 24b5659
1 parent c8cbf04
commit 24b5659
Show file tree

Hide file tree

Showing 4 changed files with 132 additions and 9 deletions.
diff --git a/.github/workflows/cypress-workflow-bundle-snapshot-based-ci-groups.yml b/.github/workflows/cypress-workflow-bundle-snapshot-based-ci-groups.yml
@@ -0,0 +1,112 @@
+name: Cypress test for core Dashboards
+on:
+  pull_request:
+    branches: ['**']
+    paths:
+      - '.github/workflows/cypress-workflow-bundle-snapshot-based-ci-groups.yml'
+      - 'cypress/**/core-opensearch-dashboards/**'
+      - 'cypress/utils/dashboards/**'
+  push:
+    branches: ['**']
+    paths:
+      - '.github/workflows/cypress-workflow-bundle-snapshot-based-ci-groups.yml'
+      - 'cypress/**/core-opensearch-dashboards/**'
+      - 'cypress/utils/dashboards/**'
+
+env:
+  CI_GROUPS: "1,2,3,4,5,6,7,8,9"
+
+jobs:
+  get_spec:
+    runs-on: 'ubuntu-latest'
+    outputs:
+      MATRIX_INCLUDES: ${{ steps.get_spec.outputs.MATRIX_INCLUDES }}
+    steps:
+      - name: Checkout Branch
+        uses: actions/checkout@v3
+      - name: Checkout cypress-test
+        uses: actions/checkout@v2
+        with:
+          repository: ${{github.repository}}
+          path: spec-detect
+      - id: get_spec
+        name: Get specs array
+        run: |
+          cd spec-detect
+          result="["
+          ## split CI_Groups into array
+          IFS="," read -a groups <<< "${{ env.CI_GROUPS }}"
+          for group in "${groups[@]}"; do
+            item="{\"ciGroup\": \"${group}\", \"specs\": "
+            IFS="," read -a SPEC_ARRAY <<< "$(npm run -s osd:ciGroup${group})"
+            FORMATTED_SPEC="\""
+            for i in "${SPEC_ARRAY[@]}"; do
+              FORMATTED_SPEC+="cypress/integration/core-opensearch-dashboards/opensearch-dashboards/${i},"
+            done
+            FORMATTED_SPEC+="\" }"
+            if [[ "${result}" != "[" ]];
+            then
+                result+=","
+            fi
+            item+="${FORMATTED_SPEC}"
+            result+="${item}"
+          done
+
+          result+=']';
+
+          echo ${result}
+          echo "MATRIX_INCLUDES=${result}" >> "$GITHUB_OUTPUT"
+
+  tests-with-security:
+    needs: ["get_spec"]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.get_spec.outputs.MATRIX_INCLUDES) }}
+    name: (security)osd:ciGroup${{ matrix.ciGroup }}
+    uses: ./.github/workflows/release-e2e-workflow-template.yml
+    with:
+      test-name: "osd:ciGroup${{ matrix.ciGroup }}"
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec '${{ matrix.specs }}'
+      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --vis_builder.enabled=true --ml_commons_dashboards.enabled=true
+
+  tests-without-security:
+    needs: ["get_spec"]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.get_spec.outputs.MATRIX_INCLUDES) }}
+    name: (non-security)osd:ciGroup${{ matrix.ciGroup }}
+    uses: ./.github/workflows/release-e2e-workflow-template.yml
+    with:
+      test-name: "osd:ciGroup${{ matrix.ciGroup }}"
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec '${{ matrix.specs }}'
+      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --vis_builder.enabled=true --ml_commons_dashboards.enabled=true
+      security-enabled: false
+
+  # Hold on windows test cases
+  # https://github.com/opensearch-project/opensearch-dashboards-functional-test/actions/runs/9377445544/job/25819022121?pr=1352
+  # tests-with-security-windows:
+  #   needs: ["get_spec"]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       include: ${{ fromJSON(needs.get_spec.outputs.MATRIX_INCLUDES) }}
+  #   name: (security)osd:ciGroup${{ matrix.ciGroup }}
+  #   uses: ./.github/workflows/release-e2e-workflow-template-windows.yml
+  #   with:
+  #     test-name: "osd:ciGroup${{ matrix.ciGroup }}"
+  #     test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chrome --spec '${{ matrix.specs }}'
+
+  # tests-without-security-windows:
+  #   needs: ["get_spec"]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       include: ${{ fromJSON(needs.get_spec.outputs.MATRIX_INCLUDES) }}
+  #   name: (non-security)osd:ciGroup${{ matrix.ciGroup }}
+  #   uses: ./.github/workflows/release-e2e-workflow-template-windows.yml
+  #   with:
+  #     test-name: "osd:ciGroup${{ matrix.ciGroup }}"
+  #     test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chrome --spec '${{ matrix.specs }}'
+  #     security-enabled: false
diff --git a/.github/workflows/cypress-workflow-bundle-snapshot-based-windows.yml b/.github/workflows/cypress-workflow-bundle-snapshot-based-windows.yml
@@ -27,4 +27,4 @@ jobs:
       test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
       # not useful now as the windows e2e template currently do not allow serving parameters
       #osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true
-      security-enabled: false
+      security-enabled: false
diff --git a/.github/workflows/cypress-workflow-bundle-snapshot-based.yml b/.github/workflows/cypress-workflow-bundle-snapshot-based.yml
@@ -26,11 +26,3 @@ jobs:
       test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_UIMETRIC_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
       osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --vis_builder.enabled=true --ml_commons_dashboards.enabled=true --usageCollection.uiMetric.enabled=true
       security-enabled: false
-
-  tests-with-multiple-data-source-and-disabled-local-cluster:
-    uses: ./.github/workflows/release-e2e-workflow-template.yml
-    with:
-      test-name: Core Dashboards using Bundle Snapshot
-      test-command: env CYPRESS_DISABLE_LOCAL_CLUSTER=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/mds*.js'
-      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --data_source.hideLocalCluster=true
-      security-enabled: true
diff --git a/.github/workflows/mds-release-e2e-workflow.yml b/.github/workflows/mds-release-e2e-workflow.yml
@@ -0,0 +1,19 @@
+name: E2E Cypress tests workflow for MDS
+on:
+  pull_request:
+    branches: ['**']
+    paths:
+      - 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/mds*.js'
+  push:
+    branches: ['**']
+    paths:
+      - 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/mds*.js'
+
+jobs:
+  tests-with-multiple-data-source-and-disabled-local-cluster:
+    uses: ./.github/workflows/release-e2e-workflow-template.yml
+    with:
+      test-name: MDS enabled and local cluster disabled
+      test-command: env CYPRESS_DISABLE_LOCAL_CLUSTER=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/mds*.js'
+      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --data_source.hideLocalCluster=true
+      security-enabled: true