Fix flaky test for vis-augmenter (#1038)

* wait for dashboard to be searchable Signed-off-by: Hailong Cui <[email protected]> * refresh private tenant kibana index also Signed-off-by: Hailong Cui <[email protected]> * Add one more filter for core ft Signed-off-by: Hailong Cui <[email protected]> * Add CYPRESS_NO_COMMAND_LOG flag Signed-off-by: Hailong Cui <[email protected]> * only run vis-augmenter for debug Signed-off-by: Hailong Cui <[email protected]> * only run vis-augmenter for debug Signed-off-by: Hailong Cui <[email protected]> * only run vis-augmenter for debug Signed-off-by: Hailong Cui <[email protected]> * wait for page load Signed-off-by: Hailong Cui <[email protected]> * firefox Signed-off-by: Hailong Cui <[email protected]> * revert test code Signed-off-by: Hailong Cui <[email protected]> * wait for page load Signed-off-by: Hailong Cui <[email protected]> * add more timeout Signed-off-by: Hailong Cui <[email protected]> * add more checks Signed-off-by: Hailong Cui <[email protected]> * add vis to dashboard Signed-off-by: Hailong Cui <[email protected]> * no command log Signed-off-by: Hailong Cui <[email protected]> * use correct flag CYPRESS_NO_COMMAND_LOG Signed-off-by: Hailong Cui <[email protected]> * set to global tenant when security enabled Signed-off-by: Hailong Cui <[email protected]> --------- Signed-off-by: Hailong Cui <[email protected]>
opensearch-project · Feb 23, 2024 · e9fa679 · e9fa679
1 parent 2784d24
commit e9fa679
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 7 deletions.
diff --git a/.github/workflows/cypress-workflow-bundle-snapshot-based-windows.yml b/.github/workflows/cypress-workflow-bundle-snapshot-based-windows.yml
@@ -17,14 +17,15 @@ jobs:
           filters: |
             tests:
               - 'cypress/**/core-opensearch-dashboards/**'
+              - 'cypress/utils/dashboards/**'
 
   tests-with-security:
     needs: changes
     if: ${{ needs.changes.outputs.tests == 'true' }}
     uses: ./.github/workflows/release-e2e-workflow-template-windows.yml
     with:
       test-name: Core Dashboards using Bundle Snapshot
-      test-command: env CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
       # not useful now as the windows e2e template currently do not allow serving parameters
       #osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true
 
@@ -34,7 +35,7 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template-windows.yml
     with:
       test-name: Core Dashboards using Bundle Snapshot
-      test-command: env CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
       # not useful now as the windows e2e template currently do not allow serving parameters
       #osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true
       security-enabled: false
diff --git a/.github/workflows/cypress-workflow-bundle-snapshot-based.yml b/.github/workflows/cypress-workflow-bundle-snapshot-based.yml
@@ -17,14 +17,15 @@ jobs:
           filters: |
             tests:
               - 'cypress/**/core-opensearch-dashboards/**'
+              - 'cypress/utils/dashboards/**'
 
   tests-with-security:
     needs: changes
     if: ${{ needs.changes.outputs.tests == 'true' }}
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Core Dashboards using Bundle Snapshot
-      test-command: env CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
       osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true --ml_commons_dashboards.enabled=true
 
   tests-without-security:
@@ -33,6 +34,6 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Core Dashboards using Bundle Snapshot
-      test-command: env CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
       osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true --ml_commons_dashboards.enabled=true
       security-enabled: false
diff --git a/...ion/core-opensearch-dashboards/opensearch-dashboards/apps/vis-augmenter/dashboard_spec.js b/...ion/core-opensearch-dashboards/opensearch-dashboards/apps/vis-augmenter/dashboard_spec.js
@@ -12,6 +12,7 @@ import {
   deleteVisAugmenterData,
   bootstrapDashboard,
 } from '../../../../../utils/dashboards/vis-augmenter/helpers';
+import { CURRENT_TENANT } from '../../../../../utils/commands';
 
 describe('Vis augmenter - existing dashboards work as expected', () => {
   describe('dashboard with ineligible, eligible, and vega visualizations', () => {
@@ -79,6 +80,7 @@ describe('Vis augmenter - existing dashboards work as expected', () => {
     );
 
     before(() => {
+      CURRENT_TENANT.newTenant = 'global';
       // Create a dashboard and add some visualizations
       bootstrapDashboard(
         INDEX_SETTINGS_FILEPATH_SIMPLE,

diff --git a/cypress/utils/dashboards/vis-augmenter/commands.js b/cypress/utils/dashboards/vis-augmenter/commands.js
@@ -32,8 +32,9 @@ Cypress.Commands.add('getMenuItems', { prevSubject: 'optional' }, (menu) =>
 );
 
 Cypress.Commands.add('visitDashboard', (dashboardName) => {
+  cy.intercept('/api/saved_objects/_find*').as('loadDashboards');
   cy.visit(`${BASE_PATH}/app/dashboards`);
-  cy.wait(2000);
+  cy.wait('@loadDashboards', { timeout: 300000 });
   cy.get('.euiFieldSearch').type(dashboardName);
   cy.wait(2000);
   cy.get('[data-test-subj="itemsInMemTable"]').contains(dashboardName).click({

diff --git a/cypress/utils/dashboards/vis-augmenter/helpers.js b/cypress/utils/dashboards/vis-augmenter/helpers.js
@@ -178,7 +178,6 @@ const addMetric = (metric, index) => {
           force: true,
         }
       );
-      cy.contains(`${metric.aggregation}`).click({ force: true });
     });
 
   // non-count aggregations will have an additional field value to set
@@ -257,7 +256,9 @@ export const bootstrapDashboard = (
     sampleDataFilepath
   );
 
+  cy.intercept('/api/saved_objects/*').as('savedObjectsApis');
   miscUtils.visitPage('app/dashboards#/create');
+  cy.wait(['@savedObjectsApis'], { timeout: 300000 });
 
   // Create several different visualizations
   visualizationSpecs.forEach((visualizationSpec) => {
@@ -267,9 +268,35 @@ export const bootstrapDashboard = (
       visualizationSpec.name,
       visualizationSpec.metrics
     );
+    // wait for vis saved
+    cy.contains(`Saved '${visualizationSpec.name}'`);
+    cy.wait(5000);
+    // cy.get(`[data-title="${visualizationSpec.name}"]`).should('have.length', 1);
+    // somehow the visualization is not added to dashboard
+    cy.get('body').then(($body) => {
+      if ($body.find(`[data-title="${visualizationSpec.name}"]`).length === 0) {
+        cy.getElementByTestId('dashboardAddPanelButton').click({ force: true });
+        cy.getElementByTestId('savedObjectFinderSearchInput')
+          .focus()
+          .clear()
+          .type(visualizationSpec.name);
+
+        cy.getElementByTestId(
+          `savedObjectTitle${visualizationSpec.name}`
+        ).click();
+
+        cy.getElementByTestId('euiFlyoutCloseButton').click();
+
+        cy.get(`[data-title="${visualizationSpec.name}"]`).should(
+          'have.length',
+          1
+        );
+      }
+    });
   });
 
-  cy.getElementByTestId('dashboardSaveMenuItem').click({
+  /// wait for page load
+  cy.getElementByTestId('dashboardSaveMenuItem', { timeout: 300000 }).click({
     force: true,
   });
 
@@ -278,6 +305,11 @@ export const bootstrapDashboard = (
   cy.getElementByTestId('confirmSaveSavedObjectButton').click({
     force: true,
   });
+
+  cy.contains(`Dashboard '${dashboardName}' was saved`);
+
+  // make newly created dashboards searchable
+  devToolsRequest('.kibana*/_refresh', 'POST');
 };
 
 export const filterByObjectType = (type) => {