From e9fa6796a84f02851f7920baed854959e3ffe9b2 Mon Sep 17 00:00:00 2001 From: Hailong Cui Date: Fri, 23 Feb 2024 09:33:44 +0800 Subject: [PATCH] Fix flaky test for vis-augmenter (#1038) * wait for dashboard to be searchable Signed-off-by: Hailong Cui * refresh private tenant kibana index also Signed-off-by: Hailong Cui * Add one more filter for core ft Signed-off-by: Hailong Cui * Add CYPRESS_NO_COMMAND_LOG flag Signed-off-by: Hailong Cui * only run vis-augmenter for debug Signed-off-by: Hailong Cui * only run vis-augmenter for debug Signed-off-by: Hailong Cui * only run vis-augmenter for debug Signed-off-by: Hailong Cui * wait for page load Signed-off-by: Hailong Cui * firefox Signed-off-by: Hailong Cui * revert test code Signed-off-by: Hailong Cui * wait for page load Signed-off-by: Hailong Cui * add more timeout Signed-off-by: Hailong Cui * add more checks Signed-off-by: Hailong Cui * add vis to dashboard Signed-off-by: Hailong Cui * no command log Signed-off-by: Hailong Cui * use correct flag CYPRESS_NO_COMMAND_LOG Signed-off-by: Hailong Cui * set to global tenant when security enabled Signed-off-by: Hailong Cui --------- Signed-off-by: Hailong Cui --- ...workflow-bundle-snapshot-based-windows.yml | 5 +-- ...cypress-workflow-bundle-snapshot-based.yml | 5 +-- .../apps/vis-augmenter/dashboard_spec.js | 2 ++ .../dashboards/vis-augmenter/commands.js | 3 +- .../utils/dashboards/vis-augmenter/helpers.js | 36 +++++++++++++++++-- 5 files changed, 44 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cypress-workflow-bundle-snapshot-based-windows.yml b/.github/workflows/cypress-workflow-bundle-snapshot-based-windows.yml index 2a8b82fdf..520383555 100644 --- a/.github/workflows/cypress-workflow-bundle-snapshot-based-windows.yml +++ b/.github/workflows/cypress-workflow-bundle-snapshot-based-windows.yml @@ -17,6 +17,7 @@ jobs: filters: | tests: - 'cypress/**/core-opensearch-dashboards/**' + - 'cypress/utils/dashboards/**' tests-with-security: needs: changes @@ -24,7 +25,7 @@ jobs: uses: ./.github/workflows/release-e2e-workflow-template-windows.yml with: test-name: Core Dashboards using Bundle Snapshot - test-command: env CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js' + test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js' # not useful now as the windows e2e template currently do not allow serving parameters #osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true @@ -34,7 +35,7 @@ jobs: uses: ./.github/workflows/release-e2e-workflow-template-windows.yml with: test-name: Core Dashboards using Bundle Snapshot - test-command: env CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js' + test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js' # not useful now as the windows e2e template currently do not allow serving parameters #osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true security-enabled: false diff --git a/.github/workflows/cypress-workflow-bundle-snapshot-based.yml b/.github/workflows/cypress-workflow-bundle-snapshot-based.yml index b1038c86d..f542887d7 100644 --- a/.github/workflows/cypress-workflow-bundle-snapshot-based.yml +++ b/.github/workflows/cypress-workflow-bundle-snapshot-based.yml @@ -17,6 +17,7 @@ jobs: filters: | tests: - 'cypress/**/core-opensearch-dashboards/**' + - 'cypress/utils/dashboards/**' tests-with-security: needs: changes @@ -24,7 +25,7 @@ jobs: uses: ./.github/workflows/release-e2e-workflow-template.yml with: test-name: Core Dashboards using Bundle Snapshot - test-command: env CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js' + test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js' osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true --ml_commons_dashboards.enabled=true tests-without-security: @@ -33,6 +34,6 @@ jobs: uses: ./.github/workflows/release-e2e-workflow-template.yml with: test-name: Core Dashboards using Bundle Snapshot - test-command: env CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js' + test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js' osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true --ml_commons_dashboards.enabled=true security-enabled: false diff --git a/cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/vis-augmenter/dashboard_spec.js b/cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/vis-augmenter/dashboard_spec.js index d97647c55..917c81288 100644 --- a/cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/vis-augmenter/dashboard_spec.js +++ b/cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/vis-augmenter/dashboard_spec.js @@ -12,6 +12,7 @@ import { deleteVisAugmenterData, bootstrapDashboard, } from '../../../../../utils/dashboards/vis-augmenter/helpers'; +import { CURRENT_TENANT } from '../../../../../utils/commands'; describe('Vis augmenter - existing dashboards work as expected', () => { describe('dashboard with ineligible, eligible, and vega visualizations', () => { @@ -79,6 +80,7 @@ describe('Vis augmenter - existing dashboards work as expected', () => { ); before(() => { + CURRENT_TENANT.newTenant = 'global'; // Create a dashboard and add some visualizations bootstrapDashboard( INDEX_SETTINGS_FILEPATH_SIMPLE, diff --git a/cypress/utils/dashboards/vis-augmenter/commands.js b/cypress/utils/dashboards/vis-augmenter/commands.js index e0ee418de..28d3b57e0 100644 --- a/cypress/utils/dashboards/vis-augmenter/commands.js +++ b/cypress/utils/dashboards/vis-augmenter/commands.js @@ -32,8 +32,9 @@ Cypress.Commands.add('getMenuItems', { prevSubject: 'optional' }, (menu) => ); Cypress.Commands.add('visitDashboard', (dashboardName) => { + cy.intercept('/api/saved_objects/_find*').as('loadDashboards'); cy.visit(`${BASE_PATH}/app/dashboards`); - cy.wait(2000); + cy.wait('@loadDashboards', { timeout: 300000 }); cy.get('.euiFieldSearch').type(dashboardName); cy.wait(2000); cy.get('[data-test-subj="itemsInMemTable"]').contains(dashboardName).click({ diff --git a/cypress/utils/dashboards/vis-augmenter/helpers.js b/cypress/utils/dashboards/vis-augmenter/helpers.js index 49103a96b..1cccea74a 100644 --- a/cypress/utils/dashboards/vis-augmenter/helpers.js +++ b/cypress/utils/dashboards/vis-augmenter/helpers.js @@ -178,7 +178,6 @@ const addMetric = (metric, index) => { force: true, } ); - cy.contains(`${metric.aggregation}`).click({ force: true }); }); // non-count aggregations will have an additional field value to set @@ -257,7 +256,9 @@ export const bootstrapDashboard = ( sampleDataFilepath ); + cy.intercept('/api/saved_objects/*').as('savedObjectsApis'); miscUtils.visitPage('app/dashboards#/create'); + cy.wait(['@savedObjectsApis'], { timeout: 300000 }); // Create several different visualizations visualizationSpecs.forEach((visualizationSpec) => { @@ -267,9 +268,35 @@ export const bootstrapDashboard = ( visualizationSpec.name, visualizationSpec.metrics ); + // wait for vis saved + cy.contains(`Saved '${visualizationSpec.name}'`); + cy.wait(5000); + // cy.get(`[data-title="${visualizationSpec.name}"]`).should('have.length', 1); + // somehow the visualization is not added to dashboard + cy.get('body').then(($body) => { + if ($body.find(`[data-title="${visualizationSpec.name}"]`).length === 0) { + cy.getElementByTestId('dashboardAddPanelButton').click({ force: true }); + cy.getElementByTestId('savedObjectFinderSearchInput') + .focus() + .clear() + .type(visualizationSpec.name); + + cy.getElementByTestId( + `savedObjectTitle${visualizationSpec.name}` + ).click(); + + cy.getElementByTestId('euiFlyoutCloseButton').click(); + + cy.get(`[data-title="${visualizationSpec.name}"]`).should( + 'have.length', + 1 + ); + } + }); }); - cy.getElementByTestId('dashboardSaveMenuItem').click({ + /// wait for page load + cy.getElementByTestId('dashboardSaveMenuItem', { timeout: 300000 }).click({ force: true, }); @@ -278,6 +305,11 @@ export const bootstrapDashboard = ( cy.getElementByTestId('confirmSaveSavedObjectButton').click({ force: true, }); + + cy.contains(`Dashboard '${dashboardName}' was saved`); + + // make newly created dashboards searchable + devToolsRequest('.kibana*/_refresh', 'POST'); }; export const filterByObjectType = (type) => {