cleanup and optimize uploading data

Signed-off-by: Qingyang(Abby) Hu <[email protected]>
opensearch-project · Oct 3, 2023 · 00a7328 · 00a7328
1 parent f8e335d
commit 00a7328
Show file tree

Hide file tree

Showing 19 changed files with 353 additions and 124 deletions.
diff --git a/...tion/core-opensearch-dashboards/opensearch-dashboards/apps/data_explorer/discover.spec.js b/...tion/core-opensearch-dashboards/opensearch-dashboards/apps/data_explorer/discover.spec.js
@@ -17,23 +17,27 @@ const testFixtureHandler = new TestFixtureHandler(
   cy,
   Cypress.env('openSearchUrl')
 );
+const indexSet = [
+  'logstash-2015.09.22',
+  'logstash-2015.09.21',
+  'logstash-2015.09.20',
+];
 
-describe('discover app', () => {
+describe('discover app', { scrollBehavior: false }, () => {
   before(() => {
-    testFixtureHandler.importJSONMapping(
-      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/discover/discover.mappings.json.txt'
-    );
-
-    testFixtureHandler.importJSONDoc(
-      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/discover/discover.json.txt'
+    // import logstash functional
+    testFixtureHandler.importJSONDocIfNeeded(
+      indexSet,
+      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.mappings.json.txt',
+      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.json.txt'
     );
 
     testFixtureHandler.importJSONMapping(
-      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.mappings.json.txt'
+      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/discover/discover.mappings.json.txt'
     );
 
     testFixtureHandler.importJSONDoc(
-      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.json.txt'
+      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/discover/discover.json.txt'
     );
 
     cy.setAdvancedSetting({
@@ -48,6 +52,8 @@ describe('discover app', () => {
     cy.waitForSearch();
   });
 
+  after(() => {});
+
   describe('save search', () => {
     const saveSearch1 = 'Save Search # 1';
     const saveSearch2 = 'Modified Save Search # 1';
@@ -119,22 +125,26 @@ describe('discover app', () => {
     });
   });
 
-  describe('save search #2 which has an empty time range', () => {
-    const fromTime = 'Jun 11, 1999 @ 09:22:11.000';
-    const toTime = 'Jun 12, 1999 @ 11:21:04.000';
+  describe(
+    'save search #2 which has an empty time range',
+    { scrollBehavior: false },
+    () => {
+      const fromTime = 'Jun 11, 1999 @ 09:22:11.000';
+      const toTime = 'Jun 12, 1999 @ 11:21:04.000';
 
-    before(() => {
-      cy.setTopNavDate(fromTime, toTime);
-    });
+      before(() => {
+        cy.setTopNavDate(fromTime, toTime);
+      });
 
-    it('should show "no results"', () => {
-      cy.getElementByTestId('discoverNoResults').should('be.exist');
-    });
+      it('should show "no results"', () => {
+        cy.getElementByTestId('discoverNoResults').should('be.exist');
+      });
 
-    it('should suggest a new time range is picked', () => {
-      cy.getElementByTestId('discoverNoResultsTimefilter').should('be.exist');
-    });
-  });
+      it('should suggest a new time range is picked', () => {
+        cy.getElementByTestId('discoverNoResultsTimefilter').should('be.exist');
+      });
+    }
+  );
 
   describe('nested query', () => {
     before(() => {
@@ -157,32 +167,44 @@ describe('discover app', () => {
 
       cy.loadSaveSearch(expected.title);
       cy.setTopNavDate(DE_DEFAULT_START_TIME, DE_DEFAULT_END_TIME);
+      cy.waitForSearch();
       cy.waitForLoader();
-      cy.getElementByTestId('docTable')
-        .should('have.attr', 'data-shared-item')
-        .should('have.attr', 'data-title', expected.title)
-        .should('have.attr', 'data-description', expected.description);
+      cy.getElementByTestId('discoverTable').should(
+        'have.attr',
+        'data-shared-item'
+      );
+      cy.getElementByTestId('discoverTable').should(
+        'have.attr',
+        'data-title',
+        expected.title
+      );
+      cy.getElementByTestId('discoverTable').should(
+        'have.attr',
+        'data-description',
+        expected.description
+      );
     });
   });
 
-  //https://github.com/opensearch-project/OpenSearch-Dashboards/issues/5057
-  describe.skip('usage of discover:searchOnPageLoad', () => {
+  describe('usage of discover:searchOnPageLoad', () => {
     it('should fetch data from OpenSearch initially when discover:searchOnPageLoad is false', function () {
       cy.setAdvancedSetting({
         'discover:searchOnPageLoad': false,
       });
       miscUtils.visitPage(`app/data-explorer/discover#/`);
       cy.waitForLoader();
-      // expect(await PageObjects.discover.getNrOfFetches()).to.be(0);
+      cy.getElementByTestId('discoverTable').should('not.exist');
     });
 
     it('should not fetch data from OpenSearch initially when discover:searchOnPageLoad is true', function () {
       cy.setAdvancedSetting({
         'discover:searchOnPageLoad': true,
       });
       miscUtils.visitPage(`app/data-explorer/discover#/`);
+      cy.setTopNavDate(DE_DEFAULT_START_TIME, DE_DEFAULT_END_TIME);
+      cy.waitForSearch();
       cy.waitForLoader();
-      // expect(await PageObjects.discover.getNrOfFetches()).to.be(1);
+      cy.getElementByTestId('discoverTable').should('be.visible');
     });
   });
 
@@ -197,7 +219,7 @@ describe('discover app', () => {
 
       cy.getElementByTestId('fieldFilterSearchInput').type('ip');
 
-      cy.getElementByTestId('fieldToggle-ip').should('be.visible').click();
+      cy.getElementByTestId('fieldToggle-ip').click({ force: true });
 
       cy.getElementByTestId('dataGridHeaderCell-ip').should('be.visible');
       cy.getElementByTestId('dataGridHeaderCellActionButton-ip').click();
@@ -210,7 +232,7 @@ describe('discover app', () => {
 
       cy.contains('button', 'Move left').click();
 
-      cy.getElementByTestId('fieldToggle-ip').should('be.visible').click();
+      cy.getElementByTestId('fieldToggle-ip').click({ force: true });
 
       cy.getElementByTestId('dataGridHeaderCell-ip').should('not.be.exist');
     });

diff --git a/...opensearch-dashboards/opensearch-dashboards/apps/data_explorer/discover_histogram.spec.js b/...opensearch-dashboards/opensearch-dashboards/apps/data_explorer/discover_histogram.spec.js
@@ -13,8 +13,13 @@ const testFixtureHandler = new TestFixtureHandler(
   cy,
   Cypress.env('openSearchUrl')
 );
+const indexSet = [
+  'logstash-2015.09.22',
+  'logstash-2015.09.21',
+  'logstash-2015.09.20',
+];
 
-describe('discover histogram', () => {
+describe('discover histogram', { scrollBehavior: false }, () => {
   before(() => {
     cy.log('load opensearch-dashboards index with default index pattern');
 
@@ -24,11 +29,9 @@ describe('discover histogram', () => {
     );
 
     // import logstash functional
-    testFixtureHandler.importJSONMapping(
-      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.mappings.json.txt'
-    );
-
-    testFixtureHandler.importJSONDoc(
+    testFixtureHandler.importJSONDocIfNeeded(
+      indexSet,
+      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.mappings.json.txt',
       'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.json.txt'
     );
 
@@ -53,6 +56,7 @@ describe('discover histogram', () => {
 
   after(() => {
     miscUtils.visitPage('app/management/opensearch-dashboards/settings');
+    cy.waitForLoader();
     cy.getElementByTestId('advancedSetting-resetField-dateFormat:tz').click();
     cy.getElementByTestId('advancedSetting-saveButton').click({ force: true });
     testFixtureHandler.clearJSONMapping(

diff --git a/...ore-opensearch-dashboards/opensearch-dashboards/apps/data_explorer/doc_navigation.spec.js b/...ore-opensearch-dashboards/opensearch-dashboards/apps/data_explorer/doc_navigation.spec.js
@@ -3,12 +3,31 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { MiscUtils } from '@opensearch-dashboards-test/opensearch-dashboards-test-library';
+import {
+  MiscUtils,
+  TestFixtureHandler,
+} from '@opensearch-dashboards-test/opensearch-dashboards-test-library';
 
 const miscUtils = new MiscUtils(cy);
+const testFixtureHandler = new TestFixtureHandler(
+  cy,
+  Cypress.env('openSearchUrl')
+);
+const indexSet = [
+  'logstash-2015.09.22',
+  'logstash-2015.09.21',
+  'logstash-2015.09.20',
+];
 
 describe('doc link in discover', () => {
   beforeEach(() => {
+    // import logstash functional
+    testFixtureHandler.importJSONDocIfNeeded(
+      indexSet,
+      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.mappings.json.txt',
+      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.json.txt'
+    );
+
     cy.setAdvancedSetting({
       defaultIndex: 'logstash-*',
     });
@@ -20,17 +39,23 @@ describe('doc link in discover', () => {
     cy.waitForSearch();
   });
 
+  after(() => {});
+
   it('should open the doc view of the selected document', function () {
     cy.getElementByTestId(`docTableExpandToggleColumn-0`)
       .should('be.visible')
       .click();
     cy.getElementByTestId(`documentDetailFlyOut`).should('be.visible');
 
     // Both actions will take to the new tab
-    cy.getElementByTestId('docTableRowAction').contains('View single document');
+    cy.getElementByTestId('docTableRowAction-0').should(
+      'have.text',
+      'View surrounding documents(opens in a new tab or window)'
+    );
 
-    cy.getElementByTestId('docTableRowAction').contains(
-      'View surrounding documents'
+    cy.getElementByTestId('docTableRowAction-1').should(
+      'have.text',
+      'View single document(opens in a new tab or window)'
     );
   });
 

diff --git a/...ion/core-opensearch-dashboards/opensearch-dashboards/apps/data_explorer/doc_table.spec.js b/...ion/core-opensearch-dashboards/opensearch-dashboards/apps/data_explorer/doc_table.spec.js
@@ -3,12 +3,31 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { MiscUtils } from '@opensearch-dashboards-test/opensearch-dashboards-test-library';
+import {
+  MiscUtils,
+  TestFixtureHandler,
+} from '@opensearch-dashboards-test/opensearch-dashboards-test-library';
 
 const miscUtils = new MiscUtils(cy);
+const testFixtureHandler = new TestFixtureHandler(
+  cy,
+  Cypress.env('openSearchUrl')
+);
+const indexSet = [
+  'logstash-2015.09.22',
+  'logstash-2015.09.21',
+  'logstash-2015.09.20',
+];
 
 describe('discover doc table', () => {
   before(() => {
+    // import logstash functional
+    testFixtureHandler.importJSONDocIfNeeded(
+      indexSet,
+      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.mappings.json.txt',
+      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.json.txt'
+    );
+
     cy.setAdvancedSetting({
       defaultIndex: 'logstash-*',
     });
@@ -20,23 +39,21 @@ describe('discover doc table', () => {
     cy.waitForSearch();
   });
 
+  after(() => {});
+
   describe('add and remove columns', function () {
     it('should add more columns to the table', function () {
       cy.getElementByTestId('fieldFilterSearchInput').type('phpmemory');
 
-      cy.getElementByTestId('fieldToggle-phpmemory')
-        .should('be.visible')
-        .click();
+      cy.getElementByTestId('fieldToggle-phpmemory').click({ force: true });
 
       cy.getElementByTestId('dataGridHeaderCell-phpmemory').should(
         'be.visible'
       );
     });
 
     it('should remove columns from the table', function () {
-      cy.getElementByTestId('fieldToggle-phpmemory')
-        .should('be.visible')
-        .click();
+      cy.getElementByTestId('fieldToggle-phpmemory').click({ force: true });
 
       cy.getElementByTestId('dataGridHeaderCell-phpmemory').should('not.exist');
     });

diff --git a/...ration/core-opensearch-dashboards/opensearch-dashboards/apps/data_explorer/errors.spec.js b/...ration/core-opensearch-dashboards/opensearch-dashboards/apps/data_explorer/errors.spec.js
@@ -35,6 +35,12 @@ describe('errors', () => {
     cy.setTopNavDate(DE_DEFAULT_START_TIME, DE_DEFAULT_END_TIME);
   });
 
+  // after(() => {
+  //   testFixtureHandler.clearJSONMapping(
+  //     'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/invalid_scripted_field/mappings.json.txt'
+  //   );
+  // });
+
   describe('invalid scripted field error', function () {
     it('is rendered', function () {
       cy.getElementByTestId('painlessStackTrace').should('be.visible');

diff --git a/...on/core-opensearch-dashboards/opensearch-dashboards/apps/data_explorer/field_data.spec.js b/...on/core-opensearch-dashboards/opensearch-dashboards/apps/data_explorer/field_data.spec.js
@@ -13,24 +13,27 @@ const testFixtureHandler = new TestFixtureHandler(
   cy,
   Cypress.env('openSearchUrl')
 );
+const indexSet = [
+  'logstash-2015.09.22',
+  'logstash-2015.09.21',
+  'logstash-2015.09.20',
+];
 
 describe('discover tab', () => {
   before(() => {
-    testFixtureHandler.importJSONMapping(
-      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/discover/discover.mappings.json.txt'
-    );
-
-    testFixtureHandler.importJSONDoc(
-      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/discover/discover.json.txt'
+    // import logstash functional
+    testFixtureHandler.importJSONDocIfNeeded(
+      indexSet,
+      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.mappings.json.txt',
+      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.json.txt'
     );
 
-    // import logstash functional
     testFixtureHandler.importJSONMapping(
-      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.mappings.json.txt'
+      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/discover/discover.mappings.json.txt'
     );
 
     testFixtureHandler.importJSONDoc(
-      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/logstash/logstash.json.txt'
+      'cypress/fixtures/dashboard/opensearch_dashboards/data_explorer/discover/discover.json.txt'
     );
 
     cy.setAdvancedSetting({
@@ -44,6 +47,8 @@ describe('discover tab', () => {
     cy.waitForSearch();
   });
 
+  after(() => {});
+
   describe('field data', function () {
     it('search php should show the correct hit count', function () {
       const expectedHitCount = '445';
@@ -54,7 +59,7 @@ describe('discover tab', () => {
     it('the search term should be highlighted in the field data', function () {
       cy.getElementByTestId('dataGridWrapper')
         .get('mark')
-        .should('have.length', 10);
+        .should('have.length', 100);
     });
 
     it('search type:apache should show the correct hit count', () => {
@@ -80,6 +85,7 @@ describe('discover tab', () => {
       const expectedError =
         'Expected ":", "<", "<=", ">", ">=", AND, OR, end of input, ' +
         'whitespace but "(" found.';
+      cy.getElementByTestId('queryInput').clear();
       cy.setTopNavQuery('xxx(yyy))');
       cy.getElementByTestId('errorToastMessage').contains(expectedError);
     });