[Reporting] Fix ability to export CSV on searched data with frozen indices

x-pack/plugins/reporting/common/constants.ts

@@ -45,6 +45,7 @@ export const KBN_SCREENSHOT_HEADER_BLOCK_LIST = [
 
 export const KBN_SCREENSHOT_HEADER_BLOCK_LIST_STARTS_WITH_PATTERN = ['proxy-'];
 
+export const UI_SETTINGS_SEARCH_INCLUDE_FROZEN = 'search:includeFrozen';
 export const UI_SETTINGS_CUSTOM_PDF_LOGO = 'xpackReporting:customPdfLogo';
 export const UI_SETTINGS_CSV_SEPARATOR = 'csv:separator';
 export const UI_SETTINGS_CSV_QUOTE_VALUES = 'csv:quoteValues';

x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts

@@ -198,7 +198,7 @@ it('calculates the bytes of the content', async () => {
     Rx.of({
       rawResponse: {
         hits: {
-          hits: range(0, HITS_TOTAL).map((hit, i) => ({
+          hits: range(0, HITS_TOTAL).map(() => ({
             fields: {
               message: ['this is a great message'],
             },
@@ -248,7 +248,7 @@ it('warns if max size was reached', async () => {
     Rx.of({
       rawResponse: {
         hits: {
-          hits: range(0, HITS_TOTAL).map((hit, i) => ({
+          hits: range(0, HITS_TOTAL).map(() => ({
             fields: {
               date: ['2020-12-31T00:14:28.000Z'],
               ip: ['110.135.176.89'],
@@ -289,7 +289,7 @@ it('uses the scrollId to page all the data', async () => {
       rawResponse: {
         _scroll_id: 'awesome-scroll-hero',
         hits: {
-          hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({
+          hits: range(0, HITS_TOTAL / 10).map(() => ({
             fields: {
               date: ['2020-12-31T00:14:28.000Z'],
               ip: ['110.135.176.89'],
@@ -304,7 +304,7 @@ it('uses the scrollId to page all the data', async () => {
   mockEsClient.asCurrentUser.scroll = jest.fn().mockResolvedValue({
     body: {
       hits: {
-        hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({
+        hits: range(0, HITS_TOTAL / 10).map(() => ({
           fields: {
             date: ['2020-12-31T00:14:28.000Z'],
             ip: ['110.135.176.89'],
@@ -337,7 +337,7 @@ it('uses the scrollId to page all the data', async () => {
 
   expect(mockDataClient.search).toHaveBeenCalledTimes(1);
   expect(mockDataClient.search).toBeCalledWith(
-    { params: { scroll: '30s', size: 500 } },
+    { params: { ignore_throttled: true, scroll: '30s', size: 500 } },
     { strategy: 'es' }
   );
 

x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts

@@ -83,8 +83,9 @@ export class CsvGenerator {
   private async scan(
     index: IndexPattern,
     searchSource: ISearchSource,
-    scrollSettings: CsvExportSettings['scroll']
+    settings: CsvExportSettings
   ) {
+    const { scroll: scrollSettings, includeFrozen } = settings;
     const searchBody = searchSource.getSearchRequestBody();
     this.logger.debug(`executing search request`);
     const searchParams = {
@@ -93,8 +94,10 @@ export class CsvGenerator {
         index: index.title,
         scroll: scrollSettings.duration,
         size: scrollSettings.size,
+        ignore_throttled: !includeFrozen,
       },
     };
+
     const results = (
       await this.clients.data.search(searchParams, { strategy: ES_SEARCH_STRATEGY }).toPromise()
     ).rawResponse as estypes.SearchResponse<unknown>;
@@ -326,7 +329,7 @@ export class CsvGenerator {
         let results: estypes.SearchResponse<unknown> | undefined;
         if (scrollId == null) {
           // open a scroll cursor in Elasticsearch
-          results = await this.scan(index, searchSource, scrollSettings);
+          results = await this.scan(index, searchSource, settings);
           scrollId = results?._scroll_id;
           if (results.hits?.total != null) {
             totalRecords = results.hits.total as number;

x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts

@@ -9,6 +9,7 @@ import {
   UI_SETTINGS_DATEFORMAT_TZ,
   UI_SETTINGS_CSV_QUOTE_VALUES,
   UI_SETTINGS_CSV_SEPARATOR,
+  UI_SETTINGS_SEARCH_INCLUDE_FROZEN,
 } from '../../../../common/constants';
 import { IUiSettingsClient } from 'kibana/server';
 import { savedObjectsClientMock, uiSettingsServiceMock } from 'src/core/server/mocks';
@@ -36,6 +37,8 @@ describe('getExportSettings', () => {
           return ',';
         case UI_SETTINGS_DATEFORMAT_TZ:
           return 'Browser';
+        case UI_SETTINGS_SEARCH_INCLUDE_FROZEN:
+          return false;
       }
 
       return 'helo world';
@@ -49,6 +52,7 @@ describe('getExportSettings', () => {
         "checkForFormulas": undefined,
         "escapeFormulaValues": undefined,
         "escapeValue": [Function],
+        "includeFrozen": false,
         "maxSizeBytes": undefined,
         "scroll": Object {
           "duration": undefined,

x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts

@@ -12,9 +12,10 @@ import { createEscapeValue } from '../../../../../../../src/plugins/data/common'
 import { ReportingConfig } from '../../../';
 import {
   CSV_BOM_CHARS,
-  UI_SETTINGS_DATEFORMAT_TZ,
   UI_SETTINGS_CSV_QUOTE_VALUES,
   UI_SETTINGS_CSV_SEPARATOR,
+  UI_SETTINGS_DATEFORMAT_TZ,
+  UI_SETTINGS_SEARCH_INCLUDE_FROZEN,
 } from '../../../../common/constants';
 import { LevelLogger } from '../../../lib';
 
@@ -30,6 +31,7 @@ export interface CsvExportSettings {
   checkForFormulas: boolean;
   escapeFormulaValues: boolean;
   escapeValue: (value: string) => string;
+  includeFrozen: boolean;
 }
 
 export const getExportSettings = async (
@@ -38,9 +40,7 @@ export const getExportSettings = async (
   timezone: string | undefined,
   logger: LevelLogger
 ): Promise<CsvExportSettings> => {
-  // Timezone
   let setTimezone: string;
-  // timezone in job params?
   if (timezone) {
     setTimezone = timezone;
   } else {
@@ -59,8 +59,9 @@ export const getExportSettings = async (
     }
   }
 
-  // Separator, QuoteValues
-  const [separator, quoteValues] = await Promise.all([
+  // Advanced Settings that affect search export + CSV
+  const [includeFrozen, separator, quoteValues] = await Promise.all([
+    client.get(UI_SETTINGS_SEARCH_INCLUDE_FROZEN),
     client.get(UI_SETTINGS_CSV_SEPARATOR),
     client.get(UI_SETTINGS_CSV_QUOTE_VALUES),
   ]);
@@ -76,6 +77,7 @@ export const getExportSettings = async (
       duration: config.get('csv', 'scroll', 'duration'),
     },
     bom,
+    includeFrozen,
     separator,
     maxSizeBytes: config.get('csv', 'maxSizeBytes'),
     checkForFormulas: config.get('csv', 'checkForFormulas'),

x-pack/test/reporting_api_integration/reporting_and_security/index.ts

@@ -29,5 +29,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./spaces'));
     loadTestFile(require.resolve('./usage'));
     loadTestFile(require.resolve('./ilm_migration_apis'));
+    loadTestFile(require.resolve('./search_frozen_indices'));
   });
 }

x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts

@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../ftr_provider_context';
+
+// eslint-disable-next-line import/no-default-export
+export default function ({ getService }: FtrProviderContext) {
+  const kibanaServer = getService('kibanaServer');
+  const supertestSvc = getService('supertest');
+  const esSupertest = getService('esSupertest');
+  const indexPatternId = 'cool-test-index-pattern';
+
+  async function callExportAPI() {
+    const job = {
+      browserTimezone: 'UTC',
+      columns: ['@timestamp', 'ip', 'utilization'],
+      searchSource: {
+        fields: [{ field: '*', include_unmapped: 'true' }],
+        filter: [
+          {
+            meta: { field: '@timestamp', index: indexPatternId, params: {} },
+            range: {
+              '@timestamp': {
+                format: 'strict_date_optional_time',
+                gte: '2020-08-24T00:00:00.000Z',
+                lte: '2022-08-24T21:40:48.346Z',
+              },
+            },
+          },
+        ],
+        index: indexPatternId,
+        parent: { filter: [], index: indexPatternId, query: { language: 'kuery', query: '' } },
+        sort: [{ '@timestamp': 'desc' }],
+        trackTotalHits: true,
+      },
+      title: 'Test search',
+    };
+
+    return await supertestSvc
+      .post(`/api/reporting/v1/generate/immediate/csv_searchsource`)
+      .set('kbn-xsrf', 'xxx')
+      .send(job);
+  }
+
+  describe('Frozen indices search', () => {
+    const reset = async () => {
+      await kibanaServer.uiSettings.replace({ 'search:includeFrozen': false });
+      try {
+        await esSupertest.delete('/test1,test2,test3');
+        await kibanaServer.savedObjects.delete({ type: 'index-pattern', id: indexPatternId });
+      } catch (err) {
+        // ignore 404 error
+      }
+    };
+
+    before(reset);
+    after(reset);
+
+    it('Search includes frozen indices based on Advanced Setting', async () => {
+      await kibanaServer.uiSettings.update({ 'csv:quoteValues': true });
+
+      // setup: add multiple indices of test data
+      await Promise.all([
+        esSupertest
+          .post('/test1/_doc')
+          .send({ '@timestamp': '2021-08-24T21:36:40Z', ip: '43.98.8.183', utilization: 18725 }),
+        esSupertest
+          .post('/test2/_doc')
+          .send({ '@timestamp': '2021-08-21T09:36:40Z', ip: '63.91.103.79', utilization: 8480 }),
+        esSupertest
+          .post('/test3/_doc')
+          .send({ '@timestamp': '2021-08-17T21:36:40Z', ip: '139.108.162.171', utilization: 3078 }),
+      ]);
+      await esSupertest.post('/test*/_refresh');
+
+      // setup: create index pattern
+      const indexPatternCreateResponse = await kibanaServer.savedObjects.create({
+        type: 'index-pattern',
+        id: indexPatternId,
+        overwrite: true,
+        attributes: { title: 'test*', timeFieldName: '@timestamp' },
+      });
+      expect(indexPatternCreateResponse.id).to.be(indexPatternId);
+
+      // 1. check the initial data with a CSV export
+      const initialSearch = await callExportAPI();
+      expectSnapshot(initialSearch.text).toMatchInline(`
+        "\\"@timestamp\\",ip,utilization
+        \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
+        \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
+        \\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\"
+        "
+      `);
+
+      // 2. freeze an index in the pattern
+      await esSupertest.post('/test3/_freeze').expect(200);
+      await esSupertest.post('/test*/_refresh').expect(200);
+
+      // 3. recheck the search results
+      const afterFreezeSearch = await callExportAPI();
+      expectSnapshot(afterFreezeSearch.text).toMatchInline(`
+        "\\"@timestamp\\",ip,utilization
+        \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
+        \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
+        "
+      `);
+
+      // 4. update setting to allow searching frozen data
+      await kibanaServer.uiSettings.update({ 'search:includeFrozen': true });
+
+      // 5. recheck the search results
+      const afterAllowSearch = await callExportAPI();
+      expectSnapshot(afterAllowSearch.text).toMatchInline(`
+        "\\"@timestamp\\",ip,utilization
+        \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
+        \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
+        \\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\"
+        "
+      `);
+    });
+  });
+}