From c8d3e2456629db9a807ac60f9d55a7bbe31eeaad Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 24 Aug 2021 13:45:00 -0700 Subject: [PATCH 1/6] use include frozen setting in csv export --- x-pack/plugins/reporting/common/constants.ts | 1 + .../csv_searchsource/generate_csv/generate_csv.ts | 2 ++ .../generate_csv/get_export_settings.test.ts | 4 ++++ .../generate_csv/get_export_settings.ts | 10 ++++++---- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/reporting/common/constants.ts b/x-pack/plugins/reporting/common/constants.ts index 842d7d1eb4b8d..0e7d8f1ffe9ca 100644 --- a/x-pack/plugins/reporting/common/constants.ts +++ b/x-pack/plugins/reporting/common/constants.ts @@ -45,6 +45,7 @@ export const KBN_SCREENSHOT_HEADER_BLOCK_LIST = [ export const KBN_SCREENSHOT_HEADER_BLOCK_LIST_STARTS_WITH_PATTERN = ['proxy-']; +export const UI_SETTINGS_SEARCH_INCLUDE_FROZEN = 'search:includeFrozen'; export const UI_SETTINGS_CUSTOM_PDF_LOGO = 'xpackReporting:customPdfLogo'; export const UI_SETTINGS_CSV_SEPARATOR = 'csv:separator'; export const UI_SETTINGS_CSV_QUOTE_VALUES = 'csv:quoteValues'; diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts index b49dbc1043135..9e8eb0319e4c3 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts @@ -66,6 +66,7 @@ function isPlainStringArray( export class CsvGenerator { private _columns?: string[]; private _formatters?: Record; + private includeFrozen: boolean; private csvContainsFormulas = false; private maxSizeReached = false; private csvRowCount = 0; @@ -93,6 +94,7 @@ export class CsvGenerator { index: index.title, scroll: scrollSettings.duration, size: scrollSettings.size, + ignore_throttled: !this.includeFrozen, }, }; const results = ( diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts index efdb583a89dc8..2ae3e5e712d31 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts @@ -9,6 +9,7 @@ import { UI_SETTINGS_DATEFORMAT_TZ, UI_SETTINGS_CSV_QUOTE_VALUES, UI_SETTINGS_CSV_SEPARATOR, + UI_SETTINGS_SEARCH_INCLUDE_FROZEN, } from '../../../../common/constants'; import { IUiSettingsClient } from 'kibana/server'; import { savedObjectsClientMock, uiSettingsServiceMock } from 'src/core/server/mocks'; @@ -36,6 +37,8 @@ describe('getExportSettings', () => { return ','; case UI_SETTINGS_DATEFORMAT_TZ: return 'Browser'; + case UI_SETTINGS_SEARCH_INCLUDE_FROZEN: + return false; } return 'helo world'; @@ -49,6 +52,7 @@ describe('getExportSettings', () => { "checkForFormulas": undefined, "escapeFormulaValues": undefined, "escapeValue": [Function], + "includeFrozen": false, "maxSizeBytes": undefined, "scroll": Object { "duration": undefined, diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts index 0b8815836367f..882c617f5aaed 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts @@ -12,9 +12,10 @@ import { createEscapeValue } from '../../../../../../../src/plugins/data/common' import { ReportingConfig } from '../../../'; import { CSV_BOM_CHARS, - UI_SETTINGS_DATEFORMAT_TZ, UI_SETTINGS_CSV_QUOTE_VALUES, UI_SETTINGS_CSV_SEPARATOR, + UI_SETTINGS_DATEFORMAT_TZ, + UI_SETTINGS_SEARCH_INCLUDE_FROZEN, } from '../../../../common/constants'; import { LevelLogger } from '../../../lib'; @@ -30,6 +31,7 @@ export interface CsvExportSettings { checkForFormulas: boolean; escapeFormulaValues: boolean; escapeValue: (value: string) => string; + includeFrozen: boolean; } export const getExportSettings = async ( @@ -38,9 +40,7 @@ export const getExportSettings = async ( timezone: string | undefined, logger: LevelLogger ): Promise => { - // Timezone let setTimezone: string; - // timezone in job params? if (timezone) { setTimezone = timezone; } else { @@ -60,7 +60,8 @@ export const getExportSettings = async ( } // Separator, QuoteValues - const [separator, quoteValues] = await Promise.all([ + const [includeFrozen, separator, quoteValues] = await Promise.all([ + client.get(UI_SETTINGS_SEARCH_INCLUDE_FROZEN), client.get(UI_SETTINGS_CSV_SEPARATOR), client.get(UI_SETTINGS_CSV_QUOTE_VALUES), ]); @@ -76,6 +77,7 @@ export const getExportSettings = async ( duration: config.get('csv', 'scroll', 'duration'), }, bom, + includeFrozen, separator, maxSizeBytes: config.get('csv', 'maxSizeBytes'), checkForFormulas: config.get('csv', 'checkForFormulas'), From a7cfc82acb9e002298f0dbe573cad67d80a271b7 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 24 Aug 2021 15:35:34 -0700 Subject: [PATCH 2/6] add api integration test --- .../generate_csv/generate_csv.ts | 2 +- .../reporting_and_security/index.ts | 1 + .../search_frozen_indices.ts | 149 ++++++++++++++++++ 3 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts index 9e8eb0319e4c3..85fc531320247 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts @@ -94,7 +94,7 @@ export class CsvGenerator { index: index.title, scroll: scrollSettings.duration, size: scrollSettings.size, - ignore_throttled: !this.includeFrozen, + ignore_throttled: this.includeFrozen, }, }; const results = ( diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts index e6fd534274df4..266fee37b288d 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts @@ -29,5 +29,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./spaces')); loadTestFile(require.resolve('./usage')); loadTestFile(require.resolve('./ilm_migration_apis')); + loadTestFile(require.resolve('./search_frozen_indices')); }); } diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts b/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts new file mode 100644 index 0000000000000..12efd84ae2826 --- /dev/null +++ b/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Filter, FilterMeta } from '@kbn/es-query'; +import { SortDirectionFormat } from 'src/plugins/data/common'; +import { JobParamsDownloadCSV } from '../../../plugins/reporting/server/export_types/csv_searchsource_immediate/types'; +import { FtrProviderContext } from '../ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default function ({ getService }: FtrProviderContext) { + const kibanaServer = getService('kibanaServer'); + const supertestSvc = getService('supertest'); + const esSupertest = getService('esSupertest'); + const log = getService('log'); + + const generateAPI = { + getCSVFromSearchSource: async (job: JobParamsDownloadCSV) => { + return await supertestSvc + .post(`/api/reporting/v1/generate/immediate/csv_searchsource`) + .set('kbn-xsrf', 'xxx') + .send(job); + }, + }; + + describe('Frozen indices search', () => { + before(async () => { + await kibanaServer.uiSettings.update({ + 'csv:quoteValues': true, + 'search:includeFrozen': false, + }); + }); + + after(async () => { + await kibanaServer.uiSettings.update({ 'search:includeFrozen': false }); + }); + + it('Search includes frozen indices based on Advanced Setting', async () => { + // setup: settings and clear data from previous testing + await kibanaServer.uiSettings.update({ 'csv:quoteValues': true }); + await esSupertest.delete('/test2,test2,test3'); + + // setup: add mwhdflaksjefhultiple indices of test data + await Promise.all([ + esSupertest + .post('/test1/_doc?refresh=true') + .send({ '@timestamp': '2021-08-24T21:36:40Z', ip: '43.98.8.183', utilization: 18725 }), + esSupertest + .post('/test2/_doc?refresh=true') + .send({ '@timestamp': '2021-08-21T09:36:40Z', ip: '63.91.103.79', utilization: 8480 }), + esSupertest + .post('/test3/_doc?refresh=true') + .send({ '@timestamp': '2021-08-17T21:36:40Z', ip: '139.108.162.171', utilization: 3078 }), + ]); + + // setup: create index pattern + const indexPatternId = 'cool-test-index-pattern'; + const indexPatternCreateResponse = await kibanaServer.savedObjects.create({ + type: 'index-pattern', + id: indexPatternId, + overwrite: true, + attributes: { title: 'test*', timeFieldName: '@timestamp' }, + }); + + log.info(`index pattern create response: ${JSON.stringify(indexPatternCreateResponse)}`); + + const callExportApi = async () => { + return await generateAPI.getCSVFromSearchSource({ + browserTimezone: 'UTC', + columns: ['@timestamp', 'ip', 'utilization'], + searchSource: { + fields: [{ field: '*', include_unmapped: 'true' }], + filter: [ + { + meta: { + field: '@timestamp', + index: indexPatternId, + params: {}, + } as FilterMeta, + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2020-08-24T00:00:00.000Z', + lte: '2022-08-24T21:40:48.346Z', + }, + }, + } as Filter, + ], + // @ts-expect-error Type 'string' is not assignable to type 'IndexPattern | undefined'. + index: indexPatternId, + parent: { + filter: [], + // @ts-expect-error Type 'string' is not assignable to type 'IndexPattern | undefined'. + index: indexPatternId, + query: { language: 'kuery', query: '' }, + }, + sort: ([{ '@timestamp': 'desc' }] as unknown) as Record, + trackTotalHits: true, + }, + title: 'Test search', + }); + }; + + // testing: call the API to export the search data as CSV + try { + // check the initial data with a CSV export + const initialResult = await callExportApi(); + expectSnapshot(initialResult.text).toMatchInline(` + "\\"@timestamp\\",ip,utilization + \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" + \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\" + \\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\" + " + `); + + // freeze an index in the pattern + // recheck the search results + const freezeResult = await esSupertest.post('/test3/_freeze'); + log.info(`freeze index response: ${JSON.stringify(freezeResult)}`); + const afterFreezeResult = await callExportApi(); + expectSnapshot(afterFreezeResult.text).toMatchInline(` + "\\"@timestamp\\",ip,utilization + \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" + \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\" + " + `); + + // allow searching frozen data + // recheck the search results + await kibanaServer.uiSettings.update({ 'search:includeFrozen': true }); + const afterAllowSearchResult = await callExportApi(); + expectSnapshot(afterAllowSearchResult.text).toMatchInline(` + "\\"@timestamp\\",ip,utilization + \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" + \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\" + \\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\" + " + `); + } finally { + // cleanup + await esSupertest.delete('/test1,test2,test3'); + await kibanaServer.savedObjects.delete({ type: 'index-pattern', id: indexPatternId }); + } + }); + }); +} From e55ba57e5c0c31b5621d6223d0449835aef85c5d Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 24 Aug 2021 16:33:09 -0700 Subject: [PATCH 3/6] add fixes --- .../generate_csv/generate_csv.test.ts | 10 +- .../generate_csv/generate_csv.ts | 9 +- .../search_frozen_indices.ts | 97 +++++++++---------- 3 files changed, 57 insertions(+), 59 deletions(-) diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts index 3e4e663733e2c..cb5670f6eafd9 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts @@ -198,7 +198,7 @@ it('calculates the bytes of the content', async () => { Rx.of({ rawResponse: { hits: { - hits: range(0, HITS_TOTAL).map((hit, i) => ({ + hits: range(0, HITS_TOTAL).map(() => ({ fields: { message: ['this is a great message'], }, @@ -248,7 +248,7 @@ it('warns if max size was reached', async () => { Rx.of({ rawResponse: { hits: { - hits: range(0, HITS_TOTAL).map((hit, i) => ({ + hits: range(0, HITS_TOTAL).map(() => ({ fields: { date: ['2020-12-31T00:14:28.000Z'], ip: ['110.135.176.89'], @@ -289,7 +289,7 @@ it('uses the scrollId to page all the data', async () => { rawResponse: { _scroll_id: 'awesome-scroll-hero', hits: { - hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({ + hits: range(0, HITS_TOTAL / 10).map(() => ({ fields: { date: ['2020-12-31T00:14:28.000Z'], ip: ['110.135.176.89'], @@ -304,7 +304,7 @@ it('uses the scrollId to page all the data', async () => { mockEsClient.asCurrentUser.scroll = jest.fn().mockResolvedValue({ body: { hits: { - hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({ + hits: range(0, HITS_TOTAL / 10).map(() => ({ fields: { date: ['2020-12-31T00:14:28.000Z'], ip: ['110.135.176.89'], @@ -337,7 +337,7 @@ it('uses the scrollId to page all the data', async () => { expect(mockDataClient.search).toHaveBeenCalledTimes(1); expect(mockDataClient.search).toBeCalledWith( - { params: { scroll: '30s', size: 500 } }, + { params: { ignore_throttled: true, scroll: '30s', size: 500 } }, { strategy: 'es' } ); diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts index 85fc531320247..7f01f9fcb297c 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts @@ -66,7 +66,6 @@ function isPlainStringArray( export class CsvGenerator { private _columns?: string[]; private _formatters?: Record; - private includeFrozen: boolean; private csvContainsFormulas = false; private maxSizeReached = false; private csvRowCount = 0; @@ -84,8 +83,9 @@ export class CsvGenerator { private async scan( index: IndexPattern, searchSource: ISearchSource, - scrollSettings: CsvExportSettings['scroll'] + settings: CsvExportSettings ) { + const { scroll: scrollSettings, includeFrozen } = settings; const searchBody = searchSource.getSearchRequestBody(); this.logger.debug(`executing search request`); const searchParams = { @@ -94,9 +94,10 @@ export class CsvGenerator { index: index.title, scroll: scrollSettings.duration, size: scrollSettings.size, - ignore_throttled: this.includeFrozen, + ignore_throttled: !includeFrozen, }, }; + const results = ( await this.clients.data.search(searchParams, { strategy: ES_SEARCH_STRATEGY }).toPromise() ).rawResponse as estypes.SearchResponse; @@ -328,7 +329,7 @@ export class CsvGenerator { let results: estypes.SearchResponse | undefined; if (scrollId == null) { // open a scroll cursor in Elasticsearch - results = await this.scan(index, searchSource, scrollSettings); + results = await this.scan(index, searchSource, settings); scrollId = results?._scroll_id; if (results.hits?.total != null) { totalRecords = results.hits.total as number; diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts b/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts index 12efd84ae2826..0947982c3e84f 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts @@ -5,6 +5,7 @@ * 2.0. */ +import expect from '@kbn/expect'; import { Filter, FilterMeta } from '@kbn/es-query'; import { SortDirectionFormat } from 'src/plugins/data/common'; import { JobParamsDownloadCSV } from '../../../plugins/reporting/server/export_types/csv_searchsource_immediate/types'; @@ -15,7 +16,6 @@ export default function ({ getService }: FtrProviderContext) { const kibanaServer = getService('kibanaServer'); const supertestSvc = getService('supertest'); const esSupertest = getService('esSupertest'); - const log = getService('log'); const generateAPI = { getCSVFromSearchSource: async (job: JobParamsDownloadCSV) => { @@ -27,16 +27,18 @@ export default function ({ getService }: FtrProviderContext) { }; describe('Frozen indices search', () => { - before(async () => { + const indexPatternId = 'cool-test-index-pattern'; + const reset = async () => { + await esSupertest.delete('/test1,test2,test3'); + await kibanaServer.savedObjects.delete({ type: 'index-pattern', id: indexPatternId }); await kibanaServer.uiSettings.update({ 'csv:quoteValues': true, 'search:includeFrozen': false, }); - }); + }; - after(async () => { - await kibanaServer.uiSettings.update({ 'search:includeFrozen': false }); - }); + before(reset); + after(reset); it('Search includes frozen indices based on Advanced Setting', async () => { // setup: settings and clear data from previous testing @@ -46,18 +48,18 @@ export default function ({ getService }: FtrProviderContext) { // setup: add mwhdflaksjefhultiple indices of test data await Promise.all([ esSupertest - .post('/test1/_doc?refresh=true') + .post('/test1/_doc') .send({ '@timestamp': '2021-08-24T21:36:40Z', ip: '43.98.8.183', utilization: 18725 }), esSupertest - .post('/test2/_doc?refresh=true') + .post('/test2/_doc') .send({ '@timestamp': '2021-08-21T09:36:40Z', ip: '63.91.103.79', utilization: 8480 }), esSupertest - .post('/test3/_doc?refresh=true') + .post('/test3/_doc') .send({ '@timestamp': '2021-08-17T21:36:40Z', ip: '139.108.162.171', utilization: 3078 }), ]); + await esSupertest.post('/test*/_refresh'); // setup: create index pattern - const indexPatternId = 'cool-test-index-pattern'; const indexPatternCreateResponse = await kibanaServer.savedObjects.create({ type: 'index-pattern', id: indexPatternId, @@ -65,9 +67,9 @@ export default function ({ getService }: FtrProviderContext) { attributes: { title: 'test*', timeFieldName: '@timestamp' }, }); - log.info(`index pattern create response: ${JSON.stringify(indexPatternCreateResponse)}`); + expect(indexPatternCreateResponse.id).to.be(indexPatternId); - const callExportApi = async () => { + const callExportAPI = async () => { return await generateAPI.getCSVFromSearchSource({ browserTimezone: 'UTC', columns: ['@timestamp', 'ip', 'utilization'], @@ -104,46 +106,41 @@ export default function ({ getService }: FtrProviderContext) { }); }; - // testing: call the API to export the search data as CSV - try { - // check the initial data with a CSV export - const initialResult = await callExportApi(); - expectSnapshot(initialResult.text).toMatchInline(` - "\\"@timestamp\\",ip,utilization - \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" - \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\" - \\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\" - " - `); + // 1. check the initial data with a CSV export + const initialSearch = await callExportAPI(); + expectSnapshot(initialSearch.text).toMatchInline(` + "\\"@timestamp\\",ip,utilization + \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" + \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\" + \\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\" + " + `); + + // 2. freeze an index in the pattern + await esSupertest.post('/test3/_freeze').expect(200); + await esSupertest.post('/test*/_refresh').expect(200); + + // 3. recheck the search results + const afterFreezeSearch = await callExportAPI(); + expectSnapshot(afterFreezeSearch.text).toMatchInline(` + "\\"@timestamp\\",ip,utilization + \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" + \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\" + " + `); - // freeze an index in the pattern - // recheck the search results - const freezeResult = await esSupertest.post('/test3/_freeze'); - log.info(`freeze index response: ${JSON.stringify(freezeResult)}`); - const afterFreezeResult = await callExportApi(); - expectSnapshot(afterFreezeResult.text).toMatchInline(` - "\\"@timestamp\\",ip,utilization - \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" - \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\" - " - `); + // 4. update setting to allow searching frozen data + await kibanaServer.uiSettings.update({ 'search:includeFrozen': true }); - // allow searching frozen data - // recheck the search results - await kibanaServer.uiSettings.update({ 'search:includeFrozen': true }); - const afterAllowSearchResult = await callExportApi(); - expectSnapshot(afterAllowSearchResult.text).toMatchInline(` - "\\"@timestamp\\",ip,utilization - \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" - \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\" - \\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\" - " - `); - } finally { - // cleanup - await esSupertest.delete('/test1,test2,test3'); - await kibanaServer.savedObjects.delete({ type: 'index-pattern', id: indexPatternId }); - } + // 5. recheck the search results + const afterAllowSearch = await callExportAPI(); + expectSnapshot(afterAllowSearch.text).toMatchInline(` + "\\"@timestamp\\",ip,utilization + \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" + \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\" + \\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\" + " + `); }); }); } From 61d4f61111623d3c3ba31058f9406bd763d919c7 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Tue, 24 Aug 2021 16:56:51 -0700 Subject: [PATCH 4/6] Update x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts --- .../reporting_and_security/search_frozen_indices.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts b/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts index 0947982c3e84f..87e3156a4e786 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts @@ -45,7 +45,7 @@ export default function ({ getService }: FtrProviderContext) { await kibanaServer.uiSettings.update({ 'csv:quoteValues': true }); await esSupertest.delete('/test2,test2,test3'); - // setup: add mwhdflaksjefhultiple indices of test data + // setup: add multiple indices of test data await Promise.all([ esSupertest .post('/test1/_doc') From bb2585bd31babe53b4a9100d834e037c8dc00e45 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 24 Aug 2021 21:19:18 -0700 Subject: [PATCH 5/6] test polish --- .../search_frozen_indices.ts | 98 ++++++++----------- 1 file changed, 41 insertions(+), 57 deletions(-) diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts b/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts index 87e3156a4e786..0cd3abb4839f7 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts @@ -6,9 +6,6 @@ */ import expect from '@kbn/expect'; -import { Filter, FilterMeta } from '@kbn/es-query'; -import { SortDirectionFormat } from 'src/plugins/data/common'; -import { JobParamsDownloadCSV } from '../../../plugins/reporting/server/export_types/csv_searchsource_immediate/types'; import { FtrProviderContext } from '../ftr_provider_context'; // eslint-disable-next-line import/no-default-export @@ -16,33 +13,58 @@ export default function ({ getService }: FtrProviderContext) { const kibanaServer = getService('kibanaServer'); const supertestSvc = getService('supertest'); const esSupertest = getService('esSupertest'); + const indexPatternId = 'cool-test-index-pattern'; - const generateAPI = { - getCSVFromSearchSource: async (job: JobParamsDownloadCSV) => { - return await supertestSvc - .post(`/api/reporting/v1/generate/immediate/csv_searchsource`) - .set('kbn-xsrf', 'xxx') - .send(job); - }, - }; + async function callExportAPI() { + const job = { + browserTimezone: 'UTC', + columns: ['@timestamp', 'ip', 'utilization'], + searchSource: { + fields: [{ field: '*', include_unmapped: 'true' }], + filter: [ + { + meta: { field: '@timestamp', index: indexPatternId, params: {} }, + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2020-08-24T00:00:00.000Z', + lte: '2022-08-24T21:40:48.346Z', + }, + }, + }, + ], + index: indexPatternId, + parent: { filter: [], index: indexPatternId, query: { language: 'kuery', query: '' } }, + sort: [{ '@timestamp': 'desc' }], + trackTotalHits: true, + }, + title: 'Test search', + }; + + return await supertestSvc + .post(`/api/reporting/v1/generate/immediate/csv_searchsource`) + .set('kbn-xsrf', 'xxx') + .send(job); + } describe('Frozen indices search', () => { - const indexPatternId = 'cool-test-index-pattern'; const reset = async () => { - await esSupertest.delete('/test1,test2,test3'); - await kibanaServer.savedObjects.delete({ type: 'index-pattern', id: indexPatternId }); - await kibanaServer.uiSettings.update({ - 'csv:quoteValues': true, - 'search:includeFrozen': false, - }); + await kibanaServer.uiSettings.replace({ 'search:includeFrozen': false }); + try { + await esSupertest.delete('/test1,test2,test3'); + await kibanaServer.savedObjects.delete({ type: 'index-pattern', id: indexPatternId }); + } catch (err) { + // ignore 404 error + } }; before(reset); after(reset); it('Search includes frozen indices based on Advanced Setting', async () => { - // setup: settings and clear data from previous testing await kibanaServer.uiSettings.update({ 'csv:quoteValues': true }); + + // setup: settings and clear data from previous testing await esSupertest.delete('/test2,test2,test3'); // setup: add multiple indices of test data @@ -66,46 +88,8 @@ export default function ({ getService }: FtrProviderContext) { overwrite: true, attributes: { title: 'test*', timeFieldName: '@timestamp' }, }); - expect(indexPatternCreateResponse.id).to.be(indexPatternId); - const callExportAPI = async () => { - return await generateAPI.getCSVFromSearchSource({ - browserTimezone: 'UTC', - columns: ['@timestamp', 'ip', 'utilization'], - searchSource: { - fields: [{ field: '*', include_unmapped: 'true' }], - filter: [ - { - meta: { - field: '@timestamp', - index: indexPatternId, - params: {}, - } as FilterMeta, - range: { - '@timestamp': { - format: 'strict_date_optional_time', - gte: '2020-08-24T00:00:00.000Z', - lte: '2022-08-24T21:40:48.346Z', - }, - }, - } as Filter, - ], - // @ts-expect-error Type 'string' is not assignable to type 'IndexPattern | undefined'. - index: indexPatternId, - parent: { - filter: [], - // @ts-expect-error Type 'string' is not assignable to type 'IndexPattern | undefined'. - index: indexPatternId, - query: { language: 'kuery', query: '' }, - }, - sort: ([{ '@timestamp': 'desc' }] as unknown) as Record, - trackTotalHits: true, - }, - title: 'Test search', - }); - }; - // 1. check the initial data with a CSV export const initialSearch = await callExportAPI(); expectSnapshot(initialSearch.text).toMatchInline(` From 9a89f14614b382c7ab6855dd0d7544f8f33004b9 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Wed, 25 Aug 2021 12:28:06 -0700 Subject: [PATCH 6/6] update per feedback --- .../csv_searchsource/generate_csv/get_export_settings.ts | 2 +- .../reporting_and_security/search_frozen_indices.ts | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts index 882c617f5aaed..7d4db38ef4aae 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts @@ -59,7 +59,7 @@ export const getExportSettings = async ( } } - // Separator, QuoteValues + // Advanced Settings that affect search export + CSV const [includeFrozen, separator, quoteValues] = await Promise.all([ client.get(UI_SETTINGS_SEARCH_INCLUDE_FROZEN), client.get(UI_SETTINGS_CSV_SEPARATOR), diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts b/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts index 0cd3abb4839f7..daa749649e250 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts @@ -64,9 +64,6 @@ export default function ({ getService }: FtrProviderContext) { it('Search includes frozen indices based on Advanced Setting', async () => { await kibanaServer.uiSettings.update({ 'csv:quoteValues': true }); - // setup: settings and clear data from previous testing - await esSupertest.delete('/test2,test2,test3'); - // setup: add multiple indices of test data await Promise.all([ esSupertest