Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Reporting] Fix ability to export CSV on searched data with frozen indices #109976

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions x-pack/plugins/reporting/common/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ export const KBN_SCREENSHOT_HEADER_BLOCK_LIST = [

export const KBN_SCREENSHOT_HEADER_BLOCK_LIST_STARTS_WITH_PATTERN = ['proxy-'];

export const UI_SETTINGS_SEARCH_INCLUDE_FROZEN = 'search:includeFrozen';
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cc @lukasolson shouldn't this setting be respected inside search service ? (and as such we should not need to set it here ?)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Filed: #110679

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like this setting is only respected with the ESE_SEARCH_STRATEGY. Unfortunately, CSV export can not use that search strategy because it doesn't support scroll queries: #110679 (comment)

export const UI_SETTINGS_CUSTOM_PDF_LOGO = 'xpackReporting:customPdfLogo';
export const UI_SETTINGS_CSV_SEPARATOR = 'csv:separator';
export const UI_SETTINGS_CSV_QUOTE_VALUES = 'csv:quoteValues';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ it('calculates the bytes of the content', async () => {
Rx.of({
rawResponse: {
hits: {
hits: range(0, HITS_TOTAL).map((hit, i) => ({
hits: range(0, HITS_TOTAL).map(() => ({
fields: {
message: ['this is a great message'],
},
Expand Down Expand Up @@ -248,7 +248,7 @@ it('warns if max size was reached', async () => {
Rx.of({
rawResponse: {
hits: {
hits: range(0, HITS_TOTAL).map((hit, i) => ({
hits: range(0, HITS_TOTAL).map(() => ({
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
Expand Down Expand Up @@ -289,7 +289,7 @@ it('uses the scrollId to page all the data', async () => {
rawResponse: {
_scroll_id: 'awesome-scroll-hero',
hits: {
hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({
hits: range(0, HITS_TOTAL / 10).map(() => ({
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
Expand All @@ -304,7 +304,7 @@ it('uses the scrollId to page all the data', async () => {
mockEsClient.asCurrentUser.scroll = jest.fn().mockResolvedValue({
body: {
hits: {
hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({
hits: range(0, HITS_TOTAL / 10).map(() => ({
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
Expand Down Expand Up @@ -337,7 +337,7 @@ it('uses the scrollId to page all the data', async () => {

expect(mockDataClient.search).toHaveBeenCalledTimes(1);
expect(mockDataClient.search).toBeCalledWith(
{ params: { scroll: '30s', size: 500 } },
{ params: { ignore_throttled: true, scroll: '30s', size: 500 } },
{ strategy: 'es' }
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,9 @@ export class CsvGenerator {
private async scan(
index: IndexPattern,
searchSource: ISearchSource,
scrollSettings: CsvExportSettings['scroll']
settings: CsvExportSettings
) {
const { scroll: scrollSettings, includeFrozen } = settings;
const searchBody = searchSource.getSearchRequestBody();
this.logger.debug(`executing search request`);
const searchParams = {
Expand All @@ -93,8 +94,10 @@ export class CsvGenerator {
index: index.title,
scroll: scrollSettings.duration,
size: scrollSettings.size,
ignore_throttled: !includeFrozen,
},
};

const results = (
await this.clients.data.search(searchParams, { strategy: ES_SEARCH_STRATEGY }).toPromise()
).rawResponse as estypes.SearchResponse<unknown>;
Expand Down Expand Up @@ -326,7 +329,7 @@ export class CsvGenerator {
let results: estypes.SearchResponse<unknown> | undefined;
if (scrollId == null) {
// open a scroll cursor in Elasticsearch
results = await this.scan(index, searchSource, scrollSettings);
results = await this.scan(index, searchSource, settings);
scrollId = results?._scroll_id;
if (results.hits?.total != null) {
totalRecords = results.hits.total as number;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
UI_SETTINGS_DATEFORMAT_TZ,
UI_SETTINGS_CSV_QUOTE_VALUES,
UI_SETTINGS_CSV_SEPARATOR,
UI_SETTINGS_SEARCH_INCLUDE_FROZEN,
} from '../../../../common/constants';
import { IUiSettingsClient } from 'kibana/server';
import { savedObjectsClientMock, uiSettingsServiceMock } from 'src/core/server/mocks';
Expand Down Expand Up @@ -36,6 +37,8 @@ describe('getExportSettings', () => {
return ',';
case UI_SETTINGS_DATEFORMAT_TZ:
return 'Browser';
case UI_SETTINGS_SEARCH_INCLUDE_FROZEN:
return false;
}

return 'helo world';
Expand All @@ -49,6 +52,7 @@ describe('getExportSettings', () => {
"checkForFormulas": undefined,
"escapeFormulaValues": undefined,
"escapeValue": [Function],
"includeFrozen": false,
"maxSizeBytes": undefined,
"scroll": Object {
"duration": undefined,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,10 @@ import { createEscapeValue } from '../../../../../../../src/plugins/data/common'
import { ReportingConfig } from '../../../';
import {
CSV_BOM_CHARS,
UI_SETTINGS_DATEFORMAT_TZ,
UI_SETTINGS_CSV_QUOTE_VALUES,
UI_SETTINGS_CSV_SEPARATOR,
UI_SETTINGS_DATEFORMAT_TZ,
UI_SETTINGS_SEARCH_INCLUDE_FROZEN,
} from '../../../../common/constants';
import { LevelLogger } from '../../../lib';

Expand All @@ -30,6 +31,7 @@ export interface CsvExportSettings {
checkForFormulas: boolean;
escapeFormulaValues: boolean;
escapeValue: (value: string) => string;
includeFrozen: boolean;
}

export const getExportSettings = async (
Expand All @@ -38,9 +40,7 @@ export const getExportSettings = async (
timezone: string | undefined,
logger: LevelLogger
): Promise<CsvExportSettings> => {
// Timezone
let setTimezone: string;
// timezone in job params?
if (timezone) {
setTimezone = timezone;
} else {
Expand All @@ -59,8 +59,9 @@ export const getExportSettings = async (
}
}

// Separator, QuoteValues
const [separator, quoteValues] = await Promise.all([
// Advanced Settings that affect search export + CSV
const [includeFrozen, separator, quoteValues] = await Promise.all([
client.get(UI_SETTINGS_SEARCH_INCLUDE_FROZEN),
client.get(UI_SETTINGS_CSV_SEPARATOR),
client.get(UI_SETTINGS_CSV_QUOTE_VALUES),
]);
Expand All @@ -76,6 +77,7 @@ export const getExportSettings = async (
duration: config.get('csv', 'scroll', 'duration'),
},
bom,
includeFrozen,
separator,
maxSizeBytes: config.get('csv', 'maxSizeBytes'),
checkForFormulas: config.get('csv', 'checkForFormulas'),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
loadTestFile(require.resolve('./spaces'));
loadTestFile(require.resolve('./usage'));
loadTestFile(require.resolve('./ilm_migration_apis'));
loadTestFile(require.resolve('./search_frozen_indices'));
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import expect from '@kbn/expect';
import { FtrProviderContext } from '../ftr_provider_context';

// eslint-disable-next-line import/no-default-export
export default function ({ getService }: FtrProviderContext) {
const kibanaServer = getService('kibanaServer');
const supertestSvc = getService('supertest');
const esSupertest = getService('esSupertest');
const indexPatternId = 'cool-test-index-pattern';

async function callExportAPI() {
const job = {
browserTimezone: 'UTC',
columns: ['@timestamp', 'ip', 'utilization'],
searchSource: {
fields: [{ field: '*', include_unmapped: 'true' }],
filter: [
{
meta: { field: '@timestamp', index: indexPatternId, params: {} },
range: {
'@timestamp': {
format: 'strict_date_optional_time',
gte: '2020-08-24T00:00:00.000Z',
lte: '2022-08-24T21:40:48.346Z',
},
},
},
],
index: indexPatternId,
parent: { filter: [], index: indexPatternId, query: { language: 'kuery', query: '' } },
sort: [{ '@timestamp': 'desc' }],
trackTotalHits: true,
},
title: 'Test search',
};

return await supertestSvc
.post(`/api/reporting/v1/generate/immediate/csv_searchsource`)
.set('kbn-xsrf', 'xxx')
.send(job);
}

describe('Frozen indices search', () => {
const reset = async () => {
await kibanaServer.uiSettings.replace({ 'search:includeFrozen': false });
try {
await esSupertest.delete('/test1,test2,test3');
await kibanaServer.savedObjects.delete({ type: 'index-pattern', id: indexPatternId });
} catch (err) {
// ignore 404 error
}
};

before(reset);
after(reset);

it('Search includes frozen indices based on Advanced Setting', async () => {
await kibanaServer.uiSettings.update({ 'csv:quoteValues': true });

// setup: add multiple indices of test data
await Promise.all([
esSupertest
.post('/test1/_doc')
.send({ '@timestamp': '2021-08-24T21:36:40Z', ip: '43.98.8.183', utilization: 18725 }),
esSupertest
.post('/test2/_doc')
.send({ '@timestamp': '2021-08-21T09:36:40Z', ip: '63.91.103.79', utilization: 8480 }),
esSupertest
.post('/test3/_doc')
.send({ '@timestamp': '2021-08-17T21:36:40Z', ip: '139.108.162.171', utilization: 3078 }),
]);
await esSupertest.post('/test*/_refresh');

// setup: create index pattern
const indexPatternCreateResponse = await kibanaServer.savedObjects.create({
type: 'index-pattern',
id: indexPatternId,
overwrite: true,
attributes: { title: 'test*', timeFieldName: '@timestamp' },
});
expect(indexPatternCreateResponse.id).to.be(indexPatternId);

// 1. check the initial data with a CSV export
const initialSearch = await callExportAPI();
expectSnapshot(initialSearch.text).toMatchInline(`
"\\"@timestamp\\",ip,utilization
\\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
\\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
\\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\"
"
`);

// 2. freeze an index in the pattern
await esSupertest.post('/test3/_freeze').expect(200);
await esSupertest.post('/test*/_refresh').expect(200);

// 3. recheck the search results
const afterFreezeSearch = await callExportAPI();
expectSnapshot(afterFreezeSearch.text).toMatchInline(`
"\\"@timestamp\\",ip,utilization
\\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
\\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
"
`);

// 4. update setting to allow searching frozen data
await kibanaServer.uiSettings.update({ 'search:includeFrozen': true });

// 5. recheck the search results
const afterAllowSearch = await callExportAPI();
expectSnapshot(afterAllowSearch.text).toMatchInline(`
"\\"@timestamp\\",ip,utilization
\\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
\\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
\\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\"
"
`);
});
});
}
Loading