diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts new file mode 100644 index 0000000000000..fa26fb68289a6 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts @@ -0,0 +1,454 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; +import moment from 'moment-timezone'; + +import type { + FetchAvailableCatIndicesResponseRequired, + IndexSearchAggregationResponse, +} from './fetch_available_indices'; +import { fetchAvailableIndices } from './fetch_available_indices'; + +function getEsClientMock() { + return { + search: jest.fn().mockResolvedValue({ + aggregations: { + index: { + buckets: [], + }, + }, + }), + cat: { + indices: jest.fn().mockResolvedValue([]), + }, + } as unknown as ElasticsearchClient & { + cat: { + indices: jest.Mock>; + }; + search: jest.Mock>; + }; +} + +// fixing timezone for both Date and moment +// so when tests are run in different timezones, the results are consistent +process.env.TZ = 'UTC'; +moment.tz.setDefault('UTC'); + +const DAY_IN_MILLIS = 24 * 60 * 60 * 1000; + +// We assume that the dates are in UTC, because es is using UTC +// It also diminishes difference date parsing by Date and moment constructors +// in different timezones, i.e. short ISO format '2021-10-01' is parsed as local +// date by moment and as UTC date by Date, whereas long ISO format '2021-10-01T00:00:00Z' +// is parsed as UTC date by both +const startDateString: string = '2021-10-01T00:00:00Z'; +const endDateString: string = '2021-10-07T00:00:00Z'; + +const startDateMillis: number = new Date(startDateString).getTime(); +const endDateMillis: number = new Date(endDateString).getTime(); + +describe('fetchAvailableIndices', () => { + afterEach(() => { + jest.clearAllMocks(); + }); + + it('aggregate search given index by startDate and endDate', async () => { + const esClientMock = getEsClientMock(); + + await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(esClientMock.search).toHaveBeenCalledWith({ + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: startDateString, + lte: endDateString, + }, + }, + }, + ], + must: [], + must_not: [], + should: [], + }, + }, + index: 'logs-*', + size: 0, + aggs: { + index: { + terms: { + field: '_index', + }, + }, + }, + }); + }); + + it('should call esClient.cat.indices for given index', async () => { + const esClientMock = getEsClientMock(); + + await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(esClientMock.cat.indices).toHaveBeenCalledWith({ + index: 'logs-*', + format: 'json', + h: 'index,creation.date', + }); + }); + + describe('when indices are created within the date range', () => { + it('returns indices within the date range', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.01', + 'creation.date': `${startDateMillis}`, + }, + { + index: 'logs-2021.10.05', + 'creation.date': `${startDateMillis + 4 * DAY_IN_MILLIS}`, + }, + { + index: 'logs-2021.09.30', + 'creation.date': `${startDateMillis - DAY_IN_MILLIS}`, + }, + ]); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual(['logs-2021.10.01', 'logs-2021.10.05']); + + expect(esClientMock.cat.indices).toHaveBeenCalledWith({ + index: 'logs-*', + format: 'json', + h: 'index,creation.date', + }); + }); + }); + + describe('when indices are outside the date range', () => { + it('returns an empty list', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.09.30', + 'creation.date': `${startDateMillis - DAY_IN_MILLIS}`, + }, + { + index: 'logs-2021.10.08', + 'creation.date': `${endDateMillis + DAY_IN_MILLIS}`, + }, + ]); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual([]); + }); + }); + + describe('when no indices match the index pattern', () => { + it('returns empty list', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([]); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'nonexistent-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual([]); + }); + }); + + describe('when indices have data in the date range', () => { + it('returns indices with data in the date range', async () => { + const esClientMock = getEsClientMock(); + + // esClient.cat.indices returns no indices + esClientMock.cat.indices.mockResolvedValue([]); + + // esClient.search returns indices with data in the date range + esClientMock.search.mockResolvedValue({ + aggregations: { + index: { + buckets: [ + { key: 'logs-2021.10.02', doc_count: 100 }, + { key: 'logs-2021.10.03', doc_count: 150 }, + ], + }, + }, + }); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual(['logs-2021.10.02', 'logs-2021.10.03']); + }); + + it('combines indices from both methods without duplicates', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.01', + 'creation.date': `${startDateMillis}`, + }, + { + index: 'logs-2021.10.03', + 'creation.date': `${startDateMillis + 2 * DAY_IN_MILLIS}`, + }, + ]); + + esClientMock.search.mockResolvedValue({ + aggregations: { + index: { + buckets: [ + { key: 'logs-2021.10.03', doc_count: 150 }, + { key: 'logs-2021.10.04', doc_count: 200 }, + ], + }, + }, + }); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual(['logs-2021.10.01', 'logs-2021.10.03', 'logs-2021.10.04']); + }); + }); + + describe('edge cases for creation dates', () => { + it('includes indices with creation date exactly at startDate and endDate', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.01', + 'creation.date': `${startDateMillis}`, + }, + { + index: 'logs-2021.10.07', + 'creation.date': `${endDateMillis}`, + }, + ]); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual(['logs-2021.10.01', 'logs-2021.10.07']); + }); + }); + + describe('when esClient.search rejects', () => { + it('throws an error', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.search.mockRejectedValue(new Error('Elasticsearch search error')); + + await expect( + fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }) + ).rejects.toThrow('Elasticsearch search error'); + }); + }); + + describe('when both esClient.cat.indices and esClient.search return empty', () => { + it('returns an empty list', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([]); + esClientMock.search.mockResolvedValue({ + aggregations: { + index: { + buckets: [], + }, + }, + }); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual([]); + }); + }); + + describe('when indices are returned with both methods and have duplicates', () => { + it('does not duplicate indices in the result', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.05', + 'creation.date': `${startDateMillis + 4 * DAY_IN_MILLIS}`, + }, + ]); + + esClientMock.search.mockResolvedValue({ + aggregations: { + index: { + buckets: [{ key: 'logs-2021.10.05', doc_count: 100 }], + }, + }, + }); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual(['logs-2021.10.05']); + }); + }); + + describe('given keyword dates', () => { + describe('given 7 days range', () => { + beforeEach(() => { + jest.useFakeTimers(); + jest.setSystemTime(new Date('2021-10-07T00:00:00Z').getTime()); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('finds indices created within the date range', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.01', + 'creation.date': `${startDateMillis}`, + }, + { + index: 'logs-2021.10.05', + 'creation.date': `${startDateMillis + 4 * DAY_IN_MILLIS}`, + }, + ]); + + const results = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: 'now-7d/d', + endDate: 'now/d', + }); + + expect(results).toEqual(['logs-2021.10.01', 'logs-2021.10.05']); + }); + + it('finds indices with end date rounded up to the end of the day', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.06', + 'creation.date': `${new Date('2021-10-06T23:59:59Z').getTime()}`, + }, + ]); + + const results = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: 'now-7d/d', + endDate: 'now-1d/d', + }); + + expect(results).toEqual(['logs-2021.10.06']); + }); + }); + }); + + describe('rejections', () => { + beforeEach(() => { + jest.spyOn(console, 'warn').mockImplementation(() => {}); + }); + afterEach(() => { + jest.restoreAllMocks(); + }); + describe('when esClient.cat.indices rejects', () => { + it('throws an error', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockRejectedValue(new Error('Elasticsearch error')); + + await expect( + fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }) + ).rejects.toThrow('Elasticsearch error'); + }); + }); + + describe('when startDate is invalid', () => { + it('throws an error', async () => { + const esClientMock = getEsClientMock(); + + await expect( + fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: 'invalid-date', + endDate: endDateString, + }) + ).rejects.toThrow('Invalid date format: invalid-date'); + }); + }); + + describe('when endDate is invalid', () => { + it('throws an error', async () => { + const esClientMock = getEsClientMock(); + + await expect( + fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: 'invalid-date', + }) + ).rejects.toThrow('Invalid date format: invalid-date'); + }); + }); + }); +}); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts index 584a261689113..32311f28d636a 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts @@ -4,18 +4,72 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + import type { ElasticsearchClient } from '@kbn/core/server'; +import type { CatIndicesIndicesRecord } from '@elastic/elasticsearch/lib/api/types'; +import dateMath from '@kbn/datemath'; + import { getRequestBody } from '../helpers/get_available_indices'; +export type FetchAvailableCatIndicesResponseRequired = Array< + Required> +>; + type AggregateName = 'index'; -interface Result { +export interface IndexSearchAggregationResponse { index: { - buckets: Array<{ key: string }>; - doc_count: number; + buckets: Array<{ key: string; doc_count: number }>; }; } -export const fetchAvailableIndices = ( +const getParsedDateMs = (dateStr: string, roundUp = false) => { + const date = dateMath.parse(dateStr, roundUp ? { roundUp: true } : undefined); + if (!date?.isValid()) { + throw new Error(`Invalid date format: ${dateStr}`); + } + return date.valueOf(); +}; + +export const fetchAvailableIndices = async ( esClient: ElasticsearchClient, params: { indexPattern: string; startDate: string; endDate: string } -) => esClient.search(getRequestBody(params)); +): Promise => { + const { indexPattern, startDate, endDate } = params; + + const startDateMs = getParsedDateMs(startDate); + const endDateMs = getParsedDateMs(endDate, true); + + const indicesCats = (await esClient.cat.indices({ + index: indexPattern, + format: 'json', + h: 'index,creation.date', + })) as FetchAvailableCatIndicesResponseRequired; + + const indicesCatsInRange = indicesCats.filter((indexInfo) => { + const creationDateMs = parseInt(indexInfo['creation.date'], 10); + return creationDateMs >= startDateMs && creationDateMs <= endDateMs; + }); + + const timeSeriesIndicesWithDataInRangeSearchResult = await esClient.search< + AggregateName, + IndexSearchAggregationResponse + >(getRequestBody(params)); + + const timeSeriesIndicesWithDataInRange = + timeSeriesIndicesWithDataInRangeSearchResult.aggregations?.index.buckets.map( + (bucket) => bucket.key + ) || []; + + // Combine indices from both sources removing duplicates + const resultingIndices = new Set(); + + for (const indicesCat of indicesCatsInRange) { + resultingIndices.add(indicesCat.index); + } + + for (const timeSeriesIndex of timeSeriesIndicesWithDataInRange) { + resultingIndices.add(timeSeriesIndex); + } + + return Array.from(resultingIndices); +}; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts index 536fd461c61c9..40fc59219342c 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts @@ -57,16 +57,16 @@ export const parseMeteringStats = (meteringStatsIndices: MeteringStatsIndex[]) = }, {}); export const pickAvailableMeteringStats = ( - indicesBuckets: Array<{ key: string }>, + indicesBuckets: string[], meteringStatsIndices: Record ) => - indicesBuckets.reduce((acc: Record, { key }: { key: string }) => { - if (meteringStatsIndices?.[key]) { - acc[key] = { - name: meteringStatsIndices?.[key].name, - num_docs: meteringStatsIndices?.[key].num_docs, + indicesBuckets.reduce((acc: Record, indexName: string) => { + if (meteringStatsIndices?.[indexName]) { + acc[indexName] = { + name: meteringStatsIndices?.[indexName].name, + num_docs: meteringStatsIndices?.[indexName].num_docs, size_in_bytes: null, // We don't have size_in_bytes intentionally when ILM is not available - data_stream: meteringStatsIndices?.[key].data_stream, + data_stream: meteringStatsIndices?.[indexName].data_stream, }; } return acc; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts index f3ff5ec256ad6..91996a4ab9f89 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts @@ -152,17 +152,7 @@ describe('getIndexStatsRoute route', () => { }, }; (fetchMeteringStats as jest.Mock).mockResolvedValue(mockMeteringStatsIndex); - (fetchAvailableIndices as jest.Mock).mockResolvedValue({ - aggregations: { - index: { - buckets: [ - { - key: 'my-index-000001', - }, - ], - }, - }, - }); + (fetchAvailableIndices as jest.Mock).mockResolvedValue(['my-index-000001']); const response = await server.inject(request, requestContextMock.convertContext(context)); expect(response.status).toEqual(200); @@ -198,7 +188,7 @@ describe('getIndexStatsRoute route', () => { ); }); - test('returns an empty object when "availableIndices" indices are not available', async () => { + test('returns an empty object when "availableIndices" indices are empty', async () => { const request = requestMock.create({ method: 'get', path: GET_INDEX_STATS, @@ -214,9 +204,7 @@ describe('getIndexStatsRoute route', () => { const mockIndices = {}; (fetchMeteringStats as jest.Mock).mockResolvedValue(mockMeteringStatsIndex); - (fetchAvailableIndices as jest.Mock).mockResolvedValue({ - aggregations: undefined, - }); + (fetchAvailableIndices as jest.Mock).mockResolvedValue([]); const response = await server.inject(request, requestContextMock.convertContext(context)); expect(response.status).toEqual(200); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts index 665c178c62cdf..0f4fbb83f71e6 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts @@ -86,7 +86,7 @@ export const getIndexStatsRoute = (router: IRouter, logger: Logger) => { endDate: decodedEndDate, }); - if (!availableIndices.aggregations?.index?.buckets) { + if (availableIndices.length === 0) { logger.warn( `No available indices found under pattern: ${decodedIndexName}, in the given date range: ${decodedStartDate} - ${decodedEndDate}` ); @@ -95,10 +95,7 @@ export const getIndexStatsRoute = (router: IRouter, logger: Logger) => { }); } - const indices = pickAvailableMeteringStats( - availableIndices.aggregations.index.buckets, - meteringStatsIndices - ); + const indices = pickAvailableMeteringStats(availableIndices, meteringStatsIndices); return response.ok({ body: indices, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json b/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json index ceb43169165b4..cf31d7461b509 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json +++ b/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json @@ -26,6 +26,7 @@ "@kbn/core-elasticsearch-server-mocks", "@kbn/core-elasticsearch-server", "@kbn/core-security-common", + "@kbn/datemath", ], "exclude": [ "target/**/*",