From 02f277efa7470035671f67d4e86e74ea5f52838e Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 7 Oct 2024 13:41:50 +0100 Subject: [PATCH] [ML] Fix query for pattern analysis and change point analysis (#194742) Fixes https://github.com/elastic/kibana/issues/190710 Adds an additional check for `query_string` to the query creating function to adjust the query if only a single `query_string` condition is being used. This function was originally only used for pattern analysis, but has been renamed and moved to a common location so change point analysis can also use it. --- .../aiops_common/create_default_query.test.ts | 156 ++++++++++++++++++ .../create_default_query.ts} | 14 +- .../create_category_request.ts | 4 +- .../aiops_log_pattern_analysis/tsconfig.json | 1 + .../change_point_detection_context.tsx | 22 +-- .../use_validate_category_field.ts | 4 +- 6 files changed, 177 insertions(+), 24 deletions(-) create mode 100644 x-pack/packages/ml/aiops_common/create_default_query.test.ts rename x-pack/packages/ml/{aiops_log_pattern_analysis/create_categorize_query.ts => aiops_common/create_default_query.ts} (76%) diff --git a/x-pack/packages/ml/aiops_common/create_default_query.test.ts b/x-pack/packages/ml/aiops_common/create_default_query.test.ts new file mode 100644 index 0000000000000..e1fa99b70dce0 --- /dev/null +++ b/x-pack/packages/ml/aiops_common/create_default_query.test.ts @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createDefaultQuery } from './create_default_query'; + +describe('createDefaultQuery', () => { + it('should create a default match_all query when no input query is provided', () => { + const result = createDefaultQuery(undefined, 'timestamp', undefined); + expect(result).toEqual({ + bool: { + must: [{ match_all: {} }], + }, + }); + }); + + it('should wrap an existing match_all query in a bool must clause', () => { + const inputQuery = { match_all: {} }; + const result = createDefaultQuery(inputQuery, 'timestamp', undefined); + expect(result).toEqual({ + bool: { + must: [{ match_all: {} }], + }, + }); + }); + + it('should wrap an existing query_string query in a bool must clause', () => { + const inputQuery = { query_string: { query: '*' } }; + const result = createDefaultQuery(inputQuery, 'timestamp', undefined); + expect(result).toEqual({ + bool: { + must: [{ query_string: { query: '*' } }], + }, + }); + }); + + it('should wrap an existing multi_match query in a bool should clause', () => { + const inputQuery = { multi_match: { query: 'test', fields: ['field1', 'field2'] } }; + const result = createDefaultQuery(inputQuery, 'timestamp', undefined); + expect(result).toEqual({ + bool: { + must: [], + should: { multi_match: { query: 'test', fields: ['field1', 'field2'] } }, + }, + }); + }); + + it('should add a time range filter to the query', () => { + const timeRange = { from: 1609459200000, to: 1609545600000 }; + const result = createDefaultQuery(undefined, 'timestamp', timeRange); + expect(result).toEqual({ + bool: { + must: [ + { match_all: {} }, + { + range: { + timestamp: { + gte: 1609459200000, + lte: 1609545600000, + format: 'epoch_millis', + }, + }, + }, + ], + }, + }); + }); + + it('should merge existing bool query with new time range filter', () => { + const inputQuery = { bool: { must: [{ term: { field: 'value' } }] } }; + const timeRange = { from: 1609459200000, to: 1609545600000 }; + const result = createDefaultQuery(inputQuery, 'timestamp', timeRange); + expect(result).toEqual({ + bool: { + must: [ + { term: { field: 'value' } }, + { + range: { + timestamp: { + gte: 1609459200000, + lte: 1609545600000, + format: 'epoch_millis', + }, + }, + }, + ], + }, + }); + }); + + it('should handle an existing bool query with must clause', () => { + const inputQuery = { bool: { must: [{ term: { field: 'value' } }] } }; + const result = createDefaultQuery(inputQuery, 'timestamp', undefined); + expect(result).toEqual({ + bool: { + must: [{ term: { field: 'value' } }], + }, + }); + }); + + it('should handle an existing bool query with should clause', () => { + const inputQuery = { bool: { should: [{ term: { field: 'value' } }] } }; + const result = createDefaultQuery(inputQuery, 'timestamp', undefined); + expect(result).toEqual({ + bool: { + must: [], + should: [{ term: { field: 'value' } }], + }, + }); + }); + + it('should handle an existing bool query with must_not clause', () => { + const inputQuery = { bool: { must_not: [{ term: { field: 'value' } }] } }; + const result = createDefaultQuery(inputQuery, 'timestamp', undefined); + expect(result).toEqual({ + bool: { + must: [], + must_not: [{ term: { field: 'value' } }], + }, + }); + }); + + it('should handle an existing bool query with filter clause', () => { + const inputQuery = { bool: { filter: [{ term: { field: 'value' } }] } }; + const result = createDefaultQuery(inputQuery, 'timestamp', undefined); + expect(result).toEqual({ + bool: { + must: [], + filter: [{ term: { field: 'value' } }], + }, + }); + }); + + it('should handle an input query with multiple clauses', () => { + const inputQuery = { + bool: { + must: [{ term: { field1: 'value1' } }], + should: [{ term: { field2: 'value2' } }], + must_not: [{ term: { field3: 'value3' } }], + filter: [{ term: { field4: 'value4' } }], + }, + }; + const result = createDefaultQuery(inputQuery, 'timestamp', undefined); + expect(result).toEqual({ + bool: { + must: [{ term: { field1: 'value1' } }], + should: [{ term: { field2: 'value2' } }], + must_not: [{ term: { field3: 'value3' } }], + filter: [{ term: { field4: 'value4' } }], + }, + }); + }); +}); diff --git a/x-pack/packages/ml/aiops_log_pattern_analysis/create_categorize_query.ts b/x-pack/packages/ml/aiops_common/create_default_query.ts similarity index 76% rename from x-pack/packages/ml/aiops_log_pattern_analysis/create_categorize_query.ts rename to x-pack/packages/ml/aiops_common/create_default_query.ts index c3289d1527f2b..39cfafff4097b 100644 --- a/x-pack/packages/ml/aiops_log_pattern_analysis/create_categorize_query.ts +++ b/x-pack/packages/ml/aiops_common/create_default_query.ts @@ -7,9 +7,12 @@ import { cloneDeep } from 'lodash'; -import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { + QueryDslBoolQuery, + QueryDslQueryContainer, +} from '@elastic/elasticsearch/lib/api/types'; -export function createCategorizeQuery( +export function createDefaultQuery( queryIn: QueryDslQueryContainer | undefined, timeField: string, timeRange: { from: number; to: number } | undefined @@ -17,14 +20,19 @@ export function createCategorizeQuery( const query = cloneDeep(queryIn ?? { match_all: {} }); if (query.bool === undefined) { - query.bool = {}; + query.bool = Object.create(null) as QueryDslBoolQuery; } + if (query.bool.must === undefined) { query.bool.must = []; if (query.match_all !== undefined) { query.bool.must.push({ match_all: query.match_all }); delete query.match_all; } + if (query.query_string !== undefined) { + query.bool.must.push({ query_string: query.query_string }); + delete query.query_string; + } } if (query.multi_match !== undefined) { query.bool.should = { diff --git a/x-pack/packages/ml/aiops_log_pattern_analysis/create_category_request.ts b/x-pack/packages/ml/aiops_log_pattern_analysis/create_category_request.ts index c3556803745a7..c1d6f82c9e582 100644 --- a/x-pack/packages/ml/aiops_log_pattern_analysis/create_category_request.ts +++ b/x-pack/packages/ml/aiops_log_pattern_analysis/create_category_request.ts @@ -14,7 +14,7 @@ import { isPopulatedObject } from '@kbn/ml-is-populated-object/src/is_populated_ import type { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils'; -import { createCategorizeQuery } from './create_categorize_query'; +import { createDefaultQuery } from '@kbn/aiops-common/create_default_query'; const CATEGORY_LIMIT = 1000; const EXAMPLE_LIMIT = 4; @@ -38,7 +38,7 @@ export function createCategoryRequest( useStandardTokenizer: boolean = true, includeSparkline: boolean = true ) { - const query = createCategorizeQuery(queryIn, timeField, timeRange); + const query = createDefaultQuery(queryIn, timeField, timeRange); const aggs = { categories: { categorize_text: { diff --git a/x-pack/packages/ml/aiops_log_pattern_analysis/tsconfig.json b/x-pack/packages/ml/aiops_log_pattern_analysis/tsconfig.json index fb51a4d8c1b30..fe2542b34c5e9 100644 --- a/x-pack/packages/ml/aiops_log_pattern_analysis/tsconfig.json +++ b/x-pack/packages/ml/aiops_log_pattern_analysis/tsconfig.json @@ -23,5 +23,6 @@ "@kbn/saved-search-plugin", "@kbn/data-views-plugin", "@kbn/ml-is-populated-object", + "@kbn/aiops-common", ] } diff --git a/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx b/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx index 2a9ab8d535fa1..45ef73c5dd7b5 100644 --- a/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx +++ b/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx @@ -16,6 +16,7 @@ import { ES_FIELD_TYPES } from '@kbn/field-types'; import { type QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; import type { TimeBuckets, TimeBucketsInterval } from '@kbn/ml-time-buckets'; import { useTimeBuckets } from '@kbn/ml-time-buckets'; +import { createDefaultQuery } from '@kbn/aiops-common/create_default_query'; import { useFilterQueryUpdates } from '../../hooks/use_filters_query'; import { type ChangePointType, DEFAULT_AGG_FUNCTION } from './constants'; import { @@ -261,23 +262,10 @@ export const ChangePointDetectionContextProvider: FC> const combinedQuery = useMemo(() => { const mergedQuery = createMergedEsQuery(resultQuery, resultFilters, dataView, uiSettings); - if (!Array.isArray(mergedQuery.bool?.filter)) { - if (!mergedQuery.bool) { - mergedQuery.bool = {}; - } - mergedQuery.bool.filter = []; - } - - mergedQuery.bool!.filter.push({ - range: { - [dataView.timeFieldName!]: { - from: searchBounds.min?.valueOf(), - to: searchBounds.max?.valueOf(), - }, - }, - }); - - return mergedQuery; + const to = searchBounds.max?.valueOf(); + const from = searchBounds.min?.valueOf(); + const timeRange = to !== undefined && from !== undefined ? { from, to } : undefined; + return createDefaultQuery(mergedQuery, dataView.timeFieldName!, timeRange); }, [resultFilters, resultQuery, uiSettings, dataView, searchBounds]); if (!bucketInterval) return null; diff --git a/x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts b/x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts index edf055635f82a..571bb3d1e0f87 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts +++ b/x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts @@ -14,7 +14,7 @@ import type { FieldValidationResults } from '@kbn/ml-category-validator'; import type { HttpFetchOptions } from '@kbn/core/public'; import { AIOPS_API_ENDPOINT } from '@kbn/aiops-common/constants'; -import { createCategorizeQuery } from '@kbn/aiops-log-pattern-analysis/create_categorize_query'; +import { createDefaultQuery } from '@kbn/aiops-common/create_default_query'; import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; @@ -32,7 +32,7 @@ export function useValidateFieldRequest() { runtimeMappings: MappingRuntimeFields | undefined, headers?: HttpFetchOptions['headers'] ) => { - const query = createCategorizeQuery(queryIn, timeField, timeRange); + const query = createDefaultQuery(queryIn, timeField, timeRange); const resp = await http.post( AIOPS_API_ENDPOINT.CATEGORIZATION_FIELD_VALIDATION, {