[ML] AIOps Fixing runtime mappings in pattern analysis (elastic#188530)

Runtime mappings need to be passed to the categorization request factory function and the field validation function. Initially they were excluded because we only allow pattern analysis on text fields and it is not possible to create a text runtime field. However it is possible to apply a filter which uses a runtime field and doing so causes pattern analysis to fail. @walterra I have not investigated log rate analysis' behaviour, in this PR I have just updated the call to `createCategoryRequest` to pass `undefined` To test, create a runtime mapping in the data view. Use this in the query bar or in a filter in Discover and ML's Log Pattern Analysis page. --------- Co-authored-by: kibanamachine <[email protected]>
ThomThomson · Jul 19, 2024 · afe3b0f · afe3b0f
1 parent 9faf2f6
commit afe3b0f
Show file tree

Hide file tree

Showing 9 changed files with 67 additions and 16 deletions.
diff --git a/x-pack/packages/ml/aiops_log_pattern_analysis/create_category_request.ts b/x-pack/packages/ml/aiops_log_pattern_analysis/create_category_request.ts
@@ -9,6 +9,8 @@ import type {
   QueryDslQueryContainer,
   AggregationsCustomCategorizeTextAnalyzer,
 } from '@elastic/elasticsearch/lib/api/types';
+import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { isPopulatedObject } from '@kbn/ml-is-populated-object/src/is_populated_object';
 
 import type { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils';
 
@@ -29,6 +31,7 @@ export function createCategoryRequest(
   timeField: string,
   timeRange: { from: number; to: number } | undefined,
   queryIn: QueryDslQueryContainer,
+  runtimeMappings: MappingRuntimeFields | undefined,
   wrap: ReturnType<typeof createRandomSamplerWrapper>['wrap'],
   intervalMs?: number,
   additionalFilter?: CategorizationAdditionalFilter,
@@ -113,6 +116,7 @@ export function createCategoryRequest(
       body: {
         query,
         aggs: wrap(aggs),
+        ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}),
       },
     },
   };

diff --git a/x-pack/packages/ml/aiops_log_pattern_analysis/tsconfig.json b/x-pack/packages/ml/aiops_log_pattern_analysis/tsconfig.json
@@ -23,5 +23,6 @@
     "@kbn/es-query",
     "@kbn/saved-search-plugin",
     "@kbn/data-views-plugin",
+    "@kbn/ml-is-populated-object",
   ]
 }
diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_categories.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_categories.ts
@@ -75,6 +75,7 @@ export const getCategoryRequest = (
     timeFieldName,
     undefined,
     query,
+    undefined,
     wrap,
     undefined,
     undefined,

diff --git a/...og_categorization/log_categorization_for_embeddable/log_categorization_for_embeddable.tsx b/...og_categorization/log_categorization_for_embeddable/log_categorization_for_embeddable.tsx
@@ -247,30 +247,41 @@ export const LogCategorizationEmbeddable: FC<LogCategorizationEmbeddableProps> =
       from: earliest,
       to: latest,
     };
+    const runtimeMappings = dataView.getRuntimeMappings();
 
     try {
       const timeRange = await getMinimumTimeRange(
         index,
         timeField,
         additionalFilter,
         minimumTimeRangeOption,
-        searchQuery
+        searchQuery,
+        runtimeMappings
       );
 
       if (mounted.current !== true) {
         return;
       }
 
       const [validationResult, categorizationResult] = await Promise.all([
-        runValidateFieldRequest(index, selectedField.name, timeField, timeRange, searchQuery, {
-          [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin,
-        }),
+        runValidateFieldRequest(
+          index,
+          selectedField.name,
+          timeField,
+          timeRange,
+          searchQuery,
+          runtimeMappings,
+          {
+            [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin,
+          }
+        ),
         runCategorizeRequest(
           index,
           selectedField.name,
           timeField,
           { to: timeRange.to, from: timeRange.from },
           searchQuery,
+          runtimeMappings,
           intervalMs,
           timeRange.useSubAgg ? additionalFilter : undefined
         ),

diff --git a/...components/log_categorization/log_categorization_for_embeddable/use_minimum_time_range.ts b/...components/log_categorization/log_categorization_for_embeddable/use_minimum_time_range.ts
@@ -12,6 +12,7 @@ import type { HttpFetchOptions } from '@kbn/core/public';
 import { getTimeFieldRange } from '@kbn/ml-date-picker';
 import moment from 'moment';
 import { useStorage } from '@kbn/ml-local-storage';
+import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { useAiopsAppContext } from '../../../hooks/use_aiops_app_context';
 import type { MinimumTimeRangeOption } from './minimum_time_range';
 import { MINIMUM_TIME_RANGE } from './minimum_time_range';
@@ -29,6 +30,7 @@ export function useMinimumTimeRange() {
       timeRange: { from: number; to: number },
       minimumTimeRangeOption: MinimumTimeRangeOption,
       queryIn: QueryDslQueryContainer,
+      runtimeMappings: MappingRuntimeFields | undefined,
       headers?: HttpFetchOptions['headers']
     ) => {
       const minimumTimeRange = MINIMUM_TIME_RANGE[minimumTimeRangeOption];
@@ -47,6 +49,7 @@ export function useMinimumTimeRange() {
         index,
         timeFieldName: timeField,
         query: queryIn,
+        runtimeMappings,
         path: '/internal/file_upload/time_field_range',
         signal: abortController.current.signal,
       });

diff --git a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx
@@ -189,17 +189,28 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
       to: latest,
     };
 
+    const runtimeMappings = dataView.getRuntimeMappings();
+
     try {
       const [validationResult, categorizationResult] = await Promise.all([
-        runValidateFieldRequest(index, selectedField.name, timeField, timeRange, searchQuery, {
-          [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin,
-        }),
+        runValidateFieldRequest(
+          index,
+          selectedField.name,
+          timeField,
+          timeRange,
+          searchQuery,
+          runtimeMappings,
+          {
+            [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin,
+          }
+        ),
         runCategorizeRequest(
           index,
           selectedField.name,
           timeField,
           timeRange,
           searchQuery,
+          runtimeMappings,
           intervalMs,
           additionalFilter
         ),

diff --git a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx
@@ -217,13 +217,31 @@ export const LogCategorizationPage: FC<LogCategorizationPageProps> = ({ embeddin
       to: latest,
     };
 
+    const runtimeMappings = dataView.getRuntimeMappings();
+
     try {
       const [validationResult, categorizationResult] = await Promise.all([
-        runValidateFieldRequest(index, selectedField, timeField, timeRange, searchQuery, {
-          [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin,
-        }),
-
-        runCategorizeRequest(index, selectedField, timeField, timeRange, searchQuery, intervalMs),
+        runValidateFieldRequest(
+          index,
+          selectedField,
+          timeField,
+          timeRange,
+          searchQuery,
+          runtimeMappings,
+          {
+            [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin,
+          }
+        ),
+
+        runCategorizeRequest(
+          index,
+          selectedField,
+          timeField,
+          timeRange,
+          searchQuery,
+          runtimeMappings,
+          intervalMs
+        ),
       ]);
 
       setFieldValidationResult(validationResult);

diff --git a/x-pack/plugins/aiops/public/components/log_categorization/use_categorize_request.ts b/x-pack/plugins/aiops/public/components/log_categorization/use_categorize_request.ts
@@ -19,6 +19,7 @@ import {
 import { processCategoryResults } from '@kbn/aiops-log-pattern-analysis/process_category_results';
 import type { CatResponse } from '@kbn/aiops-log-pattern-analysis/types';
 
+import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 import {
   type AiOpsKey,
@@ -69,6 +70,7 @@ export function useCategorizeRequest() {
       timeField: string,
       timeRange: { from: number; to: number },
       query: QueryDslQueryContainer,
+      runtimeMappings: MappingRuntimeFields | undefined,
       intervalMs?: number,
       additionalFilter?: CategorizationAdditionalFilter
     ): Promise<ReturnType<typeof processCategoryResults>> => {
@@ -83,6 +85,7 @@ export function useCategorizeRequest() {
               timeField,
               timeRange,
               query,
+              runtimeMappings,
               wrap,
               intervalMs,
               additionalFilter,

diff --git a/x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts b/x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts
@@ -8,6 +8,7 @@
 import { useRef, useCallback } from 'react';
 
 import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
+import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
 import type { FieldValidationResults } from '@kbn/ml-category-validator';
 import type { HttpFetchOptions } from '@kbn/core/public';
@@ -28,6 +29,7 @@ export function useValidateFieldRequest() {
       timeField: string,
       timeRange: { from: number; to: number },
       queryIn: QueryDslQueryContainer,
+      runtimeMappings: MappingRuntimeFields | undefined,
       headers?: HttpFetchOptions['headers']
     ) => {
       const query = createCategorizeQuery(queryIn, timeField, timeRange);
@@ -42,10 +44,7 @@ export function useValidateFieldRequest() {
             timeField,
             start: timeRange.from,
             end: timeRange.to,
-            // only text fields are supported in pattern analysis,
-            // and it is not possible to create a text runtime field
-            // so runtimeMappings are not needed
-            runtimeMappings: undefined,
+            runtimeMappings,
             indicesOptions: undefined,
             includeExamples: false,
           }),