Replace index mapping with field caps API for trace filters

common/constants/shared.ts

@@ -11,6 +11,7 @@ export const DSL_SEARCH = '/search';
 export const DSL_CAT = '/cat.indices';
 export const DSL_MAPPING = '/indices.getFieldMapping';
 export const DSL_SETTINGS = '/indices.getFieldSettings';
+export const DSL_FIELD_CAPS = '/fieldCaps';
 export const OBSERVABILITY_BASE = '/api/observability';
 export const INTEGRATIONS_BASE = '/api/integrations';
 export const JOBS_BASE = '/query/jobs';

public/components/common/types.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export interface FieldCapAttributes {
+  type: string;
+  searchable: boolean;
+  aggregatable: boolean;
+}
+
+export interface FieldCapResponse {
+  indices: string[];
+  fields: Record<string, Record<string, FieldCapAttributes>>;
+}

public/components/custom_panels/__tests__/__snapshots__/custom_panel_view.test.tsx.snap

@@ -1306,6 +1306,7 @@ exports[`Panels View Component renders panel view container with visualizations
     dslService={
       DSLService {
         "fetch": [Function],
+        "fetchFieldCaps": [Function],
         "fetchFields": [Function],
         "fetchIndices": [Function],
         "fetchSettings": [Function],
@@ -1809,6 +1810,7 @@ exports[`Panels View Component renders panel view container with visualizations
                             dslService={
                               DSLService {
                                 "fetch": [Function],
+                                "fetchFieldCaps": [Function],
                                 "fetchFields": [Function],
                                 "fetchIndices": [Function],
                                 "fetchSettings": [Function],
@@ -3672,6 +3674,7 @@ exports[`Panels View Component renders panel view container without visualizatio
     dslService={
       DSLService {
         "fetch": [Function],
+        "fetchFieldCaps": [Function],
         "fetchFields": [Function],
         "fetchIndices": [Function],
         "fetchSettings": [Function],
@@ -4173,6 +4176,7 @@ exports[`Panels View Component renders panel view container without visualizatio
                             dslService={
                               DSLService {
                                 "fetch": [Function],
+                                "fetchFieldCaps": [Function],
                                 "fetchFields": [Function],
                                 "fetchIndices": [Function],
                                 "fetchSettings": [Function],

public/components/trace_analytics/components/common/__tests__/helper_functions.test.tsx

@@ -6,12 +6,18 @@
 import { configure, mount, shallow } from 'enzyme';
 import Adapter from 'enzyme-adapter-react-16';
 import React from 'react';
-import { TEST_SERVICE_MAP, TEST_SERVICE_MAP_GRAPH } from '../../../../../../test/constants';
+import {
+  fieldCapQueryResponse1,
+  fieldCapQueryResponse2,
+  TEST_SERVICE_MAP,
+  TEST_SERVICE_MAP_GRAPH,
+} from '../../../../../../test/constants';
 import {
   calculateTicks,
   filtersToDsl,
   fixedIntervalToMilli,
   fixedIntervalToTickFormat,
+  getAttributeFieldNames,
   getPercentileFilter,
   getServiceMapGraph,
   getServiceMapScaleColor,
@@ -170,4 +176,23 @@ describe('Helper functions', () => {
       '{"query":{"bool":{"must":[],"filter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}}],"should":["test"],"must_not":[],"minimum_should_match":1}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":["test"],"minimum_should_match":1}}}}}'
     );
   });
+
+  describe('getAttributeFieldNames', () => {
+    it("should return only field names starting with 'resource.attributes' or 'span.attributes'", () => {
+      const expectedFields = [
+        'span.attributes.http@url',
+        'span.attributes.net@peer@ip',
+        'span.attributes.http@user_agent.keyword',
+        'resource.attributes.telemetry@[email protected]',
+        '[email protected]',
+      ];
+      const result = getAttributeFieldNames(fieldCapQueryResponse1);
+      expect(result).toEqual(expectedFields);
+    });
+
+    it('should return an empty array if no fields match the specified prefixes', () => {
+      const result = getAttributeFieldNames(fieldCapQueryResponse2);
+      expect(result).toEqual([]);
+    });
+  });
 });

public/components/trace_analytics/components/common/helper_functions.tsx

@@ -24,6 +24,7 @@
   TraceAnalyticsMode,
 } from '../../../../../common/types/trace_analytics';
 import { uiSettingsService } from '../../../../../common/utils';
+import { FieldCapResponse } from '../../../common/types';
 import { serviceMapColorPalette } from './color_palette';
 import { FilterType } from './filters/filters';
 import { ServiceObject } from './plots/service_map';
@@ -324,7 +325,7 @@
   percentileMaps: Array<{ traceGroupName: string; durationFilter: { gte?: number; lte?: number } }>,
   conditionString: string // >= 95th, < 95th
 ): FilterType => {
   const DSL: any = {
     query: {
       bool: {
         must: [],
@@ -369,12 +370,12 @@
   mode: TraceAnalyticsMode,
   filters: FilterType[],
   query: string,
   startTime: any,
   endTime: any,
   page?: string,
   appConfigs: FilterType[] = []
 ) => {
   const DSL: any = {
     query: {
       bool: {
         must: [],
@@ -522,55 +523,10 @@
   return DSL;
 };
 
-interface AttributeMapping {
-  properties: {
-    [key: string]: {
-      type?: string;
-      properties?: AttributeMapping['properties'];
-    };
-  };
-}
-
-interface JsonMapping {
-  [key: string]: {
-    mappings: {
-      properties: AttributeMapping['properties'];
-    };
-  };
-}
-
-export const extractAttributes = (
-  mapping: AttributeMapping['properties'],
-  prefix: string
-): string[] => {
-  let attributes: string[] = [];
-
-  for (const [key, value] of Object.entries(mapping)) {
-    if (value.properties) {
-      attributes = attributes.concat(extractAttributes(value.properties, `${prefix}.${key}`));
-    } else {
-      attributes.push(`${prefix}.${key}`);
-    }
-  }
-
-  return attributes;
-};
-
-export const getAttributes = (jsonMapping: JsonMapping): string[] => {
-  if (Object.keys(jsonMapping)[0] !== undefined) {
-    const spanMapping =
-      jsonMapping[Object.keys(jsonMapping)[0]]?.mappings?.properties?.span?.properties?.attributes
-        ?.properties;
-    const resourceMapping =
-      jsonMapping[Object.keys(jsonMapping)[0]]?.mappings?.properties?.resource?.properties
-        ?.attributes?.properties;
-
-    const spanAttributes = extractAttributes(spanMapping!, 'span.attributes');
-    const resourceAttributes = extractAttributes(resourceMapping!, 'resource.attributes');
-
-    return [...spanAttributes, ...resourceAttributes];
-  }
-  return [];
+export const getAttributeFieldNames = (response: FieldCapResponse): string[] => {
+  return Object.keys(response.fields).filter(
+    (field) => field.startsWith('resource.attributes') || field.startsWith('span.attributes')
+  );
 };
 
 export const getTraceCustomSpanIndex = () => {

public/components/trace_analytics/home.tsx

@@ -22,11 +22,12 @@
   DataSourceViewConfig,
 } from '../../../../../src/plugins/data_source_management/public';
 import { DataSourceAttributes } from '../../../../../src/plugins/data_source_management/public/types';
+import { observabilityTracesNewNavID } from '../../../common/constants/shared';
 import { TRACE_TABLE_TYPE_KEY } from '../../../common/constants/trace_analytics';
 import { TraceAnalyticsMode, TraceQueryMode } from '../../../common/types/trace_analytics';
 import { coreRefs } from '../../framework/core_refs';
 import { FilterType } from './components/common/filters/filters';
-import { getAttributes, getSpanIndices } from './components/common/helper_functions';
+import { getAttributeFieldNames, getSpanIndices } from './components/common/helper_functions';
 import { SearchBarProps } from './components/common/search_bar';
 import { ServiceView, Services } from './components/services';
 import { ServiceFlyout } from './components/services/service_flyout';
@@ -37,7 +38,6 @@
   handleJaegerIndicesExistRequest,
 } from './requests/request_handler';
 import { TraceSideBar } from './trace_side_nav';
-import { observabilityTracesNewNavID } from '../../../common/constants/shared';
 
 const newNavigation = coreRefs.chrome?.navGroup.getNavGroupEnabled();
 
@@ -74,7 +74,7 @@
   }: {
     spanId: string;
     isFlyoutVisible: boolean;
     addSpanFilter: (field: string, value: any) => void;
     spanMode: TraceAnalyticsMode;
     spanDataSourceMDSId: string;
   }) => void;
@@ -191,7 +191,7 @@
     ) : (
       <DataSourceMenuView {...sharedProps} componentType={'DataSourceView'} />
     );
   }, [
     dataSourceMDSId,
     dataSourceMenuSelectable,
     props.setActionMenu,
@@ -221,7 +221,7 @@
     if (dataSourceMDSId[0].id && dataSourceMDSId[0].label === undefined) {
       getDatasourceAttributes();
     }
   }, [dataSourceMDSId]);
 
   const modes = [
     { id: 'jaeger', title: 'Jaeger', 'data-test-subj': 'jaeger-mode' },
@@ -235,12 +235,12 @@
 
   const fetchAttributesFields = () => {
     coreRefs.dslService
-      ?.fetchFields(getSpanIndices(mode))
+      ?.fetchFieldCaps(getSpanIndices(mode), '*attributes*', dataSourceMDSId[0].id)
       .then((res) => {
-        const attributes = getAttributes(res);
+        const attributes = getAttributeFieldNames(res);
         setAttributesFilterFields(attributes);
       })
-      .catch((error) => console.error('fetching attributes field failed', error));
+      .catch((error) => console.error('Failed to fetch attribute fields', error));
   };
 
   useEffect(() => {
@@ -257,7 +257,7 @@
 
   useEffect(() => {
     if (mode === 'data_prepper' || mode === 'custom_data_prepper') fetchAttributesFields();
-  }, [mode]);
+  }, [mode, dataSourceMDSId]);
 
   const serviceBreadcrumbs = [
     ...(!isNavGroupEnabled
@@ -327,7 +327,7 @@
   }: {
     spanId: string;
     isFlyoutVisible: boolean;
     addSpanFilter: (field: string, value: any) => void;
     spanMode: TraceAnalyticsMode;
     spanDataSourceMDSId: string;
   }) => {

public/services/requests/dsl.ts

@@ -6,11 +6,13 @@
 import { CoreStart } from '../../../../../src/core/public';
 import {
   DSL_BASE,
-  DSL_SEARCH,
   DSL_CAT,
+  DSL_FIELD_CAPS,
   DSL_MAPPING,
+  DSL_SEARCH,
   DSL_SETTINGS,
 } from '../../../common/constants/shared';
+import { FieldCapResponse } from '../../components/common/types';
 
 /* eslint-disable import/no-default-export */
 export default class DSLService {
@@ -18,7 +20,7 @@
   constructor(http: CoreStart['http']) {
     this.http = http;
   }
   fetch = async (request: any) => {
     return this.http
       .post(`${DSL_BASE}${DSL_SEARCH}`, {
         body: JSON.stringify(request),
@@ -52,4 +54,18 @@
       },
     });
   };
+
+  fetchFieldCaps = async (
+    index: string,
+    fields: string,
+    dataSourceMDSId: string
+  ): Promise<FieldCapResponse> => {
+    return this.http.get(`${DSL_BASE}${DSL_FIELD_CAPS}`, {
+      query: {
+        index,
+        fields,
+        dataSourceMDSId,
+      },
+    });
+  };
 }

server/routes/dsl.ts

@@ -9,6 +9,7 @@ import { IRouter } from '../../../../src/core/server';
 import {
   DSL_BASE,
   DSL_CAT,
+  DSL_FIELD_CAPS,
   DSL_MAPPING,
   DSL_SEARCH,
   DSL_SETTINGS,
@@ -237,6 +238,47 @@ export function registerDslRoute(
     }
   );
 
+  router.get(
+    {
+      path: `${DSL_BASE}${DSL_FIELD_CAPS}`,
+      validate: {
+        query: schema.object({
+          index: schema.string(),
+          fields: schema.string(),
+          dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })),
+        }),
+      },
+    },
+    async (context, request, response) => {
+      const dataSourceMDSId = request.query.dataSourceMDSId;
+      try {
+        const requestBody = {
+          index: request.query.index,
+          fields: request.query.fields,
+        };
+        let resp;
+        if (dataSourceEnabled && dataSourceMDSId) {
+          const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId);
+          resp = await client.callAPI('fieldCaps', requestBody);
+        } else {
+          resp = await context.core.opensearch.legacy.client.callAsCurrentUser(
+            'fieldCaps',
+            requestBody
+          );
+        }
+        return response.ok({
+          body: resp,
+        });
+      } catch (error) {
+        if (error.statusCode !== 404) console.error(error);
+        return response.custom({
+          statusCode: error.statusCode === 500 ? 503 : error.statusCode || 503,
+          body: error.message,
+        });
+      }
+    }
+  );
+
   router.post(
     {
       path: `${DSL_BASE}/integrations/refresh`,

test/constants.ts

@@ -676,3 +676,71 @@ export const mockSavedObjectActions = ({ get = [], getBulk = [] }) => {
     getBulk: jest.fn().mockResolvedValue({ observabilityObjectList: getBulk }),
   };
 };
+
+export const fieldCapQueryResponse1 = {
+  indices: ['dest1:otel-v1-apm-span-000001', 'dest2:otel-v1-apm-span-000001'],
+  fields: {
+    'span.attributes.http@url': {
+      text: {
+        type: 'text',
+        searchable: true,
+        aggregatable: false,
+      },
+    },
+    'span.attributes.net@peer@ip': {
+      text: {
+        type: 'text',
+        searchable: true,
+        aggregatable: false,
+      },
+    },
+    'span.attributes.http@user_agent.keyword': {
+      keyword: {
+        type: 'keyword',
+        searchable: true,
+        aggregatable: true,
+      },
+    },
+    'resource.attributes.telemetry@[email protected]': {
+      keyword: {
+        type: 'keyword',
+        searchable: true,
+        aggregatable: true,
+      },
+    },
+    '[email protected]': {
+      keyword: {
+        type: 'keyword',
+        searchable: true,
+        aggregatable: true,
+      },
+    },
+    'unrelated.field.name': {
+      text: {
+        type: 'text',
+        searchable: true,
+        aggregatable: false,
+      },
+    },
+  },
+};
+
+export const fieldCapQueryResponse2 = {
+  indices: ['dest1:otel-v1-apm-span-000001', 'dest2:otel-v1-apm-span-000001'],
+  fields: {
+    'unrelated.field1': {
+      text: {
+        type: 'text',
+        searchable: true,
+        aggregatable: false,
+      },
+    },
+    'another.unrelated.field': {
+      keyword: {
+        type: 'keyword',
+        searchable: true,
+        aggregatable: true,
+      },
+    },
+  },
+};