From eb741235d70fc5b9ae5ef91a1380ffbb4b007443 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 7 Nov 2024 11:02:08 -0800 Subject: [PATCH] Replace index mapping with field caps API for trace filters (#2246) (#2250) * Replace fetch fields with field capabilties for trace filters * add test updates, fix mdsId --------- (cherry picked from commit 6a43f539745ad1d51d66d99962563c620b681e4b) Signed-off-by: Shenoy Pratik Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- common/constants/shared.ts | 1 + public/components/common/types.ts | 15 ++++ .../custom_panel_view.test.tsx.snap | 4 ++ .../__tests__/helper_functions.test.tsx | 27 +++++++- .../components/common/helper_functions.tsx | 54 ++------------- public/components/trace_analytics/home.tsx | 12 ++-- public/services/requests/dsl.ts | 18 ++++- server/routes/dsl.ts | 42 ++++++++++++ test/constants.ts | 68 +++++++++++++++++++ 9 files changed, 184 insertions(+), 57 deletions(-) create mode 100644 public/components/common/types.ts diff --git a/common/constants/shared.ts b/common/constants/shared.ts index 53d7af48a..2ebe0ac93 100644 --- a/common/constants/shared.ts +++ b/common/constants/shared.ts @@ -11,6 +11,7 @@ export const DSL_SEARCH = '/search'; export const DSL_CAT = '/cat.indices'; export const DSL_MAPPING = '/indices.getFieldMapping'; export const DSL_SETTINGS = '/indices.getFieldSettings'; +export const DSL_FIELD_CAPS = '/fieldCaps'; export const OBSERVABILITY_BASE = '/api/observability'; export const INTEGRATIONS_BASE = '/api/integrations'; export const JOBS_BASE = '/query/jobs'; diff --git a/public/components/common/types.ts b/public/components/common/types.ts new file mode 100644 index 000000000..e5180adca --- /dev/null +++ b/public/components/common/types.ts @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export interface FieldCapAttributes { + type: string; + searchable: boolean; + aggregatable: boolean; +} + +export interface FieldCapResponse { + indices: string[]; + fields: Record>; +} diff --git a/public/components/custom_panels/__tests__/__snapshots__/custom_panel_view.test.tsx.snap b/public/components/custom_panels/__tests__/__snapshots__/custom_panel_view.test.tsx.snap index 26834fc0e..34346b53c 100644 --- a/public/components/custom_panels/__tests__/__snapshots__/custom_panel_view.test.tsx.snap +++ b/public/components/custom_panels/__tests__/__snapshots__/custom_panel_view.test.tsx.snap @@ -1306,6 +1306,7 @@ exports[`Panels View Component renders panel view container with visualizations dslService={ DSLService { "fetch": [Function], + "fetchFieldCaps": [Function], "fetchFields": [Function], "fetchIndices": [Function], "fetchSettings": [Function], @@ -1809,6 +1810,7 @@ exports[`Panels View Component renders panel view container with visualizations dslService={ DSLService { "fetch": [Function], + "fetchFieldCaps": [Function], "fetchFields": [Function], "fetchIndices": [Function], "fetchSettings": [Function], @@ -3672,6 +3674,7 @@ exports[`Panels View Component renders panel view container without visualizatio dslService={ DSLService { "fetch": [Function], + "fetchFieldCaps": [Function], "fetchFields": [Function], "fetchIndices": [Function], "fetchSettings": [Function], @@ -4173,6 +4176,7 @@ exports[`Panels View Component renders panel view container without visualizatio dslService={ DSLService { "fetch": [Function], + "fetchFieldCaps": [Function], "fetchFields": [Function], "fetchIndices": [Function], "fetchSettings": [Function], diff --git a/public/components/trace_analytics/components/common/__tests__/helper_functions.test.tsx b/public/components/trace_analytics/components/common/__tests__/helper_functions.test.tsx index 36d4c6620..16e6d7177 100644 --- a/public/components/trace_analytics/components/common/__tests__/helper_functions.test.tsx +++ b/public/components/trace_analytics/components/common/__tests__/helper_functions.test.tsx @@ -6,12 +6,18 @@ import { configure, mount, shallow } from 'enzyme'; import Adapter from 'enzyme-adapter-react-16'; import React from 'react'; -import { TEST_SERVICE_MAP, TEST_SERVICE_MAP_GRAPH } from '../../../../../../test/constants'; +import { + fieldCapQueryResponse1, + fieldCapQueryResponse2, + TEST_SERVICE_MAP, + TEST_SERVICE_MAP_GRAPH, +} from '../../../../../../test/constants'; import { calculateTicks, filtersToDsl, fixedIntervalToMilli, fixedIntervalToTickFormat, + getAttributeFieldNames, getPercentileFilter, getServiceMapGraph, getServiceMapScaleColor, @@ -170,4 +176,23 @@ describe('Helper functions', () => { '{"query":{"bool":{"must":[],"filter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}}],"should":["test"],"must_not":[],"minimum_should_match":1}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":["test"],"minimum_should_match":1}}}}}' ); }); + + describe('getAttributeFieldNames', () => { + it("should return only field names starting with 'resource.attributes' or 'span.attributes'", () => { + const expectedFields = [ + 'span.attributes.http@url', + 'span.attributes.net@peer@ip', + 'span.attributes.http@user_agent.keyword', + 'resource.attributes.telemetry@sdk@version.keyword', + 'resource.attributes.host@hostname.keyword', + ]; + const result = getAttributeFieldNames(fieldCapQueryResponse1); + expect(result).toEqual(expectedFields); + }); + + it('should return an empty array if no fields match the specified prefixes', () => { + const result = getAttributeFieldNames(fieldCapQueryResponse2); + expect(result).toEqual([]); + }); + }); }); diff --git a/public/components/trace_analytics/components/common/helper_functions.tsx b/public/components/trace_analytics/components/common/helper_functions.tsx index 6bec57e73..92b60ae7c 100644 --- a/public/components/trace_analytics/components/common/helper_functions.tsx +++ b/public/components/trace_analytics/components/common/helper_functions.tsx @@ -24,6 +24,7 @@ import { TraceAnalyticsMode, } from '../../../../../common/types/trace_analytics'; import { uiSettingsService } from '../../../../../common/utils'; +import { FieldCapResponse } from '../../../common/types'; import { serviceMapColorPalette } from './color_palette'; import { FilterType } from './filters/filters'; import { ServiceObject } from './plots/service_map'; @@ -522,55 +523,10 @@ export const filtersToDsl = ( return DSL; }; -interface AttributeMapping { - properties: { - [key: string]: { - type?: string; - properties?: AttributeMapping['properties']; - }; - }; -} - -interface JsonMapping { - [key: string]: { - mappings: { - properties: AttributeMapping['properties']; - }; - }; -} - -export const extractAttributes = ( - mapping: AttributeMapping['properties'], - prefix: string -): string[] => { - let attributes: string[] = []; - - for (const [key, value] of Object.entries(mapping)) { - if (value.properties) { - attributes = attributes.concat(extractAttributes(value.properties, `${prefix}.${key}`)); - } else { - attributes.push(`${prefix}.${key}`); - } - } - - return attributes; -}; - -export const getAttributes = (jsonMapping: JsonMapping): string[] => { - if (Object.keys(jsonMapping)[0] !== undefined) { - const spanMapping = - jsonMapping[Object.keys(jsonMapping)[0]]?.mappings?.properties?.span?.properties?.attributes - ?.properties; - const resourceMapping = - jsonMapping[Object.keys(jsonMapping)[0]]?.mappings?.properties?.resource?.properties - ?.attributes?.properties; - - const spanAttributes = extractAttributes(spanMapping!, 'span.attributes'); - const resourceAttributes = extractAttributes(resourceMapping!, 'resource.attributes'); - - return [...spanAttributes, ...resourceAttributes]; - } - return []; +export const getAttributeFieldNames = (response: FieldCapResponse): string[] => { + return Object.keys(response.fields).filter( + (field) => field.startsWith('resource.attributes') || field.startsWith('span.attributes') + ); }; export const getTraceCustomSpanIndex = () => { diff --git a/public/components/trace_analytics/home.tsx b/public/components/trace_analytics/home.tsx index a18e2baa5..d107bfe2d 100644 --- a/public/components/trace_analytics/home.tsx +++ b/public/components/trace_analytics/home.tsx @@ -22,11 +22,12 @@ import { DataSourceViewConfig, } from '../../../../../src/plugins/data_source_management/public'; import { DataSourceAttributes } from '../../../../../src/plugins/data_source_management/public/types'; +import { observabilityTracesNewNavID } from '../../../common/constants/shared'; import { TRACE_TABLE_TYPE_KEY } from '../../../common/constants/trace_analytics'; import { TraceAnalyticsMode, TraceQueryMode } from '../../../common/types/trace_analytics'; import { coreRefs } from '../../framework/core_refs'; import { FilterType } from './components/common/filters/filters'; -import { getAttributes, getSpanIndices } from './components/common/helper_functions'; +import { getAttributeFieldNames, getSpanIndices } from './components/common/helper_functions'; import { SearchBarProps } from './components/common/search_bar'; import { ServiceView, Services } from './components/services'; import { ServiceFlyout } from './components/services/service_flyout'; @@ -37,7 +38,6 @@ import { handleJaegerIndicesExistRequest, } from './requests/request_handler'; import { TraceSideBar } from './trace_side_nav'; -import { observabilityTracesNewNavID } from '../../../common/constants/shared'; const newNavigation = coreRefs.chrome?.navGroup.getNavGroupEnabled(); @@ -235,12 +235,12 @@ export const Home = (props: HomeProps) => { const fetchAttributesFields = () => { coreRefs.dslService - ?.fetchFields(getSpanIndices(mode)) + ?.fetchFieldCaps(getSpanIndices(mode), '*attributes*', dataSourceMDSId[0].id) .then((res) => { - const attributes = getAttributes(res); + const attributes = getAttributeFieldNames(res); setAttributesFilterFields(attributes); }) - .catch((error) => console.error('fetching attributes field failed', error)); + .catch((error) => console.error('Failed to fetch attribute fields', error)); }; useEffect(() => { @@ -257,7 +257,7 @@ export const Home = (props: HomeProps) => { useEffect(() => { if (mode === 'data_prepper' || mode === 'custom_data_prepper') fetchAttributesFields(); - }, [mode]); + }, [mode, dataSourceMDSId]); const serviceBreadcrumbs = [ ...(!isNavGroupEnabled diff --git a/public/services/requests/dsl.ts b/public/services/requests/dsl.ts index db0ba00c6..f1050fddf 100644 --- a/public/services/requests/dsl.ts +++ b/public/services/requests/dsl.ts @@ -6,11 +6,13 @@ import { CoreStart } from '../../../../../src/core/public'; import { DSL_BASE, - DSL_SEARCH, DSL_CAT, + DSL_FIELD_CAPS, DSL_MAPPING, + DSL_SEARCH, DSL_SETTINGS, } from '../../../common/constants/shared'; +import { FieldCapResponse } from '../../components/common/types'; /* eslint-disable import/no-default-export */ export default class DSLService { @@ -52,4 +54,18 @@ export default class DSLService { }, }); }; + + fetchFieldCaps = async ( + index: string, + fields: string, + dataSourceMDSId: string + ): Promise => { + return this.http.get(`${DSL_BASE}${DSL_FIELD_CAPS}`, { + query: { + index, + fields, + dataSourceMDSId, + }, + }); + }; } diff --git a/server/routes/dsl.ts b/server/routes/dsl.ts index 6318dccc0..76db687ed 100644 --- a/server/routes/dsl.ts +++ b/server/routes/dsl.ts @@ -9,6 +9,7 @@ import { IRouter } from '../../../../src/core/server'; import { DSL_BASE, DSL_CAT, + DSL_FIELD_CAPS, DSL_MAPPING, DSL_SEARCH, DSL_SETTINGS, @@ -237,6 +238,47 @@ export function registerDslRoute( } ); + router.get( + { + path: `${DSL_BASE}${DSL_FIELD_CAPS}`, + validate: { + query: schema.object({ + index: schema.string(), + fields: schema.string(), + dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), + }), + }, + }, + async (context, request, response) => { + const dataSourceMDSId = request.query.dataSourceMDSId; + try { + const requestBody = { + index: request.query.index, + fields: request.query.fields, + }; + let resp; + if (dataSourceEnabled && dataSourceMDSId) { + const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); + resp = await client.callAPI('fieldCaps', requestBody); + } else { + resp = await context.core.opensearch.legacy.client.callAsCurrentUser( + 'fieldCaps', + requestBody + ); + } + return response.ok({ + body: resp, + }); + } catch (error) { + if (error.statusCode !== 404) console.error(error); + return response.custom({ + statusCode: error.statusCode === 500 ? 503 : error.statusCode || 503, + body: error.message, + }); + } + } + ); + router.post( { path: `${DSL_BASE}/integrations/refresh`, diff --git a/test/constants.ts b/test/constants.ts index fed560878..7140688aa 100644 --- a/test/constants.ts +++ b/test/constants.ts @@ -676,3 +676,71 @@ export const mockSavedObjectActions = ({ get = [], getBulk = [] }) => { getBulk: jest.fn().mockResolvedValue({ observabilityObjectList: getBulk }), }; }; + +export const fieldCapQueryResponse1 = { + indices: ['dest1:otel-v1-apm-span-000001', 'dest2:otel-v1-apm-span-000001'], + fields: { + 'span.attributes.http@url': { + text: { + type: 'text', + searchable: true, + aggregatable: false, + }, + }, + 'span.attributes.net@peer@ip': { + text: { + type: 'text', + searchable: true, + aggregatable: false, + }, + }, + 'span.attributes.http@user_agent.keyword': { + keyword: { + type: 'keyword', + searchable: true, + aggregatable: true, + }, + }, + 'resource.attributes.telemetry@sdk@version.keyword': { + keyword: { + type: 'keyword', + searchable: true, + aggregatable: true, + }, + }, + 'resource.attributes.host@hostname.keyword': { + keyword: { + type: 'keyword', + searchable: true, + aggregatable: true, + }, + }, + 'unrelated.field.name': { + text: { + type: 'text', + searchable: true, + aggregatable: false, + }, + }, + }, +}; + +export const fieldCapQueryResponse2 = { + indices: ['dest1:otel-v1-apm-span-000001', 'dest2:otel-v1-apm-span-000001'], + fields: { + 'unrelated.field1': { + text: { + type: 'text', + searchable: true, + aggregatable: false, + }, + }, + 'another.unrelated.field': { + keyword: { + type: 'keyword', + searchable: true, + aggregatable: true, + }, + }, + }, +};