Skip to content

Commit

Permalink
Replace index mapping with field caps API for trace filters (#2246) (#…
Browse files Browse the repository at this point in the history
…2250)

* Replace fetch fields with field capabilties for trace filters



* add test updates, fix mdsId



---------


(cherry picked from commit 6a43f53)

Signed-off-by: Shenoy Pratik <[email protected]>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
  • Loading branch information
1 parent 3288817 commit eb74123
Show file tree
Hide file tree
Showing 9 changed files with 184 additions and 57 deletions.
1 change: 1 addition & 0 deletions common/constants/shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ export const DSL_SEARCH = '/search';
export const DSL_CAT = '/cat.indices';
export const DSL_MAPPING = '/indices.getFieldMapping';
export const DSL_SETTINGS = '/indices.getFieldSettings';
export const DSL_FIELD_CAPS = '/fieldCaps';
export const OBSERVABILITY_BASE = '/api/observability';
export const INTEGRATIONS_BASE = '/api/integrations';
export const JOBS_BASE = '/query/jobs';
Expand Down
15 changes: 15 additions & 0 deletions public/components/common/types.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

export interface FieldCapAttributes {
type: string;
searchable: boolean;
aggregatable: boolean;
}

export interface FieldCapResponse {
indices: string[];
fields: Record<string, Record<string, FieldCapAttributes>>;
}
Original file line number Diff line number Diff line change
Expand Up @@ -1306,6 +1306,7 @@ exports[`Panels View Component renders panel view container with visualizations
dslService={
DSLService {
"fetch": [Function],
"fetchFieldCaps": [Function],
"fetchFields": [Function],
"fetchIndices": [Function],
"fetchSettings": [Function],
Expand Down Expand Up @@ -1809,6 +1810,7 @@ exports[`Panels View Component renders panel view container with visualizations
dslService={
DSLService {
"fetch": [Function],
"fetchFieldCaps": [Function],
"fetchFields": [Function],
"fetchIndices": [Function],
"fetchSettings": [Function],
Expand Down Expand Up @@ -3672,6 +3674,7 @@ exports[`Panels View Component renders panel view container without visualizatio
dslService={
DSLService {
"fetch": [Function],
"fetchFieldCaps": [Function],
"fetchFields": [Function],
"fetchIndices": [Function],
"fetchSettings": [Function],
Expand Down Expand Up @@ -4173,6 +4176,7 @@ exports[`Panels View Component renders panel view container without visualizatio
dslService={
DSLService {
"fetch": [Function],
"fetchFieldCaps": [Function],
"fetchFields": [Function],
"fetchIndices": [Function],
"fetchSettings": [Function],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,18 @@
import { configure, mount, shallow } from 'enzyme';
import Adapter from 'enzyme-adapter-react-16';
import React from 'react';
import { TEST_SERVICE_MAP, TEST_SERVICE_MAP_GRAPH } from '../../../../../../test/constants';
import {
fieldCapQueryResponse1,
fieldCapQueryResponse2,
TEST_SERVICE_MAP,
TEST_SERVICE_MAP_GRAPH,
} from '../../../../../../test/constants';
import {
calculateTicks,
filtersToDsl,
fixedIntervalToMilli,
fixedIntervalToTickFormat,
getAttributeFieldNames,
getPercentileFilter,
getServiceMapGraph,
getServiceMapScaleColor,
Expand Down Expand Up @@ -170,4 +176,23 @@ describe('Helper functions', () => {
'{"query":{"bool":{"must":[],"filter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}}],"should":["test"],"must_not":[],"minimum_should_match":1}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":["test"],"minimum_should_match":1}}}}}'
);
});

describe('getAttributeFieldNames', () => {
it("should return only field names starting with 'resource.attributes' or 'span.attributes'", () => {
const expectedFields = [
'span.attributes.http@url',
'span.attributes.net@peer@ip',
'span.attributes.http@user_agent.keyword',
'resource.attributes.telemetry@[email protected]',
'[email protected]',
];
const result = getAttributeFieldNames(fieldCapQueryResponse1);
expect(result).toEqual(expectedFields);
});

it('should return an empty array if no fields match the specified prefixes', () => {
const result = getAttributeFieldNames(fieldCapQueryResponse2);
expect(result).toEqual([]);
});
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
TraceAnalyticsMode,
} from '../../../../../common/types/trace_analytics';
import { uiSettingsService } from '../../../../../common/utils';
import { FieldCapResponse } from '../../../common/types';
import { serviceMapColorPalette } from './color_palette';
import { FilterType } from './filters/filters';
import { ServiceObject } from './plots/service_map';
Expand Down Expand Up @@ -522,55 +523,10 @@ export const filtersToDsl = (
return DSL;
};

interface AttributeMapping {
properties: {
[key: string]: {
type?: string;
properties?: AttributeMapping['properties'];
};
};
}

interface JsonMapping {
[key: string]: {
mappings: {
properties: AttributeMapping['properties'];
};
};
}

export const extractAttributes = (
mapping: AttributeMapping['properties'],
prefix: string
): string[] => {
let attributes: string[] = [];

for (const [key, value] of Object.entries(mapping)) {
if (value.properties) {
attributes = attributes.concat(extractAttributes(value.properties, `${prefix}.${key}`));
} else {
attributes.push(`${prefix}.${key}`);
}
}

return attributes;
};

export const getAttributes = (jsonMapping: JsonMapping): string[] => {
if (Object.keys(jsonMapping)[0] !== undefined) {
const spanMapping =
jsonMapping[Object.keys(jsonMapping)[0]]?.mappings?.properties?.span?.properties?.attributes
?.properties;
const resourceMapping =
jsonMapping[Object.keys(jsonMapping)[0]]?.mappings?.properties?.resource?.properties
?.attributes?.properties;

const spanAttributes = extractAttributes(spanMapping!, 'span.attributes');
const resourceAttributes = extractAttributes(resourceMapping!, 'resource.attributes');

return [...spanAttributes, ...resourceAttributes];
}
return [];
export const getAttributeFieldNames = (response: FieldCapResponse): string[] => {
return Object.keys(response.fields).filter(
(field) => field.startsWith('resource.attributes') || field.startsWith('span.attributes')
);
};

export const getTraceCustomSpanIndex = () => {
Expand Down
12 changes: 6 additions & 6 deletions public/components/trace_analytics/home.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,12 @@ import {
DataSourceViewConfig,
} from '../../../../../src/plugins/data_source_management/public';
import { DataSourceAttributes } from '../../../../../src/plugins/data_source_management/public/types';
import { observabilityTracesNewNavID } from '../../../common/constants/shared';
import { TRACE_TABLE_TYPE_KEY } from '../../../common/constants/trace_analytics';
import { TraceAnalyticsMode, TraceQueryMode } from '../../../common/types/trace_analytics';
import { coreRefs } from '../../framework/core_refs';
import { FilterType } from './components/common/filters/filters';
import { getAttributes, getSpanIndices } from './components/common/helper_functions';
import { getAttributeFieldNames, getSpanIndices } from './components/common/helper_functions';
import { SearchBarProps } from './components/common/search_bar';
import { ServiceView, Services } from './components/services';
import { ServiceFlyout } from './components/services/service_flyout';
Expand All @@ -37,7 +38,6 @@ import {
handleJaegerIndicesExistRequest,
} from './requests/request_handler';
import { TraceSideBar } from './trace_side_nav';
import { observabilityTracesNewNavID } from '../../../common/constants/shared';

const newNavigation = coreRefs.chrome?.navGroup.getNavGroupEnabled();

Expand Down Expand Up @@ -235,12 +235,12 @@ export const Home = (props: HomeProps) => {

const fetchAttributesFields = () => {
coreRefs.dslService
?.fetchFields(getSpanIndices(mode))
?.fetchFieldCaps(getSpanIndices(mode), '*attributes*', dataSourceMDSId[0].id)
.then((res) => {
const attributes = getAttributes(res);
const attributes = getAttributeFieldNames(res);
setAttributesFilterFields(attributes);
})
.catch((error) => console.error('fetching attributes field failed', error));
.catch((error) => console.error('Failed to fetch attribute fields', error));
};

useEffect(() => {
Expand All @@ -257,7 +257,7 @@ export const Home = (props: HomeProps) => {

useEffect(() => {
if (mode === 'data_prepper' || mode === 'custom_data_prepper') fetchAttributesFields();
}, [mode]);
}, [mode, dataSourceMDSId]);

const serviceBreadcrumbs = [
...(!isNavGroupEnabled
Expand Down
18 changes: 17 additions & 1 deletion public/services/requests/dsl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@
import { CoreStart } from '../../../../../src/core/public';
import {
DSL_BASE,
DSL_SEARCH,
DSL_CAT,
DSL_FIELD_CAPS,
DSL_MAPPING,
DSL_SEARCH,
DSL_SETTINGS,
} from '../../../common/constants/shared';
import { FieldCapResponse } from '../../components/common/types';

/* eslint-disable import/no-default-export */
export default class DSLService {
Expand Down Expand Up @@ -52,4 +54,18 @@ export default class DSLService {
},
});
};

fetchFieldCaps = async (
index: string,
fields: string,
dataSourceMDSId: string
): Promise<FieldCapResponse> => {
return this.http.get(`${DSL_BASE}${DSL_FIELD_CAPS}`, {
query: {
index,
fields,
dataSourceMDSId,
},
});
};
}
42 changes: 42 additions & 0 deletions server/routes/dsl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { IRouter } from '../../../../src/core/server';
import {
DSL_BASE,
DSL_CAT,
DSL_FIELD_CAPS,
DSL_MAPPING,
DSL_SEARCH,
DSL_SETTINGS,
Expand Down Expand Up @@ -237,6 +238,47 @@ export function registerDslRoute(
}
);

router.get(
{
path: `${DSL_BASE}${DSL_FIELD_CAPS}`,
validate: {
query: schema.object({
index: schema.string(),
fields: schema.string(),
dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })),
}),
},
},
async (context, request, response) => {
const dataSourceMDSId = request.query.dataSourceMDSId;
try {
const requestBody = {
index: request.query.index,
fields: request.query.fields,
};
let resp;
if (dataSourceEnabled && dataSourceMDSId) {
const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId);
resp = await client.callAPI('fieldCaps', requestBody);
} else {
resp = await context.core.opensearch.legacy.client.callAsCurrentUser(
'fieldCaps',
requestBody
);
}
return response.ok({
body: resp,
});
} catch (error) {
if (error.statusCode !== 404) console.error(error);
return response.custom({
statusCode: error.statusCode === 500 ? 503 : error.statusCode || 503,
body: error.message,
});
}
}
);

router.post(
{
path: `${DSL_BASE}/integrations/refresh`,
Expand Down
68 changes: 68 additions & 0 deletions test/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -676,3 +676,71 @@ export const mockSavedObjectActions = ({ get = [], getBulk = [] }) => {
getBulk: jest.fn().mockResolvedValue({ observabilityObjectList: getBulk }),
};
};

export const fieldCapQueryResponse1 = {
indices: ['dest1:otel-v1-apm-span-000001', 'dest2:otel-v1-apm-span-000001'],
fields: {
'span.attributes.http@url': {
text: {
type: 'text',
searchable: true,
aggregatable: false,
},
},
'span.attributes.net@peer@ip': {
text: {
type: 'text',
searchable: true,
aggregatable: false,
},
},
'span.attributes.http@user_agent.keyword': {
keyword: {
type: 'keyword',
searchable: true,
aggregatable: true,
},
},
'resource.attributes.telemetry@[email protected]': {
keyword: {
type: 'keyword',
searchable: true,
aggregatable: true,
},
},
'[email protected]': {
keyword: {
type: 'keyword',
searchable: true,
aggregatable: true,
},
},
'unrelated.field.name': {
text: {
type: 'text',
searchable: true,
aggregatable: false,
},
},
},
};

export const fieldCapQueryResponse2 = {
indices: ['dest1:otel-v1-apm-span-000001', 'dest2:otel-v1-apm-span-000001'],
fields: {
'unrelated.field1': {
text: {
type: 'text',
searchable: true,
aggregatable: false,
},
},
'another.unrelated.field': {
keyword: {
type: 'keyword',
searchable: true,
aggregatable: true,
},
},
},
};

0 comments on commit eb74123

Please sign in to comment.