From 7eaab64fdff8b1f9b0f4bbca0ede09ca523159c9 Mon Sep 17 00:00:00 2001 From: "Yuanqi(Ella) Zhu" <53279298+zhyuanqi@users.noreply.github.com> Date: Fri, 7 Jun 2024 14:08:20 -0700 Subject: [PATCH] Modify the import for timeline visualization to includes data source name in MDS scenario (#6954) Signed-off-by: Yuanqi(Ella) Zhu --- .../check_conflict_for_data_source.test.ts | 114 ++++++++++++++++++ .../import/check_conflict_for_data_source.ts | 18 +++ .../import/create_saved_objects.test.ts | 114 +++++++++++++++++- .../import/create_saved_objects.ts | 18 +++ .../import/import_saved_objects.ts | 3 +- 5 files changed, 263 insertions(+), 4 deletions(-) diff --git a/src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts b/src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts index 2b09a1c853c4..f6628df1a727 100644 --- a/src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts +++ b/src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts @@ -318,6 +318,120 @@ describe('#checkConflictsForDataSource', () => { ); }); + /* + * Timeline test cases + */ + it('will not change timeline expression when importing from datasource to different datasource', async () => { + const timelineSavedObject = createObject('visualization', 'old-datasource-id_some-object-id'); + // @ts-expect-error + timelineSavedObject.attributes.visState = + '{"title":"(Timeline) Avg bytes over time","type":"timelion","aggs":[],"params":{"expression":".opensearch(opensearch_dashboards_sample_data_logs, metric=avg:bytes, timefield=@timestamp, data_source_name=newDataSource).lines(show=true).points(show=true).yaxis(label=\\"Average bytes\\")","interval":"auto"}}'; + const params = setupParams({ + objects: [timelineSavedObject], + ignoreRegularConflicts: true, + dataSourceId: 'some-datasource-id', + savedObjectsClient: getSavedObjectClient(), + }); + const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params); + + expect(checkConflictsForDataSourceResult).toEqual( + expect.objectContaining({ + filteredObjects: [ + { + ...timelineSavedObject, + attributes: { + title: 'some-title', + visState: + '{"title":"(Timeline) Avg bytes over time","type":"timelion","aggs":[],"params":{"expression":".opensearch(opensearch_dashboards_sample_data_logs, metric=avg:bytes, timefield=@timestamp, data_source_name=newDataSource).lines(show=true).points(show=true).yaxis(label=\\"Average bytes\\")","interval":"auto"}}', + }, + id: 'some-datasource-id_some-object-id', + }, + ], + errors: [], + importIdMap: new Map([ + [ + `visualization:old-datasource-id_some-object-id`, + { id: 'some-datasource-id_some-object-id', omitOriginId: true }, + ], + ]), + }) + ); + }); + + it('will change timeline expression when importing expression does not have a datasource name', async () => { + const timelineSavedObject = createObject('visualization', 'old-datasource-id_some-object-id'); + // @ts-expect-error + timelineSavedObject.attributes.visState = + '{"title":"(Timeline) Avg bytes over time","type":"timelion","aggs":[],"params":{"expression":".opensearch(opensearch_dashboards_sample_data_logs, metric=avg:bytes, timefield=@timestamp).lines(show=true).points(show=true).yaxis(label=\\"Average bytes\\")","interval":"auto"}}'; + const params = setupParams({ + objects: [timelineSavedObject], + ignoreRegularConflicts: true, + dataSourceId: 'some-datasource-id', + savedObjectsClient: getSavedObjectClient(), + }); + const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params); + + expect(checkConflictsForDataSourceResult).toEqual( + expect.objectContaining({ + filteredObjects: [ + { + ...timelineSavedObject, + attributes: { + title: 'some-title', + visState: + '{"title":"(Timeline) Avg bytes over time","type":"timelion","aggs":[],"params":{"expression":".opensearch(opensearch_dashboards_sample_data_logs, metric=avg:bytes, timefield=@timestamp, data_source_name=\\"some-datasource-title\\").lines(show=true).points(show=true).yaxis(label=\\"Average bytes\\")","interval":"auto"}}', + }, + id: 'some-datasource-id_some-object-id', + }, + ], + errors: [], + importIdMap: new Map([ + [ + `visualization:old-datasource-id_some-object-id`, + { id: 'some-datasource-id_some-object-id', omitOriginId: true }, + ], + ]), + }) + ); + }); + + it('When there are multiple opensearch queries in the expression, it would go through each query and add data source name if it does not have any.', async () => { + const timelineSavedObject = createObject('visualization', 'old-datasource-id_some-object-id'); + // @ts-expect-error + timelineSavedObject.attributes.visState = + '{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"aos 211\\"), .elasticsearch(index=old-datasource-title, timefield=@timestamp)"},"aggs":[]}'; + const params = setupParams({ + objects: [timelineSavedObject], + ignoreRegularConflicts: true, + dataSourceId: 'some-datasource-id', + savedObjectsClient: getSavedObjectClient(), + }); + const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params); + + expect(checkConflictsForDataSourceResult).toEqual( + expect.objectContaining({ + filteredObjects: [ + { + ...timelineSavedObject, + attributes: { + title: 'some-title', + visState: + '{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"aos 211\\"), .elasticsearch(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"some-datasource-title\\")"},"aggs":[]}', + }, + id: 'some-datasource-id_some-object-id', + }, + ], + errors: [], + importIdMap: new Map([ + [ + `visualization:old-datasource-id_some-object-id`, + { id: 'some-datasource-id_some-object-id', omitOriginId: true }, + ], + ]), + }) + ); + }); + /** * TSVB test cases */ diff --git a/src/core/server/saved_objects/import/check_conflict_for_data_source.ts b/src/core/server/saved_objects/import/check_conflict_for_data_source.ts index f04dd0c6f69e..0b7f64137faf 100644 --- a/src/core/server/saved_objects/import/check_conflict_for_data_source.ts +++ b/src/core/server/saved_objects/import/check_conflict_for_data_source.ts @@ -15,6 +15,8 @@ import { getDataSourceTitleFromId, getUpdatedTSVBVisState, updateDataSourceNameInVegaSpec, + extractTimelineExpression, + updateDataSourceNameInTimeline, } from './utils'; export interface ConflictsForDataSourceParams { @@ -120,6 +122,22 @@ export async function checkConflictsForDataSource({ } } + // For timeline visualizations, update the data source name in the timeline expression + const timelineExpression = extractTimelineExpression(object); + if (!!timelineExpression && !!dataSourceTitle) { + // Get the timeline expression with the updated data source name + const modifiedExpression = updateDataSourceNameInTimeline( + timelineExpression, + dataSourceTitle + ); + + // @ts-expect-error + const timelineStateObject = JSON.parse(object.attributes?.visState); + timelineStateObject.params.expression = modifiedExpression; + // @ts-expect-error + object.attributes.visState = JSON.stringify(timelineStateObject); + } + if (!!dataSourceId) { const visualizationObject = object as VisualizationObject; const { visState, references } = getUpdatedTSVBVisState( diff --git a/src/core/server/saved_objects/import/create_saved_objects.test.ts b/src/core/server/saved_objects/import/create_saved_objects.test.ts index da7f057435ad..2237017f3400 100644 --- a/src/core/server/saved_objects/import/create_saved_objects.test.ts +++ b/src/core/server/saved_objects/import/create_saved_objects.test.ts @@ -169,6 +169,39 @@ const getVegaMDSVisualizationObj = (id: string, dataSourceId: string) => ({ }, ], }); + +const getTimelineVisualizationObj = (id: string, dataSourceId: string) => ({ + type: 'visualization', + id: dataSourceId ? `${dataSourceId}_${id}` : id, + attributes: { + title: 'some-other-title', + visState: + '{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp)"},"aggs":[]}', + }, + references: [], +}); + +const getTimelineVisualizationObjWithMultipleQueries = (id: string, dataSourceId: string) => ({ + type: 'visualization', + id: dataSourceId ? `${dataSourceId}_${id}` : id, + attributes: { + title: 'some-other-title', + visState: + '{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"aos 211\\"), .elasticsearch(index=old-datasource-title, timefield=@timestamp)"},"aggs":[]}', + }, + references: [], +}); + +const getTimelineVisualizationObjWithDataSourceName = (id: string, dataSourceId: string) => ({ + type: 'visualization', + id: dataSourceId ? `${dataSourceId}_${id}` : id, + attributes: { + title: 'some-other-title', + visState: + '{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=ds1)"},"aggs":[]}', + }, + references: [], +}); // non-multi-namespace types shouldn't have origin IDs, but we include test cases to ensure it's handled gracefully // non-multi-namespace types by definition cannot result in an unresolvable conflict, so we don't include test cases for those const importId3 = 'id-foo'; @@ -571,7 +604,7 @@ describe('#createSavedObjects', () => { expect(results).toEqual(expectedResultsWithDataSource); }; - const testVegaVisualizationsWithDataSources = async (params: { + const testVegaTimelineVisualizationsWithDataSources = async (params: { objects: SavedObject[]; expectedFilteredObjects: Array>; dataSourceId?: string; @@ -673,7 +706,7 @@ describe('#createSavedObjects', () => { ], }, ]; - await testVegaVisualizationsWithDataSources({ + await testVegaTimelineVisualizationsWithDataSources({ objects, expectedFilteredObjects, dataSourceId: 'some-datasource-id', @@ -699,7 +732,82 @@ describe('#createSavedObjects', () => { }, }, ]; - await testVegaVisualizationsWithDataSources({ + await testVegaTimelineVisualizationsWithDataSources({ + objects, + expectedFilteredObjects, + dataSourceId: 'some-datasource-id', + dataSourceTitle: 'dataSourceName', + }); + }); + }); + + describe('with a data source for timeline saved objects', () => { + test('can attach a data source name to the timeline expression', async () => { + const objects = [getTimelineVisualizationObj('some-timeline-id', 'some-datasource-id')]; + const expectedObject = getTimelineVisualizationObj('some-timeline-id', 'some-datasource-id'); + const expectedFilteredObjects = [ + { + ...expectedObject, + attributes: { + title: 'some-other-title_dataSourceName', + visState: + '{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"dataSourceName\\")"},"aggs":[]}', + }, + }, + ]; + await testVegaTimelineVisualizationsWithDataSources({ + objects, + expectedFilteredObjects, + dataSourceId: 'some-datasource-id', + dataSourceTitle: 'dataSourceName', + }); + }); + + test('will not update the data source name in the timeline expression if no local cluster queries', async () => { + const objects = [ + getTimelineVisualizationObjWithDataSourceName('some-timeline-id', 'old-datasource-id'), + ]; + const expectedObject = getTimelineVisualizationObjWithDataSourceName( + 'some-timeline-id', + 'old-datasource-id' + ); + const expectedFilteredObjects = [ + { + ...expectedObject, + attributes: { + title: 'some-other-title_dataSourceName', + visState: + '{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=ds1)"},"aggs":[]}', + }, + }, + ]; + await testVegaTimelineVisualizationsWithDataSources({ + objects, + expectedFilteredObjects, + dataSourceId: 'some-datasource-id', + dataSourceTitle: 'dataSourceName', + }); + }); + + test('When muliple opensearch query exists in expression, we can add data source name to the queries that missing data source name.', async () => { + const objects = [ + getTimelineVisualizationObjWithMultipleQueries('some-timeline-id', 'some-datasource-id'), + ]; + const expectedObject = getTimelineVisualizationObjWithMultipleQueries( + 'some-timeline-id', + 'some-datasource-id' + ); + const expectedFilteredObjects = [ + { + ...expectedObject, + attributes: { + title: 'some-other-title_dataSourceName', + visState: + '{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"aos 211\\"), .elasticsearch(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"dataSourceName\\")"},"aggs":[]}', + }, + }, + ]; + await testVegaTimelineVisualizationsWithDataSources({ objects, expectedFilteredObjects, dataSourceId: 'some-datasource-id', diff --git a/src/core/server/saved_objects/import/create_saved_objects.ts b/src/core/server/saved_objects/import/create_saved_objects.ts index 7e3854107a29..a90ad802edaa 100644 --- a/src/core/server/saved_objects/import/create_saved_objects.ts +++ b/src/core/server/saved_objects/import/create_saved_objects.ts @@ -40,6 +40,8 @@ import { extractVegaSpecFromSavedObject, getUpdatedTSVBVisState, updateDataSourceNameInVegaSpec, + extractTimelineExpression, + updateDataSourceNameInTimeline, } from './utils'; interface CreateSavedObjectsParams { @@ -130,6 +132,22 @@ export const createSavedObjects = async ({ }); } + // Some visualization types will need special modifications, like TSVB visualizations + const timelineExpression = extractTimelineExpression(object); + if (!!timelineExpression && !!dataSourceTitle) { + // Get the timeline expression with the updated data source name + const modifiedExpression = updateDataSourceNameInTimeline( + timelineExpression, + dataSourceTitle + ); + + // @ts-expect-error + const timelineStateObject = JSON.parse(object.attributes?.visState); + timelineStateObject.params.expression = modifiedExpression; + // @ts-expect-error + object.attributes.visState = JSON.stringify(timelineStateObject); + } + const visualizationObject = object as VisualizationObject; const { visState, references } = getUpdatedTSVBVisState( visualizationObject, diff --git a/src/core/server/saved_objects/import/import_saved_objects.ts b/src/core/server/saved_objects/import/import_saved_objects.ts index cfd091149004..1289af145c58 100644 --- a/src/core/server/saved_objects/import/import_saved_objects.ts +++ b/src/core/server/saved_objects/import/import_saved_objects.ts @@ -72,7 +72,8 @@ export async function importSavedObjectsFromStream({ supportedTypes, dataSourceId, }); - // if not enable data_source, throw error early + // if dataSource is not enabled, but object type is data-source, or saved object id contains datasource id + // return unsupported type error if (!dataSourceEnabled) { const notSupportedErrors: SavedObjectsImportError[] = collectSavedObjectsResult.collectedObjects.reduce( (errors: SavedObjectsImportError[], obj) => {