From 4623841558f07c8582800881558b3c7a728a1544 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Wed, 30 Oct 2024 12:04:28 -0700 Subject: [PATCH 1/6] Filtering out serverless collections Signed-off-by: Sean Li --- .../dataset_service/lib/index_type.ts | 10 +++++----- .../public/datasets/s3_type.ts | 18 +++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts index 655d3720dab2..75574952636d 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts @@ -37,10 +37,10 @@ export const indexTypeConfig: DatasetTypeConfig = { timeFieldName: indexMeta?.timeFieldName, dataSource: dataSource ? { - id: dataSource.id, - title: dataSource.title, - type: dataSource.type, - } + id: dataSource.id, + title: dataSource.title, + type: dataSource.type, + } : DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE, }; }, @@ -120,7 +120,7 @@ const fetchDataSources = async (client: SavedObjectsClientContract) => { perPage: 10000, }); const dataSources: DataStructure[] = [DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE].concat( - response.savedObjects.map((savedObject) => ({ + response.savedObjects.filter((savedObject) => savedObject.attributes.dataSourceEngineType !== 'OpenSearch Serverless').map((savedObject) => ({ id: savedObject.id, title: savedObject.attributes.title, type: 'DATA_SOURCE', diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.ts b/src/plugins/query_enhancements/public/datasets/s3_type.ts index 6aff9ce44a62..0f87cf8530e2 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.ts @@ -49,14 +49,14 @@ export const s3TypeConfig: DatasetTypeConfig = { type: DATASET.S3, dataSource: dataSource ? { - id: dataSource.id, - title: dataSource.title, - type: dataSource.type, - meta: { - ...table.meta, - supportsTimeFilter: s3TypeConfig.meta.supportsTimeFilter, - } as DataSourceMeta, - } + id: dataSource.id, + title: dataSource.title, + type: dataSource.type, + meta: { + ...table.meta, + supportsTimeFilter: s3TypeConfig.meta.supportsTimeFilter, + } as DataSourceMeta, + } : DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE, }; }, @@ -198,7 +198,7 @@ const fetchDataSources = async (client: SavedObjectsClientContract): Promise ({ + resp.savedObjects.filter((savedObject) => savedObject.attributes.dataSourceEngineType !== 'OpenSearch Serverless').map((savedObject) => ({ id: savedObject.id, title: savedObject.attributes.title, type: 'DATA_SOURCE', From ff7cad63093466be3bef42df2c6a276a9e1ec9f3 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Wed, 30 Oct 2024 12:42:36 -0700 Subject: [PATCH 2/6] rerunning linter Signed-off-by: Sean Li --- .../dataset_service/lib/index_type.ts | 22 ++++++---- .../public/datasets/s3_type.ts | 42 ++++++++++--------- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts index 75574952636d..469c2d9c224e 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts @@ -37,10 +37,10 @@ export const indexTypeConfig: DatasetTypeConfig = { timeFieldName: indexMeta?.timeFieldName, dataSource: dataSource ? { - id: dataSource.id, - title: dataSource.title, - type: dataSource.type, - } + id: dataSource.id, + title: dataSource.title, + type: dataSource.type, + } : DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE, }; }, @@ -120,11 +120,15 @@ const fetchDataSources = async (client: SavedObjectsClientContract) => { perPage: 10000, }); const dataSources: DataStructure[] = [DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE].concat( - response.savedObjects.filter((savedObject) => savedObject.attributes.dataSourceEngineType !== 'OpenSearch Serverless').map((savedObject) => ({ - id: savedObject.id, - title: savedObject.attributes.title, - type: 'DATA_SOURCE', - })) + response.savedObjects + .filter( + (savedObject) => savedObject.attributes.dataSourceEngineType !== 'OpenSearch Serverless' + ) + .map((savedObject) => ({ + id: savedObject.id, + title: savedObject.attributes.title, + type: 'DATA_SOURCE', + })) ); return injectMetaToDataStructures(dataSources); diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.ts b/src/plugins/query_enhancements/public/datasets/s3_type.ts index 0f87cf8530e2..856854d9b03c 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.ts @@ -49,14 +49,14 @@ export const s3TypeConfig: DatasetTypeConfig = { type: DATASET.S3, dataSource: dataSource ? { - id: dataSource.id, - title: dataSource.title, - type: dataSource.type, - meta: { - ...table.meta, - supportsTimeFilter: s3TypeConfig.meta.supportsTimeFilter, - } as DataSourceMeta, - } + id: dataSource.id, + title: dataSource.title, + type: dataSource.type, + meta: { + ...table.meta, + supportsTimeFilter: s3TypeConfig.meta.supportsTimeFilter, + } as DataSourceMeta, + } : DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE, }; }, @@ -198,17 +198,21 @@ const fetchDataSources = async (client: SavedObjectsClientContract): Promise savedObject.attributes.dataSourceEngineType !== 'OpenSearch Serverless').map((savedObject) => ({ - id: savedObject.id, - title: savedObject.attributes.title, - type: 'DATA_SOURCE', - meta: { - query: { - id: savedObject.id, - }, - type: DATA_STRUCTURE_META_TYPES.CUSTOM, - } as DataStructureCustomMeta, - })) + resp.savedObjects + .filter( + (savedObject) => savedObject.attributes.dataSourceEngineType !== 'OpenSearch Serverless' + ) + .map((savedObject) => ({ + id: savedObject.id, + title: savedObject.attributes.title, + type: 'DATA_SOURCE', + meta: { + query: { + id: savedObject.id, + }, + type: DATA_STRUCTURE_META_TYPES.CUSTOM, + } as DataStructureCustomMeta, + })) ); }; From ecfba301aeb2ac1e2d7129b6703feebb43c68aea Mon Sep 17 00:00:00 2001 From: Sean Li Date: Wed, 30 Oct 2024 13:52:22 -0700 Subject: [PATCH 3/6] adding test for s3 type Signed-off-by: Sean Li --- .../public/datasets/s3_type.test.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts index bbe960b5f984..95ad98abac5c 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts @@ -152,6 +152,29 @@ describe('s3TypeConfig', () => { expect(result.children?.[1].title).toBe('DataSource 1'); expect(result.hasNext).toBe(true); }); + + it('should filter out OpenSearch Serverless data sources', async () => { + mockSavedObjectsClient.find = jest.fn().mockResolvedValue({ + savedObjects: [ + { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceEngineType: 'OpenSearch' } }, + { + id: 'ds2', + attributes: { title: 'DataSource 2', dataSourceEngineType: 'OpenSearch Serverless' }, + }, + { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceEngineType: 'OpenSearch' } }, + ], + }); + + const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [ + { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' }, + ]); + + expect(result.children).toHaveLength(3); // Including DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE + expect(result.children?.[1].title).toBe('DataSource 1'); + expect(result.children?.[2].title).toBe('DataSource 3'); + expect(result.children?.some((child) => child.title === 'DataSource 2')).toBe(false); + expect(result.hasNext).toBe(true); + }); }); test('fetchFields returns table fields', async () => { From dc6cb2885bc7a448282c02f48a19cebd6cca287b Mon Sep 17 00:00:00 2001 From: Sean Li Date: Wed, 30 Oct 2024 16:39:01 -0700 Subject: [PATCH 4/6] updating filter Signed-off-by: Sean Li --- .../query/query_string/dataset_service/lib/index_type.ts | 3 ++- src/plugins/query_enhancements/public/datasets/s3_type.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts index 469c2d9c224e..e3ec419e207d 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts @@ -122,7 +122,8 @@ const fetchDataSources = async (client: SavedObjectsClientContract) => { const dataSources: DataStructure[] = [DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE].concat( response.savedObjects .filter( - (savedObject) => savedObject.attributes.dataSourceEngineType !== 'OpenSearch Serverless' + (savedObject) => + !savedObject.attributes?.dataSourceEngineType?.includes('OpenSearch Serverless') ) .map((savedObject) => ({ id: savedObject.id, diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.ts b/src/plugins/query_enhancements/public/datasets/s3_type.ts index 856854d9b03c..feea21db3737 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.ts @@ -200,7 +200,8 @@ const fetchDataSources = async (client: SavedObjectsClientContract): Promise savedObject.attributes.dataSourceEngineType !== 'OpenSearch Serverless' + (savedObject) => + !savedObject.attributes?.dataSourceEngineType?.includes('OpenSearch Serverless') ) .map((savedObject) => ({ id: savedObject.id, From 848bcdfd08911a53bd578f1ceb4c9db390075271 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Wed, 30 Oct 2024 16:44:15 -0700 Subject: [PATCH 5/6] string check Signed-off-by: Sean Li --- .../query/query_string/dataset_service/lib/index_type.ts | 1 + .../query_enhancements/public/datasets/s3_type.test.ts | 6 ++++-- src/plugins/query_enhancements/public/datasets/s3_type.ts | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts index e3ec419e207d..272db09eabe8 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts @@ -123,6 +123,7 @@ const fetchDataSources = async (client: SavedObjectsClientContract) => { response.savedObjects .filter( (savedObject) => + typeof savedObject.attributes?.dataSourceEngineType === 'string' && !savedObject.attributes?.dataSourceEngineType?.includes('OpenSearch Serverless') ) .map((savedObject) => ({ diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts index 95ad98abac5c..73073891b115 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts @@ -141,7 +141,9 @@ describe('s3TypeConfig', () => { it('should fetch data sources for unknown type', async () => { mockSavedObjectsClient.find = jest.fn().mockResolvedValue({ - savedObjects: [{ id: 'ds1', attributes: { title: 'DataSource 1' } }], + savedObjects: [ + { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceEngineType: 'OpenSearch' } }, + ], }); const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [ @@ -159,7 +161,7 @@ describe('s3TypeConfig', () => { { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceEngineType: 'OpenSearch' } }, { id: 'ds2', - attributes: { title: 'DataSource 2', dataSourceEngineType: 'OpenSearch Serverless' }, + attributes: { title: 'DataSource 2', dataSourceEngineType: 3 }, }, { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceEngineType: 'OpenSearch' } }, ], diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.ts b/src/plugins/query_enhancements/public/datasets/s3_type.ts index feea21db3737..67da4e85c2a6 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.ts @@ -201,6 +201,7 @@ const fetchDataSources = async (client: SavedObjectsClientContract): Promise + typeof savedObject.attributes?.dataSourceEngineType === 'string' && !savedObject.attributes?.dataSourceEngineType?.includes('OpenSearch Serverless') ) .map((savedObject) => ({ From 58efea4d89ee43feb28ff96b49d8ce103009419e Mon Sep 17 00:00:00 2001 From: Sean Li Date: Wed, 30 Oct 2024 16:47:58 -0700 Subject: [PATCH 6/6] update test Signed-off-by: Sean Li --- src/plugins/query_enhancements/public/datasets/s3_type.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts index 73073891b115..689f996291b8 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts @@ -161,7 +161,7 @@ describe('s3TypeConfig', () => { { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceEngineType: 'OpenSearch' } }, { id: 'ds2', - attributes: { title: 'DataSource 2', dataSourceEngineType: 3 }, + attributes: { title: 'DataSource 2', dataSourceEngineType: 'OpenSearch Serverless' }, }, { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceEngineType: 'OpenSearch' } }, ],