From 3966cee2e848004c2c1407ed4842faa03e7d3bf7 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Fri, 1 Nov 2024 00:01:02 -0700 Subject: [PATCH] use semver for filtering data sources Signed-off-by: Sean Li --- .../dataset_service/lib/index_type.test.ts | 61 +++++++++++++++++-- .../dataset_service/lib/index_type.ts | 10 +-- .../public/datasets/s3_type.test.ts | 18 +++--- .../public/datasets/s3_type.ts | 10 +-- 4 files changed, 78 insertions(+), 21 deletions(-) diff --git a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.test.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.test.ts index d6847f8d239a..7a3d9810fb7f 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.test.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.test.ts @@ -9,11 +9,21 @@ import { indexTypeConfig } from './index_type'; import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; import { DATA_STRUCTURE_META_TYPES, DataStructure, Dataset } from '../../../../../common'; import * as services from '../../../../services'; +import { IDataPluginServices } from 'src/plugins/data/public'; -jest.mock('../../../../services', () => ({ - getSearchService: jest.fn(), - getIndexPatterns: jest.fn(), -})); +jest.mock('../../../../services', () => { + return { + getSearchService: jest.fn(), + getIndexPatterns: jest.fn(), + getQueryService: () => ({ + queryString: { + getLanguageService: () => ({ + getQueryEditorExtensionMap: jest.fn().mockReturnValue({}), + }), + }, + }), + }; +}); describe('indexTypeConfig', () => { const mockSavedObjectsClient = {} as SavedObjectsClientContract; @@ -77,4 +87,47 @@ describe('indexTypeConfig', () => { const mockDataset: Dataset = { id: 'index1', title: 'Index 1', type: 'INDEX' }; expect(indexTypeConfig.supportedLanguages(mockDataset)).toEqual(['SQL', 'PPL']); }); + + test('should fetch data sources for unknown type', async () => { + mockSavedObjectsClient.find = jest.fn().mockResolvedValue({ + savedObjects: [ + { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '3.0' } }, + ], + }); + + const result = await indexTypeConfig.fetch(mockServices as IDataPluginServices, [ + { id: 'unknown', title: 'Unknown', type: 'Unknown' }, + ]); + + expect(result.children).toHaveLength(1); + expect(result.children?.[0].title).toBe('DataSource 1'); + expect(result.hasNext).toBe(true); + }); + + test('should filter out data sources with versions lower than 1.0.0', async () => { + mockSavedObjectsClient.find = jest.fn().mockResolvedValue({ + savedObjects: [ + { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '1.0' } }, + { + id: 'ds2', + attributes: { title: 'DataSource 2', dataSourceVersion: '' }, + }, + { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceVersion: '2.17.0' } }, + { + id: 'ds4', + attributes: { title: 'DataSource 4', dataSourceVersion: '.0' }, + }, + ], + }); + + const result = await indexTypeConfig.fetch(mockServices as IDataPluginServices, [ + { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' }, + ]); + + expect(result.children).toHaveLength(2); + expect(result.children?.[0].title).toBe('DataSource 1'); + expect(result.children?.[1].title).toBe('DataSource 3'); + expect(result.children?.some((child) => child.title === 'DataSource 2')).toBe(false); + expect(result.hasNext).toBe(true); + }); }); diff --git a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts index 5d81e157e413..13fc4ce14f72 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts @@ -6,6 +6,7 @@ import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; import { map } from 'rxjs/operators'; import { i18n } from '@osd/i18n'; +import semver from 'semver'; import { DEFAULT_DATA, DataStructure, @@ -120,11 +121,10 @@ const fetchDataSources = async (client: SavedObjectsClientContract) => { perPage: 10000, }); const dataSources: DataStructure[] = response.savedObjects - .filter( - (savedObject) => - typeof savedObject.attributes?.dataSourceEngineType === 'string' && - !savedObject.attributes?.dataSourceEngineType?.includes('OpenSearch Serverless') - ) + .filter((savedObject) => { + const coercedVersion = semver.coerce(savedObject.attributes.dataSourceVersion); + return coercedVersion ? semver.satisfies(coercedVersion, '>=1.0.0') : false; + }) .map((savedObject) => ({ id: savedObject.id, title: savedObject.attributes.title, diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts index eb1df6736303..6a2d5cc6182c 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts @@ -142,7 +142,7 @@ describe('s3TypeConfig', () => { it('should fetch data sources for unknown type', async () => { mockSavedObjectsClient.find = jest.fn().mockResolvedValue({ savedObjects: [ - { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceEngineType: 'OpenSearch' } }, + { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '3.0' } }, ], }); @@ -150,20 +150,24 @@ describe('s3TypeConfig', () => { { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' }, ]); - expect(result.children).toHaveLength(1); // Including DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE + expect(result.children).toHaveLength(1); expect(result.children?.[0].title).toBe('DataSource 1'); expect(result.hasNext).toBe(true); }); - it('should filter out OpenSearch Serverless data sources', async () => { + it('should filter out data sources with versions lower than 1.0.0', async () => { mockSavedObjectsClient.find = jest.fn().mockResolvedValue({ savedObjects: [ - { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceEngineType: 'OpenSearch' } }, + { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '1.0' } }, { id: 'ds2', - attributes: { title: 'DataSource 2', dataSourceEngineType: 'OpenSearch Serverless' }, + attributes: { title: 'DataSource 2', dataSourceVersion: '' }, + }, + { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceVersion: '2.17.0' } }, + { + id: 'ds4', + attributes: { title: 'DataSource 4', dataSourceVersion: '.0' }, }, - { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceEngineType: 'OpenSearch' } }, ], }); @@ -171,7 +175,7 @@ describe('s3TypeConfig', () => { { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' }, ]); - expect(result.children).toHaveLength(2); // Including DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE + expect(result.children).toHaveLength(2); expect(result.children?.[0].title).toBe('DataSource 1'); expect(result.children?.[1].title).toBe('DataSource 3'); expect(result.children?.some((child) => child.title === 'DataSource 2')).toBe(false); diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.ts b/src/plugins/query_enhancements/public/datasets/s3_type.ts index 948ed9461ac6..90660289c931 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.ts @@ -6,6 +6,7 @@ import { i18n } from '@osd/i18n'; import { trimEnd } from 'lodash'; import { HttpSetup, SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import semver from 'semver'; import { DATA_STRUCTURE_META_TYPES, DEFAULT_DATA, @@ -197,11 +198,10 @@ const fetchDataSources = async (client: SavedObjectsClientContract): Promise - typeof savedObject.attributes?.dataSourceEngineType === 'string' && - !savedObject.attributes?.dataSourceEngineType?.includes('OpenSearch Serverless') - ) + .filter((savedObject) => { + const coercedVersion = semver.coerce(savedObject.attributes.dataSourceVersion); + return coercedVersion ? semver.satisfies(coercedVersion, '>=1.0.0') : false; + }) .map((savedObject) => ({ id: savedObject.id, title: savedObject.attributes.title,