use semver for filtering data sources (#8785)

Description

    Uses semver to filter out data sources with invalid versions or versions that are lower than 1.0.0.
    Adding tests for index_type.ts

Signed-off-by: Sean Li <[email protected]>
(cherry picked from commit 9f7f8aa)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

src/plugins/data/public/query/query_string/dataset_service/lib/index_type.test.ts

@@ -9,11 +9,21 @@ import { indexTypeConfig } from './index_type';
 import { SavedObjectsClientContract } from 'opensearch-dashboards/public';
 import { DATA_STRUCTURE_META_TYPES, DataStructure, Dataset } from '../../../../../common';
 import * as services from '../../../../services';
+import { IDataPluginServices } from 'src/plugins/data/public';
 
-jest.mock('../../../../services', () => ({
-  getSearchService: jest.fn(),
-  getIndexPatterns: jest.fn(),
-}));
+jest.mock('../../../../services', () => {
+  return {
+    getSearchService: jest.fn(),
+    getIndexPatterns: jest.fn(),
+    getQueryService: () => ({
+      queryString: {
+        getLanguageService: () => ({
+          getQueryEditorExtensionMap: jest.fn().mockReturnValue({}),
+        }),
+      },
+    }),
+  };
+});
 
 describe('indexTypeConfig', () => {
   const mockSavedObjectsClient = {} as SavedObjectsClientContract;
@@ -77,4 +87,47 @@ describe('indexTypeConfig', () => {
     const mockDataset: Dataset = { id: 'index1', title: 'Index 1', type: 'INDEX' };
     expect(indexTypeConfig.supportedLanguages(mockDataset)).toEqual(['SQL', 'PPL']);
   });
+
+  test('should fetch data sources for unknown type', async () => {
+    mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
+      savedObjects: [
+        { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '3.0' } },
+      ],
+    });
+
+    const result = await indexTypeConfig.fetch(mockServices as IDataPluginServices, [
+      { id: 'unknown', title: 'Unknown', type: 'Unknown' },
+    ]);
+
+    expect(result.children).toHaveLength(1);
+    expect(result.children?.[0].title).toBe('DataSource 1');
+    expect(result.hasNext).toBe(true);
+  });
+
+  test('should filter out data sources with versions lower than 1.0.0', async () => {
+    mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
+      savedObjects: [
+        { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '1.0' } },
+        {
+          id: 'ds2',
+          attributes: { title: 'DataSource 2', dataSourceVersion: '' },
+        },
+        { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceVersion: '2.17.0' } },
+        {
+          id: 'ds4',
+          attributes: { title: 'DataSource 4', dataSourceVersion: '.0' },
+        },
+      ],
+    });
+
+    const result = await indexTypeConfig.fetch(mockServices as IDataPluginServices, [
+      { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' },
+    ]);
+
+    expect(result.children).toHaveLength(2);
+    expect(result.children?.[0].title).toBe('DataSource 1');
+    expect(result.children?.[1].title).toBe('DataSource 3');
+    expect(result.children?.some((child) => child.title === 'DataSource 2')).toBe(false);
+    expect(result.hasNext).toBe(true);
+  });
 });

src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts

@@ -6,6 +6,7 @@
 import { SavedObjectsClientContract } from 'opensearch-dashboards/public';
 import { map } from 'rxjs/operators';
 import { i18n } from '@osd/i18n';
+import semver from 'semver';
 import {
   DEFAULT_DATA,
   DataStructure,
@@ -120,11 +121,10 @@ const fetchDataSources = async (client: SavedObjectsClientContract) => {
     perPage: 10000,
   });
   const dataSources: DataStructure[] = response.savedObjects
-    .filter(
-      (savedObject) =>
-        typeof savedObject.attributes?.dataSourceEngineType === 'string' &&
-        !savedObject.attributes?.dataSourceEngineType?.includes('OpenSearch Serverless')
-    )
+    .filter((savedObject) => {
+      const coercedVersion = semver.coerce(savedObject.attributes.dataSourceVersion);
+      return coercedVersion ? semver.satisfies(coercedVersion, '>=1.0.0') : false;
+    })
     .map((savedObject) => ({
       id: savedObject.id,
       title: savedObject.attributes.title,

src/plugins/query_enhancements/public/datasets/s3_type.test.ts

@@ -142,36 +142,40 @@ describe('s3TypeConfig', () => {
     it('should fetch data sources for unknown type', async () => {
       mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
         savedObjects: [
-          { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceEngineType: 'OpenSearch' } },
+          { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '3.0' } },
         ],
       });
 
       const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [
         { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' },
       ]);
 
-      expect(result.children).toHaveLength(1); // Including DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE
+      expect(result.children).toHaveLength(1);
       expect(result.children?.[0].title).toBe('DataSource 1');
       expect(result.hasNext).toBe(true);
     });
 
-    it('should filter out OpenSearch Serverless data sources', async () => {
+    it('should filter out data sources with versions lower than 1.0.0', async () => {
       mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
         savedObjects: [
-          { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceEngineType: 'OpenSearch' } },
+          { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '1.0' } },
           {
             id: 'ds2',
-            attributes: { title: 'DataSource 2', dataSourceEngineType: 'OpenSearch Serverless' },
+            attributes: { title: 'DataSource 2', dataSourceVersion: '' },
+          },
+          { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceVersion: '2.17.0' } },
+          {
+            id: 'ds4',
+            attributes: { title: 'DataSource 4', dataSourceVersion: '.0' },
           },
-          { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceEngineType: 'OpenSearch' } },
         ],
       });
 
       const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [
         { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' },
       ]);
 
-      expect(result.children).toHaveLength(2); // Including DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE
+      expect(result.children).toHaveLength(2);
       expect(result.children?.[0].title).toBe('DataSource 1');
       expect(result.children?.[1].title).toBe('DataSource 3');
       expect(result.children?.some((child) => child.title === 'DataSource 2')).toBe(false);

src/plugins/query_enhancements/public/datasets/s3_type.ts

@@ -6,6 +6,7 @@
 import { i18n } from '@osd/i18n';
 import { trimEnd } from 'lodash';
 import { HttpSetup, SavedObjectsClientContract } from 'opensearch-dashboards/public';
+import semver from 'semver';
 import {
   DATA_STRUCTURE_META_TYPES,
   DEFAULT_DATA,
@@ -197,11 +198,10 @@ const fetchDataSources = async (client: SavedObjectsClientContract): Promise<Dat
     perPage: 10000,
   });
   const dataSources: DataStructure[] = resp.savedObjects
-    .filter(
-      (savedObject) =>
-        typeof savedObject.attributes?.dataSourceEngineType === 'string' &&
-        !savedObject.attributes?.dataSourceEngineType?.includes('OpenSearch Serverless')
-    )
+    .filter((savedObject) => {
+      const coercedVersion = semver.coerce(savedObject.attributes.dataSourceVersion);
+      return coercedVersion ? semver.satisfies(coercedVersion, '>=1.0.0') : false;
+    })
     .map((savedObject) => ({
       id: savedObject.id,
       title: savedObject.attributes.title,