From 4fce74d9074ae500437a88cb9731c9c632b4be19 Mon Sep 17 00:00:00 2001 From: Shenoy Pratik Date: Sat, 9 Mar 2024 15:30:08 -0800 Subject: [PATCH 1/5] Expose create acceleration flyout, update acceleration docs link (#1513) * expose create acceleration flyout, update docs link Signed-off-by: Shenoy Pratik * update test for data_connection page Signed-off-by: Shenoy Pratik --------- Signed-off-by: Shenoy Pratik --- common/constants/data_connections.ts | 2 +- .../__tests__/data_connection.test.tsx | 15 +- .../create_acceleration.test.tsx.snap | 249 +++------------- .../create_acceleration_header.test.tsx.snap | 2 +- .../__tests__/create_acceleration.test.tsx | 14 +- .../create/create_acceleration.tsx | 17 +- .../source_selector.test.tsx.snap | 269 ++---------------- .../__tests__/source_selector.test.tsx | 5 +- .../selectors/index_type_selector.tsx | 2 +- .../selectors/source_selector.tsx | 82 +++--- .../components/manage/data_connection.tsx | 35 ++- public/plugin.tsx | 27 +- 12 files changed, 162 insertions(+), 557 deletions(-) diff --git a/common/constants/data_connections.ts b/common/constants/data_connections.ts index 23fd9a8697..b265b2290d 100644 --- a/common/constants/data_connections.ts +++ b/common/constants/data_connections.ts @@ -12,7 +12,7 @@ export const OPENSEARCH_S3_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest/dashboards/management/S3-data-source/'; export const OPENSEARCH_ACC_DOCUMENTATION_URL = - 'https://opensearch.org/docs/latest/data-acceleration/index'; + 'https://opensearch.org/docs/latest/dashboards/management/accelerate-external-data/'; export const QUERY_RESTRICTED = 'query-restricted'; export const QUERY_ALL = 'query-all'; diff --git a/public/components/datasources/components/__tests__/data_connection.test.tsx b/public/components/datasources/components/__tests__/data_connection.test.tsx index f17ef2c828..1510a93f13 100644 --- a/public/components/datasources/components/__tests__/data_connection.test.tsx +++ b/public/components/datasources/components/__tests__/data_connection.test.tsx @@ -3,25 +3,22 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { act } from '@testing-library/react'; import { configure } from 'enzyme'; import Adapter from 'enzyme-adapter-react-16'; -import { act } from '@testing-library/react'; import React from 'react'; +import ReactDOM from 'react-dom'; +import { coreRefs } from '../../../../../public/framework/core_refs'; import { describePrometheusDataConnection, describeS3Dataconnection, } from '../../../../../test/datasources'; import { DataConnection } from '../manage/data_connection'; -import ReactDOM from 'react-dom'; -import { coreRefs } from '../../../../../public/framework/core_refs'; jest.mock('../../../../plugin', () => ({ - getRenderAccelerationDetailsFlyout: jest.fn(() => - jest.fn().mockImplementation(() => console.log('Acceleration Details Flyout Rendered')) - ), - getRenderAssociatedObjectsDetailsFlyout: jest.fn(() => - jest.fn().mockImplementation(() => console.log('Associated Objects Details Flyout Rendered')) - ), + getRenderAccelerationDetailsFlyout: jest.fn(), + getRenderAssociatedObjectsDetailsFlyout: jest.fn(), + getRenderCreateAccelerationFlyout: jest.fn(), })); jest.mock('../../../../../public/framework/core_refs', () => ({ diff --git a/public/components/datasources/components/manage/accelerations/create/__tests__/__snapshots__/create_acceleration.test.tsx.snap b/public/components/datasources/components/manage/accelerations/create/__tests__/__snapshots__/create_acceleration.test.tsx.snap index 951139f2f0..d77367d91d 100644 --- a/public/components/datasources/components/manage/accelerations/create/__tests__/__snapshots__/create_acceleration.test.tsx.snap +++ b/public/components/datasources/components/manage/accelerations/create/__tests__/__snapshots__/create_acceleration.test.tsx.snap @@ -76,7 +76,7 @@ Array [ @@ -144,102 +144,25 @@ Array [
-
-
- -
-
+
- -
+ my_glue + + +
- flint_{Datasource Name}_{Database Name}_{Table Name}_ + flint_my_glue_{Database Name}_{Table Name}_ @@ -1311,129 +1234,25 @@ Array [
,
, -
-
- -
-
+
-
-
-
-
-

- Select a data source -

-
- -
-
-
-
- -
-
-
-
-
- A data source has to be configured and active to be able to select it and index data from. -
-
-
, + my_glue + + , +
,
- flint_{Datasource Name}_{Database Name}_{Table Name}_ + flint_my_glue_{Database Name}_{Table Name}_ diff --git a/public/components/datasources/components/manage/accelerations/create/__tests__/create_acceleration.test.tsx b/public/components/datasources/components/manage/accelerations/create/__tests__/create_acceleration.test.tsx index 49cd168a3f..165527573c 100644 --- a/public/components/datasources/components/manage/accelerations/create/__tests__/create_acceleration.test.tsx +++ b/public/components/datasources/components/manage/accelerations/create/__tests__/create_acceleration.test.tsx @@ -3,7 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { EuiComboBoxOptionOption } from '@elastic/eui'; import { waitFor } from '@testing-library/dom'; import { configure, mount } from 'enzyme'; import Adapter from 'enzyme-adapter-react-16'; @@ -19,19 +18,12 @@ describe('Create acceleration flyout components', () => { configure({ adapter: new Adapter() }); it('renders acceleration flyout component with default options', async () => { - const selectedDatasource: EuiComboBoxOptionOption[] = []; + const selectedDatasource = 'my_glue'; const resetFlyout = jest.fn(); - const updateQueries = jest.fn(); - const client = coreStartMock.http; - client.get = jest.fn().mockResolvedValue(mockDatasourcesQuery); + coreStartMock.http.get = jest.fn().mockResolvedValue(mockDatasourcesQuery); const wrapper = mount( - + ); wrapper.update(); await waitFor(() => { diff --git a/public/components/datasources/components/manage/accelerations/create/create_acceleration.tsx b/public/components/datasources/components/manage/accelerations/create/create_acceleration.tsx index 6ab2805940..08a37a3bf4 100644 --- a/public/components/datasources/components/manage/accelerations/create/create_acceleration.tsx +++ b/public/components/datasources/components/manage/accelerations/create/create_acceleration.tsx @@ -6,7 +6,6 @@ import { EuiButton, EuiButtonEmpty, - EuiComboBoxOptionOption, EuiFlexGroup, EuiFlexItem, EuiFlyout, @@ -17,35 +16,31 @@ import { EuiSpacer, } from '@elastic/eui'; import React, { useState } from 'react'; -import { CoreStart } from '../../../../../../../../../src/core/public'; import { ACCELERATION_DEFUALT_SKIPPING_INDEX_NAME, ACCELERATION_TIME_INTERVAL, } from '../../../../../../../common/constants/data_sources'; import { CreateAccelerationForm } from '../../../../../../../common/types/data_connections'; +import { coreRefs } from '../../../../../../framework/core_refs'; import { DefineIndexOptions } from '../selectors/define_index_options'; import { IndexSettingOptions } from '../selectors/index_setting_options'; import { AccelerationDataSourceSelector } from '../selectors/source_selector'; -import { accelerationQueryBuilder } from '../visual_editors/query_builder'; import { QueryVisualEditor } from '../visual_editors/query_visual_editor'; import { CreateAccelerationHeader } from './create_acceleration_header'; import { formValidator, hasError } from './utils'; export interface CreateAccelerationProps { - http: CoreStart['http']; - selectedDatasource: EuiComboBoxOptionOption[]; + selectedDatasource: string; resetFlyout: () => void; - updateQueries: (query: string) => void; } export const CreateAcceleration = ({ - http, selectedDatasource, resetFlyout, - updateQueries, }: CreateAccelerationProps) => { + const http = coreRefs!.http; const [accelerationFormData, setAccelerationFormData] = useState({ - dataSource: selectedDatasource.length > 0 ? selectedDatasource[0].label : '', + dataSource: selectedDatasource, dataTable: '', database: '', dataTableFields: [], @@ -95,7 +90,7 @@ export const CreateAcceleration = ({ setAccelerationFormData({ ...accelerationFormData, formErrors: errors }); return; } - updateQueries(accelerationQueryBuilder(accelerationFormData)); + // TODO: add -> updateQueries(accelerationQueryBuilder(accelerationFormData)); resetFlyout(); }; @@ -114,7 +109,7 @@ export const CreateAcceleration = ({ id="acceleration-form" >
,
, -
-
- -
-
+
-
-
-
-
-

- Select a data source -

-
- -
-
-
-
- -
-
-
-
-
- A data source has to be configured and active to be able to select it and index data from. -
-
-
, + my_glue + + , +
,
,
, -
-
- -
-
+
-
-
-
-
- - ds - -
- -
-
-
-
- -
-
-
-
-
- A data source has to be configured and active to be able to select it and index data from. -
-
-
, + ds + + , +
,
{ it('renders source selector with default options', async () => { const accelerationFormData = createAccelerationEmptyDataMock; - const selectedDatasource: EuiComboBoxOptionOption[] = []; + const selectedDatasource = 'my_glue'; const setAccelerationFormData = jest.fn(); const client = coreStartMock.http; client.get = jest.fn().mockResolvedValue(mockDatasourcesQuery); @@ -49,7 +48,7 @@ describe('Source selector components', () => { }); it('renders source selector with different options', async () => { - const selectedDatasource: EuiComboBoxOptionOption[] = [{ label: 'ds' }]; + const selectedDatasource = 'ds'; const accelerationFormData: CreateAccelerationForm = { ...createAccelerationEmptyDataMock, dataSource: 'ds', diff --git a/public/components/datasources/components/manage/accelerations/selectors/index_type_selector.tsx b/public/components/datasources/components/manage/accelerations/selectors/index_type_selector.tsx index 1ebd110a68..b710d87794 100644 --- a/public/components/datasources/components/manage/accelerations/selectors/index_type_selector.tsx +++ b/public/components/datasources/components/manage/accelerations/selectors/index_type_selector.tsx @@ -14,7 +14,6 @@ import { AccelerationIndexType, CreateAccelerationForm, } from '../../../../../../../common/types/data_connections'; -// import { executeAsyncQuery } from '../../../../common/utils/async_query_helpers'; interface IndexTypeSelectorProps { accelerationFormData: CreateAccelerationForm; @@ -31,6 +30,7 @@ export const IndexTypeSelector = ({ const [loading, _setLoading] = useState(false); // useEffect(() => { + // TODO: Load table schema from cache // if (accelerationFormData.dataTable !== '') { // setLoading(true); // const idPrefix = htmlIdGenerator()(); diff --git a/public/components/datasources/components/manage/accelerations/selectors/source_selector.tsx b/public/components/datasources/components/manage/accelerations/selectors/source_selector.tsx index ea3859285b..6f44fa94b0 100644 --- a/public/components/datasources/components/manage/accelerations/selectors/source_selector.tsx +++ b/public/components/datasources/components/manage/accelerations/selectors/source_selector.tsx @@ -3,20 +3,29 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { EuiComboBox, EuiComboBoxOptionOption, EuiFormRow, EuiSpacer, EuiText } from '@elastic/eui'; +import { + EuiComboBox, + EuiComboBoxOptionOption, + EuiDescriptionList, + EuiDescriptionListDescription, + EuiDescriptionListTitle, + EuiFormRow, + EuiSpacer, + EuiText, +} from '@elastic/eui'; import producer from 'immer'; import React, { useEffect, useState } from 'react'; -// import { executeAsyncQuery } from '../../../../common/utils/async_query_helpers'; import { CoreStart } from '../../../../../../../../../src/core/public'; +import { DATACONNECTIONS_BASE } from '../../../../../../../common/constants/shared'; import { CreateAccelerationForm } from '../../../../../../../common/types/data_connections'; import { useToast } from '../../../../../common/toast'; -import { hasError, validateDataSource } from '../create/utils'; +import { hasError, validateDataTable, validateDatabase } from '../create/utils'; interface AccelerationDataSourceSelectorProps { http: CoreStart['http']; accelerationFormData: CreateAccelerationForm; setAccelerationFormData: React.Dispatch>; - selectedDatasource: EuiComboBoxOptionOption[]; + selectedDatasource: string; } export const AccelerationDataSourceSelector = ({ @@ -26,12 +35,6 @@ export const AccelerationDataSourceSelector = ({ selectedDatasource, }: AccelerationDataSourceSelectorProps) => { const { setToast } = useToast(); - const [dataConnections, setDataConnections] = useState>>( - [] - ); - const [selectedDataConnection, setSelectedDataConnection] = useState< - Array> - >(selectedDatasource.length > 0 ? [{ label: selectedDatasource[0].label }] : []); const [databases, _setDatabases] = useState>>([]); const [selectedDatabase, setSelectedDatabase] = useState>>( [] @@ -47,23 +50,27 @@ export const AccelerationDataSourceSelector = ({ const loadDataSource = () => { setLoadingComboBoxes({ ...loadingComboBoxes, dataSource: true }); http - .get(`/api/get_datasources`) + .get(DATACONNECTIONS_BASE) .then((res) => { - const data = res.data.resp; - setDataConnections( - data - .filter((connection: any) => connection.connector.toUpperCase() === 'S3GLUE') - .map((connection: any) => ({ label: connection.name })) + const isValidDataSource = res.some( + (connection: any) => + connection.connector.toUpperCase() === 'S3GLUE' && + connection.name === selectedDatasource ); + + if (!isValidDataSource) { + setToast(`Received an invalid datasource in create acceleration flyout`, 'danger'); + } }) .catch((err) => { console.error(err); - setToast(`ERROR: failed to load datasources`, 'danger'); + setToast(`failed to load datasources`, 'danger'); }); setLoadingComboBoxes({ ...loadingComboBoxes, dataSource: false }); }; const loadDatabases = () => { + // TODO: Load databases from cache // setLoadingComboBoxes({ ...loadingComboBoxes, database: true }); // const query = { // lang: 'sql', @@ -93,6 +100,7 @@ export const AccelerationDataSourceSelector = ({ }; const loadTables = () => { + // TODO: Load tables from cache // setLoadingComboBoxes({ ...loadingComboBoxes, dataTable: true }); // const query = { // lang: 'sql', @@ -146,36 +154,12 @@ export const AccelerationDataSourceSelector = ({ Select the data source to accelerate data from. External data sources may take time to load. - - - { - if (dataConnectionOptions.length > 0) { - setAccelerationFormData( - producer((accData) => { - accData.dataSource = dataConnectionOptions[0].label; - accData.formErrors.dataSourceError = validateDataSource( - dataConnectionOptions[0].label - ); - }) - ); - setSelectedDataConnection(dataConnectionOptions); - } - }} - isClearable={false} - isInvalid={hasError(accelerationFormData.formErrors, 'dataSourceError')} - isLoading={loadingComboBoxes.dataSource} - /> - + + + Data source + {selectedDatasource} + + { accData.database = databaseOptions[0].label; - accData.formErrors.databaseError = validateDataSource(databaseOptions[0].label); + accData.formErrors.databaseError = validateDatabase(databaseOptions[0].label); }) ); setSelectedDatabase(databaseOptions); @@ -219,7 +203,7 @@ export const AccelerationDataSourceSelector = ({ setAccelerationFormData( producer((accData) => { accData.dataTable = tableOptions[0].label; - accData.formErrors.dataTableError = validateDataSource(tableOptions[0].label); + accData.formErrors.dataTableError = validateDataTable(tableOptions[0].label); }) ); setSelectedTable(tableOptions); diff --git a/public/components/datasources/components/manage/data_connection.tsx b/public/components/datasources/components/manage/data_connection.tsx index ee10e4a53e..2b8dc5895b 100644 --- a/public/components/datasources/components/manage/data_connection.tsx +++ b/public/components/datasources/components/manage/data_connection.tsx @@ -25,14 +25,14 @@ import { observabilityIntegrationsID, observabilityLogsID, observabilityMetricsID, - queryWorkbenchPluginID, } from '../../../../../common/constants/shared'; +import { DatasourceType } from '../../../../../common/types/data_connections'; import { coreRefs } from '../../../../framework/core_refs'; +import { getRenderCreateAccelerationFlyout } from '../../../../plugin'; import { NoAccess } from '../no_access'; +import { AccelerationTable } from './accelerations/acceleration_table'; import { AccessControlTab } from './access_control_tab'; -import { DatasourceType } from '../../../../../common/types/data_connections'; import { AssociatedObjectsTab } from './associated_objects/associated_objects_tab'; -import { AccelerationTable } from './accelerations/acceleration_table'; import { mockAssociatedObjects } from './associated_objects/utils/associated_objects_tab_utils'; interface DatasourceDetails { @@ -51,6 +51,7 @@ export interface S3GlueProperties { export interface PrometheusProperties { 'prometheus.uri': string; } +const renderCreateAccelerationFlyout = getRenderCreateAccelerationFlyout(); export const DataConnection = (props: any) => { const { dataSource } = props; @@ -82,6 +83,18 @@ export const DataConnection = (props: any) => { }, ]; + const onclickIntegrationsCard = () => { + application!.navigateToApp(observabilityIntegrationsID); + }; + + const onclickAccelerationsCard = () => { + renderCreateAccelerationFlyout(dataSource); + }; + + const onclickDiscoverCard = () => { + application!.navigateToApp(observabilityLogsID); + }; + const DefaultDatasourceCards = () => { return ( @@ -90,9 +103,9 @@ export const DataConnection = (props: any) => { icon={} title={'Configure Integrations'} description="Connect to common application log types using integrations" - onClick={() => application!.navigateToApp(observabilityIntegrationsID)} + onClick={onclickIntegrationsCard} selectable={{ - onClick: () => {}, + onClick: onclickIntegrationsCard, isDisabled: false, children: 'Add Integrations', }} @@ -103,13 +116,9 @@ export const DataConnection = (props: any) => { icon={} title={'Accelerate performance'} description="Accelerate query performance through OpenSearch indexing" - onClick={() => - application!.navigateToApp(queryWorkbenchPluginID, { - path: `#/accelerate/${dataSource}`, - }) - } + onClick={onclickAccelerationsCard} selectable={{ - onClick: () => {}, + onClick: onclickAccelerationsCard, isDisabled: false, children: 'Accelerate Performance', }} @@ -120,9 +129,9 @@ export const DataConnection = (props: any) => { icon={} title={'Query data'} description="Uncover insights from your data or better understand it" - onClick={() => application!.navigateToApp(observabilityLogsID)} + onClick={onclickDiscoverCard} selectable={{ - onClick: () => {}, + onClick: onclickDiscoverCard, isDisabled: false, children: 'Query in Discover', }} diff --git a/public/plugin.tsx b/public/plugin.tsx index ff60e84306..eff8dddc86 100644 --- a/public/plugin.tsx +++ b/public/plugin.tsx @@ -58,6 +58,11 @@ import { import { DirectSearch } from './components/common/search/direct_search'; import { Search } from './components/common/search/search'; import { AccelerationDetailsFlyout } from './components/datasources/components/manage/accelerations/acceleration_details_flyout'; +import { CreateAcceleration } from './components/datasources/components/manage/accelerations/create/create_acceleration'; +import { + AssociatedObjectsDetailsFlyout, + AssociatedObjectsFlyoutProps, +} from './components/datasources/components/manage/associated_objects/associated_objects_details_flyout'; import { convertLegacyNotebooksUrl } from './components/notebooks/components/helpers/legacy_route_helpers'; import { convertLegacyTraceAnalyticsUrl } from './components/trace_analytics/components/common/legacy_route_helpers'; import { registerAsssitantDependencies } from './dependencies/register_assistant'; @@ -84,10 +89,6 @@ import { ObservabilityStart, SetupDependencies, } from './types'; -import { - AssociatedObjectsDetailsFlyout, - AssociatedObjectsFlyoutProps, -} from './components/datasources/components/manage/associated_objects/associated_objects_details_flyout'; interface PublicConfig { query_assist: { @@ -108,6 +109,11 @@ export const [ setRenderAssociatedObjectsDetailsFlyout, ] = createGetterSetter('renderAssociatedObjectsDetailsFlyout'); +export const [ + getRenderCreateAccelerationFlyout, + setRenderCreateAccelerationFlyout, +] = createGetterSetter<(dataSource: string) => void>('renderCreateAccelerationFlyout'); + export class ObservabilityPlugin implements Plugin { @@ -393,10 +399,23 @@ export class ObservabilityPlugin ); setRenderAssociatedObjectsDetailsFlyout(renderAssociatedObjectsDetailsFlyout); + const renderCreateAccelerationFlyout = (selectedDatasource: string) => { + const createAccelerationFlyout = core.overlays.openFlyout( + toMountPoint( + createAccelerationFlyout.close()} + /> + ) + ); + }; + setRenderCreateAccelerationFlyout(renderCreateAccelerationFlyout); + // Export so other plugins can use this flyout return { renderAccelerationDetailsFlyout, renderAssociatedObjectsDetailsFlyout, + renderCreateAccelerationFlyout, }; } From 41214c22a67157b7a2f1d100162b99d4d6d5d498 Mon Sep 17 00:00:00 2001 From: Shenoy Pratik Date: Sat, 9 Mar 2024 17:27:34 -0800 Subject: [PATCH 2/5] export observability startdeps (#1515) Signed-off-by: Shenoy Pratik --- public/index.ts | 2 ++ public/plugin.tsx | 7 +++++-- public/types.ts | 14 +++++++++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/public/index.ts b/public/index.ts index 919af71b25..8738c505be 100644 --- a/public/index.ts +++ b/public/index.ts @@ -13,3 +13,5 @@ export { ObservabilityPlugin as Plugin }; export const plugin = (initializerContext: PluginInitializerContext) => new ObservabilityPlugin(initializerContext); + +export { ObservabilityStart } from './types'; diff --git a/public/plugin.tsx b/public/plugin.tsx index eff8dddc86..8d4bfdea0e 100644 --- a/public/plugin.tsx +++ b/public/plugin.tsx @@ -48,6 +48,7 @@ import { observabilityTracesTitle, } from '../common/constants/shared'; import { QueryManager } from '../common/query_manager'; +import { AssociatedObject } from '../common/types/data_connections'; import { VISUALIZATION_SAVED_OBJECT } from '../common/types/observability_saved_object_attributes'; import { setOSDHttp, @@ -102,12 +103,14 @@ interface PublicConfig { export const [ getRenderAccelerationDetailsFlyout, setRenderAccelerationDetailsFlyout, -] = createGetterSetter('renderAccelerationDetailsFlyout'); +] = createGetterSetter<(acceleration: any) => void>('renderAccelerationDetailsFlyout'); export const [ getRenderAssociatedObjectsDetailsFlyout, setRenderAssociatedObjectsDetailsFlyout, -] = createGetterSetter('renderAssociatedObjectsDetailsFlyout'); +] = createGetterSetter<({ tableDetail }: { tableDetail: AssociatedObject }) => void>( + 'renderAssociatedObjectsDetailsFlyout' +); export const [ getRenderCreateAccelerationFlyout, diff --git a/public/types.ts b/public/types.ts index 4d5fbd8322..704e967f84 100644 --- a/public/types.ts +++ b/public/types.ts @@ -11,6 +11,7 @@ import { ManagementOverViewPluginSetup } from '../../../src/plugins/management_o import { NavigationPublicPluginStart } from '../../../src/plugins/navigation/public'; import { UiActionsStart } from '../../../src/plugins/ui_actions/public'; import { VisualizationsSetup } from '../../../src/plugins/visualizations/public'; +import { AssociatedObject } from '../common/types/data_connections'; import { AssistantSetup } from './types'; export interface AppPluginStartDependencies { @@ -33,8 +34,15 @@ export interface SetupDependencies { // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface ObservabilitySetup {} -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface ObservabilityStart {} +export interface ObservabilityStart { + renderAccelerationDetailsFlyout: (acceleration: any) => void; + renderAssociatedObjectsDetailsFlyout: ({ + tableDetail, + }: { + tableDetail: AssociatedObject; + }) => void; + renderCreateAccelerationFlyout: (selectedDatasource: string) => void; +} /** * Introduce a compile dependency on dashboards-assistant @@ -42,4 +50,4 @@ export interface ObservabilityStart {} * It will gives an type error when dashboards-assistant is not installed so add a ts-ignore to suppress the error. */ // @ts-ignore -export type { AssistantSetup, RenderProps, IMessage } from '../../dashboards-assistant/public'; +export type { AssistantSetup, IMessage, RenderProps } from '../../dashboards-assistant/public'; From d17cad3fbf1b6102b6354ddcfc9a17bbba4e282d Mon Sep 17 00:00:00 2001 From: Paul Sebastian Date: Mon, 11 Mar 2024 15:09:03 -0700 Subject: [PATCH 3/5] Changed Explorer Data Grid useage of timestamp (#1479) * add support to use timestamp from set default Signed-off-by: Paul Sebastian * update data grid test and snap Signed-off-by: Paul Sebastian * direct query now sets default timestamp using same logic Signed-off-by: Paul Sebastian * small linting complaints Signed-off-by: Paul Sebastian * appeasing lint pt2 Signed-off-by: Paul Sebastian * implement i18n for column names Signed-off-by: Paul Sebastian * using timestamp prop Signed-off-by: Paul Sebastian * add test with different user timestamp Signed-off-by: Paul Sebastian * remove console log Signed-off-by: Paul Sebastian * direct search to use i18n translate Signed-off-by: Paul Sebastian --------- Signed-off-by: Paul Sebastian --- common/constants/explorer.ts | 2 +- common/constants/shared.ts | 2 + .../common/search/direct_search.tsx | 58 +- .../__snapshots__/data_grid.test.tsx.snap | 545 +++++++++++++++++- .../explorer/__tests__/data_grid.test.tsx | 86 ++- .../explorer/events_views/data_grid.tsx | 31 +- 6 files changed, 688 insertions(+), 36 deletions(-) diff --git a/common/constants/explorer.ts b/common/constants/explorer.ts index d94957c3e8..fc0102d119 100644 --- a/common/constants/explorer.ts +++ b/common/constants/explorer.ts @@ -328,7 +328,7 @@ export const TYPE_TAB_MAPPING = { }; export const DEFAULT_EMPTY_EXPLORER_FIELDS = [ - { name: 'timestamp', type: 'timestamp' }, + // timestamp field will be a default but is added after finding what it is { name: '_source', type: 'string' }, ]; diff --git a/common/constants/shared.ts b/common/constants/shared.ts index bf6ada49ef..d452b14c4a 100644 --- a/common/constants/shared.ts +++ b/common/constants/shared.ts @@ -265,3 +265,5 @@ export const DIRECT_DUMMY_QUERY = 'select 1'; export const DEFAULT_START_TIME = 'now-15m'; export const QUERY_ASSIST_START_TIME = 'now-40y'; export const QUERY_ASSIST_END_TIME = 'now'; + +export const TIMESTAMP_DATETIME_TYPES = ['date', 'date_nanos']; diff --git a/public/components/common/search/direct_search.tsx b/public/components/common/search/direct_search.tsx index f1ab57e890..1aac36bb6e 100644 --- a/public/components/common/search/direct_search.tsx +++ b/public/components/common/search/direct_search.tsx @@ -18,13 +18,25 @@ import { EuiPopoverFooter, EuiToolTip, } from '@elastic/eui'; -import { isEqual } from 'lodash'; +import { isEmpty, isEqual } from 'lodash'; import React, { useEffect, useState } from 'react'; import { batch, useDispatch, useSelector } from 'react-redux'; import { ASYNC_POLLING_INTERVAL, QUERY_LANGUAGE } from '../../../../common/constants/data_sources'; -import { APP_ANALYTICS_TAB_ID_REGEX, RAW_QUERY } from '../../../../common/constants/explorer'; -import { PPL_NEWLINE_REGEX, PPL_SPAN_REGEX } from '../../../../common/constants/shared'; -import { DirectQueryLoadingStatus, DirectQueryRequest } from '../../../../common/types/explorer'; +import { + APP_ANALYTICS_TAB_ID_REGEX, + RAW_QUERY, + SELECTED_TIMESTAMP, +} from '../../../../common/constants/explorer'; +import { + PPL_NEWLINE_REGEX, + PPL_SPAN_REGEX, + TIMESTAMP_DATETIME_TYPES, +} from '../../../../common/constants/shared'; +import { + DirectQueryLoadingStatus, + DirectQueryRequest, + IDefaultTimestampState, +} from '../../../../common/types/explorer'; import { uiSettingsService } from '../../../../common/utils'; import { getAsyncSessionId, setAsyncSessionId } from '../../../../common/utils/query_session_utils'; import { get as getObjValue } from '../../../../common/utils/shared'; @@ -42,6 +54,7 @@ import { formatError } from '../../event_analytics/utils'; import { usePolling } from '../../hooks/use_polling'; import { PPLReferenceFlyout } from '../helpers'; import { Autocomplete } from './autocomplete'; +import { i18n } from '@osd/i18n'; export interface IQueryBarProps { query: string; tempQuery: string; @@ -250,6 +263,33 @@ export const DirectSearch = (props: any) => { }); }; + const getDirectQueryTimestamp = (schema: Array<{ name: string; type: string }>) => { + const timestamp: IDefaultTimestampState = { + hasSchemaConflict: false, // schema conflict bool used for OS index w/ different mappings, not needed here + default_timestamp: '', + message: i18n.translate(`discover.events.directQuery.noTimeStampFoundMessage`, { + defaultMessage: 'Index does not contain a valid time field.', + }), + }; + + for (let i = 0; i < schema.length; i++) { + const fieldMapping = schema[i]; + if (!isEmpty(fieldMapping)) { + const fieldName = fieldMapping.name; + const fieldType = fieldMapping.type; + const isValidTimeType = TIMESTAMP_DATETIME_TYPES.some((dateTimeType) => + isEqual(fieldType, dateTimeType) + ); + if (isValidTimeType && isEmpty(timestamp.default_timestamp)) { + timestamp.default_timestamp = fieldName; + timestamp.message = ''; + break; + } + } + } + return timestamp; + }; + useEffect(() => { // cancel direct query if (!pollingResult) return; @@ -258,6 +298,16 @@ export const DirectSearch = (props: any) => { if (status === DirectQueryLoadingStatus.SUCCESS || datarows) { stopPollingWithStatus(status); + // find the timestamp from results + const derivedTimestamp = getDirectQueryTimestamp(pollingResult.schema); + dispatch( + changeQuery({ + tabId, + query: { + [SELECTED_TIMESTAMP]: derivedTimestamp.default_timestamp, + }, + }) + ); // update page with data dispatchOnGettingHis(pollingResult, ''); } else if (status === DirectQueryLoadingStatus.FAILED) { diff --git a/public/components/event_analytics/explorer/__tests__/__snapshots__/data_grid.test.tsx.snap b/public/components/event_analytics/explorer/__tests__/__snapshots__/data_grid.test.tsx.snap index e21e9494f0..3818564438 100644 --- a/public/components/event_analytics/explorer/__tests__/__snapshots__/data_grid.test.tsx.snap +++ b/public/components/event_analytics/explorer/__tests__/__snapshots__/data_grid.test.tsx.snap @@ -128,11 +128,59 @@ exports[`Datagrid component Renders data grid component 1`] = ` "unselectedFields": Array [], } } - http={[MockFunction]} + http={ + Object { + "addLoadingCountSource": [MockFunction], + "anonymousPaths": Object { + "isAnonymous": [MockFunction], + "register": [MockFunction], + }, + "basePath": BasePath { + "basePath": "", + "get": [Function], + "prepend": [Function], + "remove": [Function], + "serverBasePath": "", + }, + "delete": [MockFunction], + "fetch": [MockFunction], + "get": [MockFunction], + "getLoadingCount$": [MockFunction], + "head": [MockFunction], + "intercept": [MockFunction], + "options": [MockFunction], + "patch": [MockFunction], + "post": [MockFunction], + "put": [MockFunction], + } + } pplService={ PPLService { "fetch": [Function], - "http": [MockFunction], + "http": Object { + "addLoadingCountSource": [MockFunction], + "anonymousPaths": Object { + "isAnonymous": [MockFunction], + "register": [MockFunction], + }, + "basePath": BasePath { + "basePath": "", + "get": [Function], + "prepend": [Function], + "remove": [Function], + "serverBasePath": "", + }, + "delete": [MockFunction], + "fetch": [MockFunction], + "get": [MockFunction], + "getLoadingCount$": [MockFunction], + "head": [MockFunction], + "intercept": [MockFunction], + "options": [MockFunction], + "patch": [MockFunction], + "post": [MockFunction], + "put": [MockFunction], + }, } } rawQuery="source = opensearch_dashboards_sample_data_logs | where match(request,'filebeat')" @@ -203,14 +251,9 @@ exports[`Datagrid component Renders data grid component 1`] = ` }, ] } - rowsAll={Array []} startTime="now/y" storedSelectedColumns={ Array [ - Object { - "name": "timestamp", - "type": "timestamp", - }, Object { "name": "_source", "type": "string", @@ -243,7 +286,7 @@ exports[`Datagrid component Renders data grid component 1`] = ` columns={ Array [ Object { - "display": "Time", + "display": "Time (timestamp)", "id": "timestamp", "initialWidth": 200, "isSortable": true, @@ -442,3 +485,489 @@ exports[`Datagrid component Renders data grid component 1`] = ` `; + +exports[`Datagrid component renders data grid with different timestamp 1`] = ` + + + +
+
+ + + + + +
+ +
+
+
+ +
+ + + + + +
+
+ + + +`; diff --git a/public/components/event_analytics/explorer/__tests__/data_grid.test.tsx b/public/components/event_analytics/explorer/__tests__/data_grid.test.tsx index 5a1729adf7..d9af5b86b5 100644 --- a/public/components/event_analytics/explorer/__tests__/data_grid.test.tsx +++ b/public/components/event_analytics/explorer/__tests__/data_grid.test.tsx @@ -14,6 +14,7 @@ import { UNSELECTED_FIELDS, QUERIED_FIELDS, DEFAULT_EMPTY_EXPLORER_FIELDS, + SELECTED_TIMESTAMP, } from '../../../../../common/constants/explorer'; import { AVAILABLE_FIELDS as SIDEBAR_AVAILABLE_FIELDS, @@ -21,14 +22,15 @@ import { DATA_GRID_ROWS, EXPLORER_DATA_GRID_QUERY, } from '../../../../../test/event_analytics_constants'; -import httpClientMock from '../../../../../test/__mocks__/httpClientMock'; import { sampleEmptyPanel } from '../../../../../test/panels_constants'; import { HttpResponse } from '../../../../../../../src/core/public'; import PPLService from '../../../../../public/services/requests/ppl'; -import { applyMiddleware, createStore } from 'redux'; -import { rootReducer } from '../../../../../public/framework/redux/reducers'; -import thunk from 'redux-thunk'; import { Provider } from 'react-redux'; +import { configureStore } from '@reduxjs/toolkit'; +import { queriesReducer } from '../../redux/slices/query_slice'; +import { coreMock } from '../../../../../../../src/core/public/mocks'; + +const coreStartMock = coreMock.createStart(); describe('Datagrid component', () => { configure({ adapter: new Adapter() }); @@ -41,28 +43,82 @@ describe('Datagrid component', () => { [QUERIED_FIELDS]: QUERY_FIELDS, }; - httpClientMock.get = jest.fn(() => - Promise.resolve((sampleEmptyPanel as unknown) as HttpResponse) - ); + coreStartMock.http.get = jest + .fn() + .mockResolvedValue((sampleEmptyPanel as unknown) as HttpResponse); - const http = httpClientMock; - const pplService = new PPLService(httpClientMock); - const store = createStore(rootReducer, applyMiddleware(thunk)); + const tabId = 'explorer-tab-_fbef9141-48eb-11ee-a60a-af33302cfb3c'; + + const pplService = new PPLService(coreStartMock.http); + const preloadedState = { + queries: { + [tabId]: { + [SELECTED_TIMESTAMP]: 'timestamp', + }, + }, + }; + const store = configureStore({ reducer: queriesReducer, preloadedState }); const wrapper = mount( + + ); + + wrapper.update(); + + await waitFor(() => { + expect(wrapper).toMatchSnapshot(); + }); + }); + + it('renders data grid with different timestamp', async () => { + const explorerFields = { + [SELECTED_FIELDS]: [], + [UNSELECTED_FIELDS]: [], + [AVAILABLE_FIELDS]: SIDEBAR_AVAILABLE_FIELDS, + [QUERIED_FIELDS]: QUERY_FIELDS, + }; + + coreStartMock.http.get = jest + .fn() + .mockResolvedValue((sampleEmptyPanel as unknown) as HttpResponse); + + const tabId = 'explorer-tab-_fbef9141-48eb-11ee-a60a-af33302cfb3c'; + + const pplService = new PPLService(coreStartMock.http); + const preloadedState = { + queries: { + [tabId]: { + [SELECTED_TIMESTAMP]: 'utc_time', + }, + }, + }; + const store = configureStore({ reducer: queriesReducer, preloadedState }); + + const wrapper = mount( + + 0 ? explorerFields.selectedFields - : DEFAULT_EMPTY_EXPLORER_FIELDS; + : [{ name: timeStampField, type: 'timestamp' }, ...DEFAULT_EMPTY_EXPLORER_FIELDS]; // useRef instead of useState somehow solves the issue of user triggered sorting not // having any delays const sortingFields: MutableRefObject = useRef([]); - const pageFields = useRef([0, 100]); + const pageFields = useRef([0, 100]); // page num, row length const [data, setData] = useState(rows); @@ -110,14 +112,27 @@ export function DataGrid(props: DataGridProps) { ); }; + const columnNameTranslate = (name: string) => { + return i18n.translate(`discover.events.dataGrid.${name.toLowerCase()}Column`, { + defaultMessage: name, + }); + }; + // creates the header for each column listing what that column is const dataGridColumns = () => { const columns: EuiDataGridColumn[] = []; selectedColumns.map(({ name }) => { - if (name === 'timestamp') { - columns.push(DEFAULT_TIMESTAMP_COLUMN); + if (name === timeStampField) { + columns.push({ + ...DEFAULT_TIMESTAMP_COLUMN, + display: `${columnNameTranslate('Time')} (${timeStampField})`, + id: timeStampField, + }); } else if (name === '_source') { - columns.push(DEFAULT_SOURCE_COLUMN); + columns.push({ + ...DEFAULT_SOURCE_COLUMN, + display: columnNameTranslate('Source'), + }); } else { columns.push({ id: name, @@ -183,7 +198,7 @@ export function DataGrid(props: DataGridProps) { // renders what is shown in each cell, i.e. the content of each row const dataGridCellRender = ({ rowIndex, columnId }: { rowIndex: number; columnId: string }) => { - const trueIndex = rowIndex % pageFields.current[1]; + const trueIndex = rowIndex % pageFields.current[1]; // modulo of row length, i.e. pos on current page if (trueIndex < data.length) { if (columnId === '_source') { return ( @@ -201,8 +216,8 @@ export function DataGrid(props: DataGridProps) { ); } - if (columnId === 'timestamp') { - return `${moment(data[trueIndex][columnId]).format(DATE_DISPLAY_FORMAT)}`; + if (columnId === timeStampField) { + return `${moment(data[trueIndex][timeStampField]).format(DATE_DISPLAY_FORMAT)}`; } return `${data[trueIndex][columnId]}`; } From b0f0d9b9dea6e6b853cf785fc49b2f51707e3e3e Mon Sep 17 00:00:00 2001 From: Shenoy Pratik Date: Tue, 12 Mar 2024 17:03:30 -0700 Subject: [PATCH 4/5] Add datasource field in accelerations cache (#1525) * add datasource in accelerations cache Signed-off-by: Shenoy Pratik * fixed nits Signed-off-by: Shenoy Pratik --------- Signed-off-by: Shenoy Pratik --- common/types/data_connections.ts | 17 +- .../catalog_cache/cache_loader.test.tsx | 181 ++++++++++++++---- .../framework/catalog_cache/cache_loader.tsx | 24 +-- .../catalog_cache/cache_manager.test.tsx | 94 ++++++++- .../framework/catalog_cache/cache_manager.ts | 47 ++++- 5 files changed, 298 insertions(+), 65 deletions(-) diff --git a/common/types/data_connections.ts b/common/types/data_connections.ts index ebe48afc59..99a49b48f1 100644 --- a/common/types/data_connections.ts +++ b/common/types/data_connections.ts @@ -83,19 +83,19 @@ export interface CachedColumn { export interface CachedTable { name: string; - columns: CachedColumn[]; + columns?: CachedColumn[]; } export interface CachedDatabase { name: string; tables: CachedTable[]; - lastUpdated: string; // Assuming date string in UTC format + lastUpdated: string; // date string in UTC format status: CachedDataSourceStatus; } export interface CachedDataSource { name: string; - lastUpdated: string; // Assuming date string in UTC format + lastUpdated: string; // date string in UTC format status: CachedDataSourceStatus; databases: CachedDatabase[]; } @@ -115,13 +115,18 @@ export interface CachedAccelerations { status: string; } -export interface AccelerationsCacheData { - version: string; +export interface CachedAcclerationByDataSource { + name: string; accelerations: CachedAccelerations[]; - lastUpdated: string; // Assuming date string in UTC format + lastUpdated: string; // date string in UTC format status: CachedDataSourceStatus; } +export interface AccelerationsCacheData { + version: string; + dataSources: CachedAcclerationByDataSource[]; +} + export interface PollingSuccessResult { schema: Array<{ name: string; type: string }>; datarows: Array>; diff --git a/public/framework/catalog_cache/cache_loader.test.tsx b/public/framework/catalog_cache/cache_loader.test.tsx index 063da0108b..f7ad4fcbfd 100644 --- a/public/framework/catalog_cache/cache_loader.test.tsx +++ b/public/framework/catalog_cache/cache_loader.test.tsx @@ -11,9 +11,11 @@ import { mockShowTablesPollingResult, } from '../../../test/datasources'; import { + createLoadQuery, updateAccelerationsToCache, updateDatabasesToCache, updateTablesToCache, + updateToCache, } from './cache_loader'; import { CatalogCacheManager } from './cache_manager'; @@ -146,10 +148,7 @@ describe('loadCacheTests', () => { dataSourceName, expect.objectContaining({ name: databaseName, - tables: [ - { name: 'Table1', columns: [] }, - { name: 'Table2', columns: [] }, - ], + tables: [{ name: 'Table1' }, { name: 'Table2' }], lastUpdated: expect.any(String), status: CachedDataSourceStatus.Updated, }) @@ -166,55 +165,169 @@ describe('loadCacheTests', () => { it('should save empty accelerations cache and status failed when polling result is null', () => { const pollingResult = null; - updateAccelerationsToCache(pollingResult); + updateAccelerationsToCache('sampleDS', pollingResult); // Verify that saveAccelerationsCache is called with the correct parameters expect(CatalogCacheManager.saveAccelerationsCache).toHaveBeenCalledWith({ version: CATALOG_CACHE_VERSION, - accelerations: [], - lastUpdated: expect.any(String), - status: CachedDataSourceStatus.Failed, + dataSources: [ + { + name: 'sampleDS', + accelerations: [], + lastUpdated: expect.any(String), + status: CachedDataSourceStatus.Failed, + }, + ], }); }); it('should save new accelerations cache when polling result is not null', () => { - updateAccelerationsToCache(mockShowIndexesPollingResult); + updateAccelerationsToCache('sampleDS', mockShowIndexesPollingResult); // Verify that saveAccelerationsCache is called with the correct parameters expect(CatalogCacheManager.saveAccelerationsCache).toHaveBeenCalledWith({ version: CATALOG_CACHE_VERSION, - accelerations: [ - { - flintIndexName: 'flint_mys3_default_http_logs_skipping_index', - type: 'skipping', - database: 'default', - table: 'http_logs', - indexName: 'skipping_index', - autoRefresh: false, - status: 'Active', - }, + dataSources: [ { - flintIndexName: 'flint_mys3_default_http_logs_status_clientip_and_day_index', - type: 'covering', - database: 'default', - table: 'http_logs', - indexName: 'status_clientip_and_day', - autoRefresh: true, - status: 'Active', + name: 'sampleDS', + accelerations: [ + { + flintIndexName: 'flint_mys3_default_http_logs_skipping_index', + type: 'skipping', + database: 'default', + table: 'http_logs', + indexName: 'skipping_index', + autoRefresh: false, + status: 'Active', + }, + { + flintIndexName: 'flint_mys3_default_http_logs_status_clientip_and_day_index', + type: 'covering', + database: 'default', + table: 'http_logs', + indexName: 'status_clientip_and_day', + autoRefresh: true, + status: 'Active', + }, + { + flintIndexName: 'flint_mys3_default_http_count_view', + type: 'materialized', + database: 'default', + table: '', + indexName: 'http_count_view', + autoRefresh: true, + status: 'Active', + }, + ], + lastUpdated: expect.any(String), + status: CachedDataSourceStatus.Updated, }, + ], + }); + }); + }); + + describe('updateToCache', () => { + it('should call updateDatabasesToCache when loadCacheType is "databases"', () => { + const loadCacheType = 'databases'; + const dataSourceName = 'TestDataSource'; + + updateToCache(mockShowDatabasesPollingResult, loadCacheType, dataSourceName); + + // Verify that addOrUpdateDataSource is called + expect(CatalogCacheManager.addOrUpdateDataSource).toHaveBeenCalled(); + expect(CatalogCacheManager.updateDatabase).not.toHaveBeenCalled(); + expect(CatalogCacheManager.saveAccelerationsCache).not.toHaveBeenCalled(); + }); + + it('should call updateTablesToCache when loadCacheType is "tables"', () => { + const loadCacheType = 'tables'; + const dataSourceName = 'TestDataSource'; + const databaseName = 'TestDatabase'; + + CatalogCacheManager.addOrUpdateDataSource({ + databases: [ { - flintIndexName: 'flint_mys3_default_http_count_view', - type: 'materialized', - database: 'default', - table: '', - indexName: 'http_count_view', - autoRefresh: true, - status: 'Active', + name: databaseName, + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + tables: [], }, ], - lastUpdated: expect.any(String), + name: dataSourceName, + lastUpdated: new Date().toUTCString(), status: CachedDataSourceStatus.Updated, }); + + updateToCache(mockShowTablesPollingResult, loadCacheType, dataSourceName, databaseName); + + // Verify that updateDatabase is called + expect(CatalogCacheManager.addOrUpdateDataSource).toHaveBeenCalled(); + expect(CatalogCacheManager.updateDatabase).toHaveBeenCalled(); + expect(CatalogCacheManager.saveAccelerationsCache).not.toHaveBeenCalled(); + }); + + it('should call updateAccelerationsToCache when loadCacheType is "accelerations"', () => { + const loadCacheType = 'accelerations'; + const dataSourceName = 'TestDataSource'; + + updateToCache(mockShowIndexesPollingResult, loadCacheType, dataSourceName); + + // Verify that saveAccelerationsCache is called + expect(CatalogCacheManager.addOrUpdateDataSource).not.toHaveBeenCalled(); + expect(CatalogCacheManager.updateDatabase).not.toHaveBeenCalled(); + expect(CatalogCacheManager.saveAccelerationsCache).toHaveBeenCalled(); + }); + + it('should not call any update function when loadCacheType is not recognized', () => { + const pollResults = {}; + const loadCacheType = ''; + const dataSourceName = 'TestDataSource'; + + updateToCache(pollResults, loadCacheType, dataSourceName); + + // Verify that no update function is called + expect(CatalogCacheManager.addOrUpdateDataSource).not.toHaveBeenCalled(); + expect(CatalogCacheManager.updateDatabase).not.toHaveBeenCalled(); + expect(CatalogCacheManager.saveAccelerationsCache).not.toHaveBeenCalled(); + }); + }); + + describe('createLoadQuery', () => { + it('should create a query for loading databases', () => { + const loadCacheType = 'databases'; + const dataSourceName = 'example'; + const expectedQuery = 'SHOW SCHEMAS IN `example`'; + expect(createLoadQuery(loadCacheType, dataSourceName)).toEqual(expectedQuery); + }); + + it('should create a query for loading tables', () => { + const loadCacheType = 'tables'; + const dataSourceName = 'example'; + const databaseName = 'test'; + const expectedQuery = 'SHOW TABLES IN `example`.`test`'; + expect(createLoadQuery(loadCacheType, dataSourceName, databaseName)).toEqual(expectedQuery); + }); + + it('should create a query for loading accelerations', () => { + const loadCacheType = 'accelerations'; + const dataSourceName = 'example'; + const expectedQuery = 'SHOW FLINT INDEX in `example`'; + expect(createLoadQuery(loadCacheType, dataSourceName)).toEqual(expectedQuery); + }); + + it('should return an empty string for unknown loadCacheType', () => { + const loadCacheType = 'unknownType'; + const dataSourceName = 'example'; + expect(createLoadQuery(loadCacheType, dataSourceName)).toEqual(''); + }); + + it('should properly handle backticks in database name', () => { + const loadCacheType = 'tables'; + const dataSourceName = 'example'; + const databaseName = '`sample`'; + const expectedQuery = 'SHOW TABLES IN `example`.`sample`'; + expect(createLoadQuery(loadCacheType, dataSourceName, databaseName)).toEqual(expectedQuery); }); }); }); diff --git a/public/framework/catalog_cache/cache_loader.tsx b/public/framework/catalog_cache/cache_loader.tsx index 60fc1175e2..bc0d7869e7 100644 --- a/public/framework/catalog_cache/cache_loader.tsx +++ b/public/framework/catalog_cache/cache_loader.tsx @@ -4,10 +4,7 @@ */ import { useEffect, useState } from 'react'; -import { - ASYNC_POLLING_INTERVAL, - CATALOG_CACHE_VERSION, -} from '../../../common/constants/data_sources'; +import { ASYNC_POLLING_INTERVAL } from '../../../common/constants/data_sources'; import { AsyncPollingResult, CachedDataSourceStatus, @@ -80,7 +77,6 @@ export const updateTablesToCache = ( const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); const newTables = combinedData.map((row: any) => ({ name: row.tableName, - columns: [], })); CatalogCacheManager.updateDatabase(dataSourceName, { @@ -91,12 +87,15 @@ export const updateTablesToCache = ( }); }; -export const updateAccelerationsToCache = (pollingResult: AsyncPollingResult) => { +export const updateAccelerationsToCache = ( + dataSourceName: string, + pollingResult: AsyncPollingResult +) => { const currentTime = new Date().toUTCString(); if (!pollingResult) { - CatalogCacheManager.saveAccelerationsCache({ - version: CATALOG_CACHE_VERSION, + CatalogCacheManager.addOrUpdateAccelerationsByDataSource({ + name: dataSourceName, accelerations: [], lastUpdated: currentTime, status: CachedDataSourceStatus.Failed, @@ -116,8 +115,8 @@ export const updateAccelerationsToCache = (pollingResult: AsyncPollingResult) => status: row.status, })); - CatalogCacheManager.saveAccelerationsCache({ - version: CATALOG_CACHE_VERSION, + CatalogCacheManager.addOrUpdateAccelerationsByDataSource({ + name: dataSourceName, accelerations: newAccelerations, lastUpdated: currentTime, status: CachedDataSourceStatus.Updated, @@ -138,7 +137,7 @@ export const updateToCache = ( updateTablesToCache(dataSourceName, databaseName!, pollResults); break; case 'accelerations': - updateAccelerationsToCache(pollResults); + updateAccelerationsToCache(dataSourceName, pollResults); break; default: break; @@ -189,6 +188,7 @@ export const useLoadToCache = (loadCacheType: LoadCacheType) => { }, ASYNC_POLLING_INTERVAL); const startLoading = (dataSourceName: string, databaseName?: string) => { + setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); setCurrentDataSourceName(dataSourceName); setCurrentDatabaseName(databaseName); @@ -272,7 +272,7 @@ export const useLoadTablesToCache = () => { return { loadStatus, startLoading, stopLoading }; }; -export const useAccelerationsToCache = () => { +export const useLoadAccelerationsToCache = () => { const { loadStatus, startLoading, stopLoading } = useLoadToCache('accelerations'); return { loadStatus, startLoading, stopLoading }; }; diff --git a/public/framework/catalog_cache/cache_manager.test.tsx b/public/framework/catalog_cache/cache_manager.test.tsx index 69be8be29e..f73609eda1 100644 --- a/public/framework/catalog_cache/cache_manager.test.tsx +++ b/public/framework/catalog_cache/cache_manager.test.tsx @@ -10,6 +10,7 @@ import { } from '../../../common/constants/shared'; import { AccelerationsCacheData, + CachedAcclerationByDataSource, CachedDataSource, CachedDataSourceStatus, CachedDatabase, @@ -134,9 +135,7 @@ describe('CatalogCacheManager', () => { it('should save accelerations cache to local storage', () => { const cacheData: AccelerationsCacheData = { version: CATALOG_CACHE_VERSION, - accelerations: [], - lastUpdated: '2024-03-07T12:00:00Z', - status: CachedDataSourceStatus.Empty, + dataSources: [], }; CatalogCacheManager.saveAccelerationsCache(cacheData); expect(localStorage.setItem).toHaveBeenCalledWith( @@ -150,20 +149,16 @@ describe('CatalogCacheManager', () => { it('should retrieve accelerations cache from local storage', () => { const cacheData: AccelerationsCacheData = { version: CATALOG_CACHE_VERSION, - accelerations: [], - lastUpdated: '2024-03-07T12:00:00Z', - status: CachedDataSourceStatus.Empty, + dataSources: [], }; localStorage.setItem(ASYNC_QUERY_ACCELERATIONS_CACHE, JSON.stringify(cacheData)); expect(CatalogCacheManager.getAccelerationsCache()).toEqual(cacheData); }); it('should return default cache object if cache is not found', () => { - const defaultCacheObject = { + const defaultCacheObject: AccelerationsCacheData = { version: CATALOG_CACHE_VERSION, - accelerations: [], - lastUpdated: '', - status: CachedDataSourceStatus.Empty, + dataSources: [], }; localStorage.removeItem(ASYNC_QUERY_ACCELERATIONS_CACHE); expect(CatalogCacheManager.getAccelerationsCache()).toEqual(defaultCacheObject); @@ -396,4 +391,83 @@ describe('CatalogCacheManager', () => { expect(localStorage.removeItem).toHaveBeenCalledWith(ASYNC_QUERY_ACCELERATIONS_CACHE); }); }); + + describe('addOrUpdateAccelerationsByDataSource', () => { + it('should add a new data source to the accelerations cache', () => { + const dataSource: CachedAcclerationByDataSource = { + name: 'TestDataSource', + lastUpdated: '2024-03-08T12:00:00Z', + status: CachedDataSourceStatus.Updated, + accelerations: [], + }; + + CatalogCacheManager.addOrUpdateAccelerationsByDataSource(dataSource); + + // Verify that saveAccelerationsCache is called with the updated cache data + expect(localStorage.setItem).toHaveBeenCalledWith( + ASYNC_QUERY_ACCELERATIONS_CACHE, + JSON.stringify({ + version: '1.0', + dataSources: [{ ...dataSource }], + }) + ); + }); + + it('should update an existing data source in the accelerations cache', () => { + // Set up initial cache data + const initialDataSource: CachedAcclerationByDataSource = { + name: 'TestDataSource', + lastUpdated: '2024-03-08T12:00:00Z', + status: CachedDataSourceStatus.Updated, + accelerations: [], + }; + + // Update the data source + const updatedDataSource: CachedAcclerationByDataSource = { + ...initialDataSource, + status: CachedDataSourceStatus.Failed, + }; + + CatalogCacheManager.addOrUpdateAccelerationsByDataSource(updatedDataSource); + + // Verify that saveAccelerationsCache is called with the updated cache data + expect(localStorage.setItem).toHaveBeenCalledWith( + ASYNC_QUERY_ACCELERATIONS_CACHE, + JSON.stringify({ + version: '1.0', + dataSources: [{ ...updatedDataSource }], + }) + ); + }); + }); + + describe('getOrCreateAccelerationsByDataSource', () => { + it('should return an existing data source from the accelerations cache', () => { + // Set up initial cache data + const existingDataSource: CachedAcclerationByDataSource = { + name: 'TestDataSource', + lastUpdated: '2024-03-08T12:00:00Z', + status: CachedDataSourceStatus.Updated, + accelerations: [], + }; + + CatalogCacheManager.addOrUpdateAccelerationsByDataSource(existingDataSource); + const result = CatalogCacheManager.getOrCreateAccelerationsByDataSource('TestDataSource'); + + // Verify that the existing data source is returned + expect(result).toEqual(existingDataSource); + }); + + it('should create and return a new data source if not found in the accelerations cache', () => { + const result = CatalogCacheManager.getOrCreateAccelerationsByDataSource('TestDataSource1'); + + // Verify that the new data source is created and returned + expect(result).toEqual({ + name: 'TestDataSource1', + lastUpdated: expect.any(String), + status: CachedDataSourceStatus.Empty, + accelerations: [], + }); + }); + }); }); diff --git a/public/framework/catalog_cache/cache_manager.ts b/public/framework/catalog_cache/cache_manager.ts index b183170b08..42d5b58415 100644 --- a/public/framework/catalog_cache/cache_manager.ts +++ b/public/framework/catalog_cache/cache_manager.ts @@ -10,6 +10,7 @@ import { } from '../../../common/constants/shared'; import { AccelerationsCacheData, + CachedAcclerationByDataSource, CachedDataSource, CachedDataSourceStatus, CachedDatabase, @@ -75,15 +76,55 @@ export class CatalogCacheManager { } else { const defaultCacheObject = { version: CATALOG_CACHE_VERSION, - accelerations: [], - lastUpdated: '', - status: CachedDataSourceStatus.Empty, + dataSources: [], }; this.saveAccelerationsCache(defaultCacheObject); return defaultCacheObject; } } + /** + * Adds or updates a data source in the accelerations cache. + * @param {CachedAcclerationByDataSource} dataSource - The data source to add or update. + */ + static addOrUpdateAccelerationsByDataSource(dataSource: CachedAcclerationByDataSource): void { + const accCacheData = this.getAccelerationsCache(); + const index = accCacheData.dataSources.findIndex( + (ds: CachedAcclerationByDataSource) => ds.name === dataSource.name + ); + if (index !== -1) { + accCacheData.dataSources[index] = dataSource; + } else { + accCacheData.dataSources.push(dataSource); + } + this.saveAccelerationsCache(accCacheData); + } + + /** + * Retrieves accelerations cache from local storage by the datasource name. + * @param {string} dataSourceName - The name of the data source. + * @returns {CachedAcclerationByDataSource} The retrieved accelerations by datasource in cache. + * @throws {Error} If the data source is not found. + */ + static getOrCreateAccelerationsByDataSource( + dataSourceName: string + ): CachedAcclerationByDataSource { + const accCacheData = this.getAccelerationsCache(); + const cachedDataSource = accCacheData.dataSources.find((ds) => ds.name === dataSourceName); + + if (cachedDataSource) return cachedDataSource; + else { + const defaultDataSourceObject = { + name: dataSourceName, + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + accelerations: [], + }; + this.addOrUpdateAccelerationsByDataSource(defaultDataSourceObject); + return defaultDataSourceObject; + } + } + /** * Adds or updates a data source in the cache. * @param {CachedDataSource} dataSource - The data source to add or update. From 8874c8c6ff778221e633fc8c52356a78f878c22c Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 13 Mar 2024 10:14:56 -0700 Subject: [PATCH 5/5] Add conditional installation for S3 integrations (#1518) * Add workflows to integration format Signed-off-by: Simeon Widdis * Render integration workflows on frontend Signed-off-by: Simeon Widdis * Add ability to toggle workflows to frontend Signed-off-by: Simeon Widdis * Add workflows to integration build options Signed-off-by: Simeon Widdis * Add asset workflow filtering to builder Signed-off-by: Simeon Widdis * Add enabled workflows to setup request Signed-off-by: Simeon Widdis * Don't allow integration setup if no workflows enabled Signed-off-by: Simeon Widdis * Add workflows to other integrations Signed-off-by: Simeon Widdis * Improve header for workflows section Signed-off-by: Simeon Widdis * Update snapshots Signed-off-by: Simeon Widdis --------- Signed-off-by: Simeon Widdis --- .../setup_integration.test.tsx.snap | 1247 +++++++++++++++++ .../__tests__/setup_integration.test.tsx | 15 + .../components/create_integration_helpers.ts | 5 +- .../components/setup_integration.tsx | 120 +- .../repository/aws_elb/aws_elb-1.0.0.json | 20 +- .../aws_vpc_flow/aws_vpc_flow-1.0.0.json | 20 +- .../repository/nginx/nginx-1.0.0.json | 20 +- .../integrations/__test__/builder.test.ts | 71 + .../integrations/integrations_adaptor.ts | 3 +- .../integrations/integrations_builder.ts | 59 +- .../integrations/integrations_manager.ts | 4 +- .../repository/integration_reader.ts | 2 + server/adaptors/integrations/types.ts | 13 +- server/adaptors/integrations/validators.ts | 19 + .../integrations/integrations_router.ts | 4 +- test/constants.ts | 8 + 16 files changed, 1579 insertions(+), 51 deletions(-) diff --git a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap index d23aea4e7d..c77eb11c4b 100644 --- a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap +++ b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap @@ -35,6 +35,7 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = "connectionTableName": "sample", "connectionType": "index", "displayName": "sample Integration", + "enabledWorkflows": Array [], } } integration={ @@ -719,6 +720,7 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = "connectionTableName": "sample", "connectionType": "index", "displayName": "sample Integration", + "enabledWorkflows": Array [], } } integration={ @@ -1086,6 +1088,1243 @@ exports[`Integration Setup Page Renders the S3 connector form as expected 1`] = "name": "sample", "type": "logs", "version": "2.0.0", + "workflows": Array [ + Object { + "description": "This is a test workflow.", + "enabled_by_default": true, + "label": "Workflow 1", + "name": "workflow1", + }, + ], + } + } + setupCallout={ + Object { + "show": false, + } + } + updateConfig={[Function]} +> + +
+ +

+ Set Up Integration +

+ + +
+ + +
+ + +
+

+ Integration Details +

+
+ + +
+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+
+ + +
+
+ + +
+ + +
+

+ Integration Connection +

+
+ + +
+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+ + + + + +
+ +
+
+ + + +
+ Select the type of connection to use for queries. +
+ +
+
+ + +
+
+ + + +
+
+ +
+ + +
+
+
+ + + ss4o_logs-nginx-test + + + +
+ +
+
+ +
+ +
+ + + + + + + + + + + +
+ +
+
+ + +
+ + +
+ Select a data source to pull the data from. +
+ +
+
+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+
+ + + +
+ Must be at least 1 character. +
+ + +
+ Select a table name to associate with your data. +
+ +
+
+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+
+ + +
+
+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+
+ + + +
+ The Checkpoint location must be a unique directory and not the same as the Bucket location. It will be used for caching intermediary results. +
+ +
+
+ + +
+ + +
+

+ Installation Flows +

+
+ + +
+ + +
+
+ + + +
+
+ + + <_EuiSplitPanelOuter + className="euiCheckableCard euiCheckableCard-isChecked" + direction="row" + hasBorder={true} + responsive={false} + > + +
+ <_EuiSplitPanelInner + color="primary" + grow={false} + onClick={[Function]} + > + +
+ +
+ +
+
+ +
+ + + <_EuiSplitPanelInner> + +
+ +
+ This is a test workflow. +
+
+ + +
+ + + + + +
+ Select from the available asset types based on your use case. Choose at least one. +
+ +
+
+ +
+ + +`; + +exports[`Integration Setup Page Renders the S3 connector form without workflows 1`] = ` + { expect(wrapper).toMatchSnapshot(); }); }); + + it('Renders the S3 connector form without workflows', async () => { + const wrapper = mount( + {}} + integration={{ ...TEST_INTEGRATION_CONFIG, workflows: undefined }} + setupCallout={{ show: false }} + /> + ); + + await waitFor(() => { + expect(wrapper).toMatchSnapshot(); + }); + }); }); diff --git a/public/components/integrations/components/create_integration_helpers.ts b/public/components/integrations/components/create_integration_helpers.ts index b113c6df89..527b8fdb10 100644 --- a/public/components/integrations/components/create_integration_helpers.ts +++ b/public/components/integrations/components/create_integration_helpers.ts @@ -282,7 +282,8 @@ export async function addIntegrationRequest( integration: IntegrationConfig, setToast: (title: string, color?: Color, text?: string | undefined) => void, name?: string, - dataSource?: string + dataSource?: string, + workflows?: string[] ): Promise { const http = coreRefs.http!; if (addSample) { @@ -298,7 +299,7 @@ export async function addIntegrationRequest( let response: boolean = await http .post(`${INTEGRATIONS_BASE}/store/${templateName}`, { - body: JSON.stringify({ name, dataSource }), + body: JSON.stringify({ name, dataSource, workflows }), }) .then((res) => { setToast(`${name} integration successfully added!`, 'success'); diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 9871e1670a..01d6b52a0c 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -8,6 +8,7 @@ import { EuiButton, EuiButtonEmpty, EuiCallOut, + EuiCheckableCard, EuiComboBox, EuiEmptyPrompt, EuiFieldText, @@ -42,6 +43,7 @@ export interface IntegrationSetupInputs { connectionLocation: string; checkpointLocation: string; connectionTableName: string; + enabledWorkflows: string[]; } type SetupCallout = { show: true; title: string; color?: Color; text?: string } | { show: false }; @@ -182,6 +184,38 @@ const runQuery = async ( } }; +export function SetupWorkflowSelector({ + integration, + useWorkflows, + toggleWorkflow, +}: { + integration: IntegrationConfig; + useWorkflows: Map; + toggleWorkflow: (name: string) => void; +}) { + if (!integration.workflows) { + return null; + } + + const cards = integration.workflows.map((workflow) => { + return ( + toggleWorkflow(workflow.name)} + > + {workflow.description} + + ); + }); + + return cards; +} + export function SetupIntegrationForm({ config, updateConfig, @@ -197,6 +231,25 @@ export function SetupIntegrationForm({ const [isBucketBlurred, setIsBucketBlurred] = useState(false); const [isCheckpointBlurred, setIsCheckpointBlurred] = useState(false); + const [useWorkflows, setUseWorkflows] = useState(new Map()); + const toggleWorkflow = (name: string) => { + setUseWorkflows(new Map(useWorkflows.set(name, !useWorkflows.get(name)))); + }; + + useEffect(() => { + if (integration.workflows) { + setUseWorkflows(new Map(integration.workflows.map((w) => [w.name, w.enabled_by_default]))); + } + }, [integration.workflows]); + + useEffect(() => { + updateConfig({ + enabledWorkflows: [...useWorkflows.entries()].filter((w) => w[1]).map((w) => w[0]), + }); + // If we add the updateConfig dep here, rendering crashes with "Maximum update depth exceeded" + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [useWorkflows]); + useEffect(() => { const updateDataSources = async () => { const data = await suggestDataSources(config.connectionType); @@ -339,12 +392,47 @@ export function SetupIntegrationForm({ }} /> + {integration.workflows ? ( + <> + + +

Installation Flows

+ + + + + + + ) : null} ) : null} ); } +const prepareQuery = (query: string, config: IntegrationSetupInputs): string => { + let queryStr = query.replaceAll( + '{table_name}', + `${config.connectionDataSource}.default.${config.connectionTableName}` + ); + queryStr = queryStr.replaceAll('{s3_bucket_location}', config.connectionLocation); + queryStr = queryStr.replaceAll('{s3_checkpoint_location}', config.checkpointLocation); + queryStr = queryStr.replaceAll('{object_name}', config.connectionTableName); + queryStr = queryStr.replaceAll(/\s+/g, ' '); + return queryStr; +}; + const addIntegration = async ({ config, integration, @@ -375,22 +463,20 @@ const addIntegration = async ({ } else if (config.connectionType === 's3') { const http = coreRefs.http!; - const assets = await http.get(`${INTEGRATIONS_BASE}/repository/${integration.name}/assets`); + const assets: { data: ParsedIntegrationAsset[] } = await http.get( + `${INTEGRATIONS_BASE}/repository/${integration.name}/assets` + ); - // Queries must exist because we disable s3 if they're not present for (const query of assets.data.filter( - (a: ParsedIntegrationAsset): a is { type: 'query'; query: string; language: string } => + (a: ParsedIntegrationAsset): a is ParsedIntegrationAsset & { type: 'query' } => a.type === 'query' )) { - let queryStr = (query.query as string).replaceAll( - '{table_name}', - `${config.connectionDataSource}.default.${config.connectionTableName}` - ); + // Skip any queries that have conditional workflows but aren't enabled + if (query.workflows && !query.workflows.some((w) => config.enabledWorkflows.includes(w))) { + continue; + } - queryStr = queryStr.replaceAll('{s3_bucket_location}', config.connectionLocation); - queryStr = queryStr.replaceAll('{s3_checkpoint_location}', config.checkpointLocation); - queryStr = queryStr.replaceAll('{object_name}', config.connectionTableName); - queryStr = queryStr.replaceAll(/\s+/g, ' '); + const queryStr = prepareQuery(query.query, config); const result = await runQuery(queryStr, config.connectionDataSource, sessionId); if (!result.ok) { setLoading(false); @@ -400,7 +486,6 @@ const addIntegration = async ({ sessionId = result.value.sessionId ?? sessionId; } // Once everything is ready, add the integration to the new datasource as usual - // TODO determine actual values here after more about queries is known const res = await addIntegrationRequest( false, integration.name, @@ -408,7 +493,8 @@ const addIntegration = async ({ integration, setCalloutLikeToast, config.displayName, - `flint_${config.connectionDataSource}_default_${config.connectionTableName}_mview` + `flint_${config.connectionDataSource}_default_${config.connectionTableName}_mview`, + config.enabledWorkflows ); if (!res) { setLoading(false); @@ -418,11 +504,14 @@ const addIntegration = async ({ } }; -const isConfigValid = (config: IntegrationSetupInputs): boolean => { +const isConfigValid = (config: IntegrationSetupInputs, integration: IntegrationConfig): boolean => { if (config.displayName.length < 1 || config.connectionDataSource.length < 1) { return false; } if (config.connectionType === 's3') { + if (integration.workflows && config.enabledWorkflows.length < 1) { + return false; + } return ( config.connectionLocation.startsWith('s3://') && config.checkpointLocation.startsWith('s3://') ); @@ -477,7 +566,7 @@ export function SetupBottomBar({ iconType="arrowRight" iconSide="right" isLoading={loading} - disabled={!isConfigValid(config)} + disabled={!isConfigValid(config, integration)} onClick={async () => addIntegration({ integration, config, setLoading, setCalloutLikeToast }) } @@ -511,6 +600,7 @@ export function SetupIntegrationPage({ integration }: { integration: string }) { connectionLocation: '', checkpointLocation: '', connectionTableName: integration, + enabledWorkflows: [], }); const [template, setTemplate] = useState({ diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json index bff1d30de2..f116e45d63 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json @@ -8,6 +8,20 @@ "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info", + "workflows": [ + { + "name": "queries", + "label": "Queries (recommended)", + "description": "Tables and pre-written queries for quickly getting insights on your data.", + "enabled_by_default": true + }, + { + "name": "dashboards", + "label": "Dashboards & Visualizations", + "description": "Dashboards and indices that enable you to easily visualize important metrics.", + "enabled_by_default": false + } + ], "statics": { "logo": { "annotation": "ELB Logo", @@ -51,7 +65,8 @@ "name": "aws_elb", "version": "1.0.0", "extension": "ndjson", - "type": "savedObjectBundle" + "type": "savedObjectBundle", + "workflows": ["dashboards"] }, { "name": "create_table", @@ -63,7 +78,8 @@ "name": "create_mv", "version": "1.0.0", "extension": "sql", - "type": "query" + "type": "query", + "workflows": ["dashboards"] } ], "sampleData": { diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json index a445c626ba..11f5132931 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json @@ -8,6 +8,20 @@ "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"], "author": "Haidong Wang", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info", + "workflows": [ + { + "name": "queries", + "label": "Queries (recommended)", + "description": "Tables and pre-written queries for quickly getting insights on your data.", + "enabled_by_default": true + }, + { + "name": "dashboards", + "label": "Dashboards & Visualizations", + "description": "Dashboards and indices that enable you to easily visualize important metrics.", + "enabled_by_default": false + } + ], "statics": { "logo": { "annotation": "AWS VPC Logo", @@ -47,7 +61,8 @@ "name": "aws_vpc_flow", "version": "1.0.0", "extension": "ndjson", - "type": "savedObjectBundle" + "type": "savedObjectBundle", + "workflows": ["dashboards"] }, { "name": "create_table_vpc", @@ -59,7 +74,8 @@ "name": "create_mv_vpc", "version": "1.0.0", "extension": "sql", - "type": "query" + "type": "query", + "workflows": ["dashboards"] } ], "sampleData": { diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json index ecc7cdc7b4..e04928f148 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json @@ -8,6 +8,20 @@ "labels": ["Observability", "Logs", "Flint S3"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info", + "workflows": [ + { + "name": "queries", + "label": "Queries (recommended)", + "description": "Tables and pre-written queries for quickly getting insights on your data.", + "enabled_by_default": true + }, + { + "name": "dashboards", + "label": "Dashboards & Visualizations", + "description": "Dashboards and indices that enable you to easily visualize important metrics.", + "enabled_by_default": false + } + ], "statics": { "logo": { "annotation": "NginX Logo", @@ -43,7 +57,8 @@ "name": "nginx", "version": "1.0.0", "extension": "ndjson", - "type": "savedObjectBundle" + "type": "savedObjectBundle", + "workflows": ["dashboards"] }, { "name": "create_table", @@ -55,7 +70,8 @@ "name": "create_mv", "version": "1.0.0", "extension": "sql", - "type": "query" + "type": "query", + "workflows": ["dashboards"] } ], "sampleData": { diff --git a/server/adaptors/integrations/__test__/builder.test.ts b/server/adaptors/integrations/__test__/builder.test.ts index f27f097ef9..1779c096b1 100644 --- a/server/adaptors/integrations/__test__/builder.test.ts +++ b/server/adaptors/integrations/__test__/builder.test.ts @@ -343,3 +343,74 @@ describe('IntegrationInstanceBuilder', () => { }); }); }); + +describe('getSavedObjectBundles', () => { + let builder: IntegrationInstanceBuilder; + + beforeEach(() => { + builder = new IntegrationInstanceBuilder(mockSavedObjectsClient); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should filter assets correctly without workflows and includeWorkflows', () => { + const assets = [ + { + type: 'savedObjectBundle' as const, + data: [{ id: '1', type: 'type1', attributes: { title: 'Title 1' } }], + }, + { + type: 'savedObjectBundle' as const, + data: [{ id: '2', type: 'type2', attributes: { title: 'Title 2' } }], + }, + { type: 'query' as const, query: 'query', language: 'language' }, + ]; + const result = builder.getSavedObjectBundles(assets); + expect(result.length).toBe(2); + }); + + it('should filter assets correctly with specified workflows', () => { + const assets = [ + { + type: 'savedObjectBundle' as const, + workflows: ['workflow1'], + data: [{ id: '1', type: 'type1', attributes: { title: 'Title 1' } }], + }, + { + type: 'savedObjectBundle' as const, + workflows: ['workflow2'], + data: [{ id: '2', type: 'type2', attributes: { title: 'Title 2' } }], + }, + { type: 'query' as const, query: 'query', language: 'language' }, + ]; + const result = builder.getSavedObjectBundles(assets, ['workflow1']); + expect(result.length).toBe(1); + expect(result[0].id).toBe('1'); + }); + + it('should filter assets correctly with no matching workflows', () => { + const assets = [ + { + type: 'savedObjectBundle' as const, + workflows: ['workflow1'], + data: [{ id: '1', type: 'type1', attributes: { title: 'Title 1' } }], + }, + { + type: 'savedObjectBundle' as const, + workflows: ['workflow2'], + data: [{ id: '2', type: 'type2', attributes: { title: 'Title 2' } }], + }, + { type: 'query' as const, query: 'query', language: 'language' }, + ]; + const result = builder.getSavedObjectBundles(assets, ['workflow3']); + expect(result.length).toBe(0); + }); + + it('should return an empty array if no savedObjectBundle assets are present', () => { + const assets = [{ type: 'query' as const, query: 'query', language: 'language' }]; + const result = builder.getSavedObjectBundles(assets); + expect(result.length).toBe(0); + }); +}); diff --git a/server/adaptors/integrations/integrations_adaptor.ts b/server/adaptors/integrations/integrations_adaptor.ts index 108e997b23..4b329a37c0 100644 --- a/server/adaptors/integrations/integrations_adaptor.ts +++ b/server/adaptors/integrations/integrations_adaptor.ts @@ -17,7 +17,8 @@ export interface IntegrationsAdaptor { loadIntegrationInstance: ( templateName: string, name: string, - dataSource: string + dataSource: string, + workflows?: string[] ) => Promise; deleteIntegrationInstance: (id: string) => Promise; diff --git a/server/adaptors/integrations/integrations_builder.ts b/server/adaptors/integrations/integrations_builder.ts index 36fc8f1848..fab905d1bd 100644 --- a/server/adaptors/integrations/integrations_builder.ts +++ b/server/adaptors/integrations/integrations_builder.ts @@ -12,6 +12,7 @@ import { deepCheck } from './repository/utils'; interface BuilderOptions { name: string; dataSource: string; + workflows?: string[]; } interface SavedObject { @@ -28,32 +29,44 @@ export class IntegrationInstanceBuilder { this.client = client; } - build(integration: IntegrationReader, options: BuilderOptions): Promise { - const instance = deepCheck(integration) - .then((result) => { - if (!result.ok) { - return Promise.reject(result.error); + async build( + integration: IntegrationReader, + options: BuilderOptions + ): Promise { + const instance = await deepCheck(integration); + if (!instance.ok) { + return Promise.reject(instance.error); + } + const assets = await integration.getAssets(); + if (!assets.ok) { + return Promise.reject(assets.error); + } + const remapped = this.remapIDs(this.getSavedObjectBundles(assets.value)); + const withDataSource = this.remapDataSource(remapped, options.dataSource); + const refs = await this.postAssets(withDataSource); + const builtInstance = await this.buildInstance(integration, refs, options); + return builtInstance; + } + + getSavedObjectBundles( + assets: ParsedIntegrationAsset[], + includeWorkflows?: string[] + ): SavedObject[] { + return assets + .filter((asset) => { + // At this stage we only care about installing bundles + if (asset.type !== 'savedObjectBundle') { + return false; } - return integration.getAssets(); - }) - .then((assets) => { - if (!assets.ok) { - return Promise.reject(assets.error); + // If no workflows present: default to all workflows + // Otherwise only install if workflow is present + if (!asset.workflows || !includeWorkflows) { + return true; } - return assets.value; + return includeWorkflows.some((w) => asset.workflows?.includes(w)); }) - .then((assets) => - this.remapIDs( - assets - .filter((asset) => asset.type === 'savedObjectBundle') - .map((asset) => (asset as { type: 'savedObjectBundle'; data: object[] }).data) - .flat() as SavedObject[] - ) - ) - .then((assets) => this.remapDataSource(assets, options.dataSource)) - .then((assets) => this.postAssets(assets)) - .then((refs) => this.buildInstance(integration, refs, options)); - return instance; + .map((asset) => (asset as { type: 'savedObjectBundle'; data: object[] }).data) + .flat() as SavedObject[]; } remapDataSource( diff --git a/server/adaptors/integrations/integrations_manager.ts b/server/adaptors/integrations/integrations_manager.ts index 256aa40c8e..c64f761be1 100644 --- a/server/adaptors/integrations/integrations_manager.ts +++ b/server/adaptors/integrations/integrations_manager.ts @@ -157,7 +157,8 @@ export class IntegrationsManager implements IntegrationsAdaptor { loadIntegrationInstance = async ( templateName: string, name: string, - dataSource: string + dataSource: string, + workflows?: string[] ): Promise => { const template = await this.repository.getIntegration(templateName); if (template === null) { @@ -171,6 +172,7 @@ export class IntegrationsManager implements IntegrationsAdaptor { const result = await this.instanceBuilder.build(template, { name, dataSource, + workflows, }); const test = await this.client.create('integration-instance', result); return Promise.resolve({ ...result, id: test.id }); diff --git a/server/adaptors/integrations/repository/integration_reader.ts b/server/adaptors/integrations/repository/integration_reader.ts index 9a09dc8632..98567c01ff 100644 --- a/server/adaptors/integrations/repository/integration_reader.ts +++ b/server/adaptors/integrations/repository/integration_reader.ts @@ -202,12 +202,14 @@ export class IntegrationReader { case 'savedObjectBundle': resultValue.push({ type: 'savedObjectBundle', + workflows: asset.workflows, data: JSON.parse(serializedResult.value.data), }); break; case 'query': resultValue.push({ type: 'query', + workflows: asset.workflows, query: serializedResult.value.data, language: asset.extension, }); diff --git a/server/adaptors/integrations/types.ts b/server/adaptors/integrations/types.ts index ed458b64a2..b6bf933631 100644 --- a/server/adaptors/integrations/types.ts +++ b/server/adaptors/integrations/types.ts @@ -19,6 +19,7 @@ interface IntegrationConfig { author?: string; description?: string; sourceUrl?: string; + workflows?: IntegrationWorkflow[]; statics?: IntegrationStatics; components: IntegrationComponent[]; assets: IntegrationAsset[]; @@ -57,11 +58,19 @@ interface IntegrationAsset { version: string; extension: string; type: SupportedAssetType; + workflows?: string[]; +} + +interface IntegrationWorkflow { + name: string; + label: string; + description: string; + enabled_by_default: boolean; } type ParsedIntegrationAsset = - | { type: 'savedObjectBundle'; data: object[] } - | { type: 'query'; query: string; language: string }; + | { type: 'savedObjectBundle'; workflows?: string[]; data: object[] } + | { type: 'query'; workflows?: string[]; query: string; language: string }; interface SerializedIntegrationAsset extends IntegrationAsset { data: string; diff --git a/server/adaptors/integrations/validators.ts b/server/adaptors/integrations/validators.ts index 73fb8800c9..497e380d10 100644 --- a/server/adaptors/integrations/validators.ts +++ b/server/adaptors/integrations/validators.ts @@ -31,6 +31,20 @@ const templateSchema: JSONSchemaType = { author: { type: 'string', nullable: true }, description: { type: 'string', nullable: true }, sourceUrl: { type: 'string', nullable: true }, + workflows: { + type: 'array', + items: { + type: 'object', + properties: { + name: { type: 'string' }, + label: { type: 'string' }, + description: { type: 'string' }, + enabled_by_default: { type: 'boolean' }, + }, + required: ['name', 'label', 'description', 'enabled_by_default'], + }, + nullable: true, + }, statics: { type: 'object', properties: { @@ -64,6 +78,11 @@ const templateSchema: JSONSchemaType = { extension: { type: 'string' }, type: { type: 'string' }, data: { type: 'string', nullable: true }, + workflows: { + type: 'array', + items: { type: 'string' }, + nullable: true, + }, }, required: ['name', 'version', 'extension', 'type'], additionalProperties: false, diff --git a/server/routes/integrations/integrations_router.ts b/server/routes/integrations/integrations_router.ts index fba05b7b04..ca30fa0062 100644 --- a/server/routes/integrations/integrations_router.ts +++ b/server/routes/integrations/integrations_router.ts @@ -82,6 +82,7 @@ export function registerIntegrationsRoute(router: IRouter) { body: schema.object({ name: schema.string(), dataSource: schema.string(), + workflows: schema.maybe(schema.arrayOf(schema.string())), }), }, }, @@ -91,7 +92,8 @@ export function registerIntegrationsRoute(router: IRouter) { return a.loadIntegrationInstance( request.params.templateName, request.body.name, - request.body.dataSource + request.body.dataSource, + request.body.workflows ); }); } diff --git a/test/constants.ts b/test/constants.ts index c0eece3fe6..178b65506c 100644 --- a/test/constants.ts +++ b/test/constants.ts @@ -574,6 +574,14 @@ export const TEST_INTEGRATION_CONFIG: IntegrationConfig = { version: '2.0.0', license: 'Apache-2.0', type: 'logs', + workflows: [ + { + name: 'workflow1', + label: 'Workflow 1', + description: 'This is a test workflow.', + enabled_by_default: true, + }, + ], components: [ { name: 'logs',