diff --git a/common/constants.ts b/common/constants.ts index 107227af..2d0bf50d 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -34,6 +34,7 @@ export const BASE_NODE_API_PATH = '/api/flow_framework'; // OpenSearch node APIs export const BASE_OPENSEARCH_NODE_API_PATH = `${BASE_NODE_API_PATH}/opensearch`; export const CAT_INDICES_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/catIndices`; +export const GET_MAPPINGS_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/mappings`; export const SEARCH_INDEX_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/search`; export const INGEST_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/ingest`; export const BULK_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/bulk`; diff --git a/common/interfaces.ts b/common/interfaces.ts index f0cb927b..2d46da26 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -238,12 +238,12 @@ export type NormalizationProcessor = SearchProcessor & { }; export type IndexConfiguration = { - settings: {}; + settings: { [key: string]: any }; mappings: IndexMappings; }; export type IndexMappings = { - properties: {}; + properties: { [key: string]: any }; }; export type TemplateNode = { diff --git a/common/utils.ts b/common/utils.ts index 2992ee71..dc5b780f 100644 --- a/common/utils.ts +++ b/common/utils.ts @@ -4,7 +4,7 @@ */ import moment from 'moment'; -import { DATE_FORMAT_PATTERN } from './'; +import { DATE_FORMAT_PATTERN, WORKFLOW_TYPE, Workflow } from './'; import { isEmpty } from 'lodash'; export function toFormattedDate(timestampMillis: number): String { @@ -39,3 +39,14 @@ export function getCharacterLimitedString( export function customStringify(jsonObj: {}): string { return JSON.stringify(jsonObj, undefined, 2); } + +export function isVectorSearchUseCase(workflow: Workflow | undefined): boolean { + return ( + workflow?.ui_metadata?.type !== undefined && + [ + WORKFLOW_TYPE.HYBRID_SEARCH, + WORKFLOW_TYPE.MULTIMODAL_SEARCH, + WORKFLOW_TYPE.SEMANTIC_SEARCH, + ].includes(workflow?.ui_metadata?.type) + ); +} diff --git a/public/pages/workflow_detail/resizable_workspace.tsx b/public/pages/workflow_detail/resizable_workspace.tsx index 8c089c06..5f68e2f7 100644 --- a/public/pages/workflow_detail/resizable_workspace.tsx +++ b/public/pages/workflow_detail/resizable_workspace.tsx @@ -4,7 +4,6 @@ */ import React, { useRef, useState, useEffect } from 'react'; -import { useSelector } from 'react-redux'; import { Form, Formik } from 'formik'; import * as yup from 'yup'; import { @@ -21,6 +20,7 @@ import { WorkflowConfig, WorkflowFormValues, WorkflowSchema, + customStringify, } from '../../../common'; import { isValidUiWorkflow, @@ -286,11 +286,7 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { - {JSON.stringify( - reduceToTemplate(props.workflow as Workflow), - undefined, - 2 - )} + {customStringify(reduceToTemplate(props.workflow as Workflow))} diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx index 5f1cd16b..a0ec10df 100644 --- a/public/pages/workflow_detail/workflow_detail.tsx +++ b/public/pages/workflow_detail/workflow_detail.tsx @@ -21,6 +21,7 @@ import { getCore } from '../../services'; import { WorkflowDetailHeader } from './components'; import { AppState, + catIndices, getWorkflow, searchModels, useAppDispatch, @@ -101,9 +102,11 @@ export function WorkflowDetail(props: WorkflowDetailProps) { // On initial load: // - fetch workflow // - fetch available models as their IDs may be used when building flows + // - fetch all indices useEffect(() => { dispatch(getWorkflow({ workflowId, dataSourceId })); dispatch(searchModels({ apiBody: FETCH_ALL_QUERY, dataSourceId })); + dispatch(catIndices({ pattern: '*,-.*', dataSourceId })); }, []); return errorMessage.includes(ERROR_GETTING_WORKFLOW_MSG) || diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx index 6b3c7f0d..527def78 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx @@ -8,12 +8,13 @@ import { EuiFlexGroup, EuiFlexItem, EuiHorizontalRule } from '@elastic/eui'; import { SourceData } from './source_data'; import { EnrichData } from './enrich_data'; import { IngestData } from './ingest_data'; -import { WorkflowConfig } from '../../../../../common'; +import { Workflow, WorkflowConfig } from '../../../../../common'; interface IngestInputsProps { setIngestDocs: (docs: string) => void; uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; + workflow: Workflow | undefined; } /** @@ -23,7 +24,11 @@ export function IngestInputs(props: IngestInputsProps) { return ( - + diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx index 941894b6..d8447804 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx @@ -4,7 +4,8 @@ */ import React, { useEffect, useState } from 'react'; -import { useFormikContext } from 'formik'; +import { useSelector } from 'react-redux'; +import { getIn, useFormikContext } from 'formik'; import { EuiSmallButton, EuiCompressedFilePicker, @@ -18,19 +19,56 @@ import { EuiSpacer, EuiText, EuiTitle, + EuiFilterGroup, + EuiSmallFilterButton, + EuiSuperSelectOption, + EuiCompressedSuperSelect, } from '@elastic/eui'; import { JsonField } from '../input_fields'; -import { WorkspaceFormValues } from '../../../../../common'; +import { + FETCH_ALL_QUERY, + IndexMappings, + MapEntry, + SearchHit, + Workflow, + WorkflowConfig, + WorkspaceFormValues, + customStringify, + isVectorSearchUseCase, +} from '../../../../../common'; +import { + AppState, + getMappings, + searchIndex, + useAppDispatch, +} from '../../../../store'; +import { getDataSourceId } from '../../../../utils'; interface SourceDataProps { + workflow: Workflow | undefined; + uiConfig: WorkflowConfig; setIngestDocs: (docs: string) => void; } +enum SOURCE_OPTIONS { + MANUAL = 'manual', + UPLOAD = 'upload', + EXISTING_INDEX = 'existing_index', +} + /** * Input component for configuring the source data for ingest. */ export function SourceData(props: SourceDataProps) { + const dispatch = useAppDispatch(); + const dataSourceId = getDataSourceId(); const { values, setFieldValue } = useFormikContext(); + const indices = useSelector((state: AppState) => state.opensearch.indices); + + // selected option state + const [selectedOption, setSelectedOption] = useState( + SOURCE_OPTIONS.MANUAL + ); // edit modal state const [isEditModalOpen, setIsEditModalOpen] = useState(false); @@ -43,12 +81,98 @@ export function SourceData(props: SourceDataProps) { } }; - // Hook to listen when the docs form value changes. - // Try to set the ingestDocs if possible + // selected index state. when an index is selected, update several form values (if vector search) + const [selectedIndex, setSelectedIndex] = useState( + undefined + ); + useEffect(() => { + if (selectedIndex !== undefined) { + // 1. fetch and set sample docs + dispatch( + searchIndex({ + apiBody: { + index: selectedIndex, + body: FETCH_ALL_QUERY, + searchPipeline: '_none', + }, + dataSourceId, + }) + ) + .unwrap() + .then((resp) => { + const docObjs = resp.hits?.hits + ?.slice(0, 5) + ?.map((hit: SearchHit) => hit?._source); + setFieldValue('ingest.docs', customStringify(docObjs)); + }); + + // 2. fetch index mappings, and try to set defaults for the ML processor configs, if applicable + if (isVectorSearchUseCase(props.workflow)) { + dispatch(getMappings({ index: selectedIndex, dataSourceId })) + .unwrap() + .then((resp: IndexMappings) => { + const { processorId, inputMapEntry } = getProcessorInfo( + props.uiConfig, + values + ); + if (processorId !== undefined && inputMapEntry !== undefined) { + // set/overwrite default text field for the input map. may be empty. + if (inputMapEntry !== undefined) { + const textFieldFormPath = `ingest.enrich.${processorId}.input_map.0.0.value`; + const curTextField = getIn(values, textFieldFormPath) as string; + if (!Object.keys(resp.properties).includes(curTextField)) { + const defaultTextField = + Object.keys(resp.properties).find((fieldName) => { + return resp.properties[fieldName]?.type === 'text'; + }) || ''; + setFieldValue(textFieldFormPath, defaultTextField); + } + } + } + }); + } + } + }, [selectedIndex]); + + // hook to clear out the selected index when switching options + useEffect(() => { + if (selectedOption !== SOURCE_OPTIONS.EXISTING_INDEX) { + setSelectedIndex(undefined); + } + }, [selectedOption]); + + // hook to listen when the docs form value changes. useEffect(() => { if (values?.ingest?.docs) { props.setIngestDocs(values.ingest.docs); } + + // try to clear out any default values for the ML ingest processor, if applicable + if ( + isVectorSearchUseCase(props.workflow) && + isEditModalOpen && + selectedOption !== SOURCE_OPTIONS.EXISTING_INDEX + ) { + let sampleDoc = undefined as {} | undefined; + try { + sampleDoc = JSON.parse(values.ingest.docs)[0]; + } catch (error) {} + if (sampleDoc !== undefined) { + const { processorId, inputMapEntry } = getProcessorInfo( + props.uiConfig, + values + ); + if (processorId !== undefined && inputMapEntry !== undefined) { + if (inputMapEntry !== undefined) { + const textFieldFormPath = `ingest.enrich.${processorId}.input_map.0.0.value`; + const curTextField = getIn(values, textFieldFormPath) as string; + if (!Object.keys(sampleDoc).includes(curTextField)) { + setFieldValue(textFieldFormPath, ''); + } + } + } + } + } }, [values?.ingest?.docs]); return ( @@ -65,22 +189,74 @@ export function SourceData(props: SourceDataProps) { <> - - Upload a JSON file or enter manually. - {' '} - - { - if (files && files.length > 0) { - fileReader.readAsText(files[0]); + + setSelectedOption(SOURCE_OPTIONS.MANUAL)} + > + Manual + + setSelectedOption(SOURCE_OPTIONS.UPLOAD)} + > + Upload + + - + onClick={() => + setSelectedOption(SOURCE_OPTIONS.EXISTING_INDEX) + } + > + Existing index + + + + {selectedOption === SOURCE_OPTIONS.UPLOAD && ( + <> + { + if (files && files.length > 0) { + fileReader.readAsText(files[0]); + } + }} + display="default" + /> + + + )} + {selectedOption === SOURCE_OPTIONS.EXISTING_INDEX && ( + <> + + Up to 5 sample documents will be automatically populated. + + + + ({ + value: option.name, + inputDisplay: {option.name}, + disabled: false, + } as EuiSuperSelectOption) + )} + valueOfSelected={selectedIndex} + onChange={(option) => { + setSelectedIndex(option); + }} + isInvalid={false} + /> + + + )} ); } + +// helper fn to parse out some useful info from the ML ingest processor config, if applicable +// takes on the assumption the first processor is an ML inference processor, and should +// only be executed for workflows coming from preset vector search use cases. +function getProcessorInfo( + uiConfig: WorkflowConfig, + values: WorkspaceFormValues +): { + processorId: string | undefined; + inputMapEntry: MapEntry | undefined; +} { + const ingestProcessorId = uiConfig.ingest.enrich.processors[0]?.id as + | string + | undefined; + return { + processorId: ingestProcessorId, + inputMapEntry: + (getIn( + values, + `ingest.enrich.${ingestProcessorId}.input_map.0.0`, + undefined + ) as MapEntry) || undefined, + }; +} diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/select_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/select_field.tsx index 5a4e7bbf..3146a944 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/select_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/select_field.tsx @@ -7,7 +7,7 @@ import React from 'react'; import { Field, FieldProps, getIn, useFormikContext } from 'formik'; import { EuiCompressedFormRow, - EuiSuperSelect, + EuiCompressedSuperSelect, EuiSuperSelectOption, EuiText, } from '@elastic/eui'; @@ -31,7 +31,7 @@ export function SelectField(props: SelectFieldProps) { {({ field, form }: FieldProps) => { return ( - { setSourceInput( - JSON.stringify( + customStringify( resp.hits.hits.map( (hit: SearchHit) => hit._source - ), - undefined, - 2 + ) ) ); }) diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx index 6e84bb82..393459c1 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx @@ -210,12 +210,10 @@ export function OutputTransformModal(props: OutputTransformModalProps) { .unwrap() .then(async (resp) => { setSourceInput( - JSON.stringify( + customStringify( resp.hits.hits.map( (hit: SearchHit) => hit._source - ), - undefined, - 2 + ) ) ); }) diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx index be239bfc..06772f77 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx @@ -12,20 +12,19 @@ import { EuiFlexGroup, EuiFlexItem, EuiCompressedFormRow, - EuiSuperSelect, + EuiCompressedSuperSelect, EuiSuperSelectOption, EuiText, EuiTitle, EuiSpacer, } from '@elastic/eui'; -import { SearchHit, WorkflowFormValues } from '../../../../../common'; -import { JsonField } from '../input_fields'; import { - AppState, - catIndices, - searchIndex, - useAppDispatch, -} from '../../../../store'; + SearchHit, + WorkflowFormValues, + customStringify, +} from '../../../../../common'; +import { JsonField } from '../input_fields'; +import { AppState, searchIndex, useAppDispatch } from '../../../../store'; import { getDataSourceId } from '../../../../utils/utils'; import { EditQueryModal } from './edit_query_modal'; @@ -74,14 +73,6 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) { } }, [values?.search?.request]); - // Initialization hook to fetch available indices (if applicable) - useEffect(() => { - if (!ingestEnabled) { - // Fetch all indices besides system indices - dispatch(catIndices({ pattern: '*,-.*', dataSourceId })); - } - }, []); - return ( <> {isEditModalOpen && ( @@ -104,7 +95,7 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) { readOnly={true} /> ) : ( - ({ @@ -162,10 +153,8 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) { .unwrap() .then(async (resp) => { props.setQueryResponse( - JSON.stringify( - resp.hits.hits.map((hit: SearchHit) => hit._source), - undefined, - 2 + customStringify( + resp.hits.hits.map((hit: SearchHit) => hit._source) ) ); }) diff --git a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx index e392ef41..952ab5a7 100644 --- a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx @@ -534,10 +534,8 @@ export function WorkflowInputs(props: WorkflowInputsProps) { .unwrap() .then(async (resp) => { props.setQueryResponse( - JSON.stringify( - resp.hits.hits.map((hit: SearchHit) => hit._source), - undefined, - 2 + customStringify( + resp.hits.hits.map((hit: SearchHit) => hit._source) ) ); }) @@ -727,6 +725,7 @@ export function WorkflowInputs(props: WorkflowInputsProps) { setIngestDocs={props.setIngestDocs} uiConfig={props.uiConfig} setUiConfig={props.setUiConfig} + workflow={props.workflow} /> ) : ( Promise; + getMappings: ( + index: string, + dataSourceId?: string + ) => Promise; searchIndex: ({ index, body, @@ -270,6 +275,19 @@ export function configureRoutes(core: CoreStart): RouteService { return e as HttpFetchError; } }, + getMappings: async (index: string, dataSourceId?: string) => { + try { + const url = dataSourceId + ? `${BASE_NODE_API_PATH}/${dataSourceId}/opensearch/mappings` + : GET_MAPPINGS_NODE_API_PATH; + const response = await core.http.get<{ respString: string }>( + `${url}/${index}` + ); + return response; + } catch (e: any) { + return e as HttpFetchError; + } + }, searchIndex: async ({ index, body, diff --git a/public/store/reducers/opensearch_reducer.ts b/public/store/reducers/opensearch_reducer.ts index ddffae86..d40e9396 100644 --- a/public/store/reducers/opensearch_reducer.ts +++ b/public/store/reducers/opensearch_reducer.ts @@ -20,6 +20,7 @@ const initialState = { const OPENSEARCH_PREFIX = 'opensearch'; const CAT_INDICES_ACTION = `${OPENSEARCH_PREFIX}/catIndices`; +const GET_MAPPINGS_ACTION = `${OPENSEARCH_PREFIX}/mappings`; const SEARCH_INDEX_ACTION = `${OPENSEARCH_PREFIX}/search`; const INGEST_ACTION = `${OPENSEARCH_PREFIX}/ingest`; const BULK_ACTION = `${OPENSEARCH_PREFIX}/bulk`; @@ -47,6 +48,26 @@ export const catIndices = createAsyncThunk( } ); +export const getMappings = createAsyncThunk( + GET_MAPPINGS_ACTION, + async ( + { index, dataSourceId }: { index: string; dataSourceId?: string }, + { rejectWithValue } + ) => { + const response: any | HttpFetchError = await getRouteService().getMappings( + index, + dataSourceId + ); + if (response instanceof HttpFetchError) { + return rejectWithValue( + 'Error getting index mappings: ' + response.body.message + ); + } else { + return response; + } + } +); + export const searchIndex = createAsyncThunk( SEARCH_INDEX_ACTION, async ( @@ -169,6 +190,10 @@ const opensearchSlice = createSlice({ state.loading = true; state.errorMessage = ''; }) + .addCase(getMappings.pending, (state, action) => { + state.loading = true; + state.errorMessage = ''; + }) .addCase(searchIndex.pending, (state, action) => { state.loading = true; state.errorMessage = ''; @@ -186,6 +211,10 @@ const opensearchSlice = createSlice({ state.loading = false; state.errorMessage = ''; }) + .addCase(getMappings.fulfilled, (state, action) => { + state.loading = false; + state.errorMessage = ''; + }) .addCase(searchIndex.fulfilled, (state, action) => { state.loading = false; state.errorMessage = ''; @@ -198,6 +227,10 @@ const opensearchSlice = createSlice({ state.errorMessage = action.payload as string; state.loading = false; }) + .addCase(getMappings.rejected, (state, action) => { + state.errorMessage = action.payload as string; + state.loading = false; + }) .addCase(searchIndex.rejected, (state, action) => { state.errorMessage = action.payload as string; state.loading = false; diff --git a/server/routes/opensearch_routes_service.ts b/server/routes/opensearch_routes_service.ts index dfcd61af..4f9f7c04 100644 --- a/server/routes/opensearch_routes_service.ts +++ b/server/routes/opensearch_routes_service.ts @@ -15,8 +15,10 @@ import { BASE_NODE_API_PATH, BULK_NODE_API_PATH, CAT_INDICES_NODE_API_PATH, + GET_MAPPINGS_NODE_API_PATH, INGEST_NODE_API_PATH, Index, + IndexMappings, IngestPipelineConfig, SEARCH_INDEX_NODE_API_PATH, SIMULATE_PIPELINE_NODE_API_PATH, @@ -57,6 +59,29 @@ export function registerOpenSearchRoutes( }, opensearchRoutesService.catIndices ); + router.get( + { + path: `${GET_MAPPINGS_NODE_API_PATH}/{index}`, + validate: { + params: schema.object({ + index: schema.string(), + }), + }, + }, + opensearchRoutesService.getMappings + ); + router.get( + { + path: `${BASE_NODE_API_PATH}/{data_source_id}/opensearch/mappings/{index}`, + validate: { + params: schema.object({ + index: schema.string(), + data_source_id: schema.string(), + }), + }, + }, + opensearchRoutesService.getMappings + ); router.post( { path: `${SEARCH_INDEX_NODE_API_PATH}/{index}`, @@ -252,6 +277,37 @@ export class OpenSearchRoutesService { } }; + getMappings = async ( + context: RequestHandlerContext, + req: OpenSearchDashboardsRequest, + res: OpenSearchDashboardsResponseFactory + ): Promise> => { + const { index } = req.params as { index: string }; + const { data_source_id = '' } = req.params as { data_source_id?: string }; + try { + const callWithRequest = getClientBasedOnDataSource( + context, + this.dataSourceEnabled, + req, + data_source_id, + this.client + ); + + const response = await callWithRequest('indices.getMapping', { + index, + }); + + // Response will be a dict with key being the index name. Attempt to + // pull out the mappings. If any errors found (missing index, etc.), an error + // will be thrown. + const mappings = response[index]?.mappings as IndexMappings; + + return res.ok({ body: mappings }); + } catch (err: any) { + return generateCustomError(res, err); + } + }; + searchIndex = async ( context: RequestHandlerContext, req: OpenSearchDashboardsRequest,