From 0f565a62a95f012c126a50ba1970c884b50c0ee3 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 3 Sep 2024 18:52:20 -0700 Subject: [PATCH] Improve and simplify processor form configs (#339) Signed-off-by: Tyler Ohlsen --- common/constants.ts | 1 + .../pages/workflow_detail/workflow_detail.tsx | 4 +- .../input_fields/map_field.tsx | 10 +- .../select_with_custom_options.tsx | 19 +-- .../input_transform_modal.tsx | 33 ++-- .../processor_inputs/ml_processor_inputs.tsx | 29 ++-- .../output_transform_modal.tsx | 31 ++-- .../workflow_inputs/processors_list.tsx | 74 +++++---- .../new_workflow/quick_configure_modal.tsx | 154 +++++++++++++++--- public/utils/utils.ts | 4 +- 10 files changed, 230 insertions(+), 129 deletions(-) diff --git a/common/constants.ts b/common/constants.ts index 7027e3ce..107227af 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -396,6 +396,7 @@ export const DATE_FORMAT_PATTERN = 'MM/DD/YY hh:mm A'; export const EMPTY_FIELD_STRING = '--'; export const INDEX_NOT_FOUND_EXCEPTION = 'index_not_found_exception'; export const ERROR_GETTING_WORKFLOW_MSG = 'Failed to retrieve template'; +export const NO_TEMPLATES_FOUND_MSG = 'There are no templates'; export const NO_MODIFICATIONS_FOUND_TEXT = 'Template does not contain any modifications'; export const JSONPATH_ROOT_SELECTOR = '$.'; diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx index 0008025c..5f1cd16b 100644 --- a/public/pages/workflow_detail/workflow_detail.tsx +++ b/public/pages/workflow_detail/workflow_detail.tsx @@ -30,6 +30,7 @@ import { ERROR_GETTING_WORKFLOW_MSG, FETCH_ALL_QUERY, MAX_WORKFLOW_NAME_TO_DISPLAY, + NO_TEMPLATES_FOUND_MSG, getCharacterLimitedString, } from '../../../common'; import { MountPoint } from '../../../../../src/core/public'; @@ -105,7 +106,8 @@ export function WorkflowDetail(props: WorkflowDetailProps) { dispatch(searchModels({ apiBody: FETCH_ALL_QUERY, dataSourceId })); }, []); - return errorMessage.includes(ERROR_GETTING_WORKFLOW_MSG) ? ( + return errorMessage.includes(ERROR_GETTING_WORKFLOW_MSG) || + errorMessage.includes(NO_TEMPLATES_FOUND_MSG) ? ( ) : ( @@ -137,10 +134,6 @@ export function MapField(props: MapFieldProps) { placeholder={ props.valuePlaceholder || 'Output' } - autofill={ - props.valueOptions?.length === 1 && - idx === 0 - } /> ) : ( ([]); - // update the selected option when the form is updated. if the form is empty, - // default to the top option. by default, this will re-trigger this hook with a populated - // value, to then finally update the displayed option. + // set the visible option when the underlying form is updated. useEffect(() => { - if (props.autofill) { - const formValue = getIn(values, props.fieldPath); - if (!isEmpty(formValue)) { - setSelectedOption([{ label: getIn(values, props.fieldPath) }]); - } else { - if (props.options.length > 0) { - setFieldValue(props.fieldPath, props.options[0].label); - } - } + const formValue = getIn(values, props.fieldPath); + if (!isEmpty(formValue)) { + setSelectedOption([{ label: formValue }]); } }, [getIn(values, props.fieldPath)]); @@ -73,7 +64,7 @@ export function SelectWithCustomOptions(props: SelectWithCustomOptionsProps) { return ( void; } @@ -88,6 +89,10 @@ export function InputTransformModal(props: InputTransformModalProps) { number | undefined >((outputOptions[0]?.value as number) ?? undefined); + // TODO: integrated with Ajv to fetch any model interface and perform validation + // on the produced output on-the-fly. For examples, see + // https://www.npmjs.com/package/ajv + return ( @@ -254,7 +259,7 @@ export function InputTransformModal(props: InputTransformModalProps) { ? 'Query field' : 'Document field' } - keyOptions={props.inputFields} + keyOptions={parseModelInputs(props.modelInterface)} // If the map we are adding is the first one, populate the selected option to index 0 onMapAdd={(curArray) => { if (isEmpty(curArray)) { @@ -274,15 +279,19 @@ export function InputTransformModal(props: InputTransformModalProps) { <> - Expected output for} - options={outputOptions} - value={selectedOutputOption} - onChange={(e) => { - setSelectedOutputOption(Number(e.target.value)); - setTransformedOutput('{}'); - }} - /> + {outputOptions.length === 1 ? ( + Expected output + ) : ( + Expected output for} + options={outputOptions} + value={selectedOutputOption} + onChange={(e) => { + setSelectedOutputOption(Number(e.target.value)); + setTransformedOutput('{}'); + }} + /> + )} (false); // model interface state - const [hasModelInterface, setHasModelInterface] = useState(true); - const [inputFields, setInputFields] = useState([]); - const [outputFields, setOutputFields] = useState([]); + const [modelInterface, setModelInterface] = useState< + ModelInterface | undefined + >(undefined); // Hook to listen when the selected model has changed. We do a few checks here: // 1: update model interface states @@ -136,15 +135,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { // reusable function to update interface states based on the model ID function updateModelInterfaceStates(modelId: string) { const newSelectedModel = models[modelId]; - if (newSelectedModel?.interface !== undefined) { - setInputFields(parseModelInputs(newSelectedModel.interface)); - setOutputFields(parseModelOutputs(newSelectedModel.interface)); - setHasModelInterface(true); - } else { - setInputFields([]); - setOutputFields([]); - setHasModelInterface(false); - } + setModelInterface(newSelectedModel?.interface); } return ( @@ -156,7 +147,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { context={props.context} inputMapField={inputMapField} inputMapFieldPath={inputMapFieldPath} - inputFields={inputFields} + modelInterface={modelInterface} onClose={() => setIsInputTransformModalOpen(false)} /> )} @@ -167,14 +158,14 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { context={props.context} outputMapField={outputMapField} outputMapFieldPath={outputMapFieldPath} - outputFields={outputFields} + modelInterface={modelInterface} onClose={() => setIsOutputTransformModalOpen(false)} /> )} {!isEmpty(getIn(values, modelFieldPath)?.id) && ( @@ -226,7 +217,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { ? 'Query field' : 'Document field' } - keyOptions={inputFields} + keyOptions={parseModelInputs(modelInterface)} /> @@ -274,7 +265,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { : 'Document field' } valuePlaceholder="Model output field" - valueOptions={outputFields} + valueOptions={parseModelOutputs(modelInterface)} /> {inputMapValue.length !== outputMapValue.length && diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx index 1c4a1abc..26dbf6e4 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx @@ -28,6 +28,7 @@ import { JSONPATH_ROOT_SELECTOR, ML_INFERENCE_DOCS_LINK, MapArrayFormValue, + ModelInterface, PROCESSOR_CONTEXT, SearchHit, SearchPipelineConfig, @@ -49,7 +50,7 @@ import { } from '../../../../store'; import { getCore } from '../../../../services'; import { MapArrayField } from '../input_fields'; -import { getDataSourceId } from '../../../../utils/utils'; +import { getDataSourceId, parseModelOutputs } from '../../../../utils/utils'; interface OutputTransformModalProps { uiConfig: WorkflowConfig; @@ -57,7 +58,7 @@ interface OutputTransformModalProps { context: PROCESSOR_CONTEXT; outputMapField: IConfigField; outputMapFieldPath: string; - outputFields: any[]; + modelInterface: ModelInterface | undefined; onClose: () => void; } @@ -79,7 +80,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) { // selected output state const outputOptions = map.map((_, idx) => ({ value: idx, - text: `Prediction output ${idx + 1}`, + text: `Prediction ${idx + 1}`, })) as EuiSelectOption[]; const [selectedOutputOption, setSelectedOutputOption] = useState< number | undefined @@ -237,7 +238,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) { helpLink={ML_INFERENCE_DOCS_LINK} keyPlaceholder="Document field" valuePlaceholder="Model output field" - valueOptions={props.outputFields} + valueOptions={parseModelOutputs(props.modelInterface)} // If the map we are adding is the first one, populate the selected option to index 0 onMapAdd={(curArray) => { if (isEmpty(curArray)) { @@ -257,15 +258,19 @@ export function OutputTransformModal(props: OutputTransformModalProps) { <> - Expected output for} - options={outputOptions} - value={selectedOutputOption} - onChange={(e) => { - setSelectedOutputOption(Number(e.target.value)); - setTransformedOutput('{}'); - }} - /> + {outputOptions.length === 1 ? ( + Expected output + ) : ( + Expected output for} + options={outputOptions} + value={selectedOutputOption} + onChange={(e) => { + setSelectedOutputOption(Number(e.target.value)); + setTransformedOutput('{}'); + }} + /> + )} (); + // Processor added state. Used to automatically open accordion when a new + // processor is added, assuming users want to immediately configure it. + const [processorAdded, setProcessorAdded] = useState(false); + // Popover state when adding new processors const [isPopoverOpen, setPopover] = useState(false); const closePopover = () => { @@ -75,6 +79,7 @@ export function ProcessorsList(props: ProcessorsListProps) { // (getting any updated/interim values along the way) and add to // the list of processors function addProcessor(processor: IProcessorConfig): void { + setProcessorAdded(true); const existingConfig = cloneDeep(props.uiConfig as WorkflowConfig); let newConfig = formikToUiConfig(values, existingConfig); switch (props.context) { @@ -138,36 +143,40 @@ export function ProcessorsList(props: ProcessorsListProps) { {processors.map((processor: IProcessorConfig, processorIndex) => { return ( - - - - {processor.name || 'Processor'} - - - { - deleteProcessor(processor.id); - }} - /> - - - - - + { + deleteProcessor(processor.id); + }} + /> + } + > + + + + + ); })} @@ -178,7 +187,6 @@ export function ProcessorsList(props: ProcessorsListProps) { { setPopover(!isPopoverOpen); }} diff --git a/public/pages/workflows/new_workflow/quick_configure_modal.tsx b/public/pages/workflows/new_workflow/quick_configure_modal.tsx index b1edb744..e519eb46 100644 --- a/public/pages/workflows/new_workflow/quick_configure_modal.tsx +++ b/public/pages/workflows/new_workflow/quick_configure_modal.tsx @@ -1,9 +1,9 @@ /* - * Copyright OpenSearch Contributors + * Copyright OpenSearch Contributorsd * SPDX-License-Identifier: Apache-2.0 */ -import React, { useState } from 'react'; +import React, { useState, useEffect } from 'react'; import { useHistory } from 'react-router-dom'; import { EuiSmallButton, @@ -22,6 +22,7 @@ import { MODEL_ID_PATTERN, MapArrayFormValue, MapFormValue, + ModelInterface, QuickConfigureFields, TEXT_FIELD_PATTERN, VECTOR, @@ -34,10 +35,16 @@ import { } from '../../../../common'; import { APP_PATH } from '../../../utils'; import { processWorkflowName } from './utils'; -import { createWorkflow, useAppDispatch } from '../../../store'; -import { constructUrlWithParams, getDataSourceId } from '../../../utils/utils'; +import { AppState, createWorkflow, useAppDispatch } from '../../../store'; +import { + constructUrlWithParams, + getDataSourceId, + parseModelInputs, + parseModelOutputs, +} from '../../../utils/utils'; import { QuickConfigureInputs } from './quick_configure_inputs'; import { isEmpty } from 'lodash'; +import { useSelector } from 'react-redux'; interface QuickConfigureModalProps { workflow: Workflow; @@ -51,6 +58,12 @@ export function QuickConfigureModal(props: QuickConfigureModalProps) { const dispatch = useAppDispatch(); const dataSourceId = getDataSourceId(); const history = useHistory(); + const { models } = useSelector((state: AppState) => state.ml); + + // model interface state + const [modelInterface, setModelInterface] = useState< + ModelInterface | undefined + >(undefined); // workflow name state const [workflowName, setWorkflowName] = useState( @@ -73,6 +86,14 @@ export function QuickConfigureModal(props: QuickConfigureModalProps) { ); } + // fetching model interface if available. used to prefill some + // of the input/output maps + useEffect(() => { + setModelInterface( + models[quickConfigureFields.embeddingModelId || '']?.interface + ); + }, [models, quickConfigureFields.embeddingModelId]); + return ( props.onClose()} style={{ width: '30vw' }}> @@ -115,7 +136,8 @@ export function QuickConfigureModal(props: QuickConfigureModalProps) { if (!isEmpty(quickConfigureFields)) { workflowToCreate = injectQuickConfigureFields( workflowToCreate, - quickConfigureFields + quickConfigureFields, + modelInterface ); } dispatch( @@ -155,7 +177,8 @@ export function QuickConfigureModal(props: QuickConfigureModalProps) { // helper fn to populate UI config values if there are some quick configure fields available function injectQuickConfigureFields( workflow: Workflow, - quickConfigureFields: QuickConfigureFields + quickConfigureFields: QuickConfigureFields, + modelInterface: ModelInterface | undefined ): Workflow { if (workflow.ui_metadata?.type) { switch (workflow.ui_metadata?.type) { @@ -167,7 +190,8 @@ function injectQuickConfigureFields( if (!isEmpty(quickConfigureFields) && workflow.ui_metadata?.config) { workflow.ui_metadata.config = updateIngestProcessorConfig( workflow.ui_metadata.config, - quickConfigureFields + quickConfigureFields, + modelInterface ); workflow.ui_metadata.config = updateIndexConfig( workflow.ui_metadata.config, @@ -179,7 +203,8 @@ function injectQuickConfigureFields( ); workflow.ui_metadata.config = updateSearchRequestProcessorConfig( workflow.ui_metadata.config, - quickConfigureFields + quickConfigureFields, + modelInterface ); } break; @@ -196,32 +221,56 @@ function injectQuickConfigureFields( // prefill ML ingest processor config, if applicable function updateIngestProcessorConfig( config: WorkflowConfig, - fields: QuickConfigureFields + fields: QuickConfigureFields, + modelInterface: ModelInterface | undefined ): WorkflowConfig { config.ingest.enrich.processors[0].fields.forEach((field) => { if (field.id === 'model' && fields.embeddingModelId) { field.value = { id: fields.embeddingModelId }; } - if (field.id === 'input_map' && (fields.textField || fields.imageField)) { - const inputMap = [] as MapFormValue; + if (field.id === 'input_map') { + const inputMap = generateMapFromModelInputs(modelInterface); if (fields.textField) { - inputMap.push({ - key: '', - value: fields.textField, - }); + if (inputMap.length > 0) { + inputMap[0] = { + ...inputMap[0], + value: fields.textField, + }; + } else { + inputMap.push({ + key: '', + value: fields.textField, + }); + } } if (fields.imageField) { - inputMap.push({ - key: '', - value: fields.imageField, - }); + if (inputMap.length > 1) { + inputMap[1] = { + ...inputMap[1], + value: fields.imageField, + }; + } else { + inputMap.push({ + key: '', + value: fields.imageField, + }); + } } field.value = [inputMap] as MapArrayFormValue; } - if (field.id === 'output_map' && fields.vectorField) { - field.value = [ - [{ key: fields.vectorField, value: '' }], - ] as MapArrayFormValue; + if (field.id === 'output_map') { + const outputMap = generateMapFromModelOutputs(modelInterface); + if (fields.vectorField) { + if (outputMap.length > 0) { + outputMap[0] = { + ...outputMap[0], + key: fields.vectorField, + }; + } else { + outputMap.push({ key: fields.vectorField, value: '' }); + } + } + field.value = [outputMap] as MapArrayFormValue; } }); @@ -232,20 +281,37 @@ function updateIngestProcessorConfig( // including populating placeholders in any pre-configured query_template function updateSearchRequestProcessorConfig( config: WorkflowConfig, - fields: QuickConfigureFields + fields: QuickConfigureFields, + modelInterface: ModelInterface | undefined ): WorkflowConfig { config.search.enrichRequest.processors[0].fields.forEach((field) => { if (field.id === 'model' && fields.embeddingModelId) { field.value = { id: fields.embeddingModelId }; } if (field.id === 'input_map') { + const inputMap = generateMapFromModelInputs(modelInterface); // TODO: pre-populate more if the query becomes standard - field.value = [[EMPTY_MAP_ENTRY]] as MapArrayFormValue; + field.value = + inputMap.length > 0 + ? [inputMap] + : ([[EMPTY_MAP_ENTRY]] as MapArrayFormValue); } if (field.id === 'output_map') { // prepopulate 'vector' constant as the model output transformed field, // so it is consistent and used in the downstream query_template, if configured. - field.value = [[{ key: VECTOR, value: '' }]] as MapArrayFormValue; + const outputMap = generateMapFromModelOutputs(modelInterface); + if (outputMap.length > 0) { + outputMap[0] = { + ...outputMap[0], + key: VECTOR, + }; + } else { + outputMap.push({ + key: VECTOR, + value: '', + }); + } + field.value = [outputMap]; } }); config.search.enrichRequest.processors[0].optionalFields = config.search.enrichRequest.processors[0].optionalFields?.map( @@ -348,3 +414,39 @@ function injectPlaceholderValues( return finalRequestString; } + +// generate a set of mappings s.t. each key is +// a unique model input. +function generateMapFromModelInputs( + modelInterface?: ModelInterface +): MapFormValue { + const inputMap = [] as MapFormValue; + if (modelInterface) { + const modelInputs = parseModelInputs(modelInterface); + modelInputs.forEach((modelInput) => { + inputMap.push({ + key: modelInput.label, + value: '', + }); + }); + } + return inputMap; +} + +// generate a set of mappings s.t. each value is +// a unique model output +function generateMapFromModelOutputs( + modelInterface?: ModelInterface +): MapFormValue { + const outputMap = [] as MapFormValue; + if (modelInterface) { + const modelOutputs = parseModelOutputs(modelInterface); + modelOutputs.forEach((modelOutput) => { + outputMap.push({ + key: '', + value: modelOutput.label, + }); + }); + } + return outputMap; +} diff --git a/public/utils/utils.ts b/public/utils/utils.ts index 67d9ca88..d573eb9f 100644 --- a/public/utils/utils.ts +++ b/public/utils/utils.ts @@ -203,7 +203,7 @@ export function generateTransform(input: {}, map: MapFormValue): {} { // Derive the collection of model inputs from the model interface JSONSchema into a form-ready list export function parseModelInputs( - modelInterface: ModelInterface + modelInterface: ModelInterface | undefined ): ModelInputFormField[] { const modelInputsObj = get( modelInterface, @@ -223,7 +223,7 @@ export function parseModelInputs( // Derive the collection of model outputs from the model interface JSONSchema into a form-ready list export function parseModelOutputs( - modelInterface: ModelInterface + modelInterface: ModelInterface | undefined ): ModelOutputFormField[] { const modelOutputsObj = get(modelInterface, 'output.properties', {}) as { [key: string]: ModelOutput;