diff --git a/common/interfaces.ts b/common/interfaces.ts index 3a9326d0..baa08165 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -129,6 +129,13 @@ export type InputTransformFormValues = { }; export type InputTransformSchema = WorkflowSchema; +// Form / schema interfaces for the output transform sub-form +export type OutputTransformFormValues = { + output_map: MapArrayFormValue; + full_response_path: ConfigFieldValue; +}; +export type OutputTransformSchema = WorkflowSchema; + /** ********** WORKSPACE TYPES/INTERFACES ********** */ diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/modals/output_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/modals/output_transform_modal.tsx index 4216f2b9..f491d8bd 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/modals/output_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/modals/output_transform_modal.tsx @@ -4,8 +4,9 @@ */ import React, { useState, useEffect } from 'react'; -import { useFormikContext, getIn } from 'formik'; +import { useFormikContext, getIn, Formik } from 'formik'; import { cloneDeep, isEmpty, set } from 'lodash'; +import * as yup from 'yup'; import { EuiCodeEditor, EuiFlexGroup, @@ -27,6 +28,7 @@ import { EuiCallOut, } from '@elastic/eui'; import { + IConfigField, IProcessorConfig, IngestPipelineConfig, JSONPATH_ROOT_SELECTOR, @@ -34,6 +36,8 @@ import { ML_INFERENCE_RESPONSE_DOCS_LINK, MapArrayFormValue, ModelInterface, + OutputTransformFormValues, + OutputTransformSchema, PROCESSOR_CONTEXT, SearchHit, SearchPipelineConfig, @@ -45,6 +49,8 @@ import { import { formikToPartialPipeline, generateTransform, + getFieldSchema, + getInitialValue, prepareDocsForSimulate, unwrapTransformedDocs, } from '../../../../../utils'; @@ -77,7 +83,33 @@ interface OutputTransformModalProps { export function OutputTransformModal(props: OutputTransformModalProps) { const dispatch = useAppDispatch(); const dataSourceId = getDataSourceId(); - const { values } = useFormikContext(); + const { values, setFieldValue, setFieldTouched } = useFormikContext< + WorkflowFormValues + >(); + + // sub-form values/schema + const outputTransformFormValues = { + output_map: getInitialValue('mapArray'), + full_response_path: getInitialValue('boolean'), + } as OutputTransformFormValues; + const outputTransformFormSchema = yup.object({ + output_map: getFieldSchema({ + type: 'mapArray', + } as IConfigField), + full_response_path: getFieldSchema( + { + type: 'boolean', + } as IConfigField, + true + ), + }) as OutputTransformSchema; + + // persist standalone values. update / initialize when it is first opened + const [tempErrors, setTempErrors] = useState(false); + const [tempFullResponsePath, setTempFullResponsePath] = useState( + false + ); + const [tempOutputMap, setTempOutputMap] = useState([]); // fetching input data state const [isFetching, setIsFetching] = useState(false); @@ -87,9 +119,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) { const [transformedOutput, setTransformedOutput] = useState('{}'); // get some current form values - const map = getIn(values, props.outputMapFieldPath) as MapArrayFormValue; const fullResponsePathPath = `${props.baseConfigPath}.${props.config.id}.full_response_path`; - const fullResponsePath = getIn(values, fullResponsePathPath); const docs = getIn(values, 'ingest.docs'); let docObjs = [] as {}[] | undefined; try { @@ -111,7 +141,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) { const [popoverOpen, setPopoverOpen] = useState(false); // selected transform state - const transformOptions = map.map((_, idx) => ({ + const transformOptions = tempOutputMap.map((_, idx) => ({ value: idx, text: `Prediction ${idx + 1}`, })) as EuiSelectOption[]; @@ -121,345 +151,429 @@ export function OutputTransformModal(props: OutputTransformModalProps) { // hook to re-generate the transform when any inputs to the transform are updated useEffect(() => { - if (!isEmpty(map) && !isEmpty(JSON.parse(sourceOutput))) { + if (!isEmpty(tempOutputMap) && !isEmpty(JSON.parse(sourceOutput))) { let sampleSourceOutput = {}; try { sampleSourceOutput = JSON.parse(sourceOutput); const output = generateTransform( sampleSourceOutput, - map[selectedTransformOption] + tempOutputMap[selectedTransformOption] ); setTransformedOutput(customStringify(output)); } catch {} } else { setTransformedOutput('{}'); } - }, [map, sourceOutput, selectedTransformOption]); + }, [tempOutputMap, sourceOutput, selectedTransformOption]); // hook to clear the source output when full_response_path is toggled useEffect(() => { setSourceOutput('{}'); - }, [fullResponsePath]); + }, [tempFullResponsePath]); return ( - - - -

{`Configure output`}

-
-
- - - - <> - {(onIngestAndNoDocs || onSearchAndNoQuery) && ( - <> - - - - )} - - Fetch some sample output data and see how it is transformed. - - - {(props.context === PROCESSOR_CONTEXT.INGEST || - props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE) && ( - <> - - - - )} - + {}} + validate={(values) => {}} + > + {(formikProps) => { + // override to parent form values when changes detected + useEffect(() => { + formikProps.setFieldValue( + 'output_map', + getIn(values, props.outputMapFieldPath) + ); + }, [getIn(values, props.outputMapFieldPath)]); + useEffect(() => { + formikProps.setFieldValue( + 'full_response_path', + getIn(values, fullResponsePathPath) + ); + }, [getIn(values, fullResponsePathPath)]); + + // update temp vars when form changes are detected + useEffect(() => { + setTempOutputMap(getIn(formikProps.values, 'output_map')); + }, [getIn(formikProps.values, 'output_map')]); + useEffect(() => { + setTempFullResponsePath( + getIn(formikProps.values, 'full_response_path') + ); + }, [getIn(formikProps.values, 'full_response_path')]); + + // update tempErrors if errors detected + useEffect(() => { + setTempErrors(!isEmpty(formikProps.errors)); + }, [formikProps.errors]); + + return ( + + + +

{`Configure output`}

+
+
+ + - Source output + <> + {(onIngestAndNoDocs || onSearchAndNoQuery) && ( + <> + + + + )} + + Fetch some sample output data and see how it is + transformed. + + + {(props.context === PROCESSOR_CONTEXT.INGEST || + props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE) && ( + <> + + + + )} + + + Source output + + {!isEmpty( + parseModelOutputsObj( + props.modelInterface, + tempFullResponsePath + ) + ) && ( + + setPopoverOpen(false)} + panelPaddingSize="s" + button={ + setPopoverOpen(!popoverOpen)} + > + View output schema + + } + > + + The JSON Schema defining the model's expected + output + + + {customStringify( + parseModelOutputsObj( + props.modelInterface, + tempFullResponsePath + ) + )} + + + + )} + + { + setIsFetching(true); + switch (props.context) { + // note we skip search request processor context. that is because empty output maps are not supported. + // for more details, see comment in ml_processor_inputs.tsx + case PROCESSOR_CONTEXT.INGEST: { + // get the current ingest pipeline up to, and including this processor. + // remove any currently-configured output map since we only want the transformation + // up to, and including, the input map transformations + const valuesWithoutOutputMapConfig = cloneDeep( + values + ); + set( + valuesWithoutOutputMapConfig, + `ingest.enrich.${props.config.id}.output_map`, + [] + ); + const curIngestPipeline = formikToPartialPipeline( + valuesWithoutOutputMapConfig, + props.uiConfig, + props.config.id, + true, + PROCESSOR_CONTEXT.INGEST + ) as IngestPipelineConfig; + const curDocs = prepareDocsForSimulate( + values.ingest.docs, + values.ingest.index.name + ); + await dispatch( + simulatePipeline({ + apiBody: { + pipeline: curIngestPipeline, + docs: [curDocs[0]], + }, + dataSourceId, + }) + ) + .unwrap() + .then((resp: SimulateIngestPipelineResponse) => { + try { + const docObjs = unwrapTransformedDocs(resp); + if (docObjs.length > 0) { + const sampleModelResult = + docObjs[0]?.inference_results || {}; + setSourceOutput( + customStringify(sampleModelResult) + ); + } + } catch {} + }) + .catch((error: any) => { + getCore().notifications.toasts.addDanger( + `Failed to fetch input data` + ); + }) + .finally(() => { + setIsFetching(false); + }); + break; + } + case PROCESSOR_CONTEXT.SEARCH_RESPONSE: { + // get the current search pipeline up to, and including this processor. + // remove any currently-configured output map since we only want the transformation + // up to, and including, the input map transformations + const valuesWithoutOutputMapConfig = cloneDeep( + values + ); + set( + valuesWithoutOutputMapConfig, + `search.enrichResponse.${props.config.id}.output_map`, + [] + ); + const curSearchPipeline = formikToPartialPipeline( + valuesWithoutOutputMapConfig, + props.uiConfig, + props.config.id, + true, + PROCESSOR_CONTEXT.SEARCH_RESPONSE + ) as SearchPipelineConfig; + + // Execute search. Augment the existing query with + // the partial search pipeline (inline) to get the latest transformed + // version of the request. + dispatch( + searchIndex({ + apiBody: { + index: values.ingest.index.name, + body: JSON.stringify({ + ...JSON.parse( + values.search.request as string + ), + search_pipeline: curSearchPipeline || {}, + }), + }, + dataSourceId, + }) + ) + .unwrap() + .then(async (resp) => { + const hits = resp.hits.hits.map( + (hit: SearchHit) => hit._source + ) as any[]; + if (hits.length > 0) { + const sampleModelResult = + hits[0].inference_results || {}; + setSourceOutput( + customStringify(sampleModelResult) + ); + } + }) + .catch((error: any) => { + getCore().notifications.toasts.addDanger( + `Failed to fetch source output data` + ); + }) + .finally(() => { + setIsFetching(false); + }); + break; + } + } + }} + > + Fetch + + + + - {!isEmpty( - parseModelOutputsObj(props.modelInterface, fullResponsePath) - ) && ( - - setPopoverOpen(false)} - panelPaddingSize="s" - button={ - setPopoverOpen(!popoverOpen)} - > - View output schema - + + <> + Define transform + + - - The JSON Schema defining the model's expected output - - - {customStringify( - parseModelOutputsObj( - props.modelInterface, - fullResponsePath - ) - )} - - - - )} + keyPlaceholder={ + props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST + ? 'Specify a query field' + : 'Define a document field' + } + valueTitle="Name" + valuePlaceholder="Name" + valueOptions={ + tempFullResponsePath + ? undefined + : parseModelOutputs(props.modelInterface, false) + } + // If the map we are adding is the first one, populate the selected option to index 0 + onMapAdd={(curArray) => { + if (isEmpty(curArray)) { + setSelectedTransformOption(0); + } + }} + // If the map we are deleting is the one we last used to test, reset the state and + // default to the first map in the list. + onMapDelete={(idxToDelete) => { + if (selectedTransformOption === idxToDelete) { + setSelectedTransformOption(0); + setTransformedOutput('{}'); + } + }} + addMapEntryButtonText="Add output" + addMapButtonText="(Advanced) Add output group" + /> + +
+ + <> + {transformOptions.length <= 1 ? ( + Transformed output + ) : ( + Transformed output for + } + options={transformOptions} + value={selectedTransformOption} + onChange={(e) => { + setSelectedTransformOption(Number(e.target.value)); + }} + /> + )} + + + +
+
+ { - setIsFetching(true); - switch (props.context) { - // note we skip search request processor context. that is because empty output maps are not supported. - // for more details, see comment in ml_processor_inputs.tsx - case PROCESSOR_CONTEXT.INGEST: { - // get the current ingest pipeline up to, and including this processor. - // remove any currently-configured output map since we only want the transformation - // up to, and including, the input map transformations - const valuesWithoutOutputMapConfig = cloneDeep(values); - set( - valuesWithoutOutputMapConfig, - `ingest.enrich.${props.config.id}.output_map`, - [] - ); - const curIngestPipeline = formikToPartialPipeline( - valuesWithoutOutputMapConfig, - props.uiConfig, - props.config.id, - true, - PROCESSOR_CONTEXT.INGEST - ) as IngestPipelineConfig; - const curDocs = prepareDocsForSimulate( - values.ingest.docs, - values.ingest.index.name - ); - await dispatch( - simulatePipeline({ - apiBody: { - pipeline: curIngestPipeline, - docs: [curDocs[0]], - }, - dataSourceId, - }) - ) - .unwrap() - .then((resp: SimulateIngestPipelineResponse) => { - try { - const docObjs = unwrapTransformedDocs(resp); - if (docObjs.length > 0) { - const sampleModelResult = - docObjs[0]?.inference_results || {}; - setSourceOutput( - customStringify(sampleModelResult) - ); - } - } catch {} - }) - .catch((error: any) => { - getCore().notifications.toasts.addDanger( - `Failed to fetch input data` - ); - }) - .finally(() => { - setIsFetching(false); - }); - break; - } - case PROCESSOR_CONTEXT.SEARCH_RESPONSE: { - // get the current search pipeline up to, and including this processor. - // remove any currently-configured output map since we only want the transformation - // up to, and including, the input map transformations - const valuesWithoutOutputMapConfig = cloneDeep(values); - set( - valuesWithoutOutputMapConfig, - `search.enrichResponse.${props.config.id}.output_map`, - [] - ); - const curSearchPipeline = formikToPartialPipeline( - valuesWithoutOutputMapConfig, - props.uiConfig, - props.config.id, - true, - PROCESSOR_CONTEXT.SEARCH_RESPONSE - ) as SearchPipelineConfig; + onClick={props.onClose} + fill={false} + color="primary" + data-testid="cancelOutputTransformModalButton" + > + Cancel + + { + // update the parent form values + setFieldValue( + fullResponsePathPath, + getIn(formikProps.values, 'full_response_path') + ); + setFieldTouched(fullResponsePathPath, true); - // Execute search. Augment the existing query with - // the partial search pipeline (inline) to get the latest transformed - // version of the request. - dispatch( - searchIndex({ - apiBody: { - index: values.ingest.index.name, - body: JSON.stringify({ - ...JSON.parse(values.search.request as string), - search_pipeline: curSearchPipeline || {}, - }), - }, - dataSourceId, - }) - ) - .unwrap() - .then(async (resp) => { - const hits = resp.hits.hits.map( - (hit: SearchHit) => hit._source - ) as any[]; - if (hits.length > 0) { - const sampleModelResult = - hits[0].inference_results || {}; - setSourceOutput(customStringify(sampleModelResult)); - } - }) - .catch((error: any) => { - getCore().notifications.toasts.addDanger( - `Failed to fetch source output data` - ); - }) - .finally(() => { - setIsFetching(false); - }); - break; - } - } + setFieldValue( + props.outputMapFieldPath, + getIn(formikProps.values, 'output_map') + ); + setFieldTouched(props.outputMapFieldPath, true); + props.onClose(); }} + isDisabled={tempErrors} // blocking update until valid input is given + fill={true} + color="primary" + data-testid="updateOutputTransformModalButton" > - Fetch + Update - - - - - - <> - Define transform - - { - if (isEmpty(curArray)) { - setSelectedTransformOption(0); - } - }} - // If the map we are deleting is the one we last used to test, reset the state and - // default to the first map in the list. - onMapDelete={(idxToDelete) => { - if (selectedTransformOption === idxToDelete) { - setSelectedTransformOption(0); - setTransformedOutput('{}'); - } - }} - addMapEntryButtonText="Add output" - addMapButtonText="(Advanced) Add output group" - /> - - - - <> - {transformOptions.length <= 1 ? ( - Transformed output - ) : ( - Transformed output for} - options={transformOptions} - value={selectedTransformOption} - onChange={(e) => { - setSelectedTransformOption(Number(e.target.value)); - }} - /> - )} - - - - - - - - - Close - - -
+ + + ); + }} + ); }