From 4a840ffad858aef0e3fa4cc647d88e2ae9356e0c Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 16 Jul 2024 10:36:28 -0700 Subject: [PATCH 01/10] Add ML processor inputs component; add simple/advanced toggle; remove unnecessary data from ml processor ui config; Signed-off-by: Tyler Ohlsen --- public/configs/ml_processor.ts | 10 -- .../workflow_inputs/config_field_list.tsx | 26 ++-- .../input_fields/map_field.tsx | 11 +- .../advanced_transform_modal.tsx | 45 ++++++ .../workflow_inputs/processor_inputs/index.ts | 7 + .../processor_inputs/ml_processor_inputs.tsx | 141 ++++++++++++++++++ .../processor_inputs/processor_inputs.tsx | 48 ++++++ .../workflow_inputs/processors_list.tsx | 7 +- .../workflows/workflow_list/workflow_list.tsx | 2 +- 9 files changed, 265 insertions(+), 32 deletions(-) create mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/advanced_transform_modal.tsx create mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/index.ts create mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx create mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx diff --git a/public/configs/ml_processor.ts b/public/configs/ml_processor.ts index ce797307..53a6bef2 100644 --- a/public/configs/ml_processor.ts +++ b/public/configs/ml_processor.ts @@ -21,22 +21,12 @@ export abstract class MLProcessor extends Processor { type: 'model', }, { - label: 'Input Map', id: 'inputMap', type: 'map', - // TODO: move these fields directly into the component once design is finalized - helpText: `An array specifying how to map fields from the ingested document to the model’s input.`, - helpLink: - 'https://opensearch.org/docs/latest/ingest-pipelines/processors/ml-inference/#configuration-parameters', }, { - label: 'Output Map', id: 'outputMap', type: 'map', - // TODO: move these fields directly into the component once design is finalized - helpText: `An array specifying how to map the model’s output to new fields.`, - helpLink: - 'https://opensearch.org/docs/latest/ingest-pipelines/processors/ml-inference/#configuration-parameters', }, ]; } diff --git a/public/pages/workflow_detail/workflow_inputs/config_field_list.tsx b/public/pages/workflow_detail/workflow_inputs/config_field_list.tsx index 8daf1022..01db981b 100644 --- a/public/pages/workflow_detail/workflow_inputs/config_field_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/config_field_list.tsx @@ -55,19 +55,19 @@ export function ConfigFieldList(props: ConfigFieldListProps) { ); break; } - case 'map': { - el = ( - - - - - ); - break; - } + // case 'map': { + // el = ( + // + // + // + // + // ); + // break; + // } // case 'json': { // el = ( // diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx index 23714314..a27071e3 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx @@ -25,6 +25,9 @@ import { interface MapFieldProps { field: IConfigField; fieldPath: string; // the full path in string-form to the field (e.g., 'ingest.enrich.processors.text_embedding_processor.inputField') + label: string; + helpLink?: string; + helpText?: string; onFormChange: () => void; } @@ -60,17 +63,17 @@ export function MapField(props: MapFieldProps) { return ( - + Learn more ) : undefined } - helpText={props.field.helpText || undefined} + helpText={props.helpText || undefined} error={ getIn(errors, field.name) !== undefined && getIn(errors, field.name).length > 0 diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/advanced_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/advanced_transform_modal.tsx new file mode 100644 index 00000000..4138c9f8 --- /dev/null +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/advanced_transform_modal.tsx @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { + EuiButton, + EuiButtonEmpty, + EuiModal, + EuiModalBody, + EuiModalFooter, + EuiModalHeader, + EuiModalHeaderTitle, + EuiText, +} from '@elastic/eui'; + +interface AdvancedTransformModalProps { + onClose: () => void; + onConfirm: () => void; +} + +/** + * A modal to perform advanced JSON-to-JSON transforms to/from a model's input/output, respectively + */ +export function AdvancedTransformModal(props: AdvancedTransformModalProps) { + return ( + + + +

{`Configure advanced transform`}

+ + + + TODO TODO TODO + + + Cancel + + Save + + + + ); +} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/index.ts b/public/pages/workflow_detail/workflow_inputs/processor_inputs/index.ts new file mode 100644 index 00000000..95257a19 --- /dev/null +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './ml_processor_inputs'; +export * from './processor_inputs'; diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx new file mode 100644 index 00000000..9e01ca64 --- /dev/null +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx @@ -0,0 +1,141 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useState } from 'react'; +import { getIn, useFormikContext } from 'formik'; +import { EuiButton, EuiRadioGroup, EuiSpacer } from '@elastic/eui'; +import { + WorkspaceFormValues, + IProcessorConfig, + IConfigField, +} from '../../../../../common'; +import { MapField, ModelField } from '../input_fields'; +import { isEmpty } from 'lodash'; +import { AdvancedTransformModal } from './advanced_transform_modal'; + +interface MLProcessorInputsProps { + config: IProcessorConfig; + baseConfigPath: string; // the base path of the nested config, if applicable. e.g., 'ingest.enrich' + onFormChange: () => void; +} + +enum TRANSFORM_OPTION { + SIMPLE = 'SIMPLE', + ADVANCED = 'ADVANCED', +} + +/** + * Component to render ML processor inputs. Offers a simple and advanced flow for configuring data transforms + * before and after executing an ML inference request + */ +export function MLProcessorInputs(props: MLProcessorInputsProps) { + const { values } = useFormikContext(); + + // extracting field info from the ML processor config + // TODO: have a better mechanism for guaranteeing the expected fields/config instead of hardcoding them here + const modelField = props.config.fields.find( + (field) => field.type === 'model' + ) as IConfigField; + const modelFieldPath = `${props.baseConfigPath}.${props.config.id}.${modelField.id}`; + const inputMapField = props.config.fields.find( + (field) => field.id === 'inputMap' + ) as IConfigField; + const inputMapFieldPath = `${props.baseConfigPath}.${props.config.id}.${inputMapField.id}`; + const outputMapField = props.config.fields.find( + (field) => field.id === 'outputMap' + ) as IConfigField; + const outputMapFieldPath = `${props.baseConfigPath}.${props.config.id}.${outputMapField.id}`; + + // transform option state + const [selectedOption, setSelectedOption] = useState( + TRANSFORM_OPTION.SIMPLE + ); + + // advanced transform modal state + const [ + isAdvancedTransformModalOpen, + setIsAdvancedTransformModalOpen, + ] = useState(false); + + return ( + <> + {isAdvancedTransformModalOpen && ( + setIsAdvancedTransformModalOpen(false)} + onConfirm={() => { + console.log('saving transform configuration...'); + setIsAdvancedTransformModalOpen(false); + }} + /> + )} + + {!isEmpty(getIn(values, modelFieldPath)?.id) && ( + <> + + { + setSelectedOption(option as TRANSFORM_OPTION); + }} + /> + {selectedOption === TRANSFORM_OPTION.SIMPLE && ( + <> + + + + + + )} + {selectedOption === TRANSFORM_OPTION.ADVANCED && ( + <> + + { + setIsAdvancedTransformModalOpen(true); + }} + > + Configure + + + )} + + )} + + ); +} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx new file mode 100644 index 00000000..db6d24df --- /dev/null +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { EuiFlexItem, EuiSpacer } from '@elastic/eui'; +import { IProcessorConfig, PROCESSOR_TYPE } from '../../../../../common'; +import { MLProcessorInputs } from './ml_processor_inputs'; + +/** + * Base component for rendering processor form inputs based on the processor type + */ + +interface ProcessorInputsProps { + config: IProcessorConfig; + baseConfigPath: string; // the base path of the nested config, if applicable. e.g., 'ingest.enrich' + onFormChange: () => void; +} + +const PROCESSOR_INPUTS_SPACER_SIZE = 'm'; + +export function ProcessorInputs(props: ProcessorInputsProps) { + const configType = props.config.type; + return ( + + {(() => { + let el; + switch (configType) { + case PROCESSOR_TYPE.ML: { + el = ( + + + + + ); + break; + } + } + return el; + })()} + + ); +} diff --git a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx index 225a3a17..f85eb31f 100644 --- a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx @@ -18,19 +18,18 @@ import { import { cloneDeep } from 'lodash'; import { useFormikContext } from 'formik'; import { - IConfig, IProcessorConfig, PROCESSOR_CONTEXT, WorkflowConfig, WorkflowFormValues, } from '../../../../common'; -import { ConfigFieldList } from './config_field_list'; import { formikToUiConfig } from '../../../utils'; import { MLIngestProcessor, MLSearchRequestProcessor, MLSearchResponseProcessor, } from '../../../configs'; +import { ProcessorInputs } from './processor_inputs'; interface ProcessorsListProps { onFormChange: () => void; @@ -133,7 +132,7 @@ export function ProcessorsList(props: ProcessorsListProps) { return ( - {processors.map((processor: IConfig, processorIndex) => { + {processors.map((processor: IProcessorConfig, processorIndex) => { return ( @@ -153,7 +152,7 @@ export function ProcessorsList(props: ProcessorsListProps) { - From 24afc1463d4f067afcb375030d9858009936f6bd Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 16 Jul 2024 11:26:25 -0700 Subject: [PATCH 02/10] Set up baseline modals; set up stub for fetching ingest input Signed-off-by: Tyler Ohlsen --- .../input_transform_modal.tsx | 101 ++++++++++++++++++ .../processor_inputs/ml_processor_inputs.tsx | 57 +++++++--- .../output_transform_modal.tsx | 45 ++++++++ .../processor_inputs/processor_inputs.tsx | 8 +- .../workflow_inputs/processors_list.tsx | 1 + 5 files changed, 196 insertions(+), 16 deletions(-) create mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx create mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx new file mode 100644 index 00000000..099449c6 --- /dev/null +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx @@ -0,0 +1,101 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { + EuiButton, + EuiButtonEmpty, + EuiCodeBlock, + EuiCodeEditor, + EuiFlexGroup, + EuiFlexItem, + EuiModal, + EuiModalBody, + EuiModalFooter, + EuiModalHeader, + EuiModalHeaderTitle, + EuiSpacer, + EuiText, +} from '@elastic/eui'; +import { PROCESSOR_CONTEXT } from '../../../../../common'; + +interface InputTransformModalProps { + context: PROCESSOR_CONTEXT; + onClose: () => void; + onConfirm: () => void; +} + +/** + * A modal to configure advanced JSON-to-JSON transforms into a model's expected input + */ +export function InputTransformModal(props: InputTransformModalProps) { + return ( + + + +

{`Configure input transform`}

+ + + + + + <> + { + switch (props.context) { + case PROCESSOR_CONTEXT.INGEST: { + // TODO: complete for ingest. generate and simulate an ingest pipeline up to this point + break; + } + // TODO: complete for search request / search response contexts + } + }} + > + Fetch expected input + + + + {`{"a": "b"}`} + + + + + <> + Define transform with JSONPath + + + + + + <> + Expected output + + + {`TODO: will be model input`} + + + + + + + Cancel + + Save + + + + ); +} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx index 9e01ca64..10b8cd38 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx @@ -5,20 +5,23 @@ import React, { useState } from 'react'; import { getIn, useFormikContext } from 'formik'; -import { EuiButton, EuiRadioGroup, EuiSpacer } from '@elastic/eui'; +import { EuiButton, EuiRadioGroup, EuiSpacer, EuiText } from '@elastic/eui'; import { WorkspaceFormValues, IProcessorConfig, IConfigField, + PROCESSOR_CONTEXT, } from '../../../../../common'; import { MapField, ModelField } from '../input_fields'; import { isEmpty } from 'lodash'; -import { AdvancedTransformModal } from './advanced_transform_modal'; +import { InputTransformModal } from './input_transform_modal'; +import { OutputTransformModal } from './output_transform_modal'; interface MLProcessorInputsProps { config: IProcessorConfig; baseConfigPath: string; // the base path of the nested config, if applicable. e.g., 'ingest.enrich' onFormChange: () => void; + context: PROCESSOR_CONTEXT; } enum TRANSFORM_OPTION { @@ -53,20 +56,32 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { TRANSFORM_OPTION.SIMPLE ); - // advanced transform modal state - const [ - isAdvancedTransformModalOpen, - setIsAdvancedTransformModalOpen, - ] = useState(false); + // advanced transformations modal state + const [isInputTransformModalOpen, setIsInputTransformModalOpen] = useState< + boolean + >(true); + const [isOutputTransformModalOpen, setIsOutputTransformModalOpen] = useState< + boolean + >(false); return ( <> - {isAdvancedTransformModalOpen && ( - setIsAdvancedTransformModalOpen(false)} + {isInputTransformModalOpen && ( + setIsInputTransformModalOpen(false)} onConfirm={() => { - console.log('saving transform configuration...'); - setIsAdvancedTransformModalOpen(false); + console.log('saving transform input configuration...'); + setIsInputTransformModalOpen(false); + }} + /> + )} + {isOutputTransformModalOpen && ( + setIsOutputTransformModalOpen(false)} + onConfirm={() => { + console.log('saving transform output configuration...'); + setIsOutputTransformModalOpen(false); }} /> )} @@ -77,6 +92,8 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { /> {!isEmpty(getIn(values, modelFieldPath)?.id) && ( <> + + {`Configure data transformations (optional)`} { + setIsInputTransformModalOpen(true); + }} + > + Configure input transformation + + + { - setIsAdvancedTransformModalOpen(true); + setIsOutputTransformModalOpen(true); }} > - Configure + Configure output transformation )} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx new file mode 100644 index 00000000..3e8403ee --- /dev/null +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { + EuiButton, + EuiButtonEmpty, + EuiModal, + EuiModalBody, + EuiModalFooter, + EuiModalHeader, + EuiModalHeaderTitle, + EuiText, +} from '@elastic/eui'; + +interface OutputTransformModalProps { + onClose: () => void; + onConfirm: () => void; +} + +/** + * A modal to configure advanced JSON-to-JSON transforms from a model's expected output + */ +export function OutputTransformModal(props: OutputTransformModalProps) { + return ( + + + +

{`Configure output transform`}

+ + + + TODO TODO TODO + + + Cancel + + Save + + + + ); +} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx index db6d24df..609de12c 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx @@ -5,7 +5,11 @@ import React from 'react'; import { EuiFlexItem, EuiSpacer } from '@elastic/eui'; -import { IProcessorConfig, PROCESSOR_TYPE } from '../../../../../common'; +import { + IProcessorConfig, + PROCESSOR_CONTEXT, + PROCESSOR_TYPE, +} from '../../../../../common'; import { MLProcessorInputs } from './ml_processor_inputs'; /** @@ -16,6 +20,7 @@ interface ProcessorInputsProps { config: IProcessorConfig; baseConfigPath: string; // the base path of the nested config, if applicable. e.g., 'ingest.enrich' onFormChange: () => void; + context: PROCESSOR_CONTEXT; } const PROCESSOR_INPUTS_SPACER_SIZE = 'm'; @@ -34,6 +39,7 @@ export function ProcessorInputs(props: ProcessorInputsProps) { config={props.config} baseConfigPath={props.baseConfigPath} onFormChange={props.onFormChange} + context={props.context} /> diff --git a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx index f85eb31f..0522263b 100644 --- a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx @@ -162,6 +162,7 @@ export function ProcessorsList(props: ProcessorsListProps) { : 'search.enrichResponse' } onFormChange={props.onFormChange} + context={props.context} /> From 0083b1a241ba1c6e3b4576976d86b847b0693fb2 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 16 Jul 2024 12:51:00 -0700 Subject: [PATCH 03/10] Set up conversion fns Signed-off-by: Tyler Ohlsen --- .../input_transform_modal.tsx | 7 ++++++- public/utils/form_to_pipeline_utils.ts | 21 +++++++++++++++++++ public/utils/index.ts | 1 + 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 public/utils/form_to_pipeline_utils.ts diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx index 099449c6..b6472b20 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx @@ -4,6 +4,7 @@ */ import React from 'react'; +import { useFormikContext } from 'formik'; import { EuiButton, EuiButtonEmpty, @@ -19,7 +20,8 @@ import { EuiSpacer, EuiText, } from '@elastic/eui'; -import { PROCESSOR_CONTEXT } from '../../../../../common'; +import { PROCESSOR_CONTEXT, WorkflowFormValues } from '../../../../../common'; +import { formikToIngestPipeline } from '../../../../utils'; interface InputTransformModalProps { context: PROCESSOR_CONTEXT; @@ -31,6 +33,8 @@ interface InputTransformModalProps { * A modal to configure advanced JSON-to-JSON transforms into a model's expected input */ export function InputTransformModal(props: InputTransformModalProps) { + const { values } = useFormikContext(); + return ( @@ -48,6 +52,7 @@ export function InputTransformModal(props: InputTransformModalProps) { switch (props.context) { case PROCESSOR_CONTEXT.INGEST: { // TODO: complete for ingest. generate and simulate an ingest pipeline up to this point + const curIngestPipeline = formikToIngestPipeline(values); break; } // TODO: complete for search request / search response contexts diff --git a/public/utils/form_to_pipeline_utils.ts b/public/utils/form_to_pipeline_utils.ts new file mode 100644 index 00000000..adfbae36 --- /dev/null +++ b/public/utils/form_to_pipeline_utils.ts @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { WorkflowFormValues } from '../../common'; + +/* + **************** Form -> pipeline utils ********************** + Collection of utility fns for converting the current form state + to partial/in-progress ingest and search pipelines to run + and collect their current outputs. Primarily used for determining + the input schema at a certain stage of a pipeline. + */ + +export function formikToIngestPipeline(values: WorkflowFormValues): {} { + const ingestConfig = values.ingest; + console.log('ingest config: ', ingestConfig); + // TODO: may be able to convert form -> config -> template, where template has the API-formatted configs. + return {}; +} diff --git a/public/utils/index.ts b/public/utils/index.ts index 696b7021..1e340a10 100644 --- a/public/utils/index.ts +++ b/public/utils/index.ts @@ -10,3 +10,4 @@ export * from './config_to_form_utils'; export * from './config_to_workspace_utils'; export * from './config_to_schema_utils'; export * from './form_to_config_utils'; +export * from './form_to_pipeline_utils'; From fbff5d7603d97df8b7faf56576d9eaa03c265290 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 16 Jul 2024 14:54:03 -0700 Subject: [PATCH 04/10] Get valid simulate API JSON body complete Signed-off-by: Tyler Ohlsen --- common/interfaces.ts | 4 ++ .../input_transform_modal.tsx | 18 ++++++-- .../processor_inputs/ml_processor_inputs.tsx | 8 +++- .../processor_inputs/processor_inputs.tsx | 3 ++ .../workflow_inputs/processors_list.tsx | 1 + public/utils/config_to_template_utils.ts | 2 +- public/utils/form_to_pipeline_utils.ts | 45 ++++++++++++++++--- 7 files changed, 69 insertions(+), 12 deletions(-) diff --git a/common/interfaces.ts b/common/interfaces.ts index 90311dca..e20086d7 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -46,6 +46,10 @@ export type ProcessorsConfig = { processors: IProcessorConfig[]; }; +export type IngestPipelineConfig = ProcessorsConfig; + +export type SearchPipelineConfig = ProcessorsConfig; + export type IndexConfig = { name: IConfigField; mappings: IConfigField; diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx index b6472b20..c47c04c7 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx @@ -20,10 +20,17 @@ import { EuiSpacer, EuiText, } from '@elastic/eui'; -import { PROCESSOR_CONTEXT, WorkflowFormValues } from '../../../../../common'; +import { + IProcessorConfig, + PROCESSOR_CONTEXT, + WorkflowConfig, + WorkflowFormValues, +} from '../../../../../common'; import { formikToIngestPipeline } from '../../../../utils'; interface InputTransformModalProps { + uiConfig: WorkflowConfig; + config: IProcessorConfig; context: PROCESSOR_CONTEXT; onClose: () => void; onConfirm: () => void; @@ -51,8 +58,13 @@ export function InputTransformModal(props: InputTransformModalProps) { onClick={() => { switch (props.context) { case PROCESSOR_CONTEXT.INGEST: { - // TODO: complete for ingest. generate and simulate an ingest pipeline up to this point - const curIngestPipeline = formikToIngestPipeline(values); + // TODO: simulate an ingest pipeline up to this point + const curIngestPipeline = formikToIngestPipeline( + values, + props.uiConfig, + props.config.id + ); + console.log('cur ingestpipeline: ', curIngestPipeline); break; } // TODO: complete for search request / search response contexts diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx index 10b8cd38..f817cd3d 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx @@ -11,6 +11,7 @@ import { IProcessorConfig, IConfigField, PROCESSOR_CONTEXT, + WorkflowConfig, } from '../../../../../common'; import { MapField, ModelField } from '../input_fields'; import { isEmpty } from 'lodash'; @@ -18,6 +19,7 @@ import { InputTransformModal } from './input_transform_modal'; import { OutputTransformModal } from './output_transform_modal'; interface MLProcessorInputsProps { + uiConfig: WorkflowConfig; config: IProcessorConfig; baseConfigPath: string; // the base path of the nested config, if applicable. e.g., 'ingest.enrich' onFormChange: () => void; @@ -53,13 +55,13 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { // transform option state const [selectedOption, setSelectedOption] = useState( - TRANSFORM_OPTION.SIMPLE + TRANSFORM_OPTION.ADVANCED ); // advanced transformations modal state const [isInputTransformModalOpen, setIsInputTransformModalOpen] = useState< boolean - >(true); + >(false); const [isOutputTransformModalOpen, setIsOutputTransformModalOpen] = useState< boolean >(false); @@ -68,6 +70,8 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { <> {isInputTransformModalOpen && ( setIsInputTransformModalOpen(false)} onConfirm={() => { diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx index 609de12c..ce64962b 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx @@ -9,6 +9,7 @@ import { IProcessorConfig, PROCESSOR_CONTEXT, PROCESSOR_TYPE, + WorkflowConfig, } from '../../../../../common'; import { MLProcessorInputs } from './ml_processor_inputs'; @@ -17,6 +18,7 @@ import { MLProcessorInputs } from './ml_processor_inputs'; */ interface ProcessorInputsProps { + uiConfig: WorkflowConfig; config: IProcessorConfig; baseConfigPath: string; // the base path of the nested config, if applicable. e.g., 'ingest.enrich' onFormChange: () => void; @@ -36,6 +38,7 @@ export function ProcessorInputs(props: ProcessorInputsProps) { el = ( pipeline utils ********************** @@ -13,9 +22,33 @@ import { WorkflowFormValues } from '../../common'; the input schema at a certain stage of a pipeline. */ -export function formikToIngestPipeline(values: WorkflowFormValues): {} { - const ingestConfig = values.ingest; - console.log('ingest config: ', ingestConfig); - // TODO: may be able to convert form -> config -> template, where template has the API-formatted configs. - return {}; +export function formikToIngestPipeline( + values: WorkflowFormValues, + existingConfig: WorkflowConfig, + curProcessorId: string +): IngestPipelineConfig | SearchPipelineConfig | undefined { + const uiConfig = formikToUiConfig(values, existingConfig); + const precedingProcessors = getPrecedingProcessors( + uiConfig.ingest.enrich.processors, + curProcessorId + ); + if (!isEmpty(precedingProcessors)) { + return { + processors: processorConfigsToTemplateProcessors(precedingProcessors), + } as IngestPipelineConfig | SearchPipelineConfig; + } + return undefined; +} + +function getPrecedingProcessors( + allProcessors: IProcessorConfig[], + curProcessorId: string +): IProcessorConfig[] { + const precedingProcessors = [] as IProcessorConfig[]; + allProcessors.forEach((processor) => { + if (processor.id !== curProcessorId) { + precedingProcessors.push(processor); + } + }); + return precedingProcessors; } From e4acea02294cf07a27e6550813659ef5415686fb Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 17 Jul 2024 10:42:39 -0700 Subject: [PATCH 05/10] Onboard simulate pipeline API; integrate and get working in UI Signed-off-by: Tyler Ohlsen --- common/constants.ts | 1 + common/interfaces.ts | 37 ++++++--- .../input_transform_modal.tsx | 80 +++++++++++++++++-- .../workflow_inputs/workflow_inputs.tsx | 2 +- .../workflows/workflow_list/workflow_list.tsx | 2 +- public/route_service.ts | 23 ++++++ public/store/reducers/opensearch_reducer.ts | 26 +++++- public/utils/utils.ts | 6 +- server/routes/opensearch_routes_service.ts | 37 +++++++++ 9 files changed, 193 insertions(+), 21 deletions(-) diff --git a/common/constants.ts b/common/constants.ts index ffed2a0f..2bda1ab9 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -32,6 +32,7 @@ export const BASE_OPENSEARCH_NODE_API_PATH = `${BASE_NODE_API_PATH}/opensearch`; export const CAT_INDICES_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/catIndices`; export const SEARCH_INDEX_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/search`; export const INGEST_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/ingest`; +export const SIMULATE_PIPELINE_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/simulatePipeline`; // Flow Framework node APIs export const BASE_WORKFLOW_NODE_API_PATH = `${BASE_NODE_API_PATH}/workflow`; diff --git a/common/interfaces.ts b/common/interfaces.ts index e20086d7..e0311eef 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -15,7 +15,6 @@ export type Index = { /** ********** WORKFLOW TYPES/INTERFACES ********** -TODO: over time these can become less generic as the form inputs & UX becomes finalized */ export type ConfigFieldType = 'string' | 'json' | 'select' | 'model' | 'map'; @@ -46,7 +45,9 @@ export type ProcessorsConfig = { processors: IProcessorConfig[]; }; -export type IngestPipelineConfig = ProcessorsConfig; +export type IngestPipelineConfig = ProcessorsConfig & { + description?: string; +}; export type SearchPipelineConfig = ProcessorsConfig; @@ -388,13 +389,6 @@ export type ModelFormValue = { ********** MISC TYPES/INTERFACES ************ */ -// TODO: finalize how we have the launch data model -export type WorkflowLaunch = { - id: string; - state: WORKFLOW_STATE; - lastUpdated: number; -}; - // Based off of https://github.com/opensearch-project/flow-framework/blob/main/src/main/java/org/opensearch/flowframework/model/State.java export enum WORKFLOW_STATE { NOT_STARTED = 'Not started', @@ -435,3 +429,28 @@ export enum WORKFLOW_STEP_TO_RESOURCE_TYPE_MAP { export type WorkflowDict = { [workflowId: string]: Workflow; }; + +/** + ********** OPENSEARCH TYPES/INTERFACES ************ + */ + +// from https://opensearch.org/docs/latest/ingest-pipelines/simulate-ingest/#example-specify-a-pipeline-in-the-path +export type SimulateIngestPipelineDoc = { + _index: string; + _id: string; + _source: {}; +}; + +// from https://opensearch.org/docs/latest/ingest-pipelines/simulate-ingest/#example-specify-a-pipeline-in-the-path +export type SimulateIngestPipelineDocResponse = { + doc: SimulateIngestPipelineDoc & { + _ingest: { + timestamp: string; + }; + }; +}; + +// from https://opensearch.org/docs/latest/ingest-pipelines/simulate-ingest/#example-specify-a-pipeline-in-the-path +export type SimulateIngestPipelineResponse = { + docs: SimulateIngestPipelineDocResponse[]; +}; diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx index c47c04c7..2cf6c4c6 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React from 'react'; +import React, { useState } from 'react'; import { useFormikContext } from 'formik'; import { EuiButton, @@ -22,11 +22,16 @@ import { } from '@elastic/eui'; import { IProcessorConfig, + IngestPipelineConfig, PROCESSOR_CONTEXT, + SimulateIngestPipelineDoc, + SimulateIngestPipelineResponse, WorkflowConfig, WorkflowFormValues, } from '../../../../../common'; -import { formikToIngestPipeline } from '../../../../utils'; +import { formikToIngestPipeline, generateId } from '../../../../utils'; +import { simulatePipeline, useAppDispatch } from '../../../../store'; +import { getCore } from '../../../../services'; interface InputTransformModalProps { uiConfig: WorkflowConfig; @@ -40,8 +45,13 @@ interface InputTransformModalProps { * A modal to configure advanced JSON-to-JSON transforms into a model's expected input */ export function InputTransformModal(props: InputTransformModalProps) { + const dispatch = useAppDispatch(); const { values } = useFormikContext(); + // source input / transformed output state + const [sourceInput, setSourceInput] = useState('{}'); + const [transformedOutput, setTransformedOutput] = useState('{}'); + return ( @@ -55,16 +65,45 @@ export function InputTransformModal(props: InputTransformModalProps) { <> { + onClick={async () => { switch (props.context) { case PROCESSOR_CONTEXT.INGEST: { - // TODO: simulate an ingest pipeline up to this point const curIngestPipeline = formikToIngestPipeline( values, props.uiConfig, props.config.id ); - console.log('cur ingestpipeline: ', curIngestPipeline); + // if there are preceding processors, we need to generate the ingest pipeline + // up to this point and simulate, in order to get the latest transformed + // version of the docs + if (curIngestPipeline !== undefined) { + const curDocs = prepareDocsForSimulate( + values.ingest.docs, + values.ingest.indexName + ); + await dispatch( + simulatePipeline({ + pipeline: curIngestPipeline as IngestPipelineConfig, + docs: curDocs, + }) + ) + .unwrap() + .then((resp: SimulateIngestPipelineResponse) => { + setSourceInput(unwrapTransformedDocs(resp)); + }) + .catch((error: any) => { + getCore().notifications.toasts.addDanger( + `Failed to fetch input schema` + ); + }); + } else { + // TODO: change to bulk API + const ingestDocObj = JSON.parse(values.ingest.docs); + const ingestDocsObjs = [ingestDocObj]; + setSourceInput( + JSON.stringify(ingestDocsObjs, undefined, 2) + ); + } break; } // TODO: complete for search request / search response contexts @@ -75,7 +114,7 @@ export function InputTransformModal(props: InputTransformModalProps) { - {`{"a": "b"}`} + {sourceInput} @@ -101,7 +140,7 @@ export function InputTransformModal(props: InputTransformModalProps) { Expected output - {`TODO: will be model input`} + {transformedOutput} @@ -116,3 +155,30 @@ export function InputTransformModal(props: InputTransformModalProps) { ); } + +// docs are expected to be in a certain format to be passed to the simulate ingest pipeline API. +// for details, see https://opensearch.org/docs/latest/ingest-pipelines/simulate-ingest +function prepareDocsForSimulate( + docs: string, + indexName: string +): SimulateIngestPipelineDoc[] { + // TODO: enhance to support bulk/multiple documents + return [ + { + _index: indexName, + _id: generateId(), + _source: JSON.parse(docs), + }, + ]; +} + +// docs are returned in a certain format from the simulate ingest pipeline API. We want +// to format them into a more readable string to display +function unwrapTransformedDocs( + simulatePipelineResponse: SimulateIngestPipelineResponse +) { + const transformedDocsSources = simulatePipelineResponse.docs.map( + (transformedDoc) => transformedDoc.doc._source + ); + return JSON.stringify(transformedDocsSources, undefined, 2); +} diff --git a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx index bf6b2f78..3ab8cbea 100644 --- a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx @@ -209,7 +209,7 @@ export function WorkflowInputs(props: WorkflowInputsProps) { try { let ingestDocsObjs = [] as {}[]; try { - // TODO: test with multiple objs, make sure parsing logic works + // TODO: change to bulk API const ingestDocObj = JSON.parse(props.ingestDocs); ingestDocsObjs = [ingestDocObj]; } catch (e) {} diff --git a/public/pages/workflows/workflow_list/workflow_list.tsx b/public/pages/workflows/workflow_list/workflow_list.tsx index 29ec61d6..9e7c0a2a 100644 --- a/public/pages/workflows/workflow_list/workflow_list.tsx +++ b/public/pages/workflows/workflow_list/workflow_list.tsx @@ -152,7 +152,7 @@ export function WorkflowList(props: WorkflowListProps) { ); }) .catch((err: any) => { - getCore().notifications.toasts.addSuccess( + getCore().notifications.toasts.addDanger( `Failed to delete ${selectedWorkflow.name}` ); console.error( diff --git a/public/route_service.ts b/public/route_service.ts index 782de01f..3d8ac79d 100644 --- a/public/route_service.ts +++ b/public/route_service.ts @@ -19,6 +19,9 @@ import { WorkflowTemplate, SEARCH_INDEX_NODE_API_PATH, INGEST_NODE_API_PATH, + SIMULATE_PIPELINE_NODE_API_PATH, + IngestPipelineConfig, + SimulateIngestPipelineDoc, } from '../common'; /** @@ -49,6 +52,10 @@ export interface RouteService { ) => Promise; ingest: (index: string, doc: {}) => Promise; searchModels: (body: {}) => Promise; + simulatePipeline: (body: { + pipeline: IngestPipelineConfig; + docs: SimulateIngestPipelineDoc[]; + }) => Promise; } export function configureRoutes(core: CoreStart): RouteService { @@ -205,5 +212,21 @@ export function configureRoutes(core: CoreStart): RouteService { return e as HttpFetchError; } }, + simulatePipeline: async (body: { + pipeline: IngestPipelineConfig; + docs: SimulateIngestPipelineDoc[]; + }) => { + try { + const response = await core.http.post<{ respString: string }>( + SIMULATE_PIPELINE_NODE_API_PATH, + { + body: JSON.stringify(body), + } + ); + return response; + } catch (e: any) { + return e as HttpFetchError; + } + }, }; } diff --git a/public/store/reducers/opensearch_reducer.ts b/public/store/reducers/opensearch_reducer.ts index 44a92b06..ec4f1db1 100644 --- a/public/store/reducers/opensearch_reducer.ts +++ b/public/store/reducers/opensearch_reducer.ts @@ -5,7 +5,11 @@ import { createAsyncThunk, createSlice } from '@reduxjs/toolkit'; import { getRouteService } from '../../services'; -import { Index } from '../../../common'; +import { + Index, + IngestPipelineConfig, + SimulateIngestPipelineDoc, +} from '../../../common'; import { HttpFetchError } from '../../../../../src/core/public'; const initialState = { @@ -18,6 +22,7 @@ const OPENSEARCH_PREFIX = 'opensearch'; const CAT_INDICES_ACTION = `${OPENSEARCH_PREFIX}/catIndices`; const SEARCH_INDEX_ACTION = `${OPENSEARCH_PREFIX}/search`; const INGEST_ACTION = `${OPENSEARCH_PREFIX}/ingest`; +const SIMULATE_PIPELINE_ACTION = `${OPENSEARCH_PREFIX}/simulatePipeline`; export const catIndices = createAsyncThunk( CAT_INDICES_ACTION, @@ -75,6 +80,25 @@ export const ingest = createAsyncThunk( } ); +export const simulatePipeline = createAsyncThunk( + SIMULATE_PIPELINE_ACTION, + async ( + body: { pipeline: IngestPipelineConfig; docs: SimulateIngestPipelineDoc[] }, + { rejectWithValue } + ) => { + const response: + | any + | HttpFetchError = await getRouteService().simulatePipeline(body); + if (response instanceof HttpFetchError) { + return rejectWithValue( + 'Error simulating ingest pipeline: ' + response.body.message + ); + } else { + return response; + } + } +); + const opensearchSlice = createSlice({ name: OPENSEARCH_PREFIX, initialState, diff --git a/public/utils/utils.ts b/public/utils/utils.ts index d38347a5..80eaac7d 100644 --- a/public/utils/utils.ts +++ b/public/utils/utils.ts @@ -7,12 +7,14 @@ import yaml from 'js-yaml'; import { WORKFLOW_STEP_TYPE, Workflow } from '../../common'; // Append 16 random characters -export function generateId(prefix: string): string { +export function generateId(prefix?: string): string { const uniqueChar = () => { // eslint-disable-next-line no-bitwise return (((1 + Math.random()) * 0x10000) | 0).toString(16).substring(1); }; - return `${prefix}_${uniqueChar()}${uniqueChar()}${uniqueChar()}${uniqueChar()}`; + return `${ + prefix || '' + }_${uniqueChar()}${uniqueChar()}${uniqueChar()}${uniqueChar()}`; } export function hasProvisionedIngestResources( diff --git a/server/routes/opensearch_routes_service.ts b/server/routes/opensearch_routes_service.ts index b1440f97..6d070ba0 100644 --- a/server/routes/opensearch_routes_service.ts +++ b/server/routes/opensearch_routes_service.ts @@ -15,7 +15,11 @@ import { CAT_INDICES_NODE_API_PATH, INGEST_NODE_API_PATH, Index, + IngestPipelineConfig, SEARCH_INDEX_NODE_API_PATH, + SIMULATE_PIPELINE_NODE_API_PATH, + SimulateIngestPipelineDoc, + SimulateIngestPipelineResponse, } from '../../common'; import { generateCustomError } from './helpers'; @@ -75,6 +79,18 @@ export function registerOpenSearchRoutes( }, opensearchRoutesService.ingest ); + router.post( + { + path: SIMULATE_PIPELINE_NODE_API_PATH, + validate: { + body: schema.object({ + pipeline: schema.any(), + docs: schema.any(), + }), + }, + }, + opensearchRoutesService.simulatePipeline + ); } export class OpenSearchRoutesService { @@ -156,4 +172,25 @@ export class OpenSearchRoutesService { return generateCustomError(res, err); } }; + + simulatePipeline = async ( + context: RequestHandlerContext, + req: OpenSearchDashboardsRequest, + res: OpenSearchDashboardsResponseFactory + ): Promise> => { + const { pipeline, docs } = req.body as { + pipeline: IngestPipelineConfig; + docs: SimulateIngestPipelineDoc[]; + }; + try { + const response = await this.client + .asScoped(req) + .callAsCurrentUser('ingest.simulate', { body: { pipeline, docs } }); + return res.ok({ + body: { docs: response.docs } as SimulateIngestPipelineResponse, + }); + } catch (err: any) { + return generateCustomError(res, err); + } + }; } From 042f3d318a9eb432c5a72b9a937feff6532565c7 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 17 Jul 2024 12:40:12 -0700 Subject: [PATCH 06/10] Add bulk api; refactor ingest to use bulk api; support multi-doc simulation; Signed-off-by: Tyler Ohlsen --- common/constants.ts | 1 + common/interfaces.ts | 3 ++ .../input_transform_modal.tsx | 41 +++++++++++------ .../workflow_inputs/workflow_inputs.tsx | 35 ++++++++++---- public/route_service.ts | 15 ++++++ public/store/reducers/opensearch_reducer.ts | 20 ++++++++ server/routes/opensearch_routes_service.ts | 46 +++++++++++++++++++ 7 files changed, 139 insertions(+), 22 deletions(-) diff --git a/common/constants.ts b/common/constants.ts index 2bda1ab9..10ada745 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -32,6 +32,7 @@ export const BASE_OPENSEARCH_NODE_API_PATH = `${BASE_NODE_API_PATH}/opensearch`; export const CAT_INDICES_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/catIndices`; export const SEARCH_INDEX_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/search`; export const INGEST_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/ingest`; +export const BULK_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/bulk`; export const SIMULATE_PIPELINE_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/simulatePipeline`; // Flow Framework node APIs diff --git a/common/interfaces.ts b/common/interfaces.ts index e0311eef..5dd84445 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -448,6 +448,9 @@ export type SimulateIngestPipelineDocResponse = { timestamp: string; }; }; + error?: { + reason: string; + }; }; // from https://opensearch.org/docs/latest/ingest-pipelines/simulate-ingest/#example-specify-a-pipeline-in-the-path diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx index 2cf6c4c6..7a45c1cc 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx @@ -79,7 +79,7 @@ export function InputTransformModal(props: InputTransformModalProps) { if (curIngestPipeline !== undefined) { const curDocs = prepareDocsForSimulate( values.ingest.docs, - values.ingest.indexName + values.ingest.index.name ); await dispatch( simulatePipeline({ @@ -97,12 +97,7 @@ export function InputTransformModal(props: InputTransformModalProps) { ); }); } else { - // TODO: change to bulk API - const ingestDocObj = JSON.parse(values.ingest.docs); - const ingestDocsObjs = [ingestDocObj]; - setSourceInput( - JSON.stringify(ingestDocsObjs, undefined, 2) - ); + setSourceInput(values.ingest.docs); } break; } @@ -162,14 +157,16 @@ function prepareDocsForSimulate( docs: string, indexName: string ): SimulateIngestPipelineDoc[] { - // TODO: enhance to support bulk/multiple documents - return [ - { + const preparedDocs = [] as SimulateIngestPipelineDoc[]; + const docObjs = JSON.parse(docs) as {}[]; + docObjs.forEach((doc) => { + preparedDocs.push({ _index: indexName, _id: generateId(), - _source: JSON.parse(docs), - }, - ]; + _source: doc, + }); + }); + return preparedDocs; } // docs are returned in a certain format from the simulate ingest pipeline API. We want @@ -177,8 +174,24 @@ function prepareDocsForSimulate( function unwrapTransformedDocs( simulatePipelineResponse: SimulateIngestPipelineResponse ) { + let errorDuringSimulate = undefined as string | undefined; const transformedDocsSources = simulatePipelineResponse.docs.map( - (transformedDoc) => transformedDoc.doc._source + (transformedDoc) => { + if (transformedDoc.error !== undefined) { + errorDuringSimulate = transformedDoc.error.reason || ''; + } else { + return transformedDoc.doc._source; + } + } ); + + // there is an edge case where simulate may fail if there is some server-side or OpenSearch issue when + // running ingest (e.g., hitting rate limits on remote model) + // We pull out any returned error from a document and propagate it to the user. + if (errorDuringSimulate !== undefined) { + getCore().notifications.toasts.addDanger( + `Failed to simulate ingest on all documents: ${errorDuringSimulate}` + ); + } return JSON.stringify(transformedDocsSources, undefined, 2); } diff --git a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx index 3ab8cbea..e6dc6322 100644 --- a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx @@ -36,9 +36,9 @@ import { IngestInputs } from './ingest_inputs'; import { SearchInputs } from './search_inputs'; import { AppState, + bulk, deprovisionWorkflow, getWorkflow, - ingest, provisionWorkflow, removeDirty, searchIndex, @@ -52,6 +52,7 @@ import { configToTemplateFlows, hasProvisionedIngestResources, hasProvisionedSearchResources, + generateId, } from '../../../utils'; import { BooleanField } from './input_fields'; import { ExportOptions } from './export_options'; @@ -209,16 +210,16 @@ export function WorkflowInputs(props: WorkflowInputsProps) { try { let ingestDocsObjs = [] as {}[]; try { - // TODO: change to bulk API - const ingestDocObj = JSON.parse(props.ingestDocs); - ingestDocsObjs = [ingestDocObj]; + ingestDocsObjs = JSON.parse(props.ingestDocs); } catch (e) {} if (ingestDocsObjs.length > 0 && !isEmpty(ingestDocsObjs[0])) { success = await validateAndUpdateWorkflow(); if (success) { - const indexName = values.ingest.index.name; - const doc = ingestDocsObjs[0]; - dispatch(ingest({ index: indexName, doc })) + const bulkBody = prepareBulkBody( + values.ingest.index.name, + ingestDocsObjs + ); + dispatch(bulk({ body: bulkBody })) .unwrap() .then(async (resp) => { props.setIngestResponse(JSON.stringify(resp, undefined, 2)); @@ -230,7 +231,9 @@ export function WorkflowInputs(props: WorkflowInputsProps) { }); } } else { - getCore().notifications.toasts.addDanger('No valid document provided'); + getCore().notifications.toasts.addDanger( + 'No valid document provided. Ensure it is a valid JSON array.' + ); } } catch (error) { console.error('Error ingesting documents: ', error); @@ -559,3 +562,19 @@ export function WorkflowInputs(props: WorkflowInputsProps) { ); } + +// ingesting multiple documents must follow the proper format for the bulk API. +// see https://opensearch.org/docs/latest/api-reference/document-apis/bulk/#request-body +function prepareBulkBody(indexName: string, docObjs: {}[]): {} { + const bulkBody = [] as any[]; + docObjs.forEach((doc) => { + bulkBody.push({ + index: { + _index: indexName, + _id: generateId(), + }, + }); + bulkBody.push(doc); + }); + return bulkBody; +} diff --git a/public/route_service.ts b/public/route_service.ts index 3d8ac79d..9e110c38 100644 --- a/public/route_service.ts +++ b/public/route_service.ts @@ -22,6 +22,7 @@ import { SIMULATE_PIPELINE_NODE_API_PATH, IngestPipelineConfig, SimulateIngestPipelineDoc, + BULK_NODE_API_PATH, } from '../common'; /** @@ -51,6 +52,7 @@ export interface RouteService { searchPipeline?: string ) => Promise; ingest: (index: string, doc: {}) => Promise; + bulk: (body: {}, ingestPipeline?: string) => Promise; searchModels: (body: {}) => Promise; simulatePipeline: (body: { pipeline: IngestPipelineConfig; @@ -199,6 +201,19 @@ export function configureRoutes(core: CoreStart): RouteService { return e as HttpFetchError; } }, + bulk: async (body: {}, ingestPipeline?: string) => { + try { + const path = ingestPipeline + ? `${BULK_NODE_API_PATH}/${ingestPipeline}` + : BULK_NODE_API_PATH; + const response = await core.http.post<{ respString: string }>(path, { + body: JSON.stringify(body), + }); + return response; + } catch (e: any) { + return e as HttpFetchError; + } + }, searchModels: async (body: {}) => { try { const response = await core.http.post<{ respString: string }>( diff --git a/public/store/reducers/opensearch_reducer.ts b/public/store/reducers/opensearch_reducer.ts index ec4f1db1..c977a05b 100644 --- a/public/store/reducers/opensearch_reducer.ts +++ b/public/store/reducers/opensearch_reducer.ts @@ -22,6 +22,7 @@ const OPENSEARCH_PREFIX = 'opensearch'; const CAT_INDICES_ACTION = `${OPENSEARCH_PREFIX}/catIndices`; const SEARCH_INDEX_ACTION = `${OPENSEARCH_PREFIX}/search`; const INGEST_ACTION = `${OPENSEARCH_PREFIX}/ingest`; +const BULK_ACTION = `${OPENSEARCH_PREFIX}/bulk`; const SIMULATE_PIPELINE_ACTION = `${OPENSEARCH_PREFIX}/simulatePipeline`; export const catIndices = createAsyncThunk( @@ -80,6 +81,25 @@ export const ingest = createAsyncThunk( } ); +export const bulk = createAsyncThunk( + BULK_ACTION, + async ( + bulkInfo: { body: {}; ingestPipeline?: string }, + { rejectWithValue } + ) => { + const { body, ingestPipeline } = bulkInfo; + const response: any | HttpFetchError = await getRouteService().bulk( + body, + ingestPipeline + ); + if (response instanceof HttpFetchError) { + return rejectWithValue('Error performing bulk: ' + response.body.message); + } else { + return response; + } + } +); + export const simulatePipeline = createAsyncThunk( SIMULATE_PIPELINE_ACTION, async ( diff --git a/server/routes/opensearch_routes_service.ts b/server/routes/opensearch_routes_service.ts index 6d070ba0..084e2a3b 100644 --- a/server/routes/opensearch_routes_service.ts +++ b/server/routes/opensearch_routes_service.ts @@ -12,6 +12,7 @@ import { OpenSearchDashboardsResponseFactory, } from '../../../../src/core/server'; import { + BULK_NODE_API_PATH, CAT_INDICES_NODE_API_PATH, INGEST_NODE_API_PATH, Index, @@ -79,6 +80,27 @@ export function registerOpenSearchRoutes( }, opensearchRoutesService.ingest ); + router.post( + { + path: `${BULK_NODE_API_PATH}/{pipeline}`, + validate: { + params: schema.object({ + pipeline: schema.string(), + }), + body: schema.any(), + }, + }, + opensearchRoutesService.bulk + ); + router.post( + { + path: BULK_NODE_API_PATH, + validate: { + body: schema.any(), + }, + }, + opensearchRoutesService.bulk + ); router.post( { path: SIMULATE_PIPELINE_NODE_API_PATH, @@ -173,6 +195,30 @@ export class OpenSearchRoutesService { } }; + bulk = async ( + context: RequestHandlerContext, + req: OpenSearchDashboardsRequest, + res: OpenSearchDashboardsResponseFactory + ): Promise> => { + const { pipeline } = req.params as { + pipeline: string | undefined; + }; + const body = req.body; + + try { + const response = await this.client + .asScoped(req) + .callAsCurrentUser('bulk', { + body, + pipeline, + }); + + return res.ok({ body: response }); + } catch (err: any) { + return generateCustomError(res, err); + } + }; + simulatePipeline = async ( context: RequestHandlerContext, req: OpenSearchDashboardsRequest, From b4abd93d66758784e09ebdd6dccf19d41e7870ea Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 17 Jul 2024 12:51:17 -0700 Subject: [PATCH 07/10] Update defaults/wording around ingest docs being an array Signed-off-by: Tyler Ohlsen --- .../workflow_inputs/ingest_inputs/source_data.tsx | 1 + .../processor_inputs/input_transform_modal.tsx | 4 ++-- public/utils/config_to_form_utils.ts | 2 +- public/utils/config_to_schema_utils.ts | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx index d6ba2a8e..2e8a1937 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx @@ -65,6 +65,7 @@ export function SourceData(props: SourceDataProps) { {}} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx index 7a45c1cc..670d22be 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx @@ -49,8 +49,8 @@ export function InputTransformModal(props: InputTransformModalProps) { const { values } = useFormikContext(); // source input / transformed output state - const [sourceInput, setSourceInput] = useState('{}'); - const [transformedOutput, setTransformedOutput] = useState('{}'); + const [sourceInput, setSourceInput] = useState('[]'); + const [transformedOutput, setTransformedOutput] = useState('[]'); return ( diff --git a/public/utils/config_to_form_utils.ts b/public/utils/config_to_form_utils.ts index fa6a3dc3..fb3bee5f 100644 --- a/public/utils/config_to_form_utils.ts +++ b/public/utils/config_to_form_utils.ts @@ -42,7 +42,7 @@ function ingestConfigToFormik( let ingestFormikValues = {} as FormikValues; if (ingestConfig) { ingestFormikValues['enabled'] = ingestConfig.enabled; - ingestFormikValues['docs'] = ingestDocs || getInitialValue('json'); + ingestFormikValues['docs'] = ingestDocs || '[]'; ingestFormikValues['enrich'] = processorsConfigToFormik( ingestConfig.enrich ); diff --git a/public/utils/config_to_schema_utils.ts b/public/utils/config_to_schema_utils.ts index 72d8055a..7f4a2a39 100644 --- a/public/utils/config_to_schema_utils.ts +++ b/public/utils/config_to_schema_utils.ts @@ -108,10 +108,10 @@ function getFieldSchema(fieldType: ConfigFieldType): Schema { break; } case 'json': { - baseSchema = yup.string().test('json', 'Invalid JSON', (value) => { + baseSchema = yup.string().test('json', 'Invalid JSON array', (value) => { try { // @ts-ignore - JSON.parse(value); + return Array.isArray(JSON.parse(value)); return true; } catch (error) { return false; From f728c1813a2d780c1604bfda3b2ad15509aed76b Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 17 Jul 2024 13:22:33 -0700 Subject: [PATCH 08/10] Refactor JSON array into standalone field type / validation / defaults Signed-off-by: Tyler Ohlsen --- common/interfaces.ts | 8 +++++++- public/utils/config_to_form_utils.ts | 5 ++++- public/utils/config_to_schema_utils.ts | 21 ++++++++++++++++++--- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/common/interfaces.ts b/common/interfaces.ts index 5dd84445..205707ae 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -17,7 +17,13 @@ export type Index = { ********** WORKFLOW TYPES/INTERFACES ********** */ -export type ConfigFieldType = 'string' | 'json' | 'select' | 'model' | 'map'; +export type ConfigFieldType = + | 'string' + | 'json' + | 'jsonArray' + | 'select' + | 'model' + | 'map'; export type ConfigFieldValue = string | {}; export interface IConfigField { type: ConfigFieldType; diff --git a/public/utils/config_to_form_utils.ts b/public/utils/config_to_form_utils.ts index fb3bee5f..afc51414 100644 --- a/public/utils/config_to_form_utils.ts +++ b/public/utils/config_to_form_utils.ts @@ -42,7 +42,7 @@ function ingestConfigToFormik( let ingestFormikValues = {} as FormikValues; if (ingestConfig) { ingestFormikValues['enabled'] = ingestConfig.enabled; - ingestFormikValues['docs'] = ingestDocs || '[]'; + ingestFormikValues['docs'] = ingestDocs || getInitialValue('jsonArray'); ingestFormikValues['enrich'] = processorsConfigToFormik( ingestConfig.enrich ); @@ -120,5 +120,8 @@ export function getInitialValue(fieldType: ConfigFieldType): ConfigFieldValue { case 'json': { return '{}'; } + case 'jsonArray': { + return '[]'; + } } } diff --git a/public/utils/config_to_schema_utils.ts b/public/utils/config_to_schema_utils.ts index 7f4a2a39..9ccb60c0 100644 --- a/public/utils/config_to_schema_utils.ts +++ b/public/utils/config_to_schema_utils.ts @@ -32,7 +32,7 @@ function ingestConfigToSchema( ): ObjectSchema { const ingestSchemaObj = {} as { [key: string]: Schema }; if (ingestConfig) { - ingestSchemaObj['docs'] = getFieldSchema('json'); + ingestSchemaObj['docs'] = getFieldSchema('jsonArray'); ingestSchemaObj['enrich'] = processorsConfigToSchema(ingestConfig.enrich); ingestSchemaObj['index'] = indexConfigToSchema(ingestConfig.index); } @@ -108,16 +108,31 @@ function getFieldSchema(fieldType: ConfigFieldType): Schema { break; } case 'json': { - baseSchema = yup.string().test('json', 'Invalid JSON array', (value) => { + baseSchema = yup.string().test('json', 'Invalid JSON', (value) => { try { // @ts-ignore - return Array.isArray(JSON.parse(value)); + JSON.parse(value); return true; } catch (error) { return false; } }); + break; + } + case 'jsonArray': { + baseSchema = yup + .string() + .test('jsonArray', 'Invalid JSON array', (value) => { + try { + // @ts-ignore + return Array.isArray(JSON.parse(value)); + return true; + } catch (error) { + return false; + } + }); + break; } } From 8e5f6fd926ec1f62ff46a4dcc65127d7cd21b4f9 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 17 Jul 2024 14:21:12 -0700 Subject: [PATCH 09/10] fix multi-processor editing preceding processor bug Signed-off-by: Tyler Ohlsen --- public/utils/form_to_pipeline_utils.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/public/utils/form_to_pipeline_utils.ts b/public/utils/form_to_pipeline_utils.ts index 094ec3e6..21f7f957 100644 --- a/public/utils/form_to_pipeline_utils.ts +++ b/public/utils/form_to_pipeline_utils.ts @@ -45,8 +45,10 @@ function getPrecedingProcessors( curProcessorId: string ): IProcessorConfig[] { const precedingProcessors = [] as IProcessorConfig[]; - allProcessors.forEach((processor) => { - if (processor.id !== curProcessorId) { + allProcessors.some((processor) => { + if (processor.id === curProcessorId) { + return true; + } else { precedingProcessors.push(processor); } }); From b1797c8da70649d47b5aff4a08a068caf7d85ffb Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 17 Jul 2024 14:43:26 -0700 Subject: [PATCH 10/10] Remove radio; other minor visual updates Signed-off-by: Tyler Ohlsen --- .../advanced_transform_modal.tsx | 45 ------- .../input_transform_modal.tsx | 9 +- .../processor_inputs/ml_processor_inputs.tsx | 116 +++++++----------- 3 files changed, 46 insertions(+), 124 deletions(-) delete mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/advanced_transform_modal.tsx diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/advanced_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/advanced_transform_modal.tsx deleted file mode 100644 index 4138c9f8..00000000 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/advanced_transform_modal.tsx +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React from 'react'; -import { - EuiButton, - EuiButtonEmpty, - EuiModal, - EuiModalBody, - EuiModalFooter, - EuiModalHeader, - EuiModalHeaderTitle, - EuiText, -} from '@elastic/eui'; - -interface AdvancedTransformModalProps { - onClose: () => void; - onConfirm: () => void; -} - -/** - * A modal to perform advanced JSON-to-JSON transforms to/from a model's input/output, respectively - */ -export function AdvancedTransformModal(props: AdvancedTransformModalProps) { - return ( - - - -

{`Configure advanced transform`}

- - - - TODO TODO TODO - - - Cancel - - Save - - - - ); -} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx index 670d22be..9995303e 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx @@ -50,7 +50,7 @@ export function InputTransformModal(props: InputTransformModalProps) { // source input / transformed output state const [sourceInput, setSourceInput] = useState('[]'); - const [transformedOutput, setTransformedOutput] = useState('[]'); + const [transformedOutput, setTransformedOutput] = useState('TODO'); return ( @@ -63,8 +63,9 @@ export function InputTransformModal(props: InputTransformModalProps) { <> + Expected input { switch (props.context) { case PROCESSOR_CONTEXT.INGEST: { @@ -105,7 +106,7 @@ export function InputTransformModal(props: InputTransformModalProps) { } }} > - Fetch expected input + Fetch @@ -120,7 +121,7 @@ export function InputTransformModal(props: InputTransformModalProps) { ( - TRANSFORM_OPTION.ADVANCED - ); - // advanced transformations modal state const [isInputTransformModalOpen, setIsInputTransformModalOpen] = useState< boolean @@ -99,72 +89,48 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { {`Configure data transformations (optional)`} - { - setSelectedOption(option as TRANSFORM_OPTION); + + { + setIsInputTransformModalOpen(true); }} + > + Advanced input configuration + + + + + { + setIsOutputTransformModalOpen(true); + }} + > + Advanced output configuration + + + - {selectedOption === TRANSFORM_OPTION.SIMPLE && ( - <> - - - - - - )} - {selectedOption === TRANSFORM_OPTION.ADVANCED && ( - <> - - { - setIsInputTransformModalOpen(true); - }} - > - Configure input transformation - - - { - setIsOutputTransformModalOpen(true); - }} - > - Configure output transformation - - - )} )}