Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Integrate with JSONPath; complete input transform (ingest) #229

Merged
merged 5 commits into from
Jul 22, 2024
Merged

Integrate with JSONPath; complete input transform (ingest) #229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
68427ba
lower chance of error during manual provision / deprovision
ohltyler Jul 19, 2024
315f067
Onboard jsonpath; add basic form inputs and testing functionality in …
ohltyler Jul 19, 2024
1437fea
Add autosave trigger for map add/delete
ohltyler Jul 19, 2024
05a0a60
Consistently execute dot notation of jsonpath to match ml inference p…
ohltyler Jul 22, 2024
9064590
Update buttons; refetch workflow after autosave
ohltyler Jul 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions common/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ export const FETCH_ALL_QUERY_BODY = {
size: 1000,
};
export const INDEX_NOT_FOUND_EXCEPTION = 'index_not_found_exception';
export const JSONPATH_ROOT_SELECTOR = '$.';

export enum PROCESSOR_CONTEXT {
INGEST = 'ingest',
Expand Down
4 changes: 3 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,13 @@
]
},
"dependencies": {
"@types/jsonpath": "^0.2.4",
"formik": "2.4.2",
"js-yaml": "^4.1.0",
"jsonpath": "^1.1.1",
"reactflow": "^11.8.3",
"yup": "^1.3.2"
},
"devDependencies": {},
"resolutions": {}
}
}
25 changes: 0 additions & 25 deletions public/pages/workflow_detail/workflow_inputs/config_field_list.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,31 +55,6 @@ export function ConfigFieldList(props: ConfigFieldListProps) {
);
break;
}
// case 'map': {
// el = (
// <EuiFlexItem key={idx}>
// <MapField
// field={field}
// fieldPath={`${props.baseConfigPath}.${configId}.${field.id}`}
// onFormChange={props.onFormChange}
// />
// <EuiSpacer size={CONFIG_FIELD_SPACER_SIZE} />
// </EuiFlexItem>
// );
// break;
// }
// case 'json': {
// el = (
// <EuiFlexItem key={idx}>
// <JsonField
// label={field.label}
// placeholder={field.placeholder || ''}
// />
// <EuiSpacer size={INPUT_FIELD_SPACER_SIZE} />
// </EuiFlexItem>
// );
// break;
// }
}
return el;
})}
Expand Down
13 changes: 7 additions & 6 deletions public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,21 +28,24 @@ interface MapFieldProps {
label: string;
helpLink?: string;
helpText?: string;
keyPlaceholder?: string;
valuePlaceholder?: string;
onFormChange: () => void;
}

/**
* Input component for configuring field mappings
*/
export function MapField(props: MapFieldProps) {
const { setFieldValue, errors, touched } = useFormikContext<
const { setFieldValue, setFieldTouched, errors, touched } = useFormikContext<
WorkflowFormValues
>();

// Adding a map entry to the end of the existing arr
function addMapEntry(curEntries: MapFormValue): void {
const updatedEntries = [...curEntries, { key: '', value: '' } as MapEntry];
setFieldValue(props.fieldPath, updatedEntries);
setFieldTouched(props.fieldPath, true);
props.onFormChange();
}

Expand All @@ -54,6 +57,7 @@ export function MapField(props: MapFieldProps) {
const updatedEntries = [...curEntries];
updatedEntries.splice(entryIndexToDelete, 1);
setFieldValue(props.fieldPath, updatedEntries);
setFieldTouched(props.fieldPath, true);
props.onFormChange();
}

Expand Down Expand Up @@ -97,10 +101,7 @@ export function MapField(props: MapFieldProps) {
startControl={
<input
type="string"
// TODO: find a way to config/title the placeholder text.
// For example, K/V values have different meanings if input
// map or output map for ML inference processors.
placeholder="Input"
placeholder={props.keyPlaceholder || 'Input'}
className="euiFieldText"
value={mapping.key}
onChange={(e) => {
Expand All @@ -119,7 +120,7 @@ export function MapField(props: MapFieldProps) {
endControl={
<input
type="string"
placeholder="Output"
placeholder={props.valuePlaceholder || 'Output'}
className="euiFieldText"
value={mapping.value}
onChange={(e) => {
Expand Down
128 changes: 108 additions & 20 deletions public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@
*/

import React, { useState } from 'react';
import { useFormikContext } from 'formik';
import { useFormikContext, getIn } from 'formik';
import { isEmpty, get } from 'lodash';
import jsonpath from 'jsonpath';
import {
EuiButton,
EuiButtonEmpty,
EuiCodeBlock,
EuiCodeEditor,
EuiFlexGroup,
EuiFlexItem,
EuiModal,
Expand All @@ -21,8 +21,10 @@ import {
EuiText,
} from '@elastic/eui';
import {
IConfigField,
IProcessorConfig,
IngestPipelineConfig,
JSONPATH_ROOT_SELECTOR,
PROCESSOR_CONTEXT,
SimulateIngestPipelineDoc,
SimulateIngestPipelineResponse,
Expand All @@ -32,13 +34,16 @@ import {
import { formikToIngestPipeline, generateId } from '../../../../utils';
import { simulatePipeline, useAppDispatch } from '../../../../store';
import { getCore } from '../../../../services';
import { MapField } from '../input_fields';

interface InputTransformModalProps {
uiConfig: WorkflowConfig;
config: IProcessorConfig;
context: PROCESSOR_CONTEXT;
inputMapField: IConfigField;
inputMapFieldPath: string;
onClose: () => void;
onConfirm: () => void;
onFormChange: () => void;
}

/**
Expand All @@ -50,13 +55,16 @@ export function InputTransformModal(props: InputTransformModalProps) {

// source input / transformed output state
const [sourceInput, setSourceInput] = useState<string>('[]');
const [transformedOutput, setTransformedOutput] = useState<string>('TODO');
const [transformedOutput, setTransformedOutput] = useState<string>('[]');

// parse out the values and determine if there are none/some/all valid jsonpaths
const mapValues = getIn(values, `ingest.enrich.${props.config.id}.inputMap`);

return (
<EuiModal onClose={props.onClose} style={{ width: '70vw' }}>
<EuiModalHeader>
<EuiModalHeaderTitle>
<p>{`Configure input transform`}</p>
<p>{`Configure input`}</p>
</EuiModalHeaderTitle>
</EuiModalHeader>
<EuiModalBody>
Expand Down Expand Up @@ -116,24 +124,105 @@ export function InputTransformModal(props: InputTransformModalProps) {
</EuiFlexItem>
<EuiFlexItem>
<>
<EuiText>Define transform with JSONPath</EuiText>
<EuiText>Define transform</EuiText>
<EuiText size="s" color="subdued">
{`Dot notation is used by default. To explicitly use JSONPath, please ensure to prepend with the
root object selector "${JSONPATH_ROOT_SELECTOR}"`}
</EuiText>
<EuiSpacer size="s" />
<EuiCodeEditor
mode="json"
theme="textmate"
value={`TODO`}
readOnly={false}
setOptions={{
fontSize: '12px',
autoScrollEditorIntoView: true,
}}
tabSize={2}
<MapField
field={props.inputMapField}
fieldPath={props.inputMapFieldPath}
label="Input map"
helpText={`An array specifying how to map fields from the ingested document to the model’s input.`}
helpLink={
'https://opensearch.org/docs/latest/ingest-pipelines/processors/ml-inference/#configuration-parameters'
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should have the links in a common file or the constant file

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, will move in the next PR

}
keyPlaceholder="Model input field"
valuePlaceholder="Document field"
onFormChange={props.onFormChange}
/>
</>
</EuiFlexItem>
<EuiFlexItem>
<>
<EuiText>Expected output</EuiText>
<EuiButton
style={{ width: '100px' }}
disabled={
isEmpty(mapValues) || isEmpty(JSON.parse(sourceInput))
}
onClick={async () => {
switch (props.context) {
case PROCESSOR_CONTEXT.INGEST: {
if (
!isEmpty(mapValues) &&
!isEmpty(JSON.parse(sourceInput))
) {
let output = {};
let sampleSourceInput = {};
try {
sampleSourceInput = JSON.parse(sourceInput)[0];
} catch {}

mapValues.forEach(
(mapValue: { key: string; value: string }) => {
const path = mapValue.value;
try {
let transformedResult = undefined;
// ML inference processors will use standard dot notation or JSONPath depending on the input.
// We follow the same logic here to generate consistent results.
if (
mapValue.value.startsWith(
JSONPATH_ROOT_SELECTOR
)
) {
// JSONPath transform
transformedResult = jsonpath.query(
sampleSourceInput,
path
);
// Bracket notation not supported - throw an error
} else if (
mapValue.value.includes(']') ||
mapValue.value.includes(']')
) {
throw new Error();
// Standard dot notation
} else {
transformedResult = get(
sampleSourceInput,
path
);
}

output = {
...output,
[mapValue.key]: transformedResult || '',
};

setTransformedOutput(
JSON.stringify(output, undefined, 2)
);
} catch (e: any) {
console.error(e);
getCore().notifications.toasts.addDanger(
'Error generating expected output. Ensure your inputs are valid JSONPath or dot notation syntax.',
e
);
}
}
);
}

break;
}
// TODO: complete for search request / search response contexts
}
}}
>
Generate
</EuiButton>
<EuiSpacer size="s" />
<EuiCodeBlock fontSize="m" isCopyable={false}>
{transformedOutput}
Expand All @@ -143,9 +232,8 @@ export function InputTransformModal(props: InputTransformModalProps) {
</EuiFlexGroup>
</EuiModalBody>
<EuiModalFooter>
<EuiButtonEmpty onClick={props.onClose}>Cancel</EuiButtonEmpty>
<EuiButton onClick={props.onConfirm} fill={true} color="primary">
Save
<EuiButton onClick={props.onClose} fill={false} color="primary">
Close
</EuiButton>
</EuiModalFooter>
</EuiModal>
Expand Down
Loading
Loading