Skip to content

Commit

Permalink
Standardize input/output transforms; bug fixes (opensearch-project#373)
Browse files Browse the repository at this point in the history 
Signed-off-by: Tyler Ohlsen <[email protected]>
  • Loading branch information
@ohltyler
ohltyler authored Sep 12, 2024
1 parent d020e14 commit c6d9adf
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 45 deletions.
4 changes: 2 additions & 2 deletions common/interfaces.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -394,8 +394,8 @@ export type ModelInputFormField = ModelInput & {
export type ModelOutputFormField = ModelInputFormField;

export type ModelInterface = {
input: ModelInput;
output: ModelOutput;
input?: ModelInput;
output?: ModelOutput;
};

export type ConnectorParameters = {
Expand Down
1 change: 1 addition & 0 deletions public/pages/workflow_detail/workflow_inputs/input_fields/boolean_field.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ export function BooleanField(props: BooleanFieldProps) {
}
onChange={(id) => {
form.setFieldValue(field.name, !field.value);
form.setFieldTouched(field.name, true);
}}
/>
</EuiCompressedFormRow>
Expand Down
48 changes: 30 additions & 18 deletions public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ export function InputTransformModal(props: InputTransformModalProps) {
const dataSourceId = getDataSourceId();
const { values } = useFormikContext<WorkflowFormValues>();

// fetching input data state
const [isFetching, setIsFetching] = useState<boolean>(false);

// source input / transformed output state
const [sourceInput, setSourceInput] = useState<string>('[]');
const [transformedOutput, setTransformedOutput] = useState<string>('{}');
Expand Down Expand Up @@ -116,13 +119,7 @@ export function InputTransformModal(props: InputTransformModalProps) {
) {
let sampleSourceInput = {};
try {
// In the context of ingest or search resp, this input will be an array (list of docs)
// In the context of request, it will be a single JSON
sampleSourceInput =
props.context === PROCESSOR_CONTEXT.INGEST ||
props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE
? JSON.parse(sourceInput)[0]
: JSON.parse(sourceInput);
sampleSourceInput = JSON.parse(sourceInput);
const output = generateTransform(
sampleSourceInput,
map[selectedOutputOption]
Expand Down Expand Up @@ -167,7 +164,9 @@ export function InputTransformModal(props: InputTransformModalProps) {
<EuiText>Source input</EuiText>
<EuiSmallButton
style={{ width: '100px' }}
isLoading={isFetching}
onClick={async () => {
setIsFetching(true);
switch (props.context) {
case PROCESSOR_CONTEXT.INGEST: {
// get the current ingest pipeline up to, but not including, this processor
Expand Down Expand Up @@ -196,21 +195,31 @@ export function InputTransformModal(props: InputTransformModalProps) {
)
.unwrap()
.then((resp: SimulateIngestPipelineResponse) => {
setSourceInput(unwrapTransformedDocs(resp));
const docObjs = unwrapTransformedDocs(resp);
if (docObjs.length > 0) {
setSourceInput(customStringify(docObjs[0]));
}
})
.catch((error: any) => {
getCore().notifications.toasts.addDanger(
`Failed to fetch input data`
);
})
.finally(() => {
setIsFetching(false);
});
} else {
try {
const docObjs = JSON.parse(
values.ingest.docs
) as {}[];
if (docObjs.length > 0)
setSourceInput(customStringify([docObjs[0]]));
} catch {}
if (docObjs.length > 0) {
setSourceInput(customStringify(docObjs[0]));
}
} catch {
} finally {
setIsFetching(false);
}
}
break;
}
Expand All @@ -230,6 +239,7 @@ export function InputTransformModal(props: InputTransformModalProps) {
if (curSearchPipeline === undefined) {
setSourceInput(values.search.request);
}
setIsFetching(false);
break;
}
case PROCESSOR_CONTEXT.SEARCH_RESPONSE: {
Expand Down Expand Up @@ -257,18 +267,20 @@ export function InputTransformModal(props: InputTransformModalProps) {
)
.unwrap()
.then(async (resp) => {
setSourceInput(
customStringify(
resp.hits.hits.map(
(hit: SearchHit) => hit._source
)
)
const hits = resp.hits.hits.map(
(hit: SearchHit) => hit._source
);
if (hits.length > 0) {
setSourceInput(customStringify(hits[0]));
}
})
.catch((error: any) => {
getCore().notifications.toasts.addDanger(
`Failed to fetch source input data`
);
})
.finally(() => {
setIsFetching(false);
});
break;
}
Expand Down Expand Up @@ -357,7 +369,7 @@ export function InputTransformModal(props: InputTransformModalProps) {
</EuiFlexItem>
)}
<EuiFlexItem grow={true}>
{outputOptions.length === 1 ? (
{outputOptions.length <= 1 ? (
<EuiText>Transformed input</EuiText>
) : (
<EuiCompressedSelect
Expand Down
54 changes: 36 additions & 18 deletions public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import React, { useState, useEffect } from 'react';
import { useFormikContext, getIn } from 'formik';
import { cloneDeep, isEmpty, set } from 'lodash';
import { cloneDeep, get, isEmpty, set } from 'lodash';
import {
EuiCodeEditor,
EuiFlexGroup,
Expand Down Expand Up @@ -70,6 +70,9 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
const dataSourceId = getDataSourceId();
const { values } = useFormikContext<WorkflowFormValues>();

// fetching input data state
const [isFetching, setIsFetching] = useState<boolean>(false);

// source input / transformed output state
const [sourceInput, setSourceInput] = useState<string>('[]');
const [transformedOutput, setTransformedOutput] = useState<string>('{}');
Expand All @@ -95,13 +98,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
) {
let sampleSourceInput = {};
try {
// In the context of ingest or search resp, this input will be an array (list of docs)
// In the context of request, it will be a single JSON
sampleSourceInput =
props.context === PROCESSOR_CONTEXT.INGEST ||
props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE
? JSON.parse(sourceInput)[0]
: JSON.parse(sourceInput);
sampleSourceInput = JSON.parse(sourceInput);
const output = generateTransform(
sampleSourceInput,
map[selectedOutputOption]
Expand Down Expand Up @@ -129,7 +126,9 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
<EuiText>Source output</EuiText>
<EuiSmallButton
style={{ width: '100px' }}
isLoading={isFetching}
onClick={async () => {
setIsFetching(true);
switch (props.context) {
// note we skip search request processor context. that is because empty output maps are not supported.
// for more details, see comment in ml_processor_inputs.tsx
Expand Down Expand Up @@ -165,12 +164,24 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
)
.unwrap()
.then((resp: SimulateIngestPipelineResponse) => {
setSourceInput(unwrapTransformedDocs(resp));
try {
const docObjs = unwrapTransformedDocs(resp);
if (docObjs.length > 0) {
const sampleModelResult =
docObjs[0]?.inference_results;
setSourceInput(
customStringify(sampleModelResult)
);
}
} catch {}
})
.catch((error: any) => {
getCore().notifications.toasts.addDanger(
`Failed to fetch input data`
);
})
.finally(() => {
setIsFetching(false);
});
break;
}
Expand Down Expand Up @@ -201,26 +212,33 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
index: values.ingest.index.name,
body: JSON.stringify({
...JSON.parse(values.search.request as string),
search_pipeline: curSearchPipeline,
search_pipeline: curSearchPipeline || {},
}),
},
dataSourceId,
})
)
.unwrap()
.then(async (resp) => {
setSourceInput(
customStringify(
resp.hits.hits.map(
(hit: SearchHit) => hit._source
)
)
);
const hits = resp.hits.hits.map(
(hit: SearchHit) => hit._source
) as any[];
if (hits.length > 0) {
const sampleModelResult = get(
hits,
'0.inference_results.0',
{}
);
setSourceInput(customStringify(sampleModelResult));
}
})
.catch((error: any) => {
getCore().notifications.toasts.addDanger(
`Failed to fetch source output data`
);
})
.finally(() => {
setIsFetching(false);
});
break;
}
Expand Down Expand Up @@ -281,7 +299,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
</EuiFlexItem>
<EuiFlexItem>
<>
{outputOptions.length === 1 ? (
{outputOptions.length <= 1 ? (
<EuiText>Transformed output</EuiText>
) : (
<EuiCompressedSelect
Expand Down
7 changes: 2 additions & 5 deletions public/utils/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ export function prepareDocsForSimulate(
// to format them into a more readable string to display
export function unwrapTransformedDocs(
simulatePipelineResponse: SimulateIngestPipelineResponse
) {
): any[] {
let errorDuringSimulate = undefined as string | undefined;
const transformedDocsSources = simulatePipelineResponse.docs.map(
(transformedDoc) => {
Expand All @@ -167,7 +167,7 @@ export function unwrapTransformedDocs(
`Failed to simulate ingest on all documents: ${errorDuringSimulate}`
);
}
return customStringify(transformedDocsSources);
return transformedDocsSources;
}

// ML inference processors will use standard dot notation or JSONPath depending on the input.
Expand All @@ -181,9 +181,6 @@ export function generateTransform(input: {}, map: MapFormValue): {} {
if (mapEntry.value.startsWith(JSONPATH_ROOT_SELECTOR)) {
// JSONPath transform
transformedResult = jsonpath.query(input, path);
// Non-JSONPath bracket notation not supported - throw an error
} else if (mapEntry.value.includes('[') || mapEntry.value.includes(']')) {
throw new Error();
// Standard dot notation
} else {
transformedResult = get(input, path);
Expand Down
14 changes: 12 additions & 2 deletions server/routes/helpers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@ import {
MODEL_STATE,
Model,
ModelDict,
ModelInput,
ModelInterface,
ModelOutput,
NO_MODIFICATIONS_FOUND_TEXT,
SearchHit,
WORKFLOW_RESOURCE_TYPE,
Expand Down Expand Up @@ -107,9 +109,17 @@ export function getModelsFromResponses(modelHits: SearchHit[]): ModelDict {
| undefined;
let modelInterface = undefined as ModelInterface | undefined;
if (indexedModelInterface !== undefined) {
let parsedInput = undefined as ModelInput | undefined;
let parsedOutput = undefined as ModelOutput | undefined;
try {
parsedInput = JSON.parse(indexedModelInterface.input);
} catch {}
try {
parsedOutput = JSON.parse(indexedModelInterface.output);
} catch {}
modelInterface = {
input: JSON.parse(indexedModelInterface.input),
output: JSON.parse(indexedModelInterface.output),
input: parsedInput,
output: parsedOutput,
} as ModelInterface;
}

Expand Down

0 comments on commit c6d9adf

Please sign in to comment.