Skip to content

Commit

Permalink
Add standalone form state for source data (#445)
Browse files Browse the repository at this point in the history 
Signed-off-by: Tyler Ohlsen <[email protected]>
  • Loading branch information
@ohltyler
ohltyler authored Oct 30, 2024
1 parent 0a4d768 commit a197021
Show file tree
Hide file tree
Showing 11 changed files with 361 additions and 224 deletions.
5 changes: 5 additions & 0 deletions common/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -484,3 +484,8 @@ export enum CONFIG_STEP {
INGEST = 'Ingestion pipeline',
SEARCH = 'Search pipeline',
}
export enum SOURCE_OPTIONS {
MANUAL = 'manual',
UPLOAD = 'upload',
EXISTING_INDEX = 'existing_index',
}
10 changes: 7 additions & 3 deletions common/interfaces.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,6 @@ export type IndexConfig = {
settings: IConfigField;
};

// TODO: may expand to just IndexConfig (including mappings/settings info)
// if we want to persist this for users using some existing index,
// and want to pass that index config around.
export type SearchIndexConfig = {
name: IConfigField;
};
Expand Down Expand Up @@ -113,6 +110,13 @@ export type WorkflowSchemaObj = {
};
export type WorkflowSchema = ObjectSchema<WorkflowSchemaObj>;

// Form / schema interfaces for the ingest docs sub-form
export type IngestDocsFormValues = {
docs: FormikValues;
};
export type IngestDocsSchemaObj = WorkflowSchemaObj;
export type IngestDocsSchema = WorkflowSchema;

/**
********** WORKSPACE TYPES/INTERFACES **********
*/
Expand Down
221 changes: 13 additions & 208 deletions public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,47 +4,26 @@
*/

import React, { useEffect, useState } from 'react';
import { useSelector } from 'react-redux';
import { getIn, useFormikContext } from 'formik';
import {
EuiSmallButton,
EuiCompressedFilePicker,
EuiFlexGroup,
EuiFlexItem,
EuiModal,
EuiModalBody,
EuiModalFooter,
EuiModalHeader,
EuiModalHeaderTitle,
EuiSpacer,
EuiText,
EuiFilterGroup,
EuiSmallFilterButton,
EuiSuperSelectOption,
EuiCompressedSuperSelect,
EuiCodeBlock,
EuiSmallButtonEmpty,
} from '@elastic/eui';
import { JsonField } from '../input_fields';
import {
FETCH_ALL_QUERY,
IndexMappings,
MapEntry,
SearchHit,
SOURCE_OPTIONS,
Workflow,
WorkflowConfig,
WorkspaceFormValues,
customStringify,
WorkflowFormValues,
isVectorSearchUseCase,
toFormattedDate,
} from '../../../../../common';
import {
AppState,
getMappings,
searchIndex,
useAppDispatch,
} from '../../../../store';
import { getDataSourceId } from '../../../../utils';
import { SourceDataModal } from './source_data_modal';

interface SourceDataProps {
workflow: Workflow | undefined;
Expand All @@ -53,20 +32,11 @@ interface SourceDataProps {
lastIngested: number | undefined;
}

enum SOURCE_OPTIONS {
MANUAL = 'manual',
UPLOAD = 'upload',
EXISTING_INDEX = 'existing_index',
}

/**
* Input component for configuring the source data for ingest.
*/
export function SourceData(props: SourceDataProps) {
const dispatch = useAppDispatch();
const dataSourceId = getDataSourceId();
const { values, setFieldValue } = useFormikContext<WorkspaceFormValues>();
const indices = useSelector((state: AppState) => state.opensearch.indices);
const { values, setFieldValue } = useFormikContext<WorkflowFormValues>();

// empty/populated docs state
let docs = [];
Expand All @@ -83,74 +53,6 @@ export function SourceData(props: SourceDataProps) {
// edit modal state
const [isEditModalOpen, setIsEditModalOpen] = useState<boolean>(false);

// files state. when a file is read, update the form value.
const fileReader = new FileReader();
fileReader.onload = (e) => {
if (e.target) {
setFieldValue('ingest.docs', e.target.result);
}
};

// selected index state. when an index is selected, update several form values (if vector search)
const [selectedIndex, setSelectedIndex] = useState<string | undefined>(
undefined
);
useEffect(() => {
if (selectedIndex !== undefined) {
// 1. fetch and set sample docs
dispatch(
searchIndex({
apiBody: {
index: selectedIndex,
body: FETCH_ALL_QUERY,
searchPipeline: '_none',
},
dataSourceId,
})
)
.unwrap()
.then((resp) => {
const docObjs = resp.hits?.hits
?.slice(0, 5)
?.map((hit: SearchHit) => hit?._source);
setFieldValue('ingest.docs', customStringify(docObjs));
});

// 2. fetch index mappings, and try to set defaults for the ML processor configs, if applicable
if (isVectorSearchUseCase(props.workflow)) {
dispatch(getMappings({ index: selectedIndex, dataSourceId }))
.unwrap()
.then((resp: IndexMappings) => {
const { processorId, inputMapEntry } = getProcessorInfo(
props.uiConfig,
values
);
if (processorId !== undefined && inputMapEntry !== undefined) {
// set/overwrite default text field for the input map. may be empty.
if (inputMapEntry !== undefined) {
const textFieldFormPath = `ingest.enrich.${processorId}.input_map.0.0.value`;
const curTextField = getIn(values, textFieldFormPath) as string;
if (!Object.keys(resp.properties).includes(curTextField)) {
const defaultTextField =
Object.keys(resp.properties).find((fieldName) => {
return resp.properties[fieldName]?.type === 'text';
}) || '';
setFieldValue(textFieldFormPath, defaultTextField);
}
}
}
});
}
}
}, [selectedIndex]);

// hook to clear out the selected index when switching options
useEffect(() => {
if (selectedOption !== SOURCE_OPTIONS.EXISTING_INDEX) {
setSelectedIndex(undefined);
}
}, [selectedOption]);

// hook to listen when the docs form value changes.
useEffect(() => {
if (values?.ingest?.docs) {
Expand Down Expand Up @@ -188,110 +90,13 @@ export function SourceData(props: SourceDataProps) {
return (
<>
{isEditModalOpen && (
<EuiModal
onClose={() => setIsEditModalOpen(false)}
style={{ width: '70vw' }}
>
<EuiModalHeader>
<EuiModalHeaderTitle>
<p>{`Import data`}</p>
</EuiModalHeaderTitle>
</EuiModalHeader>
<EuiModalBody>
<>
<EuiFilterGroup>
<EuiSmallFilterButton
id={SOURCE_OPTIONS.MANUAL}
hasActiveFilters={selectedOption === SOURCE_OPTIONS.MANUAL}
onClick={() => setSelectedOption(SOURCE_OPTIONS.MANUAL)}
data-testid="manualEditSourceDataButton"
>
Manual
</EuiSmallFilterButton>
<EuiSmallFilterButton
id={SOURCE_OPTIONS.UPLOAD}
hasActiveFilters={selectedOption === SOURCE_OPTIONS.UPLOAD}
onClick={() => setSelectedOption(SOURCE_OPTIONS.UPLOAD)}
data-testid="uploadSourceDataButton"
>
Upload
</EuiSmallFilterButton>
<EuiSmallFilterButton
id={SOURCE_OPTIONS.EXISTING_INDEX}
hasActiveFilters={
selectedOption === SOURCE_OPTIONS.EXISTING_INDEX
}
onClick={() =>
setSelectedOption(SOURCE_OPTIONS.EXISTING_INDEX)
}
data-testid="selectIndexSourceDataButton"
>
Existing index
</EuiSmallFilterButton>
</EuiFilterGroup>
<EuiSpacer size="m" />
{selectedOption === SOURCE_OPTIONS.UPLOAD && (
<>
<EuiCompressedFilePicker
accept="application/json"
multiple={false}
initialPromptText="Upload file"
onChange={(files) => {
if (files && files.length > 0) {
fileReader.readAsText(files[0]);
}
}}
display="default"
/>
<EuiSpacer size="s" />
</>
)}
{selectedOption === SOURCE_OPTIONS.EXISTING_INDEX && (
<>
<EuiText color="subdued" size="s">
Up to 5 sample documents will be automatically populated.
</EuiText>
<EuiSpacer size="s" />
<EuiCompressedSuperSelect
options={Object.values(indices).map(
(option) =>
({
value: option.name,
inputDisplay: (
<EuiText size="s">{option.name}</EuiText>
),
disabled: false,
} as EuiSuperSelectOption<string>)
)}
valueOfSelected={selectedIndex}
onChange={(option) => {
setSelectedIndex(option);
}}
isInvalid={false}
/>
<EuiSpacer size="xs" />
</>
)}
<JsonField
label="Documents to be imported"
fieldPath={'ingest.docs'}
helpText="Documents should be formatted as a valid JSON array."
editorHeight="25vh"
readOnly={false}
/>
</>
</EuiModalBody>
<EuiModalFooter>
<EuiSmallButton
onClick={() => setIsEditModalOpen(false)}
fill={false}
color="primary"
data-testid="closeSourceDataButton"
>
Close
</EuiSmallButton>
</EuiModalFooter>
</EuiModal>
<SourceDataModal
workflow={props.workflow}
uiConfig={props.uiConfig}
selectedOption={selectedOption}
setSelectedOption={setSelectedOption}
setIsModalOpen={setIsEditModalOpen}
/>
)}
<EuiFlexGroup direction="column" gutterSize="s">
<EuiFlexItem grow={false}>
Expand Down Expand Up @@ -361,9 +166,9 @@ export function SourceData(props: SourceDataProps) {
// helper fn to parse out some useful info from the ML ingest processor config, if applicable
// takes on the assumption the first processor is an ML inference processor, and should
// only be executed for workflows coming from preset vector search use cases.
function getProcessorInfo(
export function getProcessorInfo(
uiConfig: WorkflowConfig,
values: WorkspaceFormValues
values: WorkflowFormValues
): {
processorId: string | undefined;
inputMapEntry: MapEntry | undefined;
Expand Down
Loading

0 comments on commit a197021

Please sign in to comment.