Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add standalone form state for source data #445

Merged
merged 5 commits into from
Oct 30, 2024
Merged

Add standalone form state for source data #445

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cafdd56
Refactor modal to standalone component
ohltyler Oct 29, 2024
f17cb53
Add interim state besides existing index option
ohltyler Oct 29, 2024
68793ea
Support existing indices option, move remaining state
ohltyler Oct 29, 2024
dd51259
add loading/error state
ohltyler Oct 29, 2024
a8daf08
revert testid change
ohltyler Oct 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions common/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -484,3 +484,8 @@ export enum CONFIG_STEP {
INGEST = 'Ingestion pipeline',
SEARCH = 'Search pipeline',
}
export enum SOURCE_OPTIONS {
MANUAL = 'manual',
UPLOAD = 'upload',
EXISTING_INDEX = 'existing_index',
}
10 changes: 7 additions & 3 deletions common/interfaces.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,6 @@ export type IndexConfig = {
settings: IConfigField;
};

// TODO: may expand to just IndexConfig (including mappings/settings info)
// if we want to persist this for users using some existing index,
// and want to pass that index config around.
export type SearchIndexConfig = {
name: IConfigField;
};
Expand Down Expand Up @@ -113,6 +110,13 @@ export type WorkflowSchemaObj = {
};
export type WorkflowSchema = ObjectSchema<WorkflowSchemaObj>;

// Form / schema interfaces for the ingest docs sub-form
export type IngestDocsFormValues = {
docs: FormikValues;
};
export type IngestDocsSchemaObj = WorkflowSchemaObj;
export type IngestDocsSchema = WorkflowSchema;

/**
********** WORKSPACE TYPES/INTERFACES **********
*/
Expand Down
221 changes: 13 additions & 208 deletions public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,47 +4,26 @@
*/

import React, { useEffect, useState } from 'react';
import { useSelector } from 'react-redux';
import { getIn, useFormikContext } from 'formik';
import {
EuiSmallButton,
EuiCompressedFilePicker,
EuiFlexGroup,
EuiFlexItem,
EuiModal,
EuiModalBody,
EuiModalFooter,
EuiModalHeader,
EuiModalHeaderTitle,
EuiSpacer,
EuiText,
EuiFilterGroup,
EuiSmallFilterButton,
EuiSuperSelectOption,
EuiCompressedSuperSelect,
EuiCodeBlock,
EuiSmallButtonEmpty,
} from '@elastic/eui';
import { JsonField } from '../input_fields';
import {
FETCH_ALL_QUERY,
IndexMappings,
MapEntry,
SearchHit,
SOURCE_OPTIONS,
Workflow,
WorkflowConfig,
WorkspaceFormValues,
customStringify,
WorkflowFormValues,
isVectorSearchUseCase,
toFormattedDate,
} from '../../../../../common';
import {
AppState,
getMappings,
searchIndex,
useAppDispatch,
} from '../../../../store';
import { getDataSourceId } from '../../../../utils';
import { SourceDataModal } from './source_data_modal';

interface SourceDataProps {
workflow: Workflow | undefined;
Expand All @@ -53,20 +32,11 @@ interface SourceDataProps {
lastIngested: number | undefined;
}

enum SOURCE_OPTIONS {
MANUAL = 'manual',
UPLOAD = 'upload',
EXISTING_INDEX = 'existing_index',
}

/**
* Input component for configuring the source data for ingest.
*/
export function SourceData(props: SourceDataProps) {
const dispatch = useAppDispatch();
const dataSourceId = getDataSourceId();
const { values, setFieldValue } = useFormikContext<WorkspaceFormValues>();
const indices = useSelector((state: AppState) => state.opensearch.indices);
const { values, setFieldValue } = useFormikContext<WorkflowFormValues>();

// empty/populated docs state
let docs = [];
Expand All @@ -83,74 +53,6 @@ export function SourceData(props: SourceDataProps) {
// edit modal state
const [isEditModalOpen, setIsEditModalOpen] = useState<boolean>(false);

// files state. when a file is read, update the form value.
const fileReader = new FileReader();
fileReader.onload = (e) => {
if (e.target) {
setFieldValue('ingest.docs', e.target.result);
}
};

// selected index state. when an index is selected, update several form values (if vector search)
const [selectedIndex, setSelectedIndex] = useState<string | undefined>(
undefined
);
useEffect(() => {
if (selectedIndex !== undefined) {
// 1. fetch and set sample docs
dispatch(
searchIndex({
apiBody: {
index: selectedIndex,
body: FETCH_ALL_QUERY,
searchPipeline: '_none',
},
dataSourceId,
})
)
.unwrap()
.then((resp) => {
const docObjs = resp.hits?.hits
?.slice(0, 5)
?.map((hit: SearchHit) => hit?._source);
setFieldValue('ingest.docs', customStringify(docObjs));
});

// 2. fetch index mappings, and try to set defaults for the ML processor configs, if applicable
if (isVectorSearchUseCase(props.workflow)) {
dispatch(getMappings({ index: selectedIndex, dataSourceId }))
.unwrap()
.then((resp: IndexMappings) => {
const { processorId, inputMapEntry } = getProcessorInfo(
props.uiConfig,
values
);
if (processorId !== undefined && inputMapEntry !== undefined) {
// set/overwrite default text field for the input map. may be empty.
if (inputMapEntry !== undefined) {
const textFieldFormPath = `ingest.enrich.${processorId}.input_map.0.0.value`;
const curTextField = getIn(values, textFieldFormPath) as string;
if (!Object.keys(resp.properties).includes(curTextField)) {
const defaultTextField =
Object.keys(resp.properties).find((fieldName) => {
return resp.properties[fieldName]?.type === 'text';
}) || '';
setFieldValue(textFieldFormPath, defaultTextField);
}
}
}
});
}
}
}, [selectedIndex]);

// hook to clear out the selected index when switching options
useEffect(() => {
if (selectedOption !== SOURCE_OPTIONS.EXISTING_INDEX) {
setSelectedIndex(undefined);
}
}, [selectedOption]);

// hook to listen when the docs form value changes.
useEffect(() => {
if (values?.ingest?.docs) {
Expand Down Expand Up @@ -188,110 +90,13 @@ export function SourceData(props: SourceDataProps) {
return (
<>
{isEditModalOpen && (
<EuiModal
onClose={() => setIsEditModalOpen(false)}
style={{ width: '70vw' }}
>
<EuiModalHeader>
<EuiModalHeaderTitle>
<p>{`Import data`}</p>
</EuiModalHeaderTitle>
</EuiModalHeader>
<EuiModalBody>
<>
<EuiFilterGroup>
<EuiSmallFilterButton
id={SOURCE_OPTIONS.MANUAL}
hasActiveFilters={selectedOption === SOURCE_OPTIONS.MANUAL}
onClick={() => setSelectedOption(SOURCE_OPTIONS.MANUAL)}
data-testid="manualEditSourceDataButton"
>
Manual
</EuiSmallFilterButton>
<EuiSmallFilterButton
id={SOURCE_OPTIONS.UPLOAD}
hasActiveFilters={selectedOption === SOURCE_OPTIONS.UPLOAD}
onClick={() => setSelectedOption(SOURCE_OPTIONS.UPLOAD)}
data-testid="uploadSourceDataButton"
>
Upload
</EuiSmallFilterButton>
<EuiSmallFilterButton
id={SOURCE_OPTIONS.EXISTING_INDEX}
hasActiveFilters={
selectedOption === SOURCE_OPTIONS.EXISTING_INDEX
}
onClick={() =>
setSelectedOption(SOURCE_OPTIONS.EXISTING_INDEX)
}
data-testid="selectIndexSourceDataButton"
>
Existing index
</EuiSmallFilterButton>
</EuiFilterGroup>
<EuiSpacer size="m" />
{selectedOption === SOURCE_OPTIONS.UPLOAD && (
<>
<EuiCompressedFilePicker
accept="application/json"
multiple={false}
initialPromptText="Upload file"
onChange={(files) => {
if (files && files.length > 0) {
fileReader.readAsText(files[0]);
}
}}
display="default"
/>
<EuiSpacer size="s" />
</>
)}
{selectedOption === SOURCE_OPTIONS.EXISTING_INDEX && (
<>
<EuiText color="subdued" size="s">
Up to 5 sample documents will be automatically populated.
</EuiText>
<EuiSpacer size="s" />
<EuiCompressedSuperSelect
options={Object.values(indices).map(
(option) =>
({
value: option.name,
inputDisplay: (
<EuiText size="s">{option.name}</EuiText>
),
disabled: false,
} as EuiSuperSelectOption<string>)
)}
valueOfSelected={selectedIndex}
onChange={(option) => {
setSelectedIndex(option);
}}
isInvalid={false}
/>
<EuiSpacer size="xs" />
</>
)}
<JsonField
label="Documents to be imported"
fieldPath={'ingest.docs'}
helpText="Documents should be formatted as a valid JSON array."
editorHeight="25vh"
readOnly={false}
/>
</>
</EuiModalBody>
<EuiModalFooter>
<EuiSmallButton
onClick={() => setIsEditModalOpen(false)}
fill={false}
color="primary"
data-testid="closeSourceDataButton"
>
Close
</EuiSmallButton>
</EuiModalFooter>
</EuiModal>
<SourceDataModal
workflow={props.workflow}
uiConfig={props.uiConfig}
selectedOption={selectedOption}
setSelectedOption={setSelectedOption}
setIsModalOpen={setIsEditModalOpen}
/>
)}
<EuiFlexGroup direction="column" gutterSize="s">
<EuiFlexItem grow={false}>
Expand Down Expand Up @@ -361,9 +166,9 @@ export function SourceData(props: SourceDataProps) {
// helper fn to parse out some useful info from the ML ingest processor config, if applicable
// takes on the assumption the first processor is an ML inference processor, and should
// only be executed for workflows coming from preset vector search use cases.
function getProcessorInfo(
export function getProcessorInfo(
uiConfig: WorkflowConfig,
values: WorkspaceFormValues
values: WorkflowFormValues
): {
processorId: string | undefined;
inputMapEntry: MapEntry | undefined;
Expand Down
Loading
Loading