From e805ea79e1bee977fda30d1fb44bf4a766e662ad Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 18 Apr 2024 15:11:08 -0700 Subject: [PATCH] Add option to select and provision pretrained text embedding models (#137) (#138) Signed-off-by: Tyler Ohlsen (cherry picked from commit 76584f478dc6888e3ffa6b10e5ec50dad927b47e) Co-authored-by: Tyler Ohlsen --- common/constants.ts | 43 +++- common/interfaces.ts | 90 +++++++- public/app.tsx | 17 +- public/component_types/interfaces.ts | 2 +- .../transformer/text_embedding_transformer.ts | 10 +- .../component_details/component_details.tsx | 7 +- .../component_details/component_inputs.tsx | 3 + .../component_details/input_field_list.tsx | 15 +- .../component_details/input_fields/index.ts | 1 + .../input_fields/model_field.tsx | 215 ++++++++++++++++++ .../input_fields/select_field.tsx | 13 +- .../resources/resource_list.tsx | 10 +- .../utils/workflow_to_template_utils.ts | 94 ++++++-- .../workspace/resizable_workspace.tsx | 22 +- .../pages/workflows/new_workflow/use_case.tsx | 4 +- public/utils/constants.ts | 2 +- public/utils/utils.ts | 16 ++ server/routes/helpers.ts | 28 ++- 18 files changed, 527 insertions(+), 65 deletions(-) create mode 100644 public/pages/workflow_detail/component_details/input_fields/model_field.tsx diff --git a/common/constants.ts b/common/constants.ts index fb891729..6cdbbd63 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -3,7 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { TemplateNode, WORKFLOW_STATE } from './interfaces'; +import { + MODEL_ALGORITHM, + PRETRAINED_MODEL_FORMAT, + PretrainedSentenceTransformer, + WORKFLOW_STATE, +} from './interfaces'; export const PLUGIN_ID = 'flow-framework'; @@ -52,6 +57,42 @@ export const SEARCH_MODELS_NODE_API_PATH = `${BASE_MODEL_NODE_API_PATH}/search`; */ export const CREATE_INGEST_PIPELINE_STEP_TYPE = 'create_ingest_pipeline'; export const CREATE_INDEX_STEP_TYPE = 'create_index'; +export const REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE = + 'register_local_pretrained_model'; + +/** + * ML PLUGIN PRETRAINED MODELS + * (based off of https://opensearch.org/docs/latest/ml-commons-plugin/pretrained-models/#sentence-transformers) + */ +export const ROBERTA_SENTENCE_TRANSFORMER = { + name: 'huggingface/sentence-transformers/all-distilroberta-v1', + shortenedName: 'all-distilroberta-v1', + description: 'A sentence transformer from Hugging Face', + format: PRETRAINED_MODEL_FORMAT.TORCH_SCRIPT, + algorithm: MODEL_ALGORITHM.TEXT_EMBEDDING, + version: '1.0.1', + vectorDimensions: 768, +} as PretrainedSentenceTransformer; + +export const MPNET_SENTENCE_TRANSFORMER = { + name: 'huggingface/sentence-transformers/all-mpnet-base-v2', + shortenedName: 'all-mpnet-base-v2', + description: 'A sentence transformer from Hugging Face', + format: PRETRAINED_MODEL_FORMAT.TORCH_SCRIPT, + algorithm: MODEL_ALGORITHM.TEXT_EMBEDDING, + version: '1.0.1', + vectorDimensions: 768, +} as PretrainedSentenceTransformer; + +export const BERT_SENTENCE_TRANSFORMER = { + name: 'huggingface/sentence-transformers/msmarco-distilbert-base-tas-b', + shortenedName: 'msmarco-distilbert-base-tas-b', + description: 'A sentence transformer from Hugging Face', + format: PRETRAINED_MODEL_FORMAT.TORCH_SCRIPT, + algorithm: MODEL_ALGORITHM.TEXT_EMBEDDING, + version: '1.0.2', + vectorDimensions: 768, +} as PretrainedSentenceTransformer; /** * MISCELLANEOUS diff --git a/common/interfaces.ts b/common/interfaces.ts index d9c6bc43..858965cf 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -82,6 +82,16 @@ export type CreateIndexNode = TemplateNode & { }; }; +export type RegisterPretrainedModelNode = TemplateNode & { + user_inputs: { + name: string; + description: string; + model_format: string; + version: string; + deploy: boolean; + }; +}; + export type TemplateEdge = { source: string; dest: string; @@ -130,9 +140,83 @@ export enum USE_CASE { /** ********** ML PLUGIN TYPES/INTERFACES ********** */ + +// Based off of https://github.com/opensearch-project/ml-commons/blob/main/common/src/main/java/org/opensearch/ml/common/model/MLModelState.java +export enum MODEL_STATE { + REGISTERED = 'Registered', + REGISTERING = 'Registering', + DEPLOYING = 'Deploying', + DEPLOYED = 'Deployed', + PARTIALLY_DEPLOYED = 'Partially deployed', + UNDEPLOYED = 'Undeployed', + DEPLOY_FAILED = 'Deploy failed', +} + +// Based off of https://github.com/opensearch-project/ml-commons/blob/main/common/src/main/java/org/opensearch/ml/common/FunctionName.java +export enum MODEL_ALGORITHM { + LINEAR_REGRESSION = 'Linear regression', + KMEANS = 'K-means', + AD_LIBSVM = 'AD LIBSVM', + SAMPLE_ALGO = 'Sample algorithm', + LOCAL_SAMPLE_CALCULATOR = 'Local sample calculator', + FIT_RCF = 'Fit RCF', + BATCH_RCF = 'Batch RCF', + ANOMALY_LOCALIZATION = 'Anomaly localization', + RCF_SUMMARIZE = 'RCF summarize', + LOGISTIC_REGRESSION = 'Logistic regression', + TEXT_EMBEDDING = 'Text embedding', + METRICS_CORRELATION = 'Metrics correlation', + REMOTE = 'Remote', + SPARSE_ENCODING = 'Sparse encoding', + SPARSE_TOKENIZE = 'Sparse tokenize', + TEXT_SIMILARITY = 'Text similarity', + QUESTION_ANSWERING = 'Question answering', + AGENT = 'Agent', +} + +export enum MODEL_CATEGORY { + DEPLOYED = 'Deployed', + PRETRAINED = 'Pretrained', +} + +export enum PRETRAINED_MODEL_FORMAT { + TORCH_SCRIPT = 'TORCH_SCRIPT', +} + +export type PretrainedModel = { + name: string; + shortenedName: string; + description: string; + format: PRETRAINED_MODEL_FORMAT; + algorithm: MODEL_ALGORITHM; + version: string; +}; + +export type PretrainedSentenceTransformer = PretrainedModel & { + vectorDimensions: number; +}; + +export type ModelConfig = { + modelType?: string; + embeddingDimension?: number; +}; + export type Model = { id: string; - algorithm: string; + name: string; + algorithm: MODEL_ALGORITHM; + state: MODEL_STATE; + modelConfig?: ModelConfig; +}; + +export type ModelDict = { + [modelId: string]: Model; +}; + +export type ModelFormValue = { + id: string; + category?: MODEL_CATEGORY; + algorithm?: MODEL_ALGORITHM; }; /** @@ -171,7 +255,3 @@ export enum WORKFLOW_RESOURCE_TYPE { export type WorkflowDict = { [workflowId: string]: Workflow; }; - -export type ModelDict = { - [modelId: string]: Model; -}; diff --git a/public/app.tsx b/public/app.tsx index 2e14044d..fa80d217 100644 --- a/public/app.tsx +++ b/public/app.tsx @@ -62,12 +62,21 @@ export const FlowFrameworkDashboardsApp = (props: Props) => { )} /> - {/* Defaulting to Workflows page */} + {/* + Defaulting to Workflows page. The pathname will need to be updated + to handle the redirection and get the router props consistent. + */} ) => ( - - )} + render={(routeProps: RouteComponentProps) => { + if (props.history.location.pathname !== APP_PATH.WORKFLOWS) { + props.history.replace({ + ...history, + pathname: APP_PATH.WORKFLOWS, + }); + } + return ; + }} /> diff --git a/public/component_types/interfaces.ts b/public/component_types/interfaces.ts index 0056d700..055f44ff 100644 --- a/public/component_types/interfaces.ts +++ b/public/component_types/interfaces.ts @@ -10,7 +10,7 @@ import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../utils'; /** * ************ Types ************************* */ -export type FieldType = 'string' | 'json' | 'select'; +export type FieldType = 'string' | 'json' | 'select' | 'model'; export type SelectType = 'model'; export type FieldValue = string | {}; export type ComponentFormValues = FormikValues; diff --git a/public/component_types/transformer/text_embedding_transformer.ts b/public/component_types/transformer/text_embedding_transformer.ts index c856381e..affb996c 100644 --- a/public/component_types/transformer/text_embedding_transformer.ts +++ b/public/component_types/transformer/text_embedding_transformer.ts @@ -19,11 +19,10 @@ export class TextEmbeddingTransformer extends MLTransformer { this.inputs = []; this.createFields = [ { - label: 'Model ID', - id: 'modelId', - type: 'select', - selectType: 'model', - helpText: 'The deployed text embedding model to use for embedding.', + label: 'Text Embedding Model', + id: 'model', + type: 'model', + helpText: 'A text embedding model for embedding text.', helpLink: 'https://opensearch.org/docs/latest/ml-commons-plugin/integrating-ml-models/#choosing-a-model', }, @@ -36,7 +35,6 @@ export class TextEmbeddingTransformer extends MLTransformer { helpLink: 'https://opensearch.org/docs/latest/ingest-pipelines/processors/text-embedding/', }, - { label: 'Vector Field', id: 'vectorField', diff --git a/public/pages/workflow_detail/component_details/component_details.tsx b/public/pages/workflow_detail/component_details/component_details.tsx index 41ed85d0..1de35468 100644 --- a/public/pages/workflow_detail/component_details/component_details.tsx +++ b/public/pages/workflow_detail/component_details/component_details.tsx @@ -28,9 +28,12 @@ interface ComponentDetailsProps { export function ComponentDetails(props: ComponentDetailsProps) { return ( - {props.isDeprovisionable ? ( + {/* TODO: determine if we need this view if we want the workspace to remain + readonly once provisioned */} + {/* {props.isDeprovisionable ? ( - ) : props.selectedComponent ? ( + ) : */} + {props.selectedComponent ? (

{props.selectedComponent.data.label || ''}

+ + {props.selectedComponent.data.description} + + + + + ); + break; + } case 'json': { el = ( diff --git a/public/pages/workflow_detail/component_details/input_fields/index.ts b/public/pages/workflow_detail/component_details/input_fields/index.ts index e2edf8bd..7d0561f5 100644 --- a/public/pages/workflow_detail/component_details/input_fields/index.ts +++ b/public/pages/workflow_detail/component_details/input_fields/index.ts @@ -6,3 +6,4 @@ export { TextField } from './text_field'; export { JsonField } from './json_field'; export { SelectField } from './select_field'; +export { ModelField } from './model_field'; diff --git a/public/pages/workflow_detail/component_details/input_fields/model_field.tsx b/public/pages/workflow_detail/component_details/input_fields/model_field.tsx new file mode 100644 index 00000000..26ba8e37 --- /dev/null +++ b/public/pages/workflow_detail/component_details/input_fields/model_field.tsx @@ -0,0 +1,215 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useEffect, useState } from 'react'; +import { useSelector } from 'react-redux'; +import { Field, FieldProps, useFormikContext } from 'formik'; +import { + EuiFormRow, + EuiLink, + EuiRadioGroup, + EuiRadioGroupOption, + EuiSpacer, + EuiSuperSelect, + EuiSuperSelectOption, + EuiText, +} from '@elastic/eui'; +import { + BERT_SENTENCE_TRANSFORMER, + IComponentField, + MODEL_STATE, + ROBERTA_SENTENCE_TRANSFORMER, + WorkspaceFormValues, + isFieldInvalid, + ModelFormValue, + MODEL_CATEGORY, + MPNET_SENTENCE_TRANSFORMER, +} from '../../../../../common'; +import { AppState } from '../../../../store'; + +interface ModelFieldProps { + field: IComponentField; + componentId: string; + onFormChange: () => void; +} + +type ModelItem = ModelFormValue & { + name: string; +}; + +// TODO: there is no concrete UX for model selection and model provisioning. This component is TBD +// and simply provides the ability to select existing models, or deploy some pretrained ones, +// and persist all of this in form state. +/** + * A specific field for selecting existing deployed models, or available pretrained models. + */ +export function ModelField(props: ModelFieldProps) { + // Initial store is fetched when loading base page. We don't + // re-fetch here as it could overload client-side if user clicks back and forth / + // keeps re-rendering this component (and subsequently re-fetching data) as they're building flows + const models = useSelector((state: AppState) => state.models.models); + + const formField = `${props.componentId}.${props.field.id}`; + const { errors, touched } = useFormikContext(); + + // Deployed models state + const [deployedModels, setDeployedModels] = useState([]); + const [pretrainedModels, setPretrainedModels] = useState([]); + const [selectableModels, setSelectableModels] = useState([]); + + // Radio options state + const radioOptions = [ + { + id: MODEL_CATEGORY.DEPLOYED, + label: 'Existing deployed models', + }, + { + id: MODEL_CATEGORY.PRETRAINED, + label: 'Pretrained models', + }, + ] as EuiRadioGroupOption[]; + const [selectedRadioId, setSelectedRadioId] = useState< + MODEL_CATEGORY | undefined + >(undefined); + + // Initialize available deployed models + useEffect(() => { + if (models) { + const modelItems = [] as ModelItem[]; + Object.keys(models).forEach((modelId) => { + if (models[modelId].state === MODEL_STATE.DEPLOYED) { + modelItems.push({ + id: modelId, + name: models[modelId].name, + category: MODEL_CATEGORY.DEPLOYED, + algorithm: models[modelId].algorithm, + } as ModelItem); + } + }); + setDeployedModels(modelItems); + } + }, [models]); + + // Initialize available pretrained models + useEffect(() => { + const modelItems = [ + { + id: ROBERTA_SENTENCE_TRANSFORMER.name, + name: ROBERTA_SENTENCE_TRANSFORMER.shortenedName, + category: MODEL_CATEGORY.PRETRAINED, + algorithm: ROBERTA_SENTENCE_TRANSFORMER.algorithm, + }, + { + id: MPNET_SENTENCE_TRANSFORMER.name, + name: MPNET_SENTENCE_TRANSFORMER.shortenedName, + category: MODEL_CATEGORY.PRETRAINED, + algorithm: MPNET_SENTENCE_TRANSFORMER.algorithm, + }, + { + id: BERT_SENTENCE_TRANSFORMER.name, + name: BERT_SENTENCE_TRANSFORMER.shortenedName, + category: MODEL_CATEGORY.PRETRAINED, + algorithm: BERT_SENTENCE_TRANSFORMER.algorithm, + }, + ]; + setPretrainedModels(modelItems); + }, []); + + // Update the valid available models when the radio selection changes. + // e.g., only show deployed models when 'deployed' button is selected + useEffect(() => { + if (selectedRadioId !== undefined) { + if (selectedRadioId === MODEL_CATEGORY.DEPLOYED) { + setSelectableModels(deployedModels); + } else { + setSelectableModels(pretrainedModels); + } + } + }, [selectedRadioId, deployedModels, pretrainedModels]); + + return ( + + {({ field, form }: FieldProps) => { + // a hook to update the model category and trigger reloading + // of valid models to select from + useEffect(() => { + setSelectedRadioId(field.value.category); + }, [field.value.category]); + return ( + + + Learn more + + + ) : undefined + } + helpText={props.field.helpText || undefined} + > + <> + { + // if user selects a new category: + // 1. clear the saved ID + // 2. update the field category + form.setFieldValue(formField, { + id: '', + category: radioId, + } as ModelFormValue); + props.onFormChange(); + }} + > + + + ({ + value: option.id, + inputDisplay: ( + <> + {option.name} + + ), + dropdownDisplay: ( + <> + {option.name} + + {option.category} + + + {option.algorithm} + + + ), + disabled: false, + } as EuiSuperSelectOption) + )} + valueOfSelected={field.value.id || ''} + onChange={(option: string) => { + form.setFieldValue(formField, { + id: option, + category: selectedRadioId, + } as ModelFormValue); + props.onFormChange(); + }} + isInvalid={isFieldInvalid( + props.componentId, + props.field.id, + errors, + touched + )} + /> + + + ); + }} + + ); +} diff --git a/public/pages/workflow_detail/component_details/input_fields/select_field.tsx b/public/pages/workflow_detail/component_details/input_fields/select_field.tsx index 709fe18c..38017e04 100644 --- a/public/pages/workflow_detail/component_details/input_fields/select_field.tsx +++ b/public/pages/workflow_detail/component_details/input_fields/select_field.tsx @@ -19,7 +19,6 @@ import { getInitialValue, isFieldInvalid, } from '../../../../../common'; -import { AppState } from '../../../../store'; interface SelectFieldProps { field: IComponentField; @@ -32,21 +31,15 @@ interface SelectFieldProps { * options. */ export function SelectField(props: SelectFieldProps) { - // Redux store state - // Initial store is fetched when loading base page. We don't - // re-fetch here as it could overload client-side if user clicks back and forth / - // keeps re-rendering this component (and subsequently re-fetching data) as they're building flows - const models = useSelector((state: AppState) => state.models.models); - // Options state const [options, setOptions] = useState([]); // Populate options depending on the select type useEffect(() => { - if (props.field.selectType === 'model' && models) { - setOptions(Object.keys(models)); + // TODO: figure out how we want to utilize select types to customize the options + if (props.field.selectType === 'model') { } - }, [models]); + }, []); const formField = `${props.componentId}.${props.field.id}`; const { errors, touched } = useFormikContext(); diff --git a/public/pages/workflow_detail/resources/resource_list.tsx b/public/pages/workflow_detail/resources/resource_list.tsx index 0acb0f9f..72a10f1f 100644 --- a/public/pages/workflow_detail/resources/resource_list.tsx +++ b/public/pages/workflow_detail/resources/resource_list.tsx @@ -23,10 +23,16 @@ interface ResourceListProps { export function ResourceList(props: ResourceListProps) { const [allResources, setAllResources] = useState([]); - // Hook to initialize all resources + // Hook to initialize all resources. Reduce to unique IDs, since + // the backend resources may include the same resource multiple times + // (e.g., register and deploy steps persist the same model ID resource) useEffect(() => { if (props.workflow?.resourcesCreated) { - setAllResources(props.workflow.resourcesCreated); + const resourcesMap = {} as { [id: string]: WorkflowResource }; + props.workflow.resourcesCreated.forEach((resource) => { + resourcesMap[resource.id] = resource; + }); + setAllResources(Object.values(resourcesMap)); } }, [props.workflow?.resourcesCreated]); diff --git a/public/pages/workflow_detail/utils/workflow_to_template_utils.ts b/public/pages/workflow_detail/utils/workflow_to_template_utils.ts index 3e9ce61f..399129d8 100644 --- a/public/pages/workflow_detail/utils/workflow_to_template_utils.ts +++ b/public/pages/workflow_detail/utils/workflow_to_template_utils.ts @@ -20,6 +20,15 @@ import { CreateIndexNode, TemplateFlow, TemplateEdge, + ModelFormValue, + MODEL_CATEGORY, + RegisterPretrainedModelNode, + PretrainedSentenceTransformer, + ROBERTA_SENTENCE_TRANSFORMER, + MPNET_SENTENCE_TRANSFORMER, + BERT_SENTENCE_TRANSFORMER, + REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE, + generateId, } from '../../../../common'; /** @@ -68,13 +77,13 @@ function toProvisionTemplateFlow( edges: ReactFlowEdge[] ): TemplateFlow { const prevNodes = [] as ReactFlowComponent[]; - const templateNodes = [] as TemplateNode[]; + const finalTemplateNodes = [] as TemplateNode[]; const templateEdges = [] as TemplateEdge[]; nodes.forEach((node) => { - const templateNode = toTemplateNode(node, prevNodes, edges); + const templateNodes = toTemplateNodes(node, prevNodes, edges); // it may be undefined if the node is not convertible for some reason - if (templateNode) { - templateNodes.push(templateNode); + if (templateNodes) { + finalTemplateNodes.push(...templateNodes); prevNodes.push(node); } }); @@ -84,20 +93,20 @@ function toProvisionTemplateFlow( }); return { - nodes: templateNodes, + nodes: finalTemplateNodes, edges: templateEdges, }; } -function toTemplateNode( +function toTemplateNodes( flowNode: ReactFlowComponent, prevNodes: ReactFlowComponent[], edges: ReactFlowEdge[] -): TemplateNode | undefined { +): TemplateNode[] | undefined { if (flowNode.data.baseClasses?.includes(COMPONENT_CLASS.ML_TRANSFORMER)) { - return toIngestPipelineNode(flowNode); + return transformerToTemplateNodes(flowNode); } else if (flowNode.data.baseClasses?.includes(COMPONENT_CLASS.INDEXER)) { - return toIndexerNode(flowNode, prevNodes, edges); + return [indexerToTemplateNode(flowNode, prevNodes, edges)]; } } @@ -110,9 +119,11 @@ function toTemplateEdge(flowEdge: ReactFlowEdge): TemplateEdge { // General fn to process all ML transform nodes. Convert into a final // ingest pipeline with a processor specific to the final class of the node. -function toIngestPipelineNode( +// Optionally prepend a register pretrained model step if the selected model +// is a pretrained and undeployed one. +function transformerToTemplateNodes( flowNode: ReactFlowComponent -): CreateIngestPipelineNode { +): TemplateNode[] { // TODO a few improvements to make here: // 1. Consideration of multiple ingest processors and how to collect them all, and finally create // a single ingest pipeline with all of them, in the same order as done on the UI @@ -120,17 +131,54 @@ function toIngestPipelineNode( switch (flowNode.data.type) { case COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER: default: { - const { modelId, inputField, vectorField } = componentDataToFormik( + const { model, inputField, vectorField } = componentDataToFormik( flowNode.data - ); + ) as { + model: ModelFormValue; + inputField: string; + vectorField: string; + }; + const modelId = model.id; + const ingestPipelineName = generateId('ingest_pipeline'); - return { + let registerModelStep = undefined as + | RegisterPretrainedModelNode + | undefined; + if (model.category === MODEL_CATEGORY.PRETRAINED) { + const pretrainedModel = [ + ROBERTA_SENTENCE_TRANSFORMER, + MPNET_SENTENCE_TRANSFORMER, + BERT_SENTENCE_TRANSFORMER, + ].find( + // the model ID in the form will be the unique name of the pretrained model + (model) => model.name === modelId + ) as PretrainedSentenceTransformer; + registerModelStep = { + id: REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE, + type: REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE, + user_inputs: { + name: pretrainedModel.name, + description: pretrainedModel.description, + model_format: pretrainedModel.format, + version: pretrainedModel.version, + deploy: true, + }, + } as RegisterPretrainedModelNode; + } + + // The model ID depends on if we are consuming it from a previous pretrained model step, + // or directly from the user + const finalModelId = + registerModelStep !== undefined + ? `\${{${REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE}.model_id}}` + : modelId; + + const createIngestPipelineStep = { id: flowNode.data.id, type: CREATE_INGEST_PIPELINE_STEP_TYPE, user_inputs: { - // TODO: expose as customizable - pipeline_id: 'test-pipeline', - model_id: modelId, + pipeline_id: ingestPipelineName, + model_id: finalModelId, input_field: inputField, output_field: vectorField, configurations: { @@ -138,7 +186,7 @@ function toIngestPipelineNode( processors: [ { text_embedding: { - model_id: modelId, + model_id: finalModelId, field_map: { [inputField]: vectorField, }, @@ -147,13 +195,17 @@ function toIngestPipelineNode( ], }, }, - }; + } as CreateIngestPipelineNode; + + return registerModelStep !== undefined + ? [registerModelStep, createIngestPipelineStep] + : [createIngestPipelineStep]; } } } // General fn to convert an indexer node to a final CreateIndexNode template node. -function toIndexerNode( +function indexerToTemplateNode( flowNode: ReactFlowComponent, prevNodes: ReactFlowComponent[], edges: ReactFlowEdge[] @@ -191,6 +243,8 @@ function toIndexerNode( properties: { [vectorField]: { type: 'knn_vector', + // TODO: remove hardcoding, fetch from the selected model + // (existing or from pretrained configuration) dimension: 768, method: { engine: 'lucene', diff --git a/public/pages/workflow_detail/workspace/resizable_workspace.tsx b/public/pages/workflow_detail/workspace/resizable_workspace.tsx index 37a37599..90cba2fb 100644 --- a/public/pages/workflow_detail/workspace/resizable_workspace.tsx +++ b/public/pages/workflow_detail/workspace/resizable_workspace.tsx @@ -33,6 +33,7 @@ import { processNodes, reduceToTemplate, ReactFlowEdge, + APP_PATH, } from '../../../../common'; import { validateWorkspaceFlow, toTemplateFlows } from '../utils'; import { @@ -116,6 +117,8 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { props.workflow !== undefined && !props.isNewWorkflow && props.workflow?.state !== WORKFLOW_STATE.NOT_STARTED; + // TODO: maybe remove this field. It depends on final UX if we want the + // workspace to be readonly once provisioned or not. const readonly = props.workflow === undefined || isDeprovisionable; // Loading state @@ -168,7 +171,7 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { props.workflow && !props.workflow?.ui_metadata?.workspace_flow; const missingCachedWorkflow = props.isNewWorkflow && !props.workflow; if (missingUiFlow || missingCachedWorkflow) { - history.replace('/workflows'); + history.replace(APP_PATH.WORKFLOWS); if (missingCachedWorkflow) { getCore().notifications.toasts.addWarning('No workflow found'); } else { @@ -320,6 +323,17 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { components. )} + {isDeprovisionable && isDirty && ( + + Changes cannot be saved until the flow has first been + deprovisioned. + + )} { const { workflow } = result; - history.replace(`/workflows/${workflow.id}`); + history.replace( + `${APP_PATH.WORKFLOWS}/${workflow.id}` + ); history.go(0); }) .catch((error: any) => { @@ -455,7 +471,7 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { diff --git a/public/pages/workflows/new_workflow/use_case.tsx b/public/pages/workflows/new_workflow/use_case.tsx index 111c4f7c..a6eceacf 100644 --- a/public/pages/workflows/new_workflow/use_case.tsx +++ b/public/pages/workflows/new_workflow/use_case.tsx @@ -13,7 +13,7 @@ import { EuiHorizontalRule, EuiButton, } from '@elastic/eui'; -import { NEW_WORKFLOW_ID_URL, PLUGIN_ID } from '../../../../common'; +import { APP_PATH, NEW_WORKFLOW_ID_URL, PLUGIN_ID } from '../../../../common'; interface UseCaseProps { title: string; @@ -44,7 +44,7 @@ export function UseCase(props: UseCaseProps) { disabled={false} isLoading={false} onClick={props.onClick} - href={`${PLUGIN_ID}#/workflows/${NEW_WORKFLOW_ID_URL}`} + href={`${PLUGIN_ID}#${APP_PATH.WORKFLOWS}/${NEW_WORKFLOW_ID_URL}`} > Go diff --git a/public/utils/constants.ts b/public/utils/constants.ts index 0f3b75f6..74dd67b7 100644 --- a/public/utils/constants.ts +++ b/public/utils/constants.ts @@ -15,7 +15,7 @@ export enum APP_PATH { } export const BREADCRUMBS = Object.freeze({ - FLOW_FRAMEWORK: { text: 'Flow Framework', href: '#/' }, + FLOW_FRAMEWORK: { text: 'Flow Framework' }, WORKFLOWS: { text: 'Workflows', href: `#${APP_PATH.WORKFLOWS}` }, }); diff --git a/public/utils/utils.ts b/public/utils/utils.ts index 3f9a7b8b..76b6715e 100644 --- a/public/utils/utils.ts +++ b/public/utils/utils.ts @@ -18,6 +18,7 @@ import { ReactFlowComponent, Workflow, WorkflowTemplate, + ModelFormValue, } from '../../common'; // Append 16 random characters @@ -82,6 +83,7 @@ export function reduceToTemplate(workflow: Workflow): WorkflowTemplate { lastUpdated, lastLaunched, state, + resourcesCreated, ...workflowTemplate } = workflow; return workflowTemplate; @@ -96,6 +98,13 @@ export function getInitialValue(fieldType: FieldType): FieldValue { case 'select': { return ''; } + case 'model': { + return { + id: '', + category: undefined, + algorithm: undefined, + } as ModelFormValue; + } case 'json': { return {}; } @@ -162,6 +171,13 @@ function getFieldSchema(field: IComponentField): Schema { baseSchema = yup.string().min(1, 'Too short').max(70, 'Too long'); break; } + case 'model': { + baseSchema = yup.object().shape({ + id: yup.string().min(1, 'Too short').max(70, 'Too long').required(), + category: yup.string().required(), + }); + break; + } case 'json': { baseSchema = yup.object().json(); break; diff --git a/server/routes/helpers.ts b/server/routes/helpers.ts index b4d39991..afb404f7 100644 --- a/server/routes/helpers.ts +++ b/server/routes/helpers.ts @@ -6,6 +6,8 @@ import { DEFAULT_NEW_WORKFLOW_STATE_TYPE, INDEX_NOT_FOUND_EXCEPTION, + MODEL_ALGORITHM, + MODEL_STATE, Model, ModelDict, WORKFLOW_RESOURCE_TYPE, @@ -85,13 +87,25 @@ export function getWorkflowsFromResponses( export function getModelsFromResponses(modelHits: any[]): ModelDict { const modelDict = {} as ModelDict; modelHits.forEach((modelHit: any) => { - const modelId = modelHit._source?.model_id; - // in case of schema changes from ML plugin, this may crash. That is ok, as the error - // produced will help expose the root cause - modelDict[modelId] = { - id: modelId, - algorithm: modelHit._source?.algorithm, - } as Model; + // search model API returns hits for each deployed model chunk. ignore these hits + if (modelHit._source.chunk_number === undefined) { + const modelId = modelHit._id; + // in case of schema changes from ML plugin, this may crash. That is ok, as the error + // produced will help expose the root cause + modelDict[modelId] = { + id: modelId, + name: modelHit._source?.name, + // @ts-ignore + algorithm: MODEL_ALGORITHM[modelHit._source?.algorithm], + // @ts-ignore + state: MODEL_STATE[modelHit._source?.model_state], + modelConfig: { + modelType: modelHit._source?.model_config?.model_type, + embeddingDimension: + modelHit._source?.model_config?.embedding_dimension, + }, + } as Model; + } }); return modelDict; }