Set up basic logic for parsing template -> UI flow and vice versa (#131…

…) (#132) Signed-off-by: Tyler Ohlsen <[email protected]> (cherry picked from commit 34ef573) Co-authored-by: Tyler Ohlsen <[email protected]>
opensearch-project · Apr 10, 2024 · 98ed8fc · 98ed8fc
1 parent fa7ff85
commit 98ed8fc
Show file tree

Hide file tree

Showing 18 changed files with 568 additions and 342 deletions.
diff --git a/common/constants.ts b/common/constants.ts
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { WORKFLOW_STATE } from './interfaces';
+import { TemplateNode, WORKFLOW_STATE } from './interfaces';
 
 export const PLUGIN_ID = 'flow-framework';
 
@@ -47,6 +47,12 @@ export const GET_PRESET_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH
 export const BASE_MODEL_NODE_API_PATH = `${BASE_NODE_API_PATH}/model`;
 export const SEARCH_MODELS_NODE_API_PATH = `${BASE_MODEL_NODE_API_PATH}/search`;
 
+/**
+ * BACKEND INTERFACES
+ */
+export const CREATE_INGEST_PIPELINE_STEP_TYPE = 'create_ingest_pipeline';
+export const CREATE_INDEX_STEP_TYPE = 'create_index';
+
 /**
  * MISCELLANEOUS
  */

diff --git a/common/index.ts b/common/index.ts
@@ -8,3 +8,4 @@ export * from './interfaces';
 export * from './utils';
 export * from '../public/component_types';
 export * from '../public/utils';
+export * from '../public/pages/workflow_detail/utils';
diff --git a/common/interfaces.ts b/common/interfaces.ts
@@ -29,7 +29,7 @@ type ReactFlowViewport = {
 };
 
 export type UIState = {
-  workspaceFlow: WorkspaceFlowState;
+  workspace_flow: WorkspaceFlowState;
 };
 
 export type WorkspaceFlowState = {
@@ -42,21 +42,56 @@ export type WorkspaceFlowState = {
  ********** USE CASE TEMPLATE TYPES/INTERFACES **********
  */
 
+export type IngestProcessor = {
+  description?: string;
+};
+
+export type TextEmbeddingProcessor = IngestProcessor & {
+  text_embedding: {
+    model_id: string;
+    field_map: {};
+  };
+};
+
 export type TemplateNode = {
   id: string;
   type: string;
-  previous_node_inputs?: Map<string, any>;
-  user_inputs?: Map<string, any>;
+  previous_node_inputs?: {};
+  user_inputs?: {};
+};
+
+export type CreateIngestPipelineNode = TemplateNode & {
+  user_inputs: {
+    pipeline_id: string;
+    model_id?: string;
+    input_field?: string;
+    output_field?: string;
+    configurations: {
+      description?: string;
+      processors: IngestProcessor[];
+    };
+  };
+};
+
+export type CreateIndexNode = TemplateNode & {
+  previous_node_inputs?: {
+    [ingest_pipeline_step_id: string]: string;
+  };
+  user_inputs: {
+    index_name: string;
+    configurations: {
+      settings: {};
+      mappings: {};
+    };
+  };
 };
 
 export type TemplateEdge = {
   source: string;
-  target: string;
+  dest: string;
 };
 
 export type TemplateFlow = {
-  user_inputs?: Map<string, any>;
-  previous_node_inputs?: Map<string, any>;
   nodes: TemplateNode[];
   edges?: TemplateEdge[];
 };
@@ -91,7 +126,7 @@ export type Workflow = WorkflowTemplate & {
 };
 
 export enum USE_CASE {
-  PROVISION = 'PROVISION',
+  SEMANTIC_SEARCH = 'SEMANTIC_SEARCH',
 }
 
 /**

diff --git a/common/utils.ts b/common/utils.ts
@@ -4,217 +4,7 @@
  */
 
 import moment from 'moment';
-import { MarkerType } from 'reactflow';
-import {
-  WorkspaceFlowState,
-  ReactFlowComponent,
-  initComponentData,
-  TextEmbeddingTransformer,
-  KnnIndexer,
-  generateId,
-  ReactFlowEdge,
-  TemplateFlows,
-  WorkflowTemplate,
-  DATE_FORMAT_PATTERN,
-  COMPONENT_CATEGORY,
-  NODE_CATEGORY,
-  WorkspaceFormValues,
-} from './';
-
-// TODO: implement this and remove hardcoded return values
-/**
- * Given a ReactFlow workspace flow and the set of current form values within such flow,
- * generate a backend-compatible set of sub-workflows.
- *
- */
-export function toTemplateFlows(
-  workspaceFlow: WorkspaceFlowState,
-  formValues: WorkspaceFormValues
-): TemplateFlows {
-  const textEmbeddingTransformerNodeId = Object.keys(formValues).find((key) =>
-    key.includes('text_embedding')
-  ) as string;
-  const knnIndexerNodeId = Object.keys(formValues).find((key) =>
-    key.includes('knn')
-  ) as string;
-  const textEmbeddingFields = formValues[textEmbeddingTransformerNodeId];
-  const knnIndexerFields = formValues[knnIndexerNodeId];
-
-  return {
-    provision: {
-      nodes: [
-        {
-          id: 'create_ingest_pipeline',
-          type: 'create_ingest_pipeline',
-          user_inputs: {
-            pipeline_id: 'test-pipeline',
-            model_id: textEmbeddingFields['modelId'],
-            input_field: textEmbeddingFields['inputField'],
-            output_field: textEmbeddingFields['vectorField'],
-            configurations: {
-              description: 'A text embedding ingest pipeline',
-              processors: [
-                {
-                  text_embedding: {
-                    model_id: textEmbeddingFields['modelId'],
-                    field_map: {
-                      [textEmbeddingFields['inputField']]:
-                        textEmbeddingFields['vectorField'],
-                    },
-                  },
-                },
-              ],
-            },
-          },
-        },
-        {
-          id: 'create_index',
-          type: 'create_index',
-          previous_node_inputs: {
-            create_ingest_pipeline: 'pipeline_id',
-          },
-          user_inputs: {
-            index_name: knnIndexerFields['indexName'],
-            configurations: {
-              settings: {
-                default_pipeline: '${{create_ingest_pipeline.pipeline_id}}',
-              },
-              mappings: {
-                properties: {
-                  [textEmbeddingFields['vectorField']]: {
-                    type: 'knn_vector',
-                    dimension: 768,
-                    method: {
-                      engine: 'lucene',
-                      space_type: 'l2',
-                      name: 'hnsw',
-                      parameters: {},
-                    },
-                  },
-                  [textEmbeddingFields['inputField']]: {
-                    type: 'text',
-                  },
-                },
-              },
-            },
-          },
-        },
-      ],
-    },
-  };
-}
-
-// TODO: implement this and remove hardcoded return values
-/**
- * Converts a backend set of provision/ingest/search sub-workflows into a UI-compatible set of
- * ReactFlow nodes and edges
- */
-export function toWorkspaceFlow(
-  templateFlows: TemplateFlows
-): WorkspaceFlowState {
-  const ingestId1 = generateId('text_embedding_processor');
-  const ingestId2 = generateId('knn_index');
-  const ingestGroupId = generateId(COMPONENT_CATEGORY.INGEST);
-  const searchGroupId = generateId(COMPONENT_CATEGORY.SEARCH);
-  const edgeId = generateId('edge');
-
-  const ingestNodes = [
-    {
-      id: ingestGroupId,
-      position: { x: 400, y: 400 },
-      type: NODE_CATEGORY.INGEST_GROUP,
-      data: { label: COMPONENT_CATEGORY.INGEST },
-      style: {
-        width: 900,
-        height: 400,
-      },
-      className: 'reactflow__group-node__ingest',
-      selectable: true,
-      deletable: false,
-    },
-    {
-      id: ingestId1,
-      position: { x: 100, y: 70 },
-      data: initComponentData(
-        new TextEmbeddingTransformer().toObj(),
-        ingestId1
-      ),
-      type: NODE_CATEGORY.CUSTOM,
-      parentNode: ingestGroupId,
-      extent: 'parent',
-      draggable: true,
-      deletable: false,
-    },
-    {
-      id: ingestId2,
-      position: { x: 500, y: 70 },
-      data: initComponentData(new KnnIndexer().toObj(), ingestId2),
-      type: NODE_CATEGORY.CUSTOM,
-      parentNode: ingestGroupId,
-      extent: 'parent',
-      draggable: true,
-      deletable: false,
-    },
-  ] as ReactFlowComponent[];
-
-  const searchNodes = [
-    {
-      id: searchGroupId,
-      position: { x: 400, y: 1000 },
-      type: NODE_CATEGORY.SEARCH_GROUP,
-      data: { label: COMPONENT_CATEGORY.SEARCH },
-      style: {
-        width: 900,
-        height: 400,
-      },
-      className: 'reactflow__group-node__search',
-      selectable: true,
-      deletable: false,
-    },
-  ] as ReactFlowComponent[];
-
-  return {
-    nodes: [...ingestNodes, ...searchNodes],
-    edges: [
-      {
-        id: edgeId,
-        key: edgeId,
-        source: ingestId1,
-        target: ingestId2,
-        markerEnd: {
-          type: MarkerType.ArrowClosed,
-          width: 20,
-          height: 20,
-        },
-        zIndex: 2,
-        deletable: false,
-      },
-    ] as ReactFlowEdge[],
-  };
-}
-
-// TODO: implement this
-/**
- * Validates the UI workflow state.
- * Note we don't have to validate connections since that is done via input/output handlers.
- * But we need to validate there are no open connections
- */
-export function validateWorkspaceFlow(
-  workspaceFlow: WorkspaceFlowState
-): boolean {
-  return true;
-}
-
-// TODO: implement this
-/**
- * Validates the backend template. May be used when parsing persisted templates on server-side,
- * or when importing/exporting on the UI.
- */
-export function validateWorkflowTemplate(
-  workflowTemplate: WorkflowTemplate
-): boolean {
-  return true;
-}
+import { DATE_FORMAT_PATTERN } from './';
 
 export function toFormattedDate(timestampMillis: number): String {
   return moment(new Date(timestampMillis)).format(DATE_FORMAT_PATTERN);

diff --git a/public/component_types/indexer/knn_indexer.ts b/public/component_types/indexer/knn_indexer.ts
@@ -3,6 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { COMPONENT_CLASS } from '../../../common';
 import { Indexer } from './indexer';
 
 /**
@@ -11,8 +12,10 @@ import { Indexer } from './indexer';
 export class KnnIndexer extends Indexer {
   constructor() {
     super();
+    this.type = COMPONENT_CLASS.KNN_INDEXER;
     this.label = 'K-NN Indexer';
     this.description = 'A specialized indexer for K-NN indices';
+    this.baseClasses = [...this.baseClasses, this.type];
     this.createFields = [
       // @ts-ignore
       ...this.createFields,

diff --git a/public/component_types/transformer/text_embedding_transformer.ts b/public/component_types/transformer/text_embedding_transformer.ts
@@ -3,6 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { COMPONENT_CLASS } from '../../../common';
 import { MLTransformer } from '.';
 
 /**
@@ -11,8 +12,10 @@ import { MLTransformer } from '.';
 export class TextEmbeddingTransformer extends MLTransformer {
   constructor() {
     super();
+    this.type = COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER;
     this.label = 'Text Embedding Transformer';
     this.description = 'A specialized ML transformer for embedding text';
+    this.baseClasses = [...this.baseClasses, this.type];
     this.inputs = [];
     this.createFields = [
       {

diff --git a/public/pages/workflow_detail/utils/index.ts b/public/pages/workflow_detail/utils/index.ts
@@ -0,0 +1,7 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export * from './utils';
+export * from './workflow_to_template_utils';
diff --git a/public/pages/workflow_detail/utils/utils.ts b/public/pages/workflow_detail/utils/utils.ts
@@ -0,0 +1,18 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { WorkspaceFlowState } from '../../../../common';
+
+// TODO: implement this
+/**
+ * Validates the UI workflow state.
+ * Note we don't have to validate connections since that is done via input/output handlers.
+ * But we need to validate there are no open connections
+ */
+export function validateWorkspaceFlow(
+  workspaceFlow: WorkspaceFlowState
+): boolean {
+  return true;
+}