Onboard basic RAG preset (#369) (#372)

Signed-off-by: Tyler Ohlsen <[email protected]> (cherry picked from commit 3d08de9) Co-authored-by: Tyler Ohlsen <[email protected]>
opensearch-project · Sep 12, 2024 · c7eb16c · c7eb16c
1 parent d669c10
commit c7eb16c
Show file tree

Hide file tree

Showing 14 changed files with 313 additions and 116 deletions.
diff --git a/common/constants.ts b/common/constants.ts
@@ -91,6 +91,7 @@ export enum WORKFLOW_TYPE {
   MULTIMODAL_SEARCH = 'Multimodal search',
   HYBRID_SEARCH = 'Hybrid search',
   SENTIMENT_ANALYSIS = 'Sentiment analysis',
+  RAG = 'Retrieval-augmented generation',
   CUSTOM = 'Custom',
   UNKNOWN = 'Unknown',
 }
@@ -103,6 +104,7 @@ export enum PROCESSOR_TYPE {
   SORT = 'sort',
   TEXT_CHUNKING = 'text_chunking',
   NORMALIZATION = 'normalization-processor',
+  COLLAPSE = 'collapse',
 }
 
 export enum MODEL_TYPE {
@@ -180,12 +182,17 @@ export const DELIMITER_OPTIONAL_FIELDS = ['delimiter'];
 export const SHARED_OPTIONAL_FIELDS = ['max_chunk_limit', 'description', 'tag'];
 
 /**
- * QUERY PRESETS
+ * DEFAULT FIELD VALUES
  */
 export const DEFAULT_TEXT_FIELD = 'my_text';
 export const DEFAULT_VECTOR_FIELD = 'my_embedding';
 export const DEFAULT_IMAGE_FIELD = 'my_image';
 export const DEFAULT_LABEL_FIELD = 'label';
+export const DEFAULT_LLM_RESPONSE_FIELD = 'llm_response';
+
+/**
+ * QUERY PRESETS
+ */
 export const VECTOR_FIELD_PATTERN = `{{vector_field}}`;
 export const TEXT_FIELD_PATTERN = `{{text_field}}`;
 export const IMAGE_FIELD_PATTERN = `{{image_field}}`;

diff --git a/common/interfaces.ts b/common/interfaces.ts
@@ -489,6 +489,7 @@ export type QuickConfigureFields = {
   imageField?: string;
   labelField?: string;
   embeddingLength?: number;
+  llmResponseField?: string;
 };
 
 /**

diff --git a/public/configs/search_response_processors/collapse_processor.ts b/public/configs/search_response_processors/collapse_processor.ts
@@ -0,0 +1,43 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { PROCESSOR_TYPE } from '../../../common';
+import { Processor } from '../processor';
+
+/**
+ * The collapse processor config. Used in search flows.
+ */
+export class CollapseProcessor extends Processor {
+  constructor() {
+    super();
+    this.type = PROCESSOR_TYPE.COLLAPSE;
+    this.name = 'Collapse Processor';
+    this.fields = [
+      {
+        id: 'field',
+        type: 'string',
+      },
+    ];
+    this.optionalFields = [
+      {
+        id: 'context_prefix',
+        type: 'string',
+      },
+      {
+        id: 'tag',
+        type: 'string',
+      },
+      {
+        id: 'description',
+        type: 'string',
+      },
+      {
+        id: 'ignore_failure',
+        type: 'boolean',
+        value: false,
+      },
+    ];
+  }
+}
diff --git a/public/configs/search_response_processors/index.ts b/public/configs/search_response_processors/index.ts
@@ -7,3 +7,4 @@ export * from './ml_search_response_processor';
 export * from './split_search_response_processor';
 export * from './sort_search_response_processor';
 export * from './normalization_processor';
+export * from './collapse_processor';
diff --git a/public/general_components/general-component-styles.scss b/public/general_components/general-component-styles.scss
@@ -1,5 +1,5 @@
 .multi-select-filter {
   &--width {
-    width: 200px;
+    width: 300px;
   }
 }
diff --git a/public/pages/workflow_detail/tools/tools.tsx b/public/pages/workflow_detail/tools/tools.tsx
@@ -42,7 +42,7 @@ const inputTabs = [
   },
   {
     id: TAB_ID.QUERY,
-    name: 'Run queries',
+    name: 'Run query',
     disabled: false,
   },
   {

diff --git a/public/pages/workflow_detail/workflow_detail.test.tsx b/public/pages/workflow_detail/workflow_detail.test.tsx
@@ -87,7 +87,7 @@ describe('WorkflowDetail Page with create ingestion option', () => {
       expect(getByText('Visual')).toBeInTheDocument();
       expect(getByText('JSON')).toBeInTheDocument();
       expect(getByRole('tab', { name: 'Run ingestion' })).toBeInTheDocument();
-      expect(getByRole('tab', { name: 'Run queries' })).toBeInTheDocument();
+      expect(getByRole('tab', { name: 'Run query' })).toBeInTheDocument();
       expect(getByRole('tab', { name: 'Errors' })).toBeInTheDocument();
       expect(getByRole('tab', { name: 'Resources' })).toBeInTheDocument();
 

diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx
@@ -249,7 +249,7 @@ export function InputTransformModal(props: InputTransformModalProps) {
                             index: values.search.index.name,
                             body: JSON.stringify({
                               ...JSON.parse(values.search.request as string),
-                              search_pipeline: curSearchPipeline,
+                              search_pipeline: curSearchPipeline || {},
                             }),
                           },
                           dataSourceId,

diff --git a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx
@@ -25,6 +25,7 @@ import {
 } from '../../../../common';
 import { formikToUiConfig } from '../../../utils';
 import {
+  CollapseProcessor,
   MLIngestProcessor,
   MLSearchRequestProcessor,
   MLSearchResponseProcessor,
@@ -290,6 +291,13 @@ export function ProcessorsList(props: ProcessorsListProps) {
                               );
                             },
                           },
+                          {
+                            name: 'Collapse Processor',
+                            onClick: () => {
+                              closePopover();
+                              addProcessor(new CollapseProcessor().toObj());
+                            },
+                          },
                         ],
                 },
               ]}

diff --git a/public/pages/workflows/new_workflow/quick_configure_inputs.tsx b/public/pages/workflows/new_workflow/quick_configure_inputs.tsx
@@ -19,6 +19,7 @@ import {
   COHERE_DIMENSIONS,
   DEFAULT_IMAGE_FIELD,
   DEFAULT_LABEL_FIELD,
+  DEFAULT_LLM_RESPONSE_FIELD,
   DEFAULT_TEXT_FIELD,
   DEFAULT_VECTOR_FIELD,
   MODEL_STATE,
@@ -84,6 +85,13 @@ export function QuickConfigureInputs(props: QuickConfigureInputsProps) {
         };
         break;
       }
+      case WORKFLOW_TYPE.RAG: {
+        defaultFieldValues = {
+          textField: DEFAULT_TEXT_FIELD,
+          llmResponseField: DEFAULT_LLM_RESPONSE_FIELD,
+        };
+        break;
+      }
       case WORKFLOW_TYPE.CUSTOM:
       default:
         break;
@@ -143,10 +151,7 @@ export function QuickConfigureInputs(props: QuickConfigureInputsProps) {
 
   return (
     <>
-      {(props.workflowType === WORKFLOW_TYPE.SEMANTIC_SEARCH ||
-        props.workflowType === WORKFLOW_TYPE.MULTIMODAL_SEARCH ||
-        props.workflowType === WORKFLOW_TYPE.HYBRID_SEARCH ||
-        props.workflowType === WORKFLOW_TYPE.SENTIMENT_ANALYSIS) && (
+      {props.workflowType !== WORKFLOW_TYPE.CUSTOM ? (
         <>
           <EuiSpacer size="m" />
           <EuiAccordion
@@ -159,12 +164,16 @@ export function QuickConfigureInputs(props: QuickConfigureInputsProps) {
               label={
                 props.workflowType === WORKFLOW_TYPE.SENTIMENT_ANALYSIS
                   ? 'Model'
+                  : props.workflowType === WORKFLOW_TYPE.RAG
+                  ? 'Large language model'
                   : 'Embedding model'
               }
               isInvalid={false}
               helpText={
                 props.workflowType === WORKFLOW_TYPE.SENTIMENT_ANALYSIS
                   ? 'The sentiment analysis model'
+                  : props.workflowType === WORKFLOW_TYPE.RAG
+                  ? 'The large language model to generate user-friendly responses'
                   : 'The model to generate embeddings'
               }
             >
@@ -209,6 +218,8 @@ export function QuickConfigureInputs(props: QuickConfigureInputsProps) {
               helpText={`The name of the text document field to be ${
                 props.workflowType === WORKFLOW_TYPE.SENTIMENT_ANALYSIS
                   ? 'analyzed'
+                  : props.workflowType === WORKFLOW_TYPE.RAG
+                  ? 'used as context to the large language model (LLM)'
                   : 'embedded'
               }`}
             >
@@ -297,9 +308,26 @@ export function QuickConfigureInputs(props: QuickConfigureInputsProps) {
                 />
               </EuiCompressedFormRow>
             )}
+            {props.workflowType === WORKFLOW_TYPE.RAG && (
+              <EuiCompressedFormRow
+                label={'LLM response field'}
+                isInvalid={false}
+                helpText="The name of the field containing the large language model (LLM) response"
+              >
+                <EuiCompressedFieldText
+                  value={fieldValues?.llmResponseField || ''}
+                  onChange={(e) => {
+                    setFieldValues({
+                      ...fieldValues,
+                      llmResponseField: e.target.value,
+                    });
+                  }}
+                />
+              </EuiCompressedFormRow>
+            )}
           </EuiAccordion>
         </>
-      )}
+      ) : undefined}
     </>
   );
 }