feat: Setup assistant agent & workflow file. (#1002)

* feat: enable setup

Signed-off-by: SuZhou-Joe <[email protected]>

* feat: using a external mock service to mock a dummy LLM

Signed-off-by: SuZhou-Joe <[email protected]>

* feat: use local dummy server

Signed-off-by: SuZhou-Joe <[email protected]>

* feat: add some basic test cases

Signed-off-by: SuZhou-Joe <[email protected]>

* feat: run with server

Signed-off-by: SuZhou-Joe <[email protected]>

* feat: use ml-commons api to setup test agent manually

Signed-off-by: SuZhou-Joe <[email protected]>

* feat: remove flow_framework.enabled config

Signed-off-by: SuZhou-Joe <[email protected]>

* feat: remove visualization as skills repo is not bundled in snapshot

Signed-off-by: SuZhou-Joe <[email protected]>

* feat: add flag config

Signed-off-by: SuZhou-Joe <[email protected]>

* feat: optimize stop command

Signed-off-by: SuZhou-Joe <[email protected]>

* feat: move setup steps to support/index.js

Signed-off-by: SuZhou-Joe <[email protected]>

---------

Signed-off-by: SuZhou-Joe <[email protected]>
(cherry picked from commit cd1dd23)

.github/workflows/assistant-release-e2e-workflow.yml

@@ -0,0 +1,25 @@
+name: Assistant Release tests workflow in Bundled OpenSearch Dashboards
+on:
+  pull_request:
+    branches: ['**']
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      tests: ${{ steps.filter.outputs.tests }}
+    steps:
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            tests:
+              - 'cypress/**/dashboards-assistant/**'
+
+  tests:
+    needs: changes
+    if: ${{ needs.changes.outputs.tests == 'true' }}
+    uses: ./.github/workflows/release-e2e-workflow-template.yml
+    with:
+      test-name: dashboards assistant
+      test-command: env CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/*'
+      osd-serve-args: --assistant.chat.enabled=true --assistant.chat.rootAgentName="Cypress test agent"

cypress.json

@@ -20,6 +20,7 @@
     "VISBUILDER_ENABLED": true,
     "DATASOURCE_MANAGEMENT_ENABLED": false,
     "ML_COMMONS_DASHBOARDS_ENABLED": true,
-    "WAIT_FOR_LOADER_BUFFER_MS": 0
+    "WAIT_FOR_LOADER_BUFFER_MS": 0,
+    "DASHBOARDS_ASSISTANT_ENABLED": false
   }
 }

cypress/fixtures/plugins/dashboards-assistant/agent-framework-response.json

@@ -0,0 +1,5 @@
+{
+  "completion": " ```json\n{\n  \"thought\": \"Now I know the final answer\",\n  \"final_answer\": \"The indices in your cluster are the names listed in the response obtained from using a tool to get information about the OpenSearch indices. This included index names like .plugins-ml-model-group, security-auditlog-2024.01.16, opensearch_dashboards_sample_data_ecommerce and others along with health, status and other details.\"\n}\n```\n",
+  "stop_reason": "stop_sequence",
+  "stop": "\n\nHuman:"
+}

cypress/fixtures/plugins/dashboards-assistant/cluster_settings.json

@@ -0,0 +1,9 @@
+{
+  "persistent": {
+    "plugins.ml_commons.only_run_on_ml_node": false,
+    "plugins.ml_commons.memory_feature_enabled": true,
+    "plugins.ml_commons.trusted_connector_endpoints_regex": [
+      "^http://127.0.0.1:3000$"
+    ]
+  }
+}

cypress/fixtures/plugins/dashboards-assistant/flow-template.json

@@ -0,0 +1,157 @@
+{
+  "name": "Cypress-register-agent",
+  "description": "Cypress Flow template",
+  "use_case": "REGISTER_AGENT",
+  "version": {
+    "template": "1.0.0",
+    "compatibility": ["2.12.0", "3.0.0"]
+  },
+  "workflows": {
+    "provision": {
+      "user_params": {},
+      "nodes": [
+        {
+          "id": "create_connector_1",
+          "type": "create_connector",
+          "previous_node_inputs": {},
+          "user_inputs": {
+            "version": "1",
+            "name": "Claude instant runtime Connector",
+            "protocol": "aws_sigv4",
+            "description": "The connector to BedRock service for claude model",
+            "actions": [
+              {
+                "headers": {
+                  "x-amz-content-sha256": "required",
+                  "content-type": "application/json"
+                },
+                "method": "GET",
+                "request_body": "{\"prompt\":\"${parameters.prompt}\", \"max_tokens_to_sample\":${parameters.max_tokens_to_sample}, \"temperature\":${parameters.temperature},  \"anthropic_version\":\"${parameters.anthropic_version}\" }",
+                "action_type": "predict",
+                "url": "http://127.0.0.1:3000"
+              }
+            ],
+            "credential": {
+              "access_key": "<key>",
+              "secret_key": "<value>"
+            },
+            "parameters": {
+              "endpoint": "bedrock-runtime.us-west-2.amazonaws.com",
+              "content_type": "application/json",
+              "auth": "Sig_V4",
+              "max_tokens_to_sample": "8000",
+              "service_name": "bedrock",
+              "temperature": "0.0001",
+              "response_filter": "$.completion",
+              "region": "us-west-2",
+              "anthropic_version": "bedrock-2023-05-31"
+            }
+          }
+        },
+        {
+          "id": "register_model_2",
+          "type": "register_remote_model",
+          "previous_node_inputs": {
+            "create_connector_1": "connector_id"
+          },
+          "user_inputs": {
+            "description": "test model",
+            "deploy": true,
+            "name": "claude-instant"
+          }
+        },
+        {
+          "id": "cat_index_tool",
+          "type": "create_tool",
+          "previous_node_inputs": {},
+          "user_inputs": {
+            "type": "CatIndexTool",
+            "name": "CatIndexTool",
+            "description": "Use this tool to get OpenSearch index information: (health, status, index, uuid, primary count, replica count, docs.count, docs.deleted, store.size, primary.store.size).",
+            "parameters": {
+              "index": ".kibana"
+            }
+          }
+        },
+        {
+          "id": "sub_agent",
+          "type": "register_agent",
+          "previous_node_inputs": {
+            "cat_index_tool": "tools",
+            "register_model_2": "model_id"
+          },
+          "user_inputs": {
+            "parameters": {},
+            "app_type": "chatbot",
+            "name": "Cypress test sub Agent",
+            "description": "this is a test agent",
+            "llm.parameters": {
+              "max_iteration": "5",
+              "stop_when_no_tool_found": "true",
+              "response_filter": "$.completion"
+            },
+            "memory": {
+              "type": "conversation_index"
+            },
+            "type": "conversational"
+          }
+        },
+        {
+          "id": "agent_tool",
+          "type": "create_tool",
+          "previous_node_inputs": {
+            "sub_agent": "agent_id"
+          },
+          "user_inputs": {
+            "description": "Agent Tool",
+            "include_output_in_agent_response": true,
+            "type": "AgentTool",
+            "parameters": {
+              "max_iteration": "5"
+            },
+            "name": "AgentTool"
+          }
+        },
+        {
+          "id": "ml_model_tool",
+          "type": "create_tool",
+          "previous_node_inputs": {
+            "register_model_2": "model_id"
+          },
+          "user_inputs": {
+            "parameters": {
+              "prompt": "\n\nHuman:\" turn\" You are an AI that only speaks JSON. Do not write normal text. Output should follow example JSON format: \n\n {\"response\": [\"question1\", \"question2\"]}\n\n. \n\nHuman:\" turn\":You will be given a chat history between OpenSearch Assistant and a Human.\nUse the context provided to generate follow up questions the Human would ask to the Assistant.\nThe Assistant can answer general questions about logs, traces and metrics.\nAssistant can access a set of tools listed below to answer questions given by the Human:\nQuestion suggestions generator tool\nHere's the chat history between the human and the Assistant.\n${parameters.AgentTool.output}\nUse the following steps to generate follow up questions Human may ask after the response of the Assistant:\nStep 1. Use the chat history to understand what human is trying to search and explore.\nStep 2. Understand what capabilities the assistant has with the set of tools it has access to.\nStep 3. Use the above context and generate follow up questions.Step4:You are an AI that only speaks JSON. Do not write normal text. Output should follow example JSON format: \n\n {\"response\": [\"question1\", \"question2\"]} \n \n----------------\n\nAssistant:"
+            },
+            "description": "A general tool to answer any question.",
+            "alias": "language_model_tool",
+            "include_output_in_agent_response": true,
+            "name": "QuestionSuggestor",
+            "type": "MLModelTool"
+          }
+        },
+        {
+          "id": "root_agent",
+          "type": "register_agent",
+          "previous_node_inputs": {
+            "agent_tool": "tools",
+            "register_model_2": "model_id",
+            "ml_model_tool": "tools"
+          },
+          "user_inputs": {
+            "parameters": {
+              "prompt": "Answer the question as best you can."
+            },
+            "app_type": "chatbot",
+            "name": "Cypress test agent",
+            "description": "this is the root agent",
+            "tools_order": ["agent_tool", "ml_model_tool"],
+            "memory": {
+              "type": "conversation_index"
+            },
+            "type": "flow"
+          }
+        }
+      ]
+    }
+  }
+}

cypress/fixtures/plugins/dashboards-assistant/suggestion-response.json

@@ -0,0 +1,5 @@
+{
+  "completion": {
+    "response": ["suggestion1", "suggestion2"]
+  }
+}

cypress/integration/plugins/dashboards-assistant/chatbot_agent_framework_spec.js

@@ -0,0 +1,45 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { BASE_PATH } from '../../../utils/constants';
+
+if (Cypress.env('DASHBOARDS_ASSISTANT_ENABLED')) {
+  describe('Assistant basic spec', () => {
+    before(() => {
+      // Set welcome screen tracking to false
+      localStorage.setItem('home:welcome:show', 'false');
+      // Set new theme modal to false
+      localStorage.setItem('home:newThemeModal:show', 'false');
+    });
+
+    beforeEach(() => {
+      // Visit ISM OSD
+      cy.visit(`${BASE_PATH}/app/home`);
+
+      // Common text to wait for to confirm page loaded, give up to 60 seconds for initial load
+      cy.get(`input[placeholder="Ask question"]`, { timeout: 60000 }).should(
+        'be.length',
+        1
+      );
+    });
+
+    describe('Interact with Agent framework', () => {
+      it('toggle Chatbot and enable to interact', () => {
+        // enable to toggle and show Chatbot
+        cy.get(`img[aria-label="toggle chat flyout icon"]`).click();
+
+        // click suggestions to generate response
+        cy.contains('What are the indices in my cluster?').click();
+
+        // should have a LLM Response
+        cy.contains(
+          'The indices in your cluster are the names listed in the response obtained from using a tool to get information about the OpenSearch indices.'
+        );
+
+        // should have a suggestion section
+        cy.get(`[aria-label="chat suggestions"]`).should('be.length', 1);
+      });
+    });
+  });
+}

cypress/support/assistant-dummy-llm.js

@@ -0,0 +1,49 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+const http = require('http');
+const agentFrameworkJson = require('../fixtures/plugins/dashboards-assistant/agent-framework-response.json');
+const suggestionJson = require('../fixtures/plugins/dashboards-assistant/suggestion-response.json');
+
+const MATCH_AGENT_FRAMEWORK_PROMPT =
+  'Assistant is designed to be able to assist with a wide range of tasks';
+const MATCH_SUGGESTION_PROMPT = 'You are an AI that only speaks JSON';
+
+const server = http.createServer((req, res) => {
+  // Set the content type to JSON
+  res.setHeader('Content-Type', 'application/json');
+
+  let requestBody = '';
+
+  // Listen for data events to capture the request body
+  req.on('data', (chunk) => {
+    requestBody += chunk;
+  });
+
+  // Listen for the end of the request
+  req.on('end', () => {
+    try {
+      // Why add a delay here? reference: https://github.com/opensearch-project/ml-commons/issues/1894
+      setTimeout(() => {
+        if (requestBody.includes(MATCH_AGENT_FRAMEWORK_PROMPT)) {
+          return res.end(JSON.stringify(agentFrameworkJson));
+        } else if (requestBody.includes(MATCH_SUGGESTION_PROMPT)) {
+          return res.end(JSON.stringify(suggestionJson));
+        }
+
+        res.end('');
+      }, 100);
+    } catch (error) {
+      // Handle JSON parsing errors
+      res.statusCode = 400;
+      res.end(JSON.stringify({ error: 'Invalid JSON in the request body' }));
+    }
+  });
+});
+
+// Listen on port 3000
+const PORT = 3000;
+server.listen(PORT, () => {
+  console.log(`Server is listening on port ${PORT}`);
+});

cypress/support/index.js

@@ -30,6 +30,7 @@ import '../utils/plugins/alerting-dashboards-plugin/commands';
 import '../utils/plugins/security-analytics-dashboards-plugin/commands';
 import '../utils/plugins/ml-commons-dashboards/commands';
 import '../utils/plugins/notifications-dashboards/commands';
+import '../utils/plugins/dashboards-assistant/commands';
 
 import 'cypress-real-events';
 
@@ -56,3 +57,19 @@ if (Cypress.env('ENDPOINT_WITH_PROXY')) {
     Cypress.Cookies.preserveOnce('security_authentication');
   });
 }
+
+/**
+ * Make setup step in here so that all the test files in dashboards-assistant
+ * won't need to call these commands.
+ */
+if (Cypress.env('DASHBOARDS_ASSISTANT_ENABLED')) {
+  before(() => {
+    cy.addAssistantRequiredSettings();
+    cy.registerRootAgent();
+    cy.startDummyServer();
+  });
+  after(() => {
+    cy.cleanRootAgent();
+    cy.stopDummyServer();
+  });
+}