diff --git a/.github/workflows/assistant-release-e2e-workflow.yml b/.github/workflows/assistant-release-e2e-workflow.yml new file mode 100644 index 000000000..952fd4cdd --- /dev/null +++ b/.github/workflows/assistant-release-e2e-workflow.yml @@ -0,0 +1,25 @@ +name: Assistant Release tests workflow in Bundled OpenSearch Dashboards +on: + pull_request: + branches: ['**'] +jobs: + changes: + runs-on: ubuntu-latest + outputs: + tests: ${{ steps.filter.outputs.tests }} + steps: + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + tests: + - 'cypress/**/dashboards-assistant/**' + + tests: + needs: changes + if: ${{ needs.changes.outputs.tests == 'true' }} + uses: ./.github/workflows/release-e2e-workflow-template.yml + with: + test-name: dashboards assistant + test-command: env CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/*' + osd-serve-args: --assistant.chat.enabled=true --assistant.chat.rootAgentName="Cypress test agent" diff --git a/cypress.json b/cypress.json index 9de868129..2b6abb7db 100644 --- a/cypress.json +++ b/cypress.json @@ -20,6 +20,7 @@ "VISBUILDER_ENABLED": true, "DATASOURCE_MANAGEMENT_ENABLED": false, "ML_COMMONS_DASHBOARDS_ENABLED": true, - "WAIT_FOR_LOADER_BUFFER_MS": 0 + "WAIT_FOR_LOADER_BUFFER_MS": 0, + "DASHBOARDS_ASSISTANT_ENABLED": false } } diff --git a/cypress/fixtures/plugins/dashboards-assistant/agent-framework-response.json b/cypress/fixtures/plugins/dashboards-assistant/agent-framework-response.json new file mode 100644 index 000000000..202214c63 --- /dev/null +++ b/cypress/fixtures/plugins/dashboards-assistant/agent-framework-response.json @@ -0,0 +1,5 @@ +{ + "completion": " ```json\n{\n \"thought\": \"Now I know the final answer\",\n \"final_answer\": \"The indices in your cluster are the names listed in the response obtained from using a tool to get information about the OpenSearch indices. This included index names like .plugins-ml-model-group, security-auditlog-2024.01.16, opensearch_dashboards_sample_data_ecommerce and others along with health, status and other details.\"\n}\n```\n", + "stop_reason": "stop_sequence", + "stop": "\n\nHuman:" +} diff --git a/cypress/fixtures/plugins/dashboards-assistant/cluster_settings.json b/cypress/fixtures/plugins/dashboards-assistant/cluster_settings.json new file mode 100644 index 000000000..29e65ef6f --- /dev/null +++ b/cypress/fixtures/plugins/dashboards-assistant/cluster_settings.json @@ -0,0 +1,9 @@ +{ + "persistent": { + "plugins.ml_commons.only_run_on_ml_node": false, + "plugins.ml_commons.memory_feature_enabled": true, + "plugins.ml_commons.trusted_connector_endpoints_regex": [ + "^http://127.0.0.1:3000$" + ] + } +} diff --git a/cypress/fixtures/plugins/dashboards-assistant/flow-template.json b/cypress/fixtures/plugins/dashboards-assistant/flow-template.json new file mode 100644 index 000000000..1abc02395 --- /dev/null +++ b/cypress/fixtures/plugins/dashboards-assistant/flow-template.json @@ -0,0 +1,157 @@ +{ + "name": "Cypress-register-agent", + "description": "Cypress Flow template", + "use_case": "REGISTER_AGENT", + "version": { + "template": "1.0.0", + "compatibility": ["2.12.0", "3.0.0"] + }, + "workflows": { + "provision": { + "user_params": {}, + "nodes": [ + { + "id": "create_connector_1", + "type": "create_connector", + "previous_node_inputs": {}, + "user_inputs": { + "version": "1", + "name": "Claude instant runtime Connector", + "protocol": "aws_sigv4", + "description": "The connector to BedRock service for claude model", + "actions": [ + { + "headers": { + "x-amz-content-sha256": "required", + "content-type": "application/json" + }, + "method": "GET", + "request_body": "{\"prompt\":\"${parameters.prompt}\", \"max_tokens_to_sample\":${parameters.max_tokens_to_sample}, \"temperature\":${parameters.temperature}, \"anthropic_version\":\"${parameters.anthropic_version}\" }", + "action_type": "predict", + "url": "http://127.0.0.1:3000" + } + ], + "credential": { + "access_key": "", + "secret_key": "" + }, + "parameters": { + "endpoint": "bedrock-runtime.us-west-2.amazonaws.com", + "content_type": "application/json", + "auth": "Sig_V4", + "max_tokens_to_sample": "8000", + "service_name": "bedrock", + "temperature": "0.0001", + "response_filter": "$.completion", + "region": "us-west-2", + "anthropic_version": "bedrock-2023-05-31" + } + } + }, + { + "id": "register_model_2", + "type": "register_remote_model", + "previous_node_inputs": { + "create_connector_1": "connector_id" + }, + "user_inputs": { + "description": "test model", + "deploy": true, + "name": "claude-instant" + } + }, + { + "id": "cat_index_tool", + "type": "create_tool", + "previous_node_inputs": {}, + "user_inputs": { + "type": "CatIndexTool", + "name": "CatIndexTool", + "description": "Use this tool to get OpenSearch index information: (health, status, index, uuid, primary count, replica count, docs.count, docs.deleted, store.size, primary.store.size).", + "parameters": { + "index": ".kibana" + } + } + }, + { + "id": "sub_agent", + "type": "register_agent", + "previous_node_inputs": { + "cat_index_tool": "tools", + "register_model_2": "model_id" + }, + "user_inputs": { + "parameters": {}, + "app_type": "chatbot", + "name": "Cypress test sub Agent", + "description": "this is a test agent", + "llm.parameters": { + "max_iteration": "5", + "stop_when_no_tool_found": "true", + "response_filter": "$.completion" + }, + "memory": { + "type": "conversation_index" + }, + "type": "conversational" + } + }, + { + "id": "agent_tool", + "type": "create_tool", + "previous_node_inputs": { + "sub_agent": "agent_id" + }, + "user_inputs": { + "description": "Agent Tool", + "include_output_in_agent_response": true, + "type": "AgentTool", + "parameters": { + "max_iteration": "5" + }, + "name": "AgentTool" + } + }, + { + "id": "ml_model_tool", + "type": "create_tool", + "previous_node_inputs": { + "register_model_2": "model_id" + }, + "user_inputs": { + "parameters": { + "prompt": "\n\nHuman:\" turn\" You are an AI that only speaks JSON. Do not write normal text. Output should follow example JSON format: \n\n {\"response\": [\"question1\", \"question2\"]}\n\n. \n\nHuman:\" turn\":You will be given a chat history between OpenSearch Assistant and a Human.\nUse the context provided to generate follow up questions the Human would ask to the Assistant.\nThe Assistant can answer general questions about logs, traces and metrics.\nAssistant can access a set of tools listed below to answer questions given by the Human:\nQuestion suggestions generator tool\nHere's the chat history between the human and the Assistant.\n${parameters.AgentTool.output}\nUse the following steps to generate follow up questions Human may ask after the response of the Assistant:\nStep 1. Use the chat history to understand what human is trying to search and explore.\nStep 2. Understand what capabilities the assistant has with the set of tools it has access to.\nStep 3. Use the above context and generate follow up questions.Step4:You are an AI that only speaks JSON. Do not write normal text. Output should follow example JSON format: \n\n {\"response\": [\"question1\", \"question2\"]} \n \n----------------\n\nAssistant:" + }, + "description": "A general tool to answer any question.", + "alias": "language_model_tool", + "include_output_in_agent_response": true, + "name": "QuestionSuggestor", + "type": "MLModelTool" + } + }, + { + "id": "root_agent", + "type": "register_agent", + "previous_node_inputs": { + "agent_tool": "tools", + "register_model_2": "model_id", + "ml_model_tool": "tools" + }, + "user_inputs": { + "parameters": { + "prompt": "Answer the question as best you can." + }, + "app_type": "chatbot", + "name": "Cypress test agent", + "description": "this is the root agent", + "tools_order": ["agent_tool", "ml_model_tool"], + "memory": { + "type": "conversation_index" + }, + "type": "flow" + } + } + ] + } + } +} diff --git a/cypress/fixtures/plugins/dashboards-assistant/suggestion-response.json b/cypress/fixtures/plugins/dashboards-assistant/suggestion-response.json new file mode 100644 index 000000000..f30bb088a --- /dev/null +++ b/cypress/fixtures/plugins/dashboards-assistant/suggestion-response.json @@ -0,0 +1,5 @@ +{ + "completion": { + "response": ["suggestion1", "suggestion2"] + } +} diff --git a/cypress/integration/plugins/dashboards-assistant/chatbot_agent_framework_spec.js b/cypress/integration/plugins/dashboards-assistant/chatbot_agent_framework_spec.js new file mode 100644 index 000000000..f56244401 --- /dev/null +++ b/cypress/integration/plugins/dashboards-assistant/chatbot_agent_framework_spec.js @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +import { BASE_PATH } from '../../../utils/constants'; + +if (Cypress.env('DASHBOARDS_ASSISTANT_ENABLED')) { + describe('Assistant basic spec', () => { + before(() => { + // Set welcome screen tracking to false + localStorage.setItem('home:welcome:show', 'false'); + // Set new theme modal to false + localStorage.setItem('home:newThemeModal:show', 'false'); + }); + + beforeEach(() => { + // Visit ISM OSD + cy.visit(`${BASE_PATH}/app/home`); + + // Common text to wait for to confirm page loaded, give up to 60 seconds for initial load + cy.get(`input[placeholder="Ask question"]`, { timeout: 60000 }).should( + 'be.length', + 1 + ); + }); + + describe('Interact with Agent framework', () => { + it('toggle Chatbot and enable to interact', () => { + // enable to toggle and show Chatbot + cy.get(`img[aria-label="toggle chat flyout icon"]`).click(); + + // click suggestions to generate response + cy.contains('What are the indices in my cluster?').click(); + + // should have a LLM Response + cy.contains( + 'The indices in your cluster are the names listed in the response obtained from using a tool to get information about the OpenSearch indices.' + ); + + // should have a suggestion section + cy.get(`[aria-label="chat suggestions"]`).should('be.length', 1); + }); + }); + }); +} diff --git a/cypress/support/assistant-dummy-llm.js b/cypress/support/assistant-dummy-llm.js new file mode 100644 index 000000000..1000453f8 --- /dev/null +++ b/cypress/support/assistant-dummy-llm.js @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +const http = require('http'); +const agentFrameworkJson = require('../fixtures/plugins/dashboards-assistant/agent-framework-response.json'); +const suggestionJson = require('../fixtures/plugins/dashboards-assistant/suggestion-response.json'); + +const MATCH_AGENT_FRAMEWORK_PROMPT = + 'Assistant is designed to be able to assist with a wide range of tasks'; +const MATCH_SUGGESTION_PROMPT = 'You are an AI that only speaks JSON'; + +const server = http.createServer((req, res) => { + // Set the content type to JSON + res.setHeader('Content-Type', 'application/json'); + + let requestBody = ''; + + // Listen for data events to capture the request body + req.on('data', (chunk) => { + requestBody += chunk; + }); + + // Listen for the end of the request + req.on('end', () => { + try { + // Why add a delay here? reference: https://github.com/opensearch-project/ml-commons/issues/1894 + setTimeout(() => { + if (requestBody.includes(MATCH_AGENT_FRAMEWORK_PROMPT)) { + return res.end(JSON.stringify(agentFrameworkJson)); + } else if (requestBody.includes(MATCH_SUGGESTION_PROMPT)) { + return res.end(JSON.stringify(suggestionJson)); + } + + res.end(''); + }, 100); + } catch (error) { + // Handle JSON parsing errors + res.statusCode = 400; + res.end(JSON.stringify({ error: 'Invalid JSON in the request body' })); + } + }); +}); + +// Listen on port 3000 +const PORT = 3000; +server.listen(PORT, () => { + console.log(`Server is listening on port ${PORT}`); +}); diff --git a/cypress/support/index.js b/cypress/support/index.js index cd0c48704..56efee283 100644 --- a/cypress/support/index.js +++ b/cypress/support/index.js @@ -30,6 +30,7 @@ import '../utils/plugins/alerting-dashboards-plugin/commands'; import '../utils/plugins/security-analytics-dashboards-plugin/commands'; import '../utils/plugins/ml-commons-dashboards/commands'; import '../utils/plugins/notifications-dashboards/commands'; +import '../utils/plugins/dashboards-assistant/commands'; import 'cypress-real-events'; @@ -56,3 +57,19 @@ if (Cypress.env('ENDPOINT_WITH_PROXY')) { Cypress.Cookies.preserveOnce('security_authentication'); }); } + +/** + * Make setup step in here so that all the test files in dashboards-assistant + * won't need to call these commands. + */ +if (Cypress.env('DASHBOARDS_ASSISTANT_ENABLED')) { + before(() => { + cy.addAssistantRequiredSettings(); + cy.registerRootAgent(); + cy.startDummyServer(); + }); + after(() => { + cy.cleanRootAgent(); + cy.stopDummyServer(); + }); +} diff --git a/cypress/utils/plugins/dashboards-assistant/commands.js b/cypress/utils/plugins/dashboards-assistant/commands.js new file mode 100644 index 000000000..51df1be8a --- /dev/null +++ b/cypress/utils/plugins/dashboards-assistant/commands.js @@ -0,0 +1,154 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import FlowTemplateJSON from '../../../fixtures/plugins/dashboards-assistant/flow-template.json'; +import { BACKEND_BASE_PATH } from '../../constants'; +import { ML_COMMONS_API } from './constants'; +import clusterSettings from '../../../fixtures/plugins/dashboards-assistant/cluster_settings.json'; + +Cypress.Commands.add('addAssistantRequiredSettings', () => { + cy.request('PUT', `${BACKEND_BASE_PATH}/_cluster/settings`, clusterSettings); +}); + +const agentParameters = { + connectorId: '', + modelId: '', + conversationalAgentId: '', + rootAgentId: '', +}; + +Cypress.Commands.add('registerRootAgent', () => { + // ML needs 10 seconds to initialize its master key + // The ML encryption master key has not been initialized yet. Please retry after waiting for 10 seconds. + cy.wait(10000); + /** + * TODO use flow framework if the plugin get integrate in the future. + */ + // cy.request( + // 'POST', + // `${BACKEND_BASE_PATH}${FLOW_FRAMEWORK_API.CREATE_FLOW_TEMPLATE}`, + // FlowTemplateJSON + // ).then((resp) => { + // usingWorkflowId = resp.body.workflow_id; + // if (usingWorkflowId) { + // cy.request( + // 'POST', + // `${BACKEND_BASE_PATH}${FLOW_FRAMEWORK_API.CREATE_FLOW_TEMPLATE}/${usingWorkflowId}/_provision` + // ); + // /** + // * wait for 2s + // */ + // cy.wait(2000); + // } else { + // throw new Error(resp); + // } + // }); + const nodesMap = {}; + FlowTemplateJSON.workflows.provision.nodes.forEach((node) => { + nodesMap[node.type] = nodesMap[node.type] || []; + nodesMap[node.type].push(node); + }); + cy.request( + 'POST', + `${BACKEND_BASE_PATH}${ML_COMMONS_API.CREATE_CONNECTOR}`, + nodesMap.create_connector[0].user_inputs + ) + .then((resp) => { + agentParameters.connectorId = resp.body.connector_id; + return cy.request( + 'POST', + `${BACKEND_BASE_PATH}${ML_COMMONS_API.CREATE_MODEL}?deploy=true`, + { + ...nodesMap.register_remote_model[0].user_inputs, + connector_id: agentParameters.connectorId, + function_name: 'remote', + } + ); + }) + .then((resp) => { + agentParameters.modelId = resp.body.model_id; + return cy.request( + 'POST', + `${BACKEND_BASE_PATH}${ML_COMMONS_API.CREATE_AGENT}`, + { + ...nodesMap.register_agent[0].user_inputs, + llm: { + parameters: + nodesMap.register_agent[0].user_inputs['llm.parameters'], + model_id: agentParameters.modelId, + }, + tools: [nodesMap.create_tool[0]].map((item) => item.user_inputs), + } + ); + }) + .then((resp) => { + agentParameters.conversationalAgentId = resp.body.agent_id; + return cy.request( + 'POST', + `${BACKEND_BASE_PATH}${ML_COMMONS_API.CREATE_AGENT}`, + { + ...nodesMap.register_agent[1].user_inputs, + tools: [ + { + ...nodesMap.create_tool[1].user_inputs, + parameters: { + ...nodesMap.create_tool[1].user_inputs.parameters, + agent_id: agentParameters.conversationalAgentId, + }, + }, + { + ...nodesMap.create_tool[2].user_inputs, + parameters: { + ...nodesMap.create_tool[2].user_inputs.parameters, + model_id: agentParameters.modelId, + }, + }, + ], + } + ); + }) + .then((resp) => { + agentParameters.rootAgentId = resp.body.agent_id; + }); +}); + +Cypress.Commands.add('cleanRootAgent', () => { + return; + /** + * TODO wait for flow framework to be built into snapshot. + */ + // cy.request( + // 'POST', + // `${BACKEND_BASE_PATH}${FLOW_FRAMEWORK_API.CREATE_FLOW_TEMPLATE}/${usingWorkflowId}/_deprovision` + // ); + // /** + // * wait for 2s + // */ + // cy.wait(2000); +}); + +Cypress.Commands.add('startDummyServer', () => { + // Not a good practice to start a server inside Cypress https://docs.cypress.io/guides/references/best-practices#Web-Servers + // But in out case, we need to reuse release e2e template and let's make it a tradeoff. + cy.exec(`nohup yarn start-dummy-llm-server > /dev/null 2>&1 &`); +}); + +Cypress.Commands.add('stopDummyServer', () => { + /** + * For windows and Linux + */ + cy.exec(`netstat -ano | grep "3000" | awk '{print $5}' | xargs kill -9`, { + failOnNonZeroExit: false, + }).then((result) => { + if (result.stderr) { + /** + * For Macos + */ + cy.exec(`lsof -ti :3000 -sTCP:LISTEN | xargs kill`, { + failOnNonZeroExit: false, + }); + } + }); +}); diff --git a/cypress/utils/plugins/dashboards-assistant/constants.js b/cypress/utils/plugins/dashboards-assistant/constants.js new file mode 100644 index 000000000..16592accb --- /dev/null +++ b/cypress/utils/plugins/dashboards-assistant/constants.js @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +// export const WORKFLOW_API_PREFIX = '/_plugins/_flow_framework/workflow'; +export const ML_COMMONS_API_PREFIX = '/_plugins/_ml'; + +// export const FLOW_FRAMEWORK_API = { +// CREATE_FLOW_TEMPLATE: WORKFLOW_API_PREFIX, +// }; + +export const ML_COMMONS_API = { + CREATE_CONNECTOR: `${ML_COMMONS_API_PREFIX}/connectors/_create`, + CREATE_MODEL: `${ML_COMMONS_API_PREFIX}/models/_register`, + CREATE_AGENT: `${ML_COMMONS_API_PREFIX}/agents/_register`, +}; diff --git a/package.json b/package.json index 0e46dbb96..d1af653f2 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,8 @@ "osd:ciGroup6": "echo \"apps/data_explorer/aaa_before.spec.js,apps/data_explorer/date_nanos_mixed.spec.js,apps/data_explorer/date_nanos.spec.js,apps/data_explorer/discover_histogram.spec.js,apps/data_explorer/discover.spec.js,apps/data_explorer/zzz_after.spec.js\"", "osd:ciGroup7": "echo \"apps/data_explorer/aaa_before.spec.js,apps/data_explorer/doc_navigation.spec.js,apps/data_explorer/doc_table.spec.js,apps/data_explorer/errors.spec.js,apps/data_explorer/field_data.spec.js,apps/data_explorer/zzz_after.spec.js\"", "osd:ciGroup8": "echo \"apps/data_explorer/aaa_before.spec.js,apps/data_explorer/field_visualize.spec.js,apps/data_explorer/filter_editor.spec.js,apps/data_explorer/index_pattern_with_encoded_id.spec.js,apps/data_explorer/index_pattern_without_field.spec.js,apps/data_explorer/zzz_after.spec.js\"", - "osd:ciGroup9": "echo \"apps/data_explorer/aaa_before.spec.js,apps/data_explorer/inspector.spec.js,apps/data_explorer/large_string.spec.js,apps/data_explorer/saved_queries.spec.js,apps/data_explorer/shared_links.spec.js,apps/data_explorer/sidebar.spec.js,apps/data_explorer/source_filter.spec.js,apps/data_explorer/zzz_after.spec.js\"" + "osd:ciGroup9": "echo \"apps/data_explorer/aaa_before.spec.js,apps/data_explorer/inspector.spec.js,apps/data_explorer/large_string.spec.js,apps/data_explorer/saved_queries.spec.js,apps/data_explorer/shared_links.spec.js,apps/data_explorer/sidebar.spec.js,apps/data_explorer/source_filter.spec.js,apps/data_explorer/zzz_after.spec.js\"", + "start-dummy-llm-server": "node ./cypress/support/assistant-dummy-llm.js" }, "repository": { "type": "git",