From a50d2c4a65e8b1482b476c1fba6a6168e5ad0001 Mon Sep 17 00:00:00 2001 From: Hailong Cui Date: Tue, 30 Jan 2024 14:44:43 +0800 Subject: [PATCH] Add functional test for dashboard assistant trace page (#1012) * Add functional test for trace page Signed-off-by: Hailong Cui * update to ask question direclty instead of click Signed-off-by: Hailong Cui * replace focus with click Signed-off-by: Hailong Cui * add wait before input Signed-off-by: Hailong Cui * address review comments Signed-off-by: Hailong Cui --------- Signed-off-by: Hailong Cui (cherry picked from commit 32cba8323cc07910b2e9c7babc8dc3b424adc684) --- .../agent-framework-thought-response.json | 5 + .../chatbot_interaction_trace_spec.js | 104 ++++++++++++++++++ cypress/support/assistant-dummy-llm.js | 8 +- 3 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 cypress/fixtures/plugins/dashboards-assistant/agent-framework-thought-response.json create mode 100644 cypress/integration/plugins/dashboards-assistant/chatbot_interaction_trace_spec.js diff --git a/cypress/fixtures/plugins/dashboards-assistant/agent-framework-thought-response.json b/cypress/fixtures/plugins/dashboards-assistant/agent-framework-thought-response.json new file mode 100644 index 000000000..3ce8aa6b0 --- /dev/null +++ b/cypress/fixtures/plugins/dashboards-assistant/agent-framework-thought-response.json @@ -0,0 +1,5 @@ +{ + "completion": " ```json\n{\n \"thought\": \"Thought: Let me use tool to figure out\",\n \"action\": \"CatIndexTool\",\n \"action_input\": \"\"}\n```\n", + "stop_reason": "stop_sequence", + "stop": "\n\nHuman:" +} \ No newline at end of file diff --git a/cypress/integration/plugins/dashboards-assistant/chatbot_interaction_trace_spec.js b/cypress/integration/plugins/dashboards-assistant/chatbot_interaction_trace_spec.js new file mode 100644 index 000000000..3b11bb8da --- /dev/null +++ b/cypress/integration/plugins/dashboards-assistant/chatbot_interaction_trace_spec.js @@ -0,0 +1,104 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { BASE_PATH } from '../../../utils/constants'; + +if (Cypress.env('DASHBOARDS_ASSISTANT_ENABLED')) { + describe('Interaction trace spec', () => { + before(() => { + // Set welcome screen tracking to false + localStorage.setItem('home:welcome:show', 'false'); + // Set new theme modal to false + localStorage.setItem('home:newThemeModal:show', 'false'); + + cy.visit(`${BASE_PATH}/app/home`); + // cy.waitForLoader(); + + // Common text to wait for to confirm page loaded, give up to 120 seconds for initial load + cy.get(`input[placeholder="Ask question"]`, { timeout: 120000 }).as( + 'chatInput' + ); + cy.get('@chatInput').should('be.length', 1); + + cy.wait(1000); + + cy.get('@chatInput') + .click() + .type('What are the indices in my cluster?{enter}'); + + // should have a LLM Response + cy.contains( + 'The indices in your cluster are the names listed in the response obtained from using a tool to get information about the OpenSearch indices.' + ); + }); + + // clean up localStorage items + after(() => { + localStorage.removeItem('home:welcome:show'); + localStorage.removeItem('home:newThemeModal:show'); + }); + + describe('Trace page', () => { + it('open trace page and verify page content', () => { + // click How was this generated? to view trace + cy.contains('How was this generated?').click(); + + cy.get(`.llm-chat-flyout .llm-chat-flyout-body`).as('tracePage'); + cy.get('@tracePage') + .find(`button[aria-label="back"]`) + .should('have.length', 1); + + cy.get('@tracePage') + .find(`button[aria-label="close"]`) + .should('have.length', 0); + + // title + cy.get('@tracePage').contains('h1', 'How was this generated'); + + // question + cy.get('@tracePage').contains('What are the indices in my cluster?'); + + // result + cy.get('@tracePage').contains( + 'The indices in your cluster are the names listed in the response obtained from using a tool to get information about the OpenSearch indices.' + ); + }); + + it('tools invocation displayed in trace steps', () => { + // trace + cy.get(`.llm-chat-flyout .llm-chat-flyout-body`).as('tracePage'); + cy.get('@tracePage').find('.euiAccordion').should('have.length', 1); + + cy.get('@tracePage') + .find('.euiAccordion') + // tool name + .contains('Step 1 - CatIndexTool') + .click({ force: true }); + + // tool output + cy.contains('Output: health status index'); + }); + + it('trace page display correctly in fullscreen mode', () => { + cy.get(`.llm-chat-flyout-header`) + .find(`button[aria-label="fullScreen"]`) + .click({ force: true }); + + // show close button + cy.get(`.llm-chat-flyout .llm-chat-flyout-body`).as('tracePage'); + cy.get('@tracePage') + .find(`button[aria-label="close"]`) + .should('have.length', 1); + + cy.get('@tracePage') + .find(`button[aria-label="back"]`) + .should('have.length', 0); + + // both chat and trace are both displayed + cy.contains('How was this generated?').click(); + }); + }); + }); +} diff --git a/cypress/support/assistant-dummy-llm.js b/cypress/support/assistant-dummy-llm.js index 1000453f8..930f6b223 100644 --- a/cypress/support/assistant-dummy-llm.js +++ b/cypress/support/assistant-dummy-llm.js @@ -4,11 +4,13 @@ */ const http = require('http'); const agentFrameworkJson = require('../fixtures/plugins/dashboards-assistant/agent-framework-response.json'); +const agentFrameworkThoughtJson = require('../fixtures/plugins/dashboards-assistant/agent-framework-thought-response.json'); const suggestionJson = require('../fixtures/plugins/dashboards-assistant/suggestion-response.json'); const MATCH_AGENT_FRAMEWORK_PROMPT = 'Assistant is designed to be able to assist with a wide range of tasks'; const MATCH_SUGGESTION_PROMPT = 'You are an AI that only speaks JSON'; +const TOOL_RESPONSE = 'TOOL RESPONSE:'; const server = http.createServer((req, res) => { // Set the content type to JSON @@ -27,7 +29,11 @@ const server = http.createServer((req, res) => { // Why add a delay here? reference: https://github.com/opensearch-project/ml-commons/issues/1894 setTimeout(() => { if (requestBody.includes(MATCH_AGENT_FRAMEWORK_PROMPT)) { - return res.end(JSON.stringify(agentFrameworkJson)); + if (requestBody.includes(TOOL_RESPONSE)) { + return res.end(JSON.stringify(agentFrameworkJson)); + } else { + return res.end(JSON.stringify(agentFrameworkThoughtJson)); + } } else if (requestBody.includes(MATCH_SUGGESTION_PROMPT)) { return res.end(JSON.stringify(suggestionJson)); }