Add functional test for dashboard assistant trace page (#1012) (#1019)

* Add functional test for trace page

Signed-off-by: Hailong Cui <[email protected]>

* update to ask question direclty instead of click

Signed-off-by: Hailong Cui <[email protected]>

* replace focus with click

Signed-off-by: Hailong Cui <[email protected]>

* add wait before input

Signed-off-by: Hailong Cui <[email protected]>

* address review comments

Signed-off-by: Hailong Cui <[email protected]>

---------

Signed-off-by: Hailong Cui <[email protected]>
(cherry picked from commit 32cba83)

cypress/fixtures/plugins/dashboards-assistant/agent-framework-thought-response.json

@@ -0,0 +1,5 @@
+{
+  "completion": " ```json\n{\n  \"thought\": \"Thought: Let me use tool to figure out\",\n  \"action\": \"CatIndexTool\",\n  \"action_input\": \"\"}\n```\n",
+  "stop_reason": "stop_sequence",
+  "stop": "\n\nHuman:"
+}

cypress/integration/plugins/dashboards-assistant/chatbot_interaction_trace_spec.js

@@ -0,0 +1,104 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { BASE_PATH } from '../../../utils/constants';
+
+if (Cypress.env('DASHBOARDS_ASSISTANT_ENABLED')) {
+  describe('Interaction trace spec', () => {
+    before(() => {
+      // Set welcome screen tracking to false
+      localStorage.setItem('home:welcome:show', 'false');
+      // Set new theme modal to false
+      localStorage.setItem('home:newThemeModal:show', 'false');
+
+      cy.visit(`${BASE_PATH}/app/home`);
+      // cy.waitForLoader();
+
+      // Common text to wait for to confirm page loaded, give up to 120 seconds for initial load
+      cy.get(`input[placeholder="Ask question"]`, { timeout: 120000 }).as(
+        'chatInput'
+      );
+      cy.get('@chatInput').should('be.length', 1);
+
+      cy.wait(1000);
+
+      cy.get('@chatInput')
+        .click()
+        .type('What are the indices in my cluster?{enter}');
+
+      // should have a LLM Response
+      cy.contains(
+        'The indices in your cluster are the names listed in the response obtained from using a tool to get information about the OpenSearch indices.'
+      );
+    });
+
+    // clean up localStorage items
+    after(() => {
+      localStorage.removeItem('home:welcome:show');
+      localStorage.removeItem('home:newThemeModal:show');
+    });
+
+    describe('Trace page', () => {
+      it('open trace page and verify page content', () => {
+        // click How was this generated? to view trace
+        cy.contains('How was this generated?').click();
+
+        cy.get(`.llm-chat-flyout .llm-chat-flyout-body`).as('tracePage');
+        cy.get('@tracePage')
+          .find(`button[aria-label="back"]`)
+          .should('have.length', 1);
+
+        cy.get('@tracePage')
+          .find(`button[aria-label="close"]`)
+          .should('have.length', 0);
+
+        // title
+        cy.get('@tracePage').contains('h1', 'How was this generated');
+
+        // question
+        cy.get('@tracePage').contains('What are the indices in my cluster?');
+
+        // result
+        cy.get('@tracePage').contains(
+          'The indices in your cluster are the names listed in the response obtained from using a tool to get information about the OpenSearch indices.'
+        );
+      });
+
+      it('tools invocation displayed in trace steps', () => {
+        // trace
+        cy.get(`.llm-chat-flyout .llm-chat-flyout-body`).as('tracePage');
+        cy.get('@tracePage').find('.euiAccordion').should('have.length', 1);
+
+        cy.get('@tracePage')
+          .find('.euiAccordion')
+          // tool name
+          .contains('Step 1 - CatIndexTool')
+          .click({ force: true });
+
+        // tool output
+        cy.contains('Output: health	status	index');
+      });
+
+      it('trace page display correctly in fullscreen mode', () => {
+        cy.get(`.llm-chat-flyout-header`)
+          .find(`button[aria-label="fullScreen"]`)
+          .click({ force: true });
+
+        // show close button
+        cy.get(`.llm-chat-flyout .llm-chat-flyout-body`).as('tracePage');
+        cy.get('@tracePage')
+          .find(`button[aria-label="close"]`)
+          .should('have.length', 1);
+
+        cy.get('@tracePage')
+          .find(`button[aria-label="back"]`)
+          .should('have.length', 0);
+
+        // both chat and trace are both displayed
+        cy.contains('How was this generated?').click();
+      });
+    });
+  });
+}

cypress/support/assistant-dummy-llm.js

@@ -4,11 +4,13 @@
  */
 const http = require('http');
 const agentFrameworkJson = require('../fixtures/plugins/dashboards-assistant/agent-framework-response.json');
+const agentFrameworkThoughtJson = require('../fixtures/plugins/dashboards-assistant/agent-framework-thought-response.json');
 const suggestionJson = require('../fixtures/plugins/dashboards-assistant/suggestion-response.json');
 
 const MATCH_AGENT_FRAMEWORK_PROMPT =
   'Assistant is designed to be able to assist with a wide range of tasks';
 const MATCH_SUGGESTION_PROMPT = 'You are an AI that only speaks JSON';
+const TOOL_RESPONSE = 'TOOL RESPONSE:';
 
 const server = http.createServer((req, res) => {
   // Set the content type to JSON
@@ -27,7 +29,11 @@ const server = http.createServer((req, res) => {
       // Why add a delay here? reference: https://github.com/opensearch-project/ml-commons/issues/1894
       setTimeout(() => {
         if (requestBody.includes(MATCH_AGENT_FRAMEWORK_PROMPT)) {
-          return res.end(JSON.stringify(agentFrameworkJson));
+          if (requestBody.includes(TOOL_RESPONSE)) {
+            return res.end(JSON.stringify(agentFrameworkJson));
+          } else {
+            return res.end(JSON.stringify(agentFrameworkThoughtJson));
+          }
         } else if (requestBody.includes(MATCH_SUGGESTION_PROMPT)) {
           return res.end(JSON.stringify(suggestionJson));
         }