From a66ac5d0295ea0d61e18d1266d292feeb9864a2a Mon Sep 17 00:00:00 2001 From: SuZhou-Joe Date: Mon, 22 Jan 2024 12:28:03 +0800 Subject: [PATCH] feat: add some basic test cases Signed-off-by: SuZhou-Joe --- .../assistant-release-e2e-workflow.yml | 4 +-- DEVELOPER_GUIDE.md | 5 ++++ .../dashboards-assistant/basic_spec.js | 25 ++++++++++++++++++- cypress/support/assistant-dummy-llm.js | 15 ++++++----- cypress/support/index.js | 7 ------ .../plugins/dashboards-assistant/commands.js | 25 ++++++++++++++----- package.json | 3 ++- 7 files changed, 60 insertions(+), 24 deletions(-) diff --git a/.github/workflows/assistant-release-e2e-workflow.yml b/.github/workflows/assistant-release-e2e-workflow.yml index 1038de547..877a9cd99 100644 --- a/.github/workflows/assistant-release-e2e-workflow.yml +++ b/.github/workflows/assistant-release-e2e-workflow.yml @@ -2,8 +2,6 @@ name: Assistant Release tests workflow in Bundled OpenSearch Dashboards on: pull_request: branches: ['**'] -env: - CYPRESS_ASSISTANT_AGENT_INITIALIZE_REQUIRED: true jobs: changes: runs-on: ubuntu-latest @@ -23,5 +21,5 @@ jobs: uses: ./.github/workflows/release-e2e-workflow-template.yml with: test-name: dashboards assistant - test-command: node ./cypress/support/assistant-dummy-llm.js & yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/*' + test-command: yarn start-dummy-llm-server & yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/*' osd-serve-args: --assistant.chat.enabled=true --assistant.chat.rootAgentName="Cypress test agent" diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 61b451ef7..d1d91a7ba 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -145,6 +145,11 @@ Create a new workflow by referring to [this template](https://github.com/opensea To make the build repo enable your experimental feature when spinning up OSD service, make sure that you update [this file](https://github.com/opensearch-project/opensearch-build/blob/main/src/test_workflow/integ_test/service_opensearch_dashboards.py) You could either modify the start command or the yml file. To avoid a potentially long start command, it is preferred to modify the yml file to turn on the feature. +### Develop test cases for dashboards-assistant +dashboards-assistant relies on a dummy LLM server to run its integration test so when it comes to the repo you need to prepare some extra prerequisites. + +- run `yarn start-dummy-llm-server` to start a dummy LLM server in your env on the port of 3000. + ## General ### Formatting diff --git a/cypress/integration/plugins/dashboards-assistant/basic_spec.js b/cypress/integration/plugins/dashboards-assistant/basic_spec.js index 46eeab279..bf78704ed 100644 --- a/cypress/integration/plugins/dashboards-assistant/basic_spec.js +++ b/cypress/integration/plugins/dashboards-assistant/basic_spec.js @@ -8,6 +8,11 @@ describe('Assistant basic spec', () => { before(() => { // Set welcome screen tracking to false localStorage.setItem('home:welcome:show', 'false'); + // Set new theme modal to false + localStorage.setItem('home:newThemeModal:show', 'false'); + + cy.addAssistantRequiredSettings(); + cy.registerRootAgent(); }); beforeEach(() => { @@ -22,6 +27,24 @@ describe('Assistant basic spec', () => { }); describe('show up', () => { - it.only('successfully', () => {}); + it('toggle Chatbot and enable to interact', () => { + // enable to toggle and show Chatbot + cy.get(`img[aria-label="toggle chat flyout icon"]`).click(); + + // click suggestions to generate response + cy.contains('What are the indices in my cluster?').click(); + + // should have a LLM Response + cy.contains( + 'The indices in your cluster are the names listed in the response obtained from using a tool to get information about the OpenSearch indices.' + ); + + // should have a suggestion section + cy.get(`[aria-label="chat suggestions"]`).should('be.length', 1); + }); + }); + + after(() => { + cy.cleanRootAgent(); }); }); diff --git a/cypress/support/assistant-dummy-llm.js b/cypress/support/assistant-dummy-llm.js index a410569db..1000453f8 100644 --- a/cypress/support/assistant-dummy-llm.js +++ b/cypress/support/assistant-dummy-llm.js @@ -24,13 +24,16 @@ const server = http.createServer((req, res) => { // Listen for the end of the request req.on('end', () => { try { - if (requestBody.includes(MATCH_AGENT_FRAMEWORK_PROMPT)) { - res.end(JSON.stringify(agentFrameworkJson)); - } else if (requestBody.includes(MATCH_SUGGESTION_PROMPT)) { - res.end(JSON.stringify(suggestionJson)); - } + // Why add a delay here? reference: https://github.com/opensearch-project/ml-commons/issues/1894 + setTimeout(() => { + if (requestBody.includes(MATCH_AGENT_FRAMEWORK_PROMPT)) { + return res.end(JSON.stringify(agentFrameworkJson)); + } else if (requestBody.includes(MATCH_SUGGESTION_PROMPT)) { + return res.end(JSON.stringify(suggestionJson)); + } - res.end(''); + res.end(''); + }, 100); } catch (error) { // Handle JSON parsing errors res.statusCode = 400; diff --git a/cypress/support/index.js b/cypress/support/index.js index e34617270..aa91ea743 100644 --- a/cypress/support/index.js +++ b/cypress/support/index.js @@ -57,10 +57,3 @@ if (Cypress.env('ENDPOINT_WITH_PROXY')) { Cypress.Cookies.preserveOnce('security_authentication'); }); } - -if (Cypress.env('CYPRESS_ASSISTANT_AGENT_INITIALIZE_REQUIRED')) { - before(() => { - cy.addAssistantRequiredSettings(); - cy.registerRootAgent(); - }); -} diff --git a/cypress/utils/plugins/dashboards-assistant/commands.js b/cypress/utils/plugins/dashboards-assistant/commands.js index 7e64f1d9e..f9a4ffb96 100644 --- a/cypress/utils/plugins/dashboards-assistant/commands.js +++ b/cypress/utils/plugins/dashboards-assistant/commands.js @@ -20,20 +20,22 @@ Cypress.Commands.add('addAssistantRequiredSettings', () => { }); }); +let usingWorkflowId = ''; + Cypress.Commands.add('registerRootAgent', () => { + // ML needs 10 seconds to initialize its master key + // The ML encryption master key has not been initialized yet. Please retry after waiting for 10 seconds. + cy.wait(10000); cy.request( 'POST', `${BACKEND_BASE_PATH}${FLOW_FRAMEWORK_API.CREATE_FLOW_TEMPLATE}`, FlowTemplateJSON ).then((resp) => { - // ML needs 10 seconds to initialize its master key - // The ML encryption master key has not been initialized yet. Please retry after waiting for 10 seconds. - cy.wait(10000); - const workflowId = resp.body.workflow_id; - if (workflowId) { + usingWorkflowId = resp.body.workflow_id; + if (usingWorkflowId) { cy.request( 'POST', - `${BACKEND_BASE_PATH}${FLOW_FRAMEWORK_API.CREATE_FLOW_TEMPLATE}/${workflowId}/_provision` + `${BACKEND_BASE_PATH}${FLOW_FRAMEWORK_API.CREATE_FLOW_TEMPLATE}/${usingWorkflowId}/_provision` ); /** * wait for 2s @@ -44,3 +46,14 @@ Cypress.Commands.add('registerRootAgent', () => { } }); }); + +Cypress.Commands.add('cleanRootAgent', () => { + cy.request( + 'POST', + `${BACKEND_BASE_PATH}${FLOW_FRAMEWORK_API.CREATE_FLOW_TEMPLATE}/${usingWorkflowId}/_deprovision` + ); + /** + * wait for 2s + */ + cy.wait(2000); +}); diff --git a/package.json b/package.json index 19b3ea8e9..ee6bd4bc8 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,8 @@ "osd:ciGroup6": "echo \"apps/data_explorer/aaa_before.spec.js,apps/data_explorer/date_nanos_mixed.spec.js,apps/data_explorer/date_nanos.spec.js,apps/data_explorer/discover_histogram.spec.js,apps/data_explorer/discover.spec.js,apps/data_explorer/zzz_after.spec.js\"", "osd:ciGroup7": "echo \"apps/data_explorer/aaa_before.spec.js,apps/data_explorer/doc_navigation.spec.js,apps/data_explorer/doc_table.spec.js,apps/data_explorer/errors.spec.js,apps/data_explorer/field_data.spec.js,apps/data_explorer/zzz_after.spec.js\"", "osd:ciGroup8": "echo \"apps/data_explorer/aaa_before.spec.js,apps/data_explorer/field_visualize.spec.js,apps/data_explorer/filter_editor.spec.js,apps/data_explorer/index_pattern_with_encoded_id.spec.js,apps/data_explorer/index_pattern_without_field.spec.js,apps/data_explorer/zzz_after.spec.js\"", - "osd:ciGroup9": "echo \"apps/data_explorer/aaa_before.spec.js,apps/data_explorer/inspector.spec.js,apps/data_explorer/large_string.spec.js,apps/data_explorer/saved_queries.spec.js,apps/data_explorer/shared_links.spec.js,apps/data_explorer/sidebar.spec.js,apps/data_explorer/source_filter.spec.js,apps/data_explorer/zzz_after.spec.js\"" + "osd:ciGroup9": "echo \"apps/data_explorer/aaa_before.spec.js,apps/data_explorer/inspector.spec.js,apps/data_explorer/large_string.spec.js,apps/data_explorer/saved_queries.spec.js,apps/data_explorer/shared_links.spec.js,apps/data_explorer/sidebar.spec.js,apps/data_explorer/source_filter.spec.js,apps/data_explorer/zzz_after.spec.js\"", + "start-dummy-llm-server": "node ./cypress/support/assistant-dummy-llm.js" }, "repository": { "type": "git",