feat: failure analysis (#251)

test-results-reporter · Oct 13, 2024 · 8ad86bf · 8ad86bf
1 parent 6a38155
commit 8ad86bf
Show file tree

Hide file tree

Showing 11 changed files with 235 additions and 29 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,5 +1,6 @@
 {
   "cSpell.words": [
+    "analysed",
     "mstest",
     "nunit",
     "pactum",

diff --git a/src/beats/beats.js b/src/beats/beats.js
@@ -1,7 +1,7 @@
 const { getCIInformation } = require('../helpers/ci');
 const logger = require('../utils/logger');
 const { BeatsApi } = require('./beats.api');
-const { HOOK } = require('../helpers/constants');
+const { HOOK, PROCESS_STATUS } = require('../helpers/constants');
 const TestResult = require('test-results-parser/src/models/TestResult');
 const { BeatsAttachments } = require('./beats.attachments');
 
@@ -31,9 +31,7 @@ class Beats {
     await this.#publishTestResults();
     await this.#uploadAttachments();
     this.#updateTitleLink();
-    await this.#attachFailureSummary();
-    await this.#attachSmartAnalysis();
-    await this.#attachErrorClusters();
+    await this.#attachExtensions();
   }
 
   #setCIInfo() {
@@ -104,13 +102,20 @@ class Beats {
     }
   }
 
-  async #attachFailureSummary() {
+  async #attachExtensions() {
     if (!this.test_run_id) {
       return;
     }
     if (!this.config.targets) {
       return;
     }
+    await this.#attachFailureSummary();
+    await this.#attachFailureAnalysis();
+    await this.#attachSmartAnalysis();
+    await this.#attachErrorClusters();
+  }
+
+  async #attachFailureSummary() {
     if (this.result.status !== 'FAIL') {
       return;
     }
@@ -132,13 +137,29 @@ class Beats {
     }
   }
 
-  async #attachSmartAnalysis() {
-    if (!this.test_run_id) {
+  async #attachFailureAnalysis() {
+    if (this.result.status !== 'FAIL') {
       return;
     }
-    if (!this.config.targets) {
+    if (this.config.show_failure_analysis === false) {
       return;
     }
+    try {
+      logger.info('🪄 Fetching Failure Analysis...');
+      await this.#setTestRun('Failure Analysis Status', 'failure_analysis_status');
+      this.config.extensions.push({
+        name: 'failure-analysis',
+        hook: HOOK.AFTER_SUMMARY,
+        inputs: {
+          data: this.test_run
+        }
+      });
+    } catch (error) {
+      logger.error(`❌ Unable to attach failure analysis: ${error.message}`, error);
+    }
+  }
+
+  async #attachSmartAnalysis() {
     if (this.config.show_smart_analysis === false) {
       return;
     }
@@ -165,7 +186,7 @@ class Beats {
   }
 
   async #setTestRun(text, wait_for = 'smart_analysis_status') {
-    if (this.test_run && this.test_run[wait_for] === 'COMPLETED') {
+    if (this.test_run && this.test_run[wait_for] === PROCESS_STATUS.COMPLETED) {
       return;
     }
     let retry = 3;
@@ -175,13 +196,13 @@ class Beats {
       this.test_run = await this.api.getTestRun(this.test_run_id);
       const status = this.test_run && this.test_run[wait_for];
       switch (status) {
-        case 'COMPLETED':
+        case PROCESS_STATUS.COMPLETED:
           logger.debug(`☑️ ${text} generated successfully`);
           return;
-        case 'FAILED':
+        case PROCESS_STATUS.FAILED:
           logger.error(`❌ Failed to generate ${text}`);
           return;
-        case 'SKIPPED':
+        case PROCESS_STATUS.SKIPPED:
           logger.warn(`❗ Skipped generating ${text}`);
           return;
       }
@@ -191,12 +212,6 @@ class Beats {
   }
 
   async #attachErrorClusters() {
-    if (!this.test_run_id) {
-      return;
-    }
-    if (!this.config.targets) {
-      return;
-    }
     if (this.result.status !== 'FAIL') {
       return;
     }

diff --git a/src/beats/beats.types.d.ts b/src/beats/beats.types.d.ts
@@ -9,6 +9,12 @@ export type IBeatExecutionMetric = {
   added: number
   removed: number
   flaky: number
+  product_bugs: number
+  environment_issues: number
+  automation_bugs: number
+  not_a_defects: number
+  to_investigate: number
+  auto_analysed: number
   failure_summary: any
   failure_summary_provider: any
   failure_summary_model: any

diff --git a/src/extensions/failure-analysis.extension.js b/src/extensions/failure-analysis.extension.js
@@ -0,0 +1,58 @@
+const { BaseExtension } = require('./base.extension');
+const { STATUS, HOOK } = require("../helpers/constants");
+
+class FailureAnalysisExtension extends BaseExtension {
+
+  constructor(target, extension, result, payload, root_payload) {
+    super(target, extension, result, payload, root_payload);
+    this.#setDefaultOptions();
+    this.#setDefaultInputs();
+    this.updateExtensionInputs();
+  }
+
+  #setDefaultOptions() {
+    this.default_options.hook = HOOK.AFTER_SUMMARY,
+    this.default_options.condition = STATUS.PASS_OR_FAIL;
+  }
+
+  #setDefaultInputs() {
+    this.default_inputs.title = '';
+    this.default_inputs.title_link = '';
+  }
+
+  run() {
+    this.#setText();
+    this.attach();
+  }
+
+  #setText() {
+    const data = this.extension.inputs.data;
+    if (!data) {
+      return;
+    }
+
+    /**
+     * @type {import('../beats/beats.types').IBeatExecutionMetric}
+     */
+    const execution_metrics = data.execution_metrics[0];
+
+    if (!execution_metrics) {
+      logger.warn('⚠️ No execution metrics found. Skipping.');
+      return;
+    }
+
+    const failure_analysis = [];
+
+    if (execution_metrics.to_investigate) {
+      failure_analysis.push(`🔎 To Investigate: ${execution_metrics.to_investigate}`);
+    }
+    if (execution_metrics.auto_analysed) {
+      failure_analysis.push(`🪄 Auto Analysed: ${execution_metrics.auto_analysed}`);
+    }
+
+    this.text = failure_analysis.join('  •  ');
+  }
+
+}
+
+module.exports =  { FailureAnalysisExtension };
diff --git a/src/extensions/index.js b/src/extensions/index.js
@@ -13,6 +13,7 @@ const { EXTENSION } = require('../helpers/constants');
 const { checkCondition } = require('../helpers/helper');
 const logger = require('../utils/logger');
 const { ErrorClustersExtension } = require('./error-clusters.extension');
+const { FailureAnalysisExtension } = require('./failure-analysis.extension');
 
 async function run(options) {
   const { target, result, hook } = options;
@@ -59,6 +60,8 @@ function getExtensionRunner(extension, options) {
       return new CIInfoExtension(options.target, extension, options.result, options.payload, options.root_payload);
     case EXTENSION.AI_FAILURE_SUMMARY:
       return new AIFailureSummaryExtension(options.target, extension, options.result, options.payload, options.root_payload);
+    case EXTENSION.FAILURE_ANALYSIS:
+      return new FailureAnalysisExtension(options.target, extension, options.result, options.payload, options.root_payload);
     case EXTENSION.SMART_ANALYSIS:
       return new SmartAnalysisExtension(options.target, extension, options.result, options.payload, options.root_payload);
     case EXTENSION.ERROR_CLUSTERS:

diff --git a/src/extensions/smart-analysis.extension.js b/src/extensions/smart-analysis.extension.js
@@ -65,13 +65,13 @@ class SmartAnalysisExtension extends BaseExtension {
     for (const item of smart_analysis) {
       rows.push(item);
       if (rows.length === 3) {
-        texts.push(rows.join(' ｜ '));
+        texts.push(rows.join('  •  '));
         rows.length = 0;
       }
     }
 
     if (rows.length > 0) {
-      texts.push(rows.join(' ｜ '));
+      texts.push(rows.join('  •  '));
     }
 
     this.text = this.mergeTexts(texts);

diff --git a/src/helpers/constants.js b/src/helpers/constants.js
@@ -21,6 +21,7 @@ const TARGET = Object.freeze({
 
 const EXTENSION = Object.freeze({
   AI_FAILURE_SUMMARY: 'ai-failure-summary',
+  FAILURE_ANALYSIS: 'failure-analysis',
   SMART_ANALYSIS: 'smart-analysis',
   ERROR_CLUSTERS: 'error-clusters',
   HYPERLINKS: 'hyperlinks',
@@ -39,6 +40,13 @@ const URLS = Object.freeze({
   QUICK_CHART: 'https://quickchart.io'
 });
 
+const PROCESS_STATUS = Object.freeze({
+  RUNNING: 'RUNNING',
+  COMPLETED: 'COMPLETED',
+  FAILED: 'FAILED',
+  SKIPPED: 'SKIPPED',
+});
+
 const MIN_NODE_VERSION = 14;
 
 module.exports = Object.freeze({
@@ -47,5 +55,6 @@ module.exports = Object.freeze({
   TARGET,
   EXTENSION,
   URLS,
+  PROCESS_STATUS,
   MIN_NODE_VERSION
 });
diff --git a/src/index.d.ts b/src/index.d.ts
@@ -229,6 +229,7 @@ export interface PublishReport {
   project?: string;
   run?: string;
   show_failure_summary?: boolean;
+  show_failure_analysis?: boolean;
   show_smart_analysis?: boolean;
   show_error_clusters?: boolean;
   targets?: Target[];

diff --git a/test/beats.spec.js b/test/beats.spec.js
@@ -230,4 +230,38 @@ describe('TestBeats', () => {
     assert.equal(mock.getInteraction(id4).exercised, true);
   });
 
+  it('should send results with failure analysis to beats', async () => {
+    const id1 = mock.addInteraction('post test results to beats');
+    const id2 = mock.addInteraction('get test results with failure analysis from beats');
+    const id3 = mock.addInteraction('get empty error clusters from beats');
+    const id4 = mock.addInteraction('post test-summary with beats to teams with ai failure summary and smart analysis and failure analysis');
+    await publish({
+      config: {
+        api_key: 'api-key',
+        project: 'project-name',
+        run: 'build-name',
+        targets: [
+          {
+            name: 'teams',
+            inputs: {
+              url: 'http://localhost:9393/message'
+            }
+          }
+        ],
+        results: [
+          {
+            type: 'testng',
+            files: [
+              'test/data/testng/single-suite-failures.xml'
+            ]
+          }
+        ]
+      }
+    });
+    assert.equal(mock.getInteraction(id1).exercised, true);
+    assert.equal(mock.getInteraction(id2).exercised, true);
+    assert.equal(mock.getInteraction(id3).exercised, true);
+    assert.equal(mock.getInteraction(id4).exercised, true);
+  });
+
 });
diff --git a/test/mocks/beats.mock.js b/test/mocks/beats.mock.js
@@ -29,6 +29,7 @@ addInteractionHandler('get test results from beats', () => {
       body: {
         id: 'test-run-id',
         "failure_summary_status": "COMPLETED",
+        "failure_analysis_status": "COMPLETED",
         "smart_analysis_status": "COMPLETED",
         "execution_metrics": [
           {
@@ -69,6 +70,42 @@ addInteractionHandler('get test results with smart analysis from beats', () => {
   }
 });
 
+addInteractionHandler('get test results with failure analysis from beats', () => {
+  return {
+    strict: false,
+    request: {
+      method: 'GET',
+      path: '/api/core/v1/test-runs/test-run-id'
+    },
+    response: {
+      status: 200,
+      body: {
+        id: 'test-run-id',
+        "failure_summary_status": "COMPLETED",
+        "failure_analysis_status": "COMPLETED",
+        "smart_analysis_status": "SKIPPED",
+        "execution_metrics": [
+          {
+            "failure_summary": "",
+            "newly_failed": 1,
+            "always_failing": 1,
+            "recovered": 1,
+            "added": 0,
+            "removed": 0,
+            "flaky": 1,
+            "product_bugs": 1,
+            "environment_issues": 1,
+            "automation_bugs": 1,
+            "not_a_defects": 1,
+            "to_investigate": 1,
+            "auto_analysed": 1
+          }
+        ]
+      }
+    }
+  }
+});
+
 addInteractionHandler('get error clusters from beats', () => {
   return {
     strict: false,