From 851e6902b12d12335cdc0017caf408476fbfe89d Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sun, 19 Jan 2025 06:44:46 +0200 Subject: [PATCH] improved handling information between workers and main process --- lib/command/run-workers.js | 9 ------ lib/command/workers/runTests.js | 4 +-- lib/listener/result.js | 1 - lib/mocha/hooks.js | 12 +++++++ lib/mocha/test.js | 19 +++++++----- lib/plugin/analyze.js | 55 +++++++++++++++++++++------------ lib/plugin/heal.js | 4 +-- lib/plugin/pageInfo.js | 2 -- lib/result.js | 21 ++++++++++++- lib/step/base.js | 13 ++++---- lib/workers.js | 8 ++++- 11 files changed, 97 insertions(+), 51 deletions(-) diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index 4d597f7b0..ac956d291 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -51,15 +51,6 @@ module.exports = async function (workerCount, selectedRuns, options) { }) workers.on(event.all.result, result => { - event.dispatcher.emit(event.workers.result, { - ...result?.simplify(), - suites: suiteArr, - tests: { - passed: passedTestArr, - failed: failedTestArr, - skipped: skippedTestArr, - }, - }) workers.printResults() }) diff --git a/lib/command/workers/runTests.js b/lib/command/workers/runTests.js index 09900f075..3df4b1d9a 100644 --- a/lib/command/workers/runTests.js +++ b/lib/command/workers/runTests.js @@ -98,8 +98,8 @@ function initializeListeners() { event.dispatcher.on(event.step.passed, step => sendToParentThread({ event: event.step.passed, workerIndex, data: step.simplify() })) event.dispatcher.on(event.step.failed, step => sendToParentThread({ event: event.step.failed, workerIndex, data: step.simplify() })) - event.dispatcher.on(event.hook.failed, (test, err) => sendToParentThread({ event: event.hook.failed, workerIndex, data: { ...test.simplify(), err } })) - event.dispatcher.on(event.hook.passed, (test, err) => sendToParentThread({ event: event.hook.passed, workerIndex, data: { ...test.simplify(), err } })) + event.dispatcher.on(event.hook.failed, (hook, err) => sendToParentThread({ event: event.hook.failed, workerIndex, data: { ...hook.simplify(), err } })) + event.dispatcher.on(event.hook.passed, (hook, err) => sendToParentThread({ event: event.hook.passed, workerIndex, data: { ...hook.simplify(), err } })) event.dispatcher.once(event.all.after, () => { sendToParentThread({ event: event.all.after, workerIndex, data: container.result().simplify() }) diff --git a/lib/listener/result.js b/lib/listener/result.js index fe9762017..7df7d771a 100644 --- a/lib/listener/result.js +++ b/lib/listener/result.js @@ -7,7 +7,6 @@ module.exports = function () { }) event.dispatcher.on(event.test.started, test => { - container.result().addStats({ tests: 1 }) container.result().addTest(test) }) } diff --git a/lib/mocha/hooks.js b/lib/mocha/hooks.js index 3c9aa7d4d..8ac00a7fb 100644 --- a/lib/mocha/hooks.js +++ b/lib/mocha/hooks.js @@ -1,4 +1,6 @@ const event = require('../event') +const { serializeError } = require('../utils') +// const { serializeTest } = require('./test') class Hook { constructor(context, error) { @@ -13,6 +15,16 @@ class Hook { return this.constructor.name.replace('Hook', '') } + simplify() { + return { + hookName: this.hookName, + title: this.title, + // test: this.test ? serializeTest(this.test) : null, + // suite: this.suite ? serializeSuite(this.suite) : null, + error: this.error ? serializeError(this.error) : null, + } + } + toString() { return this.hookName } diff --git a/lib/mocha/test.js b/lib/mocha/test.js index bc648c281..c0c3d6f48 100644 --- a/lib/mocha/test.js +++ b/lib/mocha/test.js @@ -1,9 +1,9 @@ const Test = require('mocha/lib/test') const Suite = require('mocha/lib/suite') const { test: testWrapper } = require('./asyncWrapper') -const { enhanceMochaSuite } = require('./suite') +const { enhanceMochaSuite, createSuite } = require('./suite') const { genTestId, serializeError, clearString } = require('../utils') - +const Step = require('../step/base') /** * Factory function to create enhanced tests * @param {string} title - Test title @@ -67,15 +67,18 @@ function enhanceMochaTest(test) { } function deserializeTest(test) { - test = Object.assign(new Test(test.title || '', () => {}), test) + test = Object.assign( + createTest(test.title || '', () => {}), + test, + ) test.parent = Object.assign(new Suite(test.parent.title), test.parent) - enhanceMochaTest(test) enhanceMochaSuite(test.parent) + test.steps = test.steps.map(step => Object.assign(new Step(step.title), step)) return test } function serializeTest(test, err = null) { - test = { ...test } + // test = { ...test } if (test.start && !test.duration) { const end = +new Date() @@ -107,14 +110,14 @@ function serializeTest(test, err = null) { uid: test.uid, retries: test._retries, title: test.title, - status: test.status, + state: test.state, notes: test.notes || [], meta: test.meta || {}, - artifacts: test.artifacts || [], + artifacts: test.artifacts || {}, duration: test.duration || 0, err, parent, - steps: [...test.steps].map(step => step.simplify()), + steps: [...test.steps].map(step => (step.simplify ? step.simplify() : step)), } } diff --git a/lib/plugin/analyze.js b/lib/plugin/analyze.js index 710a70e1f..3c3bc6748 100644 --- a/lib/plugin/analyze.js +++ b/lib/plugin/analyze.js @@ -1,5 +1,6 @@ const debug = require('debug')('codeceptjs:analyze') const { isMainThread } = require('node:worker_threads') +const { arrowRight } = require('figures') const container = require('../container') const ai = require('../ai') const colors = require('chalk') @@ -16,10 +17,11 @@ const defaultConfig = { prompts: { clusterize: testsAndErrors => { const serializedFailedTests = testsAndErrors - .map(({ test, error }, index) => { + .map((test, index) => { + if (!test || !test.err) return return ` TEST #${index + 1}: ${serializeTest(test)} - ERROR: ${serializeError(error).slice(0, MAX_DATA_LENGTH / testsAndErrors.length)}` + ERROR: ${serializeError(test.err).slice(0, MAX_DATA_LENGTH / testsAndErrors.length)}` }) .join('\n\n---\n\n') @@ -96,7 +98,7 @@ const defaultConfig = { Pick one of the categories of failures and explain it. - Common causes of failures: + Common causes of failures in order of priority: * Browser connection error / browser crash * Network errors (server error, timeout, etc) @@ -118,9 +120,9 @@ const defaultConfig = { If you have suggestions for the test, write them in SUMMARY section. Inside SUMMARY write exact values, if you have suggestions, explain which information you used to suggest. Be concise, each section should not take more than one sentence. - + Response format: - + CATEGORY STEPS SUMMARY @@ -141,30 +143,38 @@ const defaultConfig = { module.exports = function (config = {}) { config = Object.assign(defaultConfig, config) - let failedTestsAndErrors = [] + event.dispatcher.on(event.all.result, async result => { + if (!isMainThread) return // run only on main thread + if (!ai.isEnabled) { + console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.') + return + } - event.dispatcher.on(event.test.failed, (test, error) => { - if (!ai.isEnabled) return - failedTestsAndErrors.push({ test, error }) + printReport(result) }) - event.dispatcher.on(event.all.result, async () => { + event.dispatcher.on(event.workers.result, async result => { if (!ai.isEnabled) { console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.') return } + printReport(result) + }) + + async function printReport(result) { + const failedTestsAndErrors = result.tests.filter(t => t.state === 'failed' && t.err) + if (!failedTestsAndErrors.length) return - if (!isMainThread) return // run only on main thread - debug(failedTestsAndErrors.map(e => serializeTest(e.test) + '\n' + serializeError(e.error))) + debug(failedTestsAndErrors.map(t => serializeTest(t) + '\n' + serializeError(t.err))) console.log() console.log(colors.bold.white('🪄 AI REPORT:')) try { if (failedTestsAndErrors.length >= config.clusterize) { - const response = await clusterize() + const response = await clusterize(failedTestsAndErrors) console.log(response) return } @@ -172,7 +182,7 @@ module.exports = function (config = {}) { output.plugin('analyze', `Analyzing first ${config.analyze} failed tests...`) const uniqueErrors = failedTestsAndErrors.filter((item, index, array) => { - return array.findIndex(t => serializeError(t.error) === serializeError(item.error)) === index + return array.findIndex(t => serializeError(t.err) === serializeError(item.err)) === index }) for (let i = 0; i < config.analyze; i++) { @@ -185,19 +195,20 @@ module.exports = function (config = {}) { console.log() console.log('--------------------------------') - console.log(colors.bold.white(uniqueErrors[i].test.fullTitle())) + console.log(arrowRight, colors.bold.white(uniqueErrors[i].fullTitle())) + console.log() console.log(response) } } catch (err) { console.error('Error analyzing failed tests', err) } - if (!container.plugins('pageInfo')) { + if (!Object.keys(container.plugins()).includes('pageInfo')) { console.log('To improve analysis, enable pageInfo plugin to get more context for failed tests.') } - }) + } - async function clusterize() { + async function clusterize(failedTestsAndErrors) { const spinner = ora('Clusterizing failures...').start() const prompt = config.prompts.clusterize(failedTestsAndErrors) try { @@ -212,7 +223,7 @@ module.exports = function (config = {}) { async function analyze(failedTestAndError) { const spinner = ora('Analyzing failure...').start() - const prompt = config.prompts.analyze(failedTestAndError.test, failedTestAndError.error) + const prompt = config.prompts.analyze(failedTestAndError, failedTestAndError.err) try { const response = await ai.createCompletion(prompt) spinner.stop() @@ -225,6 +236,12 @@ module.exports = function (config = {}) { } function serializeError(error) { + if (typeof error === 'string') { + return error + } + + if (!error) return + let errorMessage = 'ERROR: ' + error.message if (error.inspect) { diff --git a/lib/plugin/heal.js b/lib/plugin/heal.js index 35644f875..abf788122 100644 --- a/lib/plugin/heal.js +++ b/lib/plugin/heal.js @@ -117,10 +117,10 @@ module.exports = function (config = {}) { } }) - event.dispatcher.on(event.workers.result, ({ tests }) => { + event.dispatcher.on(event.workers.result, result => { const { print } = output - const healedTests = Object.values(tests) + const healedTests = Object.values(result.tests) .flat() .filter(test => test.notes.some(note => note.type === 'heal')) if (!healedTests.length) return diff --git a/lib/plugin/pageInfo.js b/lib/plugin/pageInfo.js index d350f8c68..c04c8efcf 100644 --- a/lib/plugin/pageInfo.js +++ b/lib/plugin/pageInfo.js @@ -125,8 +125,6 @@ function pageStateToMarkdown(pageState) { markdown += '\n\n' } - markdown += `### ${ucfirst(humanizeString(key))}\n\n` - return markdown } diff --git a/lib/result.js b/lib/result.js index a9b6ff447..9fcd5d204 100644 --- a/lib/result.js +++ b/lib/result.js @@ -1,5 +1,6 @@ const fs = require('fs') const path = require('path') +const { serializeTest } = require('./mocha/test') class Result { constructor() { @@ -52,6 +53,12 @@ class Result { } addTest(test) { + const existingTestIndex = this._tests.findIndex(t => !!t.uid && t.uid === test.uid) + if (existingTestIndex >= 0) { + this._tests[existingTestIndex] = test + return + } + this._tests.push(test) } @@ -72,12 +79,24 @@ class Result { return this._endTime ? +this._endTime - +this._startTime : 0 } + get failedTests() { + return this._tests.filter(test => test.state === 'failed') + } + + get passedTests() { + return this._tests.filter(test => test.state === 'passed') + } + + get skippedTests() { + return this._tests.filter(test => test.state === 'skipped' || test.state === 'pending') + } + simplify() { return { hasFailed: this.hasFailed, stats: this.stats, duration: this.duration, - tests: this._tests.map(test => test.simplify()), + tests: this._tests.map(test => serializeTest(test)), failures: this._failures, } } diff --git a/lib/step/base.js b/lib/step/base.js index 85b66626e..08d72e9e5 100644 --- a/lib/step/base.js +++ b/lib/step/base.js @@ -185,12 +185,12 @@ class Step { if (step.args) { for (const arg of step.args) { // check if arg is a JOI object - if (arg && arg.$_root) { - args.push(JSON.stringify(arg).slice(0, 300)) - } else if (arg && typeof arg === 'function') { + if (arg && typeof arg === 'function') { args.push(arg.name) - } else { + } else if (typeof arg == 'string') { args.push(arg) + } else { + args.push(JSON.stringify(arg).slice(0, 300)) } } } @@ -198,9 +198,10 @@ class Step { return { opts: step.opts || {}, title: step.name, - args: JSON.stringify(args), + args: args, status: step.status, - duration: step.duration || 0, + startTime: step.startTime, + endTime: step.endTime, parent, } } diff --git a/lib/workers.js b/lib/workers.js index 74ab182b6..1e96beeee 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -387,8 +387,12 @@ class Workers extends EventEmitter { switch (message.event) { case event.all.result: + // we ensure consistency of result by adding tests in the very end Container.result().addFailures(message.data.failures) Container.result().addStats(message.data.stats) + message.data.tests.forEach(test => { + Container.result().addTest(deserializeTest(test)) + }) break case event.suite.before: this.emit(event.suite.before, deserializeSuite(message.data)) @@ -424,7 +428,7 @@ class Workers extends EventEmitter { this.emit(event.step.passed, message.data) break case event.step.failed: - this.emit(event.step.failed, message.data) + this.emit(event.step.failed, message.data, message.data.error) break } }) @@ -450,6 +454,7 @@ class Workers extends EventEmitter { } this.emit(event.all.result, Container.result()) + event.dispatcher.emit(event.workers.result, Container.result()) this.emit('end') // internal event } @@ -473,6 +478,7 @@ class Workers extends EventEmitter { } output.result(result.stats.passes, result.stats.failures, result.stats.pending, ms(result.duration), result.stats.failedHooks) + process.env.RUNS_WITH_WORKERS = 'false' } }