diff --git a/doc/api/errors.md b/doc/api/errors.md index 45da62b3e0608a..06b2cf8c344ecd 100644 --- a/doc/api/errors.md +++ b/doc/api/errors.md @@ -2548,6 +2548,14 @@ An unspecified or non-specific system error has occurred within the Node.js process. The error object will have an `err.info` object property with additional details. + + +### `ERR_TEST_FAILURE` + +This error represents a failed test. Additional information about the failure +is available via the `cause` property. The `failureType` property specifies +what the test was doing when the failure occurred. + ### `ERR_TLS_CERT_ALTNAME_FORMAT` diff --git a/doc/api/index.md b/doc/api/index.md index cd8e2cd83f6792..c999258769c95b 100644 --- a/doc/api/index.md +++ b/doc/api/index.md @@ -54,6 +54,7 @@ * [Report](report.md) * [Stream](stream.md) * [String decoder](string_decoder.md) +* [Test runner](test.md) * [Timers](timers.md) * [TLS/SSL](tls.md) * [Trace events](tracing.md) diff --git a/doc/api/test.md b/doc/api/test.md new file mode 100644 index 00000000000000..86cd144763329f --- /dev/null +++ b/doc/api/test.md @@ -0,0 +1,316 @@ +# Test runner + + + +> Stability: 1 - Experimental + + + +The `node:test` module facilitates the creation of JavaScript tests that +report results in [TAP][] format. To access it: + +```mjs +import test from 'node:test'; +``` + +```cjs +const test = require('node:test'); +``` + +This module is only available under the `node:` scheme. The following will not +work: + +```mjs +import test from 'test'; +``` + +```cjs +const test = require('test'); +``` + +Tests created via the `test` module consist of a single function that is +processed in one of three ways: + +1. A synchronous function that is considered failing if it throws an exception, + and is considered passing otherwise. +2. A function that returns a `Promise` that is considered failing if the + `Promise` rejects, and is considered passing if the `Promise` resolves. +3. A function that receives a callback function. If the callback receives any + truthy value as its first argument, the test is considered failing. If a + falsy value is passed as the first argument to the callback, the test is + considered passing. If the test function receives a callback function and + also returns a `Promise`, the test will fail. + +The following example illustrates how tests are written using the +`test` module. + +```js +test('synchronous passing test', (t) => { + // This test passes because it does not throw an exception. + assert.strictEqual(1, 1); +}); + +test('synchronous failing test', (t) => { + // This test fails because it throws an exception. + assert.strictEqual(1, 2); +}); + +test('asynchronous passing test', async (t) => { + // This test passes because the Promise returned by the async + // function is not rejected. + assert.strictEqual(1, 1); +}); + +test('asynchronous failing test', async (t) => { + // This test fails because the Promise returned by the async + // function is rejected. + assert.strictEqual(1, 2); +}); + +test('failing test using Promises', (t) => { + // Promises can be used directly as well. + return new Promise((resolve, reject) => { + setImmediate(() => { + reject(new Error('this will cause the test to fail')); + }); + }); +}); + +test('callback passing test', (t, done) => { + // done() is the callback function. When the setImmediate() runs, it invokes + // done() with no arguments. + setImmediate(done); +}); + +test('callback failing test', (t, done) => { + // When the setImmediate() runs, done() is invoked with an Error object and + // the test fails. + setImmediate(() => { + done(new Error('callback failure')); + }); +}); +``` + +As a test file executes, TAP is written to the standard output of the Node.js +process. This output can be interpreted by any test harness that understands +the TAP format. If any tests fail, the process exit code is set to `1`. + +## Subtests + +The test context's `test()` method allows subtests to be created. This method +behaves identically to the top level `test()` function. The following example +demonstrates the creation of a top level test with two subtests. + +```js +test('top level test', async (t) => { + await t.test('subtest 1', (t) => { + assert.strictEqual(1, 1); + }); + + await t.test('subtest 2', (t) => { + assert.strictEqual(2, 2); + }); +}); +``` + +In this example, `await` is used to ensure that both subtests have completed. +This is necessary because parent tests do not wait for their subtests to +complete. Any subtests that are still outstanding when their parent finishes +are cancelled and treated as failures. Any subtest failures cause the parent +test to fail. + +## Skipping tests + +Individual tests can be skipped by passing the `skip` option to the test, or by +calling the test context's `skip()` method. Both of these options support +including a message that is displayed in the TAP output as shown in the +following example. + +```js +// The skip option is used, but no message is provided. +test('skip option', { skip: true }, (t) => { + // This code is never executed. +}); + +// The skip option is used, and a message is provided. +test('skip option with message', { skip: 'this is skipped' }, (t) => { + // This code is never executed. +}); + +test('skip() method', (t) => { + // Make sure to return here as well if the test contains additional logic. + t.skip(); +}); + +test('skip() method with message', (t) => { + // Make sure to return here as well if the test contains additional logic. + t.skip('this is skipped'); +}); +``` + +## Extraneous asynchronous activity + +Once a test function finishes executing, the TAP results are output as quickly +as possible while maintaining the order of the tests. However, it is possible +for the test function to generate asynchronous activity that outlives the test +itself. The test runner handles this type of activity, but does not delay the +reporting of test results in order to accommodate it. + +In the following example, a test completes with two `setImmediate()` +operations still outstanding. The first `setImmediate()` attempts to create a +new subtest. Because the parent test has already finished and output its +results, the new subtest is immediately marked as failed, and reported in the +top level of the file's TAP output. + +The second `setImmediate()` creates an `uncaughtException` event. +`uncaughtException` and `unhandledRejection` events originating from a completed +test are handled by the `test` module and reported as diagnostic warnings in +the top level of the file's TAP output. + +```js +test('a test that creates asynchronous activity', (t) => { + setImmediate(() => { + t.test('subtest that is created too late', (t) => { + throw new Error('error1'); + }); + }); + + setImmediate(() => { + throw new Error('error2'); + }); + + // The test finishes after this line. +}); +``` + +## `test([name][, options][, fn])` + + + +* `name` {string} The name of the test, which is displayed when reporting test + results. **Default:** The `name` property of `fn`, or `''` if `fn` + does not have a name. +* `options` {Object} Configuration options for the test. The following + properties are supported: + * `concurrency` {number} The number of tests that can be run at the same time. + If unspecified, subtests inherit this value from their parent. + **Default:** `1`. + * `skip` {boolean|string} If truthy, the test is skipped. If a string is + provided, that string is displayed in the test results as the reason for + skipping the test. **Default:** `false`. + * `todo` {boolean|string} If truthy, the test marked as `TODO`. If a string + is provided, that string is displayed in the test results as the reason why + the test is `TODO`. **Default:** `false`. +* `fn` {Function|AsyncFunction} The function under test. This first argument + to this function is a [`TestContext`][] object. If the test uses callbacks, + the callback function is passed as the second argument. **Default:** A no-op + function. +* Returns: {Promise} Resolved with `undefined` once the test completes. + +The `test()` function is the value imported from the `test` module. Each +invocation of this function results in the creation of a test point in the TAP +output. + +The `TestContext` object passed to the `fn` argument can be used to perform +actions related to the current test. Examples include skipping the test, adding +additional TAP diagnostic information, or creating subtests. + +`test()` returns a `Promise` that resolves once the test completes. The return +value can usually be discarded for top level tests. However, the return value +from subtests should be used to prevent the parent test from finishing first +and cancelling the subtest as shown in the following example. + +```js +test('top level test', async (t) => { + // The setTimeout() in the following subtest would cause it to outlive its + // parent test if 'await' is removed on the next line. Once the parent test + // completes, it will cancel any outstanding subtests. + await t.test('longer running subtest', async (t) => { + return new Promise((resolve, reject) => { + setTimeout(resolve, 1000); + }); + }); +}); +``` + +## Class: `TestContext` + + + +An instance of `TestContext` is passed to each test function in order to +interact with the test runner. However, the `TestContext` constructor is not +exposed as part of the API. + +### `context.diagnostic(message)` + + + +* `message` {string} Message to be displayed as a TAP diagnostic. + +This function is used to write TAP diagnostics to the output. Any diagnostic +information is included at the end of the test's results. This function does +not return a value. + +### `context.skip([message])` + + + +* `message` {string} Optional skip message to be displayed in TAP output. + +This function causes the test's output to indicate the test as skipped. If +`message` is provided, it is included in the TAP output. Calling `skip()` does +not terminate execution of the test function. This function does not return a +value. + +### `context.todo([message])` + + + +* `message` {string} Optional `TODO` message to be displayed in TAP output. + +This function adds a `TODO` directive to the test's output. If `message` is +provided, it is included in the TAP output. Calling `todo()` does not terminate +execution of the test function. This function does not return a value. + +### `context.test([name][, options][, fn])` + + + +* `name` {string} The name of the subtest, which is displayed when reporting + test results. **Default:** The `name` property of `fn`, or `''` if + `fn` does not have a name. +* `options` {Object} Configuration options for the subtest. The following + properties are supported: + * `concurrency` {number} The number of tests that can be run at the same time. + If unspecified, subtests inherit this value from their parent. + **Default:** `1`. + * `skip` {boolean|string} If truthy, the test is skipped. If a string is + provided, that string is displayed in the test results as the reason for + skipping the test. **Default:** `false`. + * `todo` {boolean|string} If truthy, the test marked as `TODO`. If a string + is provided, that string is displayed in the test results as the reason why + the test is `TODO`. **Default:** `false`. +* `fn` {Function|AsyncFunction} The function under test. This first argument + to this function is a [`TestContext`][] object. If the test uses callbacks, + the callback function is passed as the second argument. **Default:** A no-op + function. +* Returns: {Promise} Resolved with `undefined` once the test completes. + +This function is used to create subtests under the current test. This function +behaves in the same fashion as the top level [`test()`][] function. + +[TAP]: https://testanything.org/ +[`TestContext`]: #class-testcontext +[`test()`]: #testname-options-fn diff --git a/lib/internal/errors.js b/lib/internal/errors.js index 6184ca245127b1..4b3a6eba852cef 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -1552,6 +1552,18 @@ E('ERR_STREAM_WRAP', 'Stream has StringDecoder set or is in objectMode', Error); E('ERR_STREAM_WRITE_AFTER_END', 'write after end', Error); E('ERR_SYNTHETIC', 'JavaScript Callstack', Error); E('ERR_SYSTEM_ERROR', 'A system error occurred', SystemError); +E('ERR_TEST_FAILURE', function(error, failureType) { + hideInternalStackFrames(this); + assert(typeof failureType === 'string', + "The 'failureType' argument must be of type string."); + + const msg = error?.message ?? lazyInternalUtilInspect().inspect(error); + + this.failureType = error?.failureType ?? failureType; + this.cause = error; + + return msg; +}, Error); E('ERR_TLS_CERT_ALTNAME_FORMAT', 'Invalid subject alternative name string', SyntaxError); E('ERR_TLS_CERT_ALTNAME_INVALID', function(reason, host, cert) { diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js new file mode 100644 index 00000000000000..66544d91522495 --- /dev/null +++ b/lib/internal/test_runner/harness.js @@ -0,0 +1,131 @@ +'use strict'; +const { FunctionPrototypeBind, SafeMap } = primordials; +const { + createHook, + executionAsyncId, +} = require('async_hooks'); +const { + codes: { + ERR_TEST_FAILURE, + }, +} = require('internal/errors'); +const { Test } = require('internal/test_runner/test'); + +function createProcessEventHandler(eventName, rootTest, testResources) { + return (err) => { + // Check if this error is coming from a test. If it is, fail the test. + const test = testResources.get(executionAsyncId()); + + if (test !== undefined) { + if (test.finished) { + // If the test is already finished, report this as a top level + // diagnostic since this is a malformed test. + const msg = `Warning: Test "${test.name}" generated asynchronous ` + + 'activity after the test ended. This activity created the error ' + + `"${err}" and would have caused the test to fail, but instead ` + + `triggered an ${eventName} event.`; + + rootTest.diagnostic(msg); + return; + } + + test.fail(new ERR_TEST_FAILURE(err, eventName)); + test.postRun(); + } + }; +} + +function setup(root) { + const testResources = new SafeMap(); + const hook = createHook({ + init(asyncId, type, triggerAsyncId, resource) { + if (resource instanceof Test) { + testResources.set(asyncId, resource); + return; + } + + const parent = testResources.get(triggerAsyncId); + + if (parent !== undefined) { + testResources.set(asyncId, parent); + } + }, + destroy(asyncId) { + testResources.delete(asyncId); + } + }); + + hook.enable(); + + const exceptionHandler = + createProcessEventHandler('uncaughtException', root, testResources); + const rejectionHandler = + createProcessEventHandler('unhandledRejection', root, testResources); + + process.on('uncaughtException', exceptionHandler); + process.on('unhandledRejection', rejectionHandler); + process.on('beforeExit', () => { + root.postRun(); + + let passCount = 0; + let failCount = 0; + let skipCount = 0; + let todoCount = 0; + + for (let i = 0; i < root.subtests.length; i++) { + const test = root.subtests[i]; + + // Check SKIP and TODO tests first, as those should not be counted as + // failures. + if (test.skipped) { + skipCount++; + } else if (test.isTodo) { + todoCount++; + } else if (!test.passed) { + failCount++; + } else { + passCount++; + } + } + + root.reporter.plan(root.indent, root.subtests.length); + + for (let i = 0; i < root.diagnostics.length; i++) { + root.reporter.diagnostic(root.indent, root.diagnostics[i]); + } + + root.reporter.diagnostic(root.indent, `tests ${root.subtests.length}`); + root.reporter.diagnostic(root.indent, `pass ${passCount}`); + root.reporter.diagnostic(root.indent, `fail ${failCount}`); + root.reporter.diagnostic(root.indent, `skipped ${skipCount}`); + root.reporter.diagnostic(root.indent, `todo ${todoCount}`); + root.reporter.diagnostic(root.indent, `duration_ms ${process.uptime()}`); + + root.reporter.push(null); + hook.disable(); + process.removeListener('unhandledRejection', rejectionHandler); + process.removeListener('uncaughtException', exceptionHandler); + + if (failCount > 0) { + process.exitCode = 1; + } + }); + + root.reporter.pipe(process.stdout); + root.reporter.version(); +} + +function test(name, options, fn) { + // If this is the first test encountered, bootstrap the test harness. + if (this.subtests.length === 0) { + setup(this); + } + + const subtest = this.createSubtest(name, options, fn); + + return subtest.start(); +} + +const root = new Test({ name: '' }); + +module.exports = FunctionPrototypeBind(test, root); diff --git a/lib/internal/test_runner/tap_stream.js b/lib/internal/test_runner/tap_stream.js new file mode 100644 index 00000000000000..d5e095991c7009 --- /dev/null +++ b/lib/internal/test_runner/tap_stream.js @@ -0,0 +1,193 @@ +'use strict'; +const { + ArrayPrototypeForEach, + ArrayPrototypeJoin, + ArrayPrototypePush, + ArrayPrototypeShift, + ObjectEntries, + StringPrototypeReplace, + StringPrototypeReplaceAll, + StringPrototypeSplit, +} = primordials; +const Readable = require('internal/streams/readable'); +const { isError } = require('internal/util'); +const { inspect } = require('internal/util/inspect'); +const kFrameStartRegExp = /^ {4}at /; +const kLineBreakRegExp = /\n|\r\n/; +const inspectOptions = { colors: false, breakLength: Infinity }; +let testModule; // Lazy loaded due to circular dependency. + +function lazyLoadTest() { + testModule ??= require('internal/test_runner/test'); + + return testModule; +} + +class TapStream extends Readable { + #buffer; + #canPush; + + constructor() { + super(); + this.#buffer = []; + this.#canPush = true; + } + + _read() { + this.#canPush = true; + + while (this.#buffer.length > 0) { + const line = ArrayPrototypeShift(this.#buffer); + + if (!this.#tryPush(line)) { + return; + } + } + } + + bail(message) { + this.#tryPush(`Bail out!${message ? ` ${tapEscape(message)}` : ''}\n`); + } + + fail(indent, testNumber, description, directive) { + this.#test(indent, testNumber, 'not ok', description, directive); + } + + ok(indent, testNumber, description, directive) { + this.#test(indent, testNumber, 'ok', description, directive); + } + + plan(indent, count, explanation) { + const exp = `${explanation ? ` # ${tapEscape(explanation)}` : ''}`; + + this.#tryPush(`${indent}1..${count}${exp}\n`); + } + + getSkip(reason) { + return `SKIP${reason ? ` ${tapEscape(reason)}` : ''}`; + } + + getTodo(reason) { + return `TODO${reason ? ` ${tapEscape(reason)}` : ''}`; + } + + details(indent, duration, error) { + let details = `${indent} ---\n`; + + details += `${indent} duration_ms: ${duration}\n`; + + if (error !== null && typeof error === 'object') { + const entries = ObjectEntries(error); + const isErrorObj = isError(error); + + for (let i = 0; i < entries.length; i++) { + const { 0: key, 1: value } = entries[i]; + + if (isError && (key === 'cause' || key === 'code')) { + continue; + } + + details += `${indent} ${key}: ${inspect(value, inspectOptions)}\n`; + } + + if (isErrorObj) { + const { kTestCodeFailure } = lazyLoadTest(); + const { + cause, + code, + failureType, + message, + stack, + } = error; + let errMsg = message ?? ''; + let errStack = stack; + let errCode = code; + + // If the ERR_TEST_FAILURE came from an error provided by user code, + // then try to unwrap the original error message and stack. + if (code === 'ERR_TEST_FAILURE' && failureType === kTestCodeFailure) { + errMsg = cause?.message ?? errMsg; + errStack = cause?.stack ?? errStack; + errCode = cause?.code ?? errCode; + } + + details += `${indent} error: ${inspect(errMsg, inspectOptions)}\n`; + + if (errCode) { + details += `${indent} code: ${errCode}\n`; + } + + if (typeof errStack === 'string') { + const frames = []; + + ArrayPrototypeForEach( + StringPrototypeSplit(errStack, kLineBreakRegExp), + (frame) => { + const processed = StringPrototypeReplace( + frame, kFrameStartRegExp, '' + ); + + if (processed.length > 0 && processed.length !== frame.length) { + ArrayPrototypePush(frames, processed); + } + } + ); + + if (frames.length > 0) { + const frameDelimiter = `\n${indent} `; + + details += `${indent} stack: |-${frameDelimiter}`; + details += `${ArrayPrototypeJoin(frames, `${frameDelimiter}`)}\n`; + } + } + } + } else if (error !== null && error !== undefined) { + details += `${indent} error: ${inspect(error, inspectOptions)}\n`; + } + + details += `${indent} ...\n`; + this.#tryPush(details); + } + + diagnostic(indent, message) { + this.#tryPush(`${indent}# ${tapEscape(message)}\n`); + } + + version() { + this.#tryPush('TAP version 13\n'); + } + + #test(indent, testNumber, status, description, directive) { + let line = `${indent}${status} ${testNumber}`; + + if (description) { + line += ` ${tapEscape(description)}`; + } + + if (directive) { + line += ` # ${directive}`; + } + + line += '\n'; + this.#tryPush(line); + } + + #tryPush(message) { + if (this.#canPush) { + this.#canPush = this.push(message); + } else { + ArrayPrototypePush(this.#buffer, message); + } + + return this.#canPush; + } +} + +// In certain places, # and \ need to be escaped as \# and \\. +function tapEscape(input) { + return StringPrototypeReplaceAll( + StringPrototypeReplaceAll(input, '\\', '\\\\'), '#', '\\#' + ); +} + +module.exports = { TapStream }; diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js new file mode 100644 index 00000000000000..e2dddf5cc20675 --- /dev/null +++ b/lib/internal/test_runner/test.js @@ -0,0 +1,426 @@ +'use strict'; +const { + ArrayPrototypePush, + ArrayPrototypeShift, + FunctionPrototype, + Number, + ObjectCreate, + SafeMap, +} = primordials; +const { AsyncResource } = require('async_hooks'); +const { + codes: { + ERR_TEST_FAILURE, + }, +} = require('internal/errors'); +const { TapStream } = require('internal/test_runner/tap_stream'); +const { createDeferredPromise } = require('internal/util'); +const { isPromise } = require('internal/util/types'); +const { isUint32 } = require('internal/validators'); +const { bigint: hrtime } = process.hrtime; +const kCallbackAndPromisePresent = 'callbackAndPromisePresent'; +const kCancelledByParent = 'cancelledByParent'; +const kMultipleCallbackInvocations = 'multipleCallbackInvocations'; +const kParentAlreadyFinished = 'parentAlreadyFinished'; +const kSubtestsFailed = 'subtestsFailed'; +const kTestCodeFailure = 'testCodeFailure'; +const kDefaultIndent = ' '; +const noop = FunctionPrototype; + +class TestContext { + #test; + + constructor(test) { + this.#test = test; + } + + diagnostic(message) { + this.#test.diagnostic(message); + } + + skip(message) { + this.#test.skip(message); + } + + todo(message) { + this.#test.todo(message); + } + + test(name, options, fn) { + const subtest = this.#test.createSubtest(name, options, fn); + + return subtest.start(); + } +} + +class Test extends AsyncResource { + constructor(options) { + super('Test'); + + let { fn, name, parent } = options; + const { concurrency, skip, todo } = options; + + if (typeof fn !== 'function') { + fn = noop; + } + + if (typeof name !== 'string' || name === '') { + name = fn.name || ''; + } + + if (!(parent instanceof Test)) { + parent = null; + } + + if (skip) { + fn = noop; + } + + this.fn = fn; + this.name = name; + this.parent = parent; + + if (parent === null) { + this.concurrency = 1; + this.indent = ''; + this.indentString = kDefaultIndent; + this.reporter = new TapStream(); + this.testNumber = 0; + } else { + const indent = parent.parent === null ? parent.indent : + parent.indent + parent.indentString; + + this.concurrency = parent.concurrency; + this.indent = indent; + this.indentString = parent.indentString; + this.reporter = parent.reporter; + this.testNumber = parent.subtests.length + 1; + } + + if (isUint32(concurrency) && concurrency !== 0) { + this.concurrency = concurrency; + } + + this.cancelled = false; + this.skipped = !!skip; + this.isTodo = !!todo; + this.startTime = null; + this.endTime = null; + this.passed = false; + this.error = null; + this.diagnostics = []; + this.message = typeof skip === 'string' ? skip : + typeof todo === 'string' ? todo : null; + this.activeSubtests = 0; + this.pendingSubtests = []; + this.readySubtests = new SafeMap(); + this.subtests = []; + this.waitingOn = 0; + this.finished = false; + } + + hasConcurrency() { + return this.concurrency > this.activeSubtests; + } + + addPendingSubtest(deferred) { + this.pendingSubtests.push(deferred); + } + + async processPendingSubtests() { + while (this.pendingSubtests.length > 0 && this.hasConcurrency()) { + const deferred = ArrayPrototypeShift(this.pendingSubtests); + await deferred.test.run(); + deferred.resolve(); + } + } + + addReadySubtest(subtest) { + this.readySubtests.set(subtest.testNumber, subtest); + } + + processReadySubtestRange(canSend) { + const start = this.waitingOn; + const end = start + this.readySubtests.size; + + for (let i = start; i < end; i++) { + const subtest = this.readySubtests.get(i); + + // Check if the specified subtest is in the map. If it is not, return + // early to avoid trying to process any more tests since they would be + // out of order. + if (subtest === undefined) { + return; + } + + // Call isClearToSend() in the loop so that it is: + // - Only called if there are results to report in the correct order. + // - Guaranteed to only be called a maximum of once per call to + // processReadySubtestRange(). + canSend = canSend || this.isClearToSend(); + + if (!canSend) { + return; + } + + // Report the subtest's results and remove it from the ready map. + subtest.finalize(); + this.readySubtests.delete(i); + } + } + + createSubtest(name, options, fn) { + if (typeof name === 'function') { + fn = name; + } else if (name !== null && typeof name === 'object') { + fn = options; + options = name; + } else if (typeof options === 'function') { + fn = options; + } + + if (options === null || typeof options !== 'object') { + options = ObjectCreate(null); + } + + let parent = this; + + // If this test has already ended, attach this test to the root test so + // that the error can be properly reported. + if (this.finished) { + while (parent.parent !== null) { + parent = parent.parent; + } + } + + const test = new Test({ fn, name, parent, ...options }); + + if (parent.waitingOn === 0) { + parent.waitingOn = test.testNumber; + } + + if (this.finished) { + test.fail( + new ERR_TEST_FAILURE( + 'test could not be started because its parent finished', + kParentAlreadyFinished + ) + ); + } + + ArrayPrototypePush(parent.subtests, test); + return test; + } + + cancel() { + if (this.endTime !== null) { + return; + } + + this.fail( + new ERR_TEST_FAILURE( + 'test did not finish before its parent and was cancelled', + kCancelledByParent + ) + ); + this.cancelled = true; + } + + fail(err) { + if (this.error !== null) { + return; + } + + this.endTime = hrtime(); + this.passed = false; + this.error = err; + } + + pass() { + if (this.endTime !== null) { + return; + } + + this.endTime = hrtime(); + this.passed = true; + } + + skip(message) { + this.skipped = true; + this.message = message; + } + + todo(message) { + this.isTodo = true; + this.message = message; + } + + diagnostic(message) { + ArrayPrototypePush(this.diagnostics, message); + } + + start() { + // If there is enough available concurrency to run the test now, then do + // it. Otherwise, return a Promise to the caller and mark the test as + // pending for later execution. + if (!this.parent.hasConcurrency()) { + const deferred = createDeferredPromise(); + + deferred.test = this; + this.parent.addPendingSubtest(deferred); + return deferred.promise; + } + + return this.run(); + } + + async run() { + this.parent.activeSubtests++; + this.startTime = hrtime(); + + try { + const ctx = new TestContext(this); + + if (this.fn.length === 2) { + // This test is using legacy Node.js error first callbacks. + const { promise, resolve, reject } = createDeferredPromise(); + let calledCount = 0; + const ret = this.runInAsyncScope(this.fn, ctx, ctx, (err) => { + calledCount++; + + // If the callback is called a second time, let the user know, but + // don't let them know more than once. + if (calledCount === 2) { + throw new ERR_TEST_FAILURE( + 'callback invoked multiple times', + kMultipleCallbackInvocations + ); + } else if (calledCount > 2) { + return; + } + + if (err) { + return reject(err); + } + + resolve(); + }); + + if (isPromise(ret)) { + this.fail(new ERR_TEST_FAILURE( + 'passed a callback but also returned a Promise', + kCallbackAndPromisePresent + )); + await ret; + } else { + await promise; + } + } else { + // This test is synchronous or using Promises. + await this.runInAsyncScope(this.fn, ctx, ctx); + } + + this.pass(); + } catch (err) { + this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure)); + } + + // Clean up the test. Then, try to report the results and execute any + // tests that were pending due to available concurrency. + this.postRun(); + } + + postRun() { + let failedSubtests = 0; + + // If the test was failed before it even started, then the end time will + // be earlier than the start time. Correct that here. + if (this.endTime < this.startTime) { + this.endTime = hrtime(); + } + + // The test has run, so recursively cancel any outstanding subtests and + // mark this test as failed if any subtests failed. + for (let i = 0; i < this.subtests.length; i++) { + const subtest = this.subtests[i]; + + if (!subtest.finished) { + subtest.cancel(); + subtest.postRun(); + } + + if (!subtest.passed) { + failedSubtests++; + } + } + + if (this.passed && failedSubtests > 0) { + const subtestString = `subtest${failedSubtests > 1 ? 's' : ''}`; + const msg = `${failedSubtests} ${subtestString} failed`; + + this.fail(new ERR_TEST_FAILURE(msg, kSubtestsFailed)); + } + + if (this.parent !== null) { + this.parent.activeSubtests--; + this.parent.addReadySubtest(this); + this.parent.processReadySubtestRange(false); + this.parent.processPendingSubtests(); + } + } + + isClearToSend() { + return this.parent === null || + ( + this.parent.waitingOn === this.testNumber && this.parent.isClearToSend() + ); + } + + finalize() { + // By the time this function is called, the following can be relied on: + // - The current test has completed or been cancelled. + // - All of this test's subtests have completed or been cancelled. + // - It is the current test's turn to report its results. + + // Report any subtests that have not been reported yet. Since all of the + // subtests have finished, it's safe to pass true to + // processReadySubtestRange(), which will finalize all remaining subtests. + this.processReadySubtestRange(true); + + // Output this test's results and update the parent's waiting counter. + if (this.subtests.length > 0) { + this.reporter.plan(this.subtests[0].indent, this.subtests.length); + } + + this.report(); + this.parent.waitingOn++; + this.finished = true; + } + + report() { + // Duration is recorded in BigInt nanoseconds. Convert to seconds. + const duration = Number(this.endTime - this.startTime) / 1_000_000_000; + const message = `- ${this.name}`; + let directive; + + if (this.skipped) { + directive = this.reporter.getSkip(this.message); + } else if (this.isTodo) { + directive = this.reporter.getTodo(this.message); + } + + if (this.passed) { + this.reporter.ok(this.indent, this.testNumber, message, directive); + } else { + this.reporter.fail(this.indent, this.testNumber, message, directive); + } + + this.reporter.details(this.indent, duration, this.error); + + for (let i = 0; i < this.diagnostics.length; i++) { + this.reporter.diagnostic(this.indent, this.diagnostics[i]); + } + } +} + +module.exports = { kDefaultIndent, kTestCodeFailure, Test }; diff --git a/lib/test.js b/lib/test.js new file mode 100644 index 00000000000000..fa319fa17b37bd --- /dev/null +++ b/lib/test.js @@ -0,0 +1,8 @@ +'use strict'; +const test = require('internal/test_runner/harness'); +const { emitExperimentalWarning } = require('internal/util'); + +emitExperimentalWarning('The test runner'); + +module.exports = test; +module.exports.test = test; diff --git a/test/message/test_runner_no_refs.js b/test/message/test_runner_no_refs.js new file mode 100644 index 00000000000000..8f2815f067af15 --- /dev/null +++ b/test/message/test_runner_no_refs.js @@ -0,0 +1,13 @@ +// Flags: --no-warnings +'use strict'; +require('../common'); +const test = require('node:test'); + +// When run alone, the test below does not keep the event loop alive. +test('does not keep event loop alive', async (t) => { + await t.test('+does not keep event loop alive', async (t) => { + return new Promise((resolve) => { + setTimeout(resolve, 1000).unref(); + }); + }); +}); diff --git a/test/message/test_runner_no_refs.out b/test/message/test_runner_no_refs.out new file mode 100644 index 00000000000000..0379ff8ca7496e --- /dev/null +++ b/test/message/test_runner_no_refs.out @@ -0,0 +1,27 @@ +TAP version 13 + not ok 1 - +does not keep event loop alive + --- + duration_ms: * + failureType: 'cancelledByParent' + error: "'test did not finish before its parent and was cancelled'" + code: ERR_TEST_FAILURE + stack: |- + * + ... + 1..1 +not ok 1 - does not keep event loop alive + --- + duration_ms: * + failureType: 'cancelledByParent' + error: "'test did not finish before its parent and was cancelled'" + code: ERR_TEST_FAILURE + stack: |- + * + ... +1..1 +# tests 1 +# pass 0 +# fail 1 +# skipped 0 +# todo 0 +# duration_ms * diff --git a/test/message/test_runner_no_tests.js b/test/message/test_runner_no_tests.js new file mode 100644 index 00000000000000..c4d226c0bd27d6 --- /dev/null +++ b/test/message/test_runner_no_tests.js @@ -0,0 +1,7 @@ +// Flags: --no-warnings +'use strict'; +require('../common'); +const test = require('node:test'); + +// No TAP output should be generated. +console.log(test.name); diff --git a/test/message/test_runner_no_tests.out b/test/message/test_runner_no_tests.out new file mode 100644 index 00000000000000..9f84e58dc125f8 --- /dev/null +++ b/test/message/test_runner_no_tests.out @@ -0,0 +1 @@ +bound test diff --git a/test/message/test_runner_output.js b/test/message/test_runner_output.js new file mode 100644 index 00000000000000..1f3e61fb2eb7f5 --- /dev/null +++ b/test/message/test_runner_output.js @@ -0,0 +1,288 @@ +// Flags: --no-warnings +'use strict'; +require('../common'); +const assert = require('node:assert'); +const test = require('node:test'); + +test('sync pass todo', (t) => { + t.todo(); +}); + +test('sync pass todo with message', (t) => { + t.todo('this is a passing todo'); +}); + +test('sync fail todo', (t) => { + t.todo(); + throw new Error('thrown from sync fail todo'); +}); + +test('sync fail todo with message', (t) => { + t.todo('this is a failing todo'); + throw new Error('thrown from sync fail todo with message'); +}); + +test('sync skip pass', (t) => { + t.skip(); +}); + +test('sync skip pass with message', (t) => { + t.skip('this is skipped'); +}); + +test('sync pass', (t) => { + t.diagnostic('this test should pass'); +}); + +test('sync throw fail', () => { + throw new Error('thrown from sync throw fail'); +}); + +test('async skip pass', async (t) => { + t.skip(); +}); + +test('async pass', async () => { + +}); + +test('async throw fail', async () => { + throw new Error('thrown from async throw fail'); +}); + +test('async skip fail', async (t) => { + t.skip(); + throw new Error('thrown from async throw fail'); +}); + +test('async assertion fail', async () => { + // Make sure the assert module is handled. + assert.strictEqual(true, false); +}); + +test('resolve pass', () => { + return Promise.resolve(); +}); + +test('reject fail', () => { + return Promise.reject(new Error('rejected from reject fail')); +}); + +test('unhandled rejection - passes but warns', () => { + Promise.reject(new Error('rejected from unhandled rejection fail')); +}); + +test('async unhandled rejection - passes but warns', async () => { + Promise.reject(new Error('rejected from async unhandled rejection fail')); +}); + +test('immediate throw - passes but warns', () => { + setImmediate(() => { + throw new Error('thrown from immediate throw fail'); + }); +}); + +test('immediate reject - passes but warns', () => { + setImmediate(() => { + Promise.reject(new Error('rejected from immediate reject fail')); + }); +}); + +test('immediate resolve pass', () => { + return new Promise((resolve) => { + setImmediate(() => { + resolve(); + }); + }); +}); + +test('subtest sync throw fail', async (t) => { + await t.test('+sync throw fail', (t) => { + t.diagnostic('this subtest should make its parent test fail'); + throw new Error('thrown from subtest sync throw fail'); + }); +}); + +test('sync throw non-error fail', async (t) => { + throw Symbol('thrown symbol from sync throw non-error fail'); +}); + +test('level 0a', { concurrency: 4 }, async (t) => { + t.test('level 1a', async (t) => { + const p1a = new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, 1000); + }); + + return p1a; + }); + + t.test('level 1b', async (t) => { + const p1b = new Promise((resolve) => { + resolve(); + }); + + return p1b; + }); + + t.test('level 1c', async (t) => { + const p1c = new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, 2000); + }); + + return p1c; + }); + + t.test('level 1d', async (t) => { + const p1c = new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, 1500); + }); + + return p1c; + }); + + const p0a = new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, 3000); + }); + + return p0a; +}); + +test('top level', { concurrency: 2 }, async (t) => { + t.test('+long running', async (t) => { + return new Promise((resolve, reject) => { + setTimeout(resolve, 3000).unref(); + }); + }); + + t.test('+short running', async (t) => { + t.test('++short running', async (t) => {}); + }); +}); + +test('invalid subtest - pass but subtest fails', (t) => { + setImmediate(() => { + t.test('invalid subtest fail', () => { + throw new Error('this should not be thrown'); + }); + }); +}); + +test('sync skip option', { skip: true }, (t) => { + throw new Error('this should not be executed'); +}); + +test('sync skip option with message', { skip: 'this is skipped' }, (t) => { + throw new Error('this should not be executed'); +}); + +test('sync skip option is false fail', { skip: false }, (t) => { + throw new Error('this should be executed'); +}); + +// A test with no arguments provided. +test(); + +// A test with only a named function provided. +test(function functionOnly() {}); + +// A test with only an anonymous function provided. +test(() => {}); + +// A test with only a name provided. +test('test with only a name provided'); + +// A test with an empty string name. +test(''); + +// A test with only options provided. +test({ skip: true }); + +// A test with only a name and options provided. +test('test with a name and options provided', { skip: true }); + +// A test with only options and a function provided. +test({ skip: true }, function functionAndOptions() {}); + +// A test whose description needs to be escaped. +test('escaped description \\ # \\#\\'); + +// A test whose skip message needs to be escaped. +test('escaped skip message', { skip: '#skip' }); + +// A test whose todo message needs to be escaped. +test('escaped todo message', { todo: '#todo' }); + +// A test with a diagnostic message that needs to be escaped. +test('escaped diagnostic', (t) => { + t.diagnostic('#diagnostic'); +}); + +test('callback pass', (t, done) => { + setImmediate(done); +}); + +test('callback fail', (t, done) => { + setImmediate(() => { + done(new Error('callback failure')); + }); +}); + +test('sync t is this in test', function(t) { + assert.strictEqual(this, t); +}); + +test('async t is this in test', async function(t) { + assert.strictEqual(this, t); +}); + +test('callback t is this in test', function(t, done) { + assert.strictEqual(this, t); + done(); +}); + +test('callback also returns a Promise', async (t, done) => { + throw new Error('thrown from callback also returns a Promise'); +}); + +test('callback throw', (t, done) => { + throw new Error('thrown from callback throw'); +}); + +test('callback called twice', (t, done) => { + done(); + done(); +}); + +test('callback called twice in different ticks', (t, done) => { + setImmediate(done); + done(); +}); + +test('callback called twice in future tick', (t, done) => { + setImmediate(() => { + done(); + done(); + }); +}); + +test('callback async throw', (t, done) => { + setImmediate(() => { + throw new Error('thrown from callback async throw'); + }); +}); + +test('callback async throw after done', (t, done) => { + setImmediate(() => { + throw new Error('thrown from callback async throw after done'); + }); + + done(); +}); diff --git a/test/message/test_runner_output.out b/test/message/test_runner_output.out new file mode 100644 index 00000000000000..ffcecba65f7d00 --- /dev/null +++ b/test/message/test_runner_output.out @@ -0,0 +1,426 @@ +TAP version 13 +ok 1 - sync pass todo # TODO + --- + duration_ms: * + ... +ok 2 - sync pass todo with message # TODO this is a passing todo + --- + duration_ms: * + ... +not ok 3 - sync fail todo # TODO + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from sync fail todo' + code: ERR_TEST_FAILURE + stack: |- + * + * + * + * + * + * + * + * + ... +not ok 4 - sync fail todo with message # TODO this is a failing todo + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from sync fail todo with message' + code: ERR_TEST_FAILURE + stack: |- + * + * + * + * + * + * + * + * + * + * + ... +ok 5 - sync skip pass # SKIP + --- + duration_ms: * + ... +ok 6 - sync skip pass with message # SKIP this is skipped + --- + duration_ms: * + ... +ok 7 - sync pass + --- + duration_ms: * + ... +# this test should pass +not ok 8 - sync throw fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from sync throw fail' + code: ERR_TEST_FAILURE + stack: |- + * + * + * + * + * + * + * + * + ... +ok 9 - async skip pass # SKIP + --- + duration_ms: * + ... +ok 10 - async pass + --- + duration_ms: * + ... +not ok 11 - async throw fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from async throw fail' + code: ERR_TEST_FAILURE + stack: |- + * + * + * + * + * + * + * + * + ... +not ok 12 - async skip fail # SKIP + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from async throw fail' + code: ERR_TEST_FAILURE + stack: |- + * + * + * + * + * + * + * + * + ... +not ok 13 - async assertion fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'Expected values to be strictly equal:\n\ntrue !== false\n' + code: ERR_ASSERTION + stack: |- + * + * + * + * + * + * + * + * + ... +ok 14 - resolve pass + --- + duration_ms: * + ... +not ok 15 - reject fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'rejected from reject fail' + code: ERR_TEST_FAILURE + stack: |- + * + * + * + * + * + * + * + * + ... +ok 16 - unhandled rejection - passes but warns + --- + duration_ms: * + ... +ok 17 - async unhandled rejection - passes but warns + --- + duration_ms: * + ... +ok 18 - immediate throw - passes but warns + --- + duration_ms: * + ... +ok 19 - immediate reject - passes but warns + --- + duration_ms: * + ... +ok 20 - immediate resolve pass + --- + duration_ms: * + ... + not ok 1 - +sync throw fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from subtest sync throw fail' + code: ERR_TEST_FAILURE + stack: |- + * + * + * + * + * + * + * + * + * + * + ... + # this subtest should make its parent test fail + 1..1 +not ok 21 - subtest sync throw fail + --- + duration_ms: * + failureType: 'subtestsFailed' + error: "'1 subtest failed'" + code: ERR_TEST_FAILURE + ... +not ok 22 - sync throw non-error fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'Symbol(thrown symbol from sync throw non-error fail)' + code: ERR_TEST_FAILURE + ... + ok 1 - level 1a + --- + duration_ms: * + ... + ok 2 - level 1b + --- + duration_ms: * + ... + ok 3 - level 1c + --- + duration_ms: * + ... + ok 4 - level 1d + --- + duration_ms: * + ... + 1..4 +ok 23 - level 0a + --- + duration_ms: * + ... + not ok 1 - +long running + --- + duration_ms: * + failureType: 'cancelledByParent' + error: "'test did not finish before its parent and was cancelled'" + code: ERR_TEST_FAILURE + ... + ok 1 - ++short running + --- + duration_ms: * + ... + 1..1 + ok 2 - +short running + --- + duration_ms: * + ... + 1..2 +not ok 24 - top level + --- + duration_ms: * + failureType: 'subtestsFailed' + error: "'1 subtest failed'" + code: ERR_TEST_FAILURE + ... +ok 25 - invalid subtest - pass but subtest fails + --- + duration_ms: * + ... +ok 26 - sync skip option # SKIP + --- + duration_ms: * + ... +ok 27 - sync skip option with message # SKIP this is skipped + --- + duration_ms: * + ... +not ok 28 - sync skip option is false fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'this should be executed' + code: ERR_TEST_FAILURE + stack: |- + * + * + * + * + * + * + * + ... +ok 29 - + --- + duration_ms: * + ... +ok 30 - functionOnly + --- + duration_ms: * + ... +ok 31 - + --- + duration_ms: * + ... +ok 32 - test with only a name provided + --- + duration_ms: * + ... +ok 33 - + --- + duration_ms: * + ... +ok 34 - # SKIP + --- + duration_ms: * + ... +ok 35 - test with a name and options provided # SKIP + --- + duration_ms: * + ... +ok 36 - functionAndOptions # SKIP + --- + duration_ms: * + ... +ok 37 - escaped description \\ \# \\\#\\ + --- + duration_ms: * + ... +ok 38 - escaped skip message # SKIP \#skip + --- + duration_ms: * + ... +ok 39 - escaped todo message # TODO \#todo + --- + duration_ms: * + ... +ok 40 - escaped diagnostic + --- + duration_ms: * + ... +# \#diagnostic +ok 41 - callback pass + --- + duration_ms: * + ... +not ok 42 - callback fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'callback failure' + code: ERR_TEST_FAILURE + stack: |- + * + * + ... +ok 43 - sync t is this in test + --- + duration_ms: * + ... +ok 44 - async t is this in test + --- + duration_ms: * + ... +ok 45 - callback t is this in test + --- + duration_ms: * + ... +not ok 46 - callback also returns a Promise + --- + duration_ms: * + failureType: 'callbackAndPromisePresent' + error: "'passed a callback but also returned a Promise'" + code: ERR_TEST_FAILURE + ... +not ok 47 - callback throw + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from callback throw' + code: ERR_TEST_FAILURE + stack: |- + * + * + * + * + * + * + * + ... +not ok 48 - callback called twice + --- + duration_ms: * + failureType: 'multipleCallbackInvocations' + error: "'callback invoked multiple times'" + code: ERR_TEST_FAILURE + ... +ok 49 - callback called twice in different ticks + --- + duration_ms: * + ... +not ok 50 - callback called twice in future tick + --- + duration_ms: * + failureType: 'multipleCallbackInvocations' + error: "'callback invoked multiple times'" + code: ERR_TEST_FAILURE + stack: |- + * + ... +not ok 51 - callback async throw + --- + duration_ms: * + failureType: 'uncaughtException' + error: 'thrown from callback async throw' + code: ERR_TEST_FAILURE + stack: |- + * + ... +ok 52 - callback async throw after done + --- + duration_ms: * + ... +not ok 53 - invalid subtest fail + --- + duration_ms: * + failureType: 'parentAlreadyFinished' + error: "'test could not be started because its parent finished'" + code: ERR_TEST_FAILURE + stack: |- + * + ... +1..53 +# Warning: Test "unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. +# Warning: Test "async unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from async unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. +# Warning: Test "immediate throw - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from immediate throw fail" and would have caused the test to fail, but instead triggered an uncaughtException event. +# Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. +# Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: 'callback invoked multiple times'" and would have caused the test to fail, but instead triggered an uncaughtException event. +# Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. +# tests 53 +# pass 23 +# fail 15 +# skipped 10 +# todo 5 +# duration_ms * diff --git a/test/message/test_runner_unresolved_promise.js b/test/message/test_runner_unresolved_promise.js new file mode 100644 index 00000000000000..00d231be34b6df --- /dev/null +++ b/test/message/test_runner_unresolved_promise.js @@ -0,0 +1,8 @@ +// Flags: --no-warnings +'use strict'; +require('../common'); +const test = require('node:test'); + +test('pass'); +test('never resolving promise', () => new Promise(() => {})); +test('fail'); diff --git a/test/message/test_runner_unresolved_promise.out b/test/message/test_runner_unresolved_promise.out new file mode 100644 index 00000000000000..263b2411c85565 --- /dev/null +++ b/test/message/test_runner_unresolved_promise.out @@ -0,0 +1,30 @@ +TAP version 13 +ok 1 - pass + --- + duration_ms: * + ... +not ok 2 - never resolving promise + --- + duration_ms: * + failureType: 'cancelledByParent' + error: "'test did not finish before its parent and was cancelled'" + code: ERR_TEST_FAILURE + stack: |- + * + ... +not ok 3 - fail + --- + duration_ms: * + failureType: 'cancelledByParent' + error: "'test did not finish before its parent and was cancelled'" + code: ERR_TEST_FAILURE + stack: |- + * + ... +1..3 +# tests 3 +# pass 1 +# fail 2 +# skipped 0 +# todo 0 +# duration_ms * diff --git a/test/parallel/test-runner-exit-code.js b/test/parallel/test-runner-exit-code.js new file mode 100644 index 00000000000000..0e72f77783e9a9 --- /dev/null +++ b/test/parallel/test-runner-exit-code.js @@ -0,0 +1,27 @@ +'use strict'; +require('../common'); +const assert = require('assert'); +const { spawnSync } = require('child_process'); + +if (process.argv[2] === 'child') { + const test = require('node:test'); + + if (process.argv[3] === 'pass') { + test('passing test', () => { + assert.strictEqual(true, true); + }); + } else { + assert.strictEqual(process.argv[3], 'fail'); + test('failing test', () => { + assert.strictEqual(true, false); + }); + } +} else { + let child = spawnSync(process.execPath, [__filename, 'child', 'pass']); + assert.strictEqual(child.status, 0); + assert.strictEqual(child.signal, null); + + child = spawnSync(process.execPath, [__filename, 'child', 'fail']); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); +}