test: skip some timing-sensitive tests on Windows CI to avoid flaky t…

…ests (#3650) Experience shows that Windows CI builds flake out frequently in timing-sensitive tests, then let's just skip them. Windows isn't important enough. Refs: #3313
elastic · Sep 29, 2023 · 97b8803 · 97b8803
1 parent ac614dc
commit 97b8803
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 32 deletions.
diff --git a/test/instrumentation/drop-fast-exit-spans.test.js b/test/instrumentation/drop-fast-exit-spans.test.js
@@ -18,6 +18,7 @@ const agent = require('../..').start({
 const Transaction = require('../../lib/instrumentation/transaction');
 const Span = require('../../lib/instrumentation/span');
 const { OUTCOME_FAILURE } = require('../../lib/constants');
+const { TIMING_SENSITIVE_TEST_OPTS } = require('../testconsts');
 const mockClient = require('../_mock_http_client');
 const tape = require('tape');
 
@@ -87,43 +88,47 @@ tape.test('end to end test', function (t) {
 });
 
 // Test that a composite span faster than `exitSpanMinDuration` is dropped.
-tape.test('end to end test with compression', function (t) {
-  resetAgent(function (data) {
-    t.equals(
-      data.spans.length,
-      0,
-      `the composite span was dropped (exitSpanMinDuration=${
-        agent._conf.exitSpanMinDuration * 1000
-      }ms, data.spans=${JSON.stringify(data.spans)})`,
-    );
-    t.end();
-  });
+tape.test(
+  'end to end test with compression',
+  TIMING_SENSITIVE_TEST_OPTS,
+  function (t) {
+    resetAgent(function (data) {
+      t.equals(
+        data.spans.length,
+        0,
+        `the composite span was dropped (exitSpanMinDuration=${
+          agent._conf.exitSpanMinDuration * 1000
+        }ms, data.spans=${JSON.stringify(data.spans)})`,
+      );
+      t.end();
+    });
 
-  agent.startTransaction('test');
-  let firstSpan, finalSpan;
-  setTimeout(function () {
-    firstSpan = agent.startSpan('name1', 'db', 'mysql', { exitSpan: true });
+    agent.startTransaction('test');
+    let firstSpan, finalSpan;
     setTimeout(function () {
-      firstSpan.end();
+      firstSpan = agent.startSpan('name1', 'db', 'mysql', { exitSpan: true });
+      setTimeout(function () {
+        firstSpan.end();
+      }, 1);
     }, 1);
-  }, 1);
 
-  setTimeout(function () {
-    const span = agent.startSpan('name1', 'db', 'mysql', { exitSpan: true });
     setTimeout(function () {
-      span.end();
-    }, 1);
-  }, 2);
+      const span = agent.startSpan('name1', 'db', 'mysql', { exitSpan: true });
+      setTimeout(function () {
+        span.end();
+      }, 1);
+    }, 2);
 
-  setTimeout(function () {
-    finalSpan = agent.startSpan('name1', 'db', 'mysql', { exitSpan: true });
     setTimeout(function () {
-      finalSpan.end();
-      agent.endTransaction();
-      agent.flush();
-    }, 1);
-  }, 3);
-});
+      finalSpan = agent.startSpan('name1', 'db', 'mysql', { exitSpan: true });
+      setTimeout(function () {
+        finalSpan.end();
+        agent.endTransaction();
+        agent.flush();
+      }, 1);
+    }, 3);
+  },
+);
 
 function resetAgent(/* numExpected, */ cb) {
   agent._instrumentation.testReset();

diff --git a/test/instrumentation/timer.test.js b/test/instrumentation/timer.test.js
@@ -9,6 +9,7 @@
 var test = require('tape');
 
 var Timer = require('../../lib/instrumentation/timer');
+const { TIMING_SENSITIVE_TEST_OPTS } = require('../testconsts');
 
 test('started', function (t) {
   var timer = new Timer();
@@ -47,7 +48,7 @@ test('elapsed', function (t) {
   });
 });
 
-test('custom start time', function (t) {
+test('custom start time', TIMING_SENSITIVE_TEST_OPTS, function (t) {
   var startTime = Date.now() - 1000;
   var timer = new Timer(null, startTime);
   t.strictEqual(timer.start, startTime * 1000);

diff --git a/test/metrics/index.test.js b/test/metrics/index.test.js
@@ -12,6 +12,7 @@ const semver = require('semver');
 const test = require('tape');
 
 const Metrics = require('../../lib/metrics');
+const { TIMING_SENSITIVE_TEST_OPTS } = require('../testconsts');
 
 const delayMs = 500;
 const delayDeviationMs = (delayMs / 100) * 10;
@@ -41,7 +42,7 @@ function isRoughlyAbsolute(received, expected, range) {
   return received >= lower && received < upper;
 }
 
-test('reports expected metrics', function (t) {
+test('reports expected metrics', TIMING_SENSITIVE_TEST_OPTS, function (t) {
   let count = 0;
   let last;
 

diff --git a/test/testconsts.js b/test/testconsts.js
@@ -6,6 +6,8 @@
 
 'use strict';
 
+const os = require('os');
+
 // Supported Node.js version range for import-in-the-middle usage.
 // - v12.20.0 add "named exports for CJS via static analysis"
 //   https://nodejs.org/en/blog/release/v12.20.0
@@ -18,7 +20,17 @@
 const NODE_VER_RANGE_IITM = '^12.20.0 || ^14.13.1 || ^16.0.0 || ^18.1.0 <20';
 const NODE_VER_RANGE_IITM_GE14 = '^14.13.1 || ^16.0.0 || ^18.1.0 <20'; // NODE_VER_RANGE_IITM minus node v12
 
+// This can be passed as tape test options for tests that are timing sensitive,
+// to *skip* those tests on Windows CI.
+const TIMING_SENSITIVE_TEST_OPTS = {
+  skip:
+    os.platform() === 'win32' && process.env.CI === 'true'
+      ? '(skip timing-sensitive test on Windows CI)'
+      : false,
+};
+
 module.exports = {
   NODE_VER_RANGE_IITM,
   NODE_VER_RANGE_IITM_GE14,
+  TIMING_SENSITIVE_TEST_OPTS,
 };