Merge branch 'master' into sabrenner/openai-streaming

.github/workflows/appsec.yml

@@ -228,6 +228,6 @@ jobs:
       - run: yarn install
       - uses: ./.github/actions/node/oldest
       - run: yarn test:appsec:plugins:ci
-      - uses: ./.github/actions/node/21
+      - uses: ./.github/actions/node/latest
       - run: yarn test:appsec:plugins:ci
       - uses: codecov/codecov-action@v3

.github/workflows/profiling.yml

@@ -34,7 +34,7 @@ jobs:
       - uses: ./.github/actions/node/20
       - run: yarn test:profiler:ci
       - run: yarn test:integration:profiler
-      - uses: ./.github/actions/node/21
+      - uses: ./.github/actions/node/latest
       - run: yarn test:profiler:ci
       - run: yarn test:integration:profiler
       - uses: codecov/codecov-action@v3

.github/workflows/tracing.yml

@@ -31,7 +31,7 @@ jobs:
       - run: yarn test:trace:core:ci
       - uses: ./.github/actions/node/20
       - run: yarn test:trace:core:ci
-      - uses: ./.github/actions/node/21
+      - uses: ./.github/actions/node/latest
       - run: yarn test:trace:core:ci
       - uses: codecov/codecov-action@v3
 

README.md

@@ -108,4 +108,11 @@ If you would like to trace your bundled application then please read this page o
 
 ## Security Vulnerabilities
 
-Please refer to the [SECURITY.md](https://github.com/DataDog/dd-trace-js/blob/master/SECURITY.md) document if you have found a security issue.
+Please refer to the [SECURITY.md](https://github.com/DataDog/dd-trace-js/blob/master/SECURITY.md) document if you have found a security issue.
+
+## Datadog With OpenTelemetery
+
+Please refer to the [Node.js Custom Instrumentation using OpenTelemetry API](https://docs.datadoghq.com/tracing/trace_collection/custom_instrumentation/nodejs/otel/) document. It includes information on how to use the OpenTelemetry API with dd-trace-js
+
+Note that our internal implementation of the OpenTelemetry API is currently set within the version range `>=1.0.0 <1.9.0`. This range will be updated at a regular cadence therefore, we recommend updating your tracer to the latest release to ensure up to date support.
+

docs/test.ts

@@ -260,6 +260,7 @@ tracer.use('aws-sdk', awsSdkOptions);
 tracer.use('bunyan');
 tracer.use('couchbase');
 tracer.use('cassandra-driver');
+tracer.use('child_process');
 tracer.use('connect');
 tracer.use('connect', httpServerOptions);
 tracer.use('cypress');

index.d.ts

@@ -144,6 +144,7 @@ interface Plugins {
   "aws-sdk": tracer.plugins.aws_sdk;
   "bunyan": tracer.plugins.bunyan;
   "cassandra-driver": tracer.plugins.cassandra_driver;
+  "child_process": tracer.plugins.child_process;
   "connect": tracer.plugins.connect;
   "couchbase": tracer.plugins.couchbase;
   "cucumber": tracer.plugins.cucumber;
@@ -1207,6 +1208,12 @@ declare namespace tracer {
      */
     interface cassandra_driver extends Instrumentation {}
 
+    /**
+     * This plugin automatically instruments the
+     * [child_process](https://nodejs.org/api/child_process.html) module.
+     */
+    interface child_process extends Instrumentation {}
+
     /**
      * This plugin automatically instruments the
      * [connect](https://github.com/senchalabs/connect) module.

init.js

@@ -1,7 +1,31 @@
 'use strict'
 
-const tracer = require('.')
+const path = require('path')
+const Module = require('module')
 
-tracer.init()
+let initBailout = false
 
-module.exports = tracer
+if (process.env.DD_INJECTION_ENABLED) {
+  // If we're running via single-step install, and we're not in the app's
+  // node_modules, then we should not initialize the tracer. This prevents
+  // single-step-installed tracer from clobbering the manually-installed tracer.
+  let resolvedInApp
+  const entrypoint = process.argv[1]
+  try {
+    resolvedInApp = Module.createRequire(entrypoint).resolve('dd-trace')
+  } catch (e) {
+    // Ignore. If we can't resolve the module, we assume it's not in the app.
+  }
+  if (resolvedInApp) {
+    const ourselves = path.join(__dirname, 'index.js')
+    if (ourselves !== resolvedInApp) {
+      initBailout = true
+    }
+  }
+}
+
+if (!initBailout) {
+  const tracer = require('.')
+  tracer.init()
+  module.exports = tracer
+}

integration-tests/ci-visibility.spec.js

@@ -45,7 +45,7 @@ const mochaCommonOptions = {
 
 const jestCommonOptions = {
   name: 'jest',
-  dependencies: ['jest', 'chai@v4', 'jest-jasmine2'],
+  dependencies: ['jest', 'chai@v4', 'jest-jasmine2', 'jest-environment-jsdom'],
   expectedStdout: 'Test Suites: 2 passed',
   expectedCoverageFiles: [
     'ci-visibility/test/sum.js',
@@ -328,6 +328,7 @@ testFrameworks.forEach(({
             done()
           }).catch(done)
         })
+
         it('reports tests when using agentless', (done) => {
           childProcess = fork(testFile, {
             cwd,
@@ -347,6 +348,7 @@ testFrameworks.forEach(({
             done()
           }).catch(done)
         })
+
         it('reports tests when using evp proxy', (done) => {
           childProcess = fork(testFile, {
             cwd,
@@ -1414,6 +1416,88 @@ testFrameworks.forEach(({
               eventsPromise.then(() => done()).catch(done)
             })
           })
+
+          it('works with jsdom', (done) => {
+            const envVars = reportingOption === 'agentless'
+              ? getCiVisAgentlessConfig(receiver.port)
+              : getCiVisEvpProxyConfig(receiver.port)
+            if (reportingOption === 'evp proxy') {
+              receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
+            }
+            // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new
+            receiver.setKnownTests({
+              [name]: {
+                'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests']
+              }
+            })
+            const NUM_RETRIES_EFD = 3
+            receiver.setSettings({
+              itr_enabled: false,
+              code_coverage: false,
+              tests_skipping: false,
+              early_flake_detection: {
+                enabled: true,
+                slow_test_retries: {
+                  '5s': NUM_RETRIES_EFD
+                },
+                faulty_session_threshold: 100
+              }
+            })
+
+            const eventsPromise = receiver
+              .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+                const events = payloads.flatMap(({ payload }) => payload.events)
+
+                const tests = events.filter(event => event.type === 'test').map(event => event.content)
+
+                // no other tests are considered new
+                const oldTests = tests.filter(test =>
+                  test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js'
+                )
+                oldTests.forEach(test => {
+                  assert.notProperty(test.meta, TEST_IS_NEW)
+                })
+                assert.equal(oldTests.length, 1)
+
+                const newTests = tests.filter(test =>
+                  test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js'
+                )
+                newTests.forEach(test => {
+                  assert.propertyVal(test.meta, TEST_IS_NEW, 'true')
+                })
+                const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
+                // all but one has been retried
+                assert.equal(
+                  newTests.length - 1,
+                  retriedTests.length
+                )
+                assert.equal(retriedTests.length, NUM_RETRIES_EFD)
+                // Test name does not change
+                newTests.forEach(test => {
+                  assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2')
+                })
+              })
+
+            childProcess = exec(
+              runTestsWithCoverageCommand,
+              {
+                cwd,
+                env: {
+                  ...envVars,
+                  TESTS_TO_RUN: 'test/ci-visibility-test',
+                  ENABLE_JSDOM: true,
+                  DD_TRACE_DEBUG: 1,
+                  DD_TRACE_LOG_LEVEL: 'warn'
+                },
+                stdio: 'inherit'
+              }
+            )
+            childProcess.on('exit', () => {
+              eventsPromise.then(() => {
+                done()
+              }).catch(done)
+            })
+          })
         }
       })
     })

integration-tests/ci-visibility/run-jest.js

@@ -20,6 +20,10 @@ if (process.env.OLD_RUNNER) {
   options.testRunner = 'jest-jasmine2'
 }
 
+if (process.env.ENABLE_JSDOM) {
+  options.testEnvironment = 'jsdom'
+}
+
 jest.runCLI(
   options,
   options.projects

integration-tests/ci-visibility/run-jest.mjs

@@ -22,6 +22,10 @@ if (process.env.OLD_RUNNER) {
   options.testRunner = 'jest-jasmine2'
 }
 
+if (process.env.ENABLE_JSDOM) {
+  options.testEnvironment = 'jsdom'
+}
+
 jest.runCLI(
   options,
   options.projects

integration-tests/init.spec.js

@@ -0,0 +1,57 @@
+const {
+  createSandbox,
+  spawnProc
+} = require('./helpers')
+const { assert } = require('chai')
+const path = require('path')
+
+const DD_INJECTION_ENABLED = 'tracing'
+
+describe('init.js', () => {
+  let cwd, proc, sandbox
+
+  async function runTest (cwd, env, expected) {
+    return new Promise((resolve, reject) => {
+      spawnProc(path.join(cwd, 'init/index.js'), { cwd, env }, data => {
+        try {
+          assert.strictEqual(data.toString(), expected)
+          resolve()
+        } catch (e) {
+          reject(e)
+        }
+      }).then(subproc => {
+        proc = subproc
+      })
+    })
+  }
+
+  before(async () => {
+    sandbox = await createSandbox()
+    cwd = sandbox.folder
+  })
+  afterEach(() => {
+    proc && proc.kill()
+  })
+  after(() => {
+    return sandbox.remove()
+  })
+
+  context('when dd-trace is not in the app dir', () => {
+    const NODE_OPTIONS = `--require ${path.join(__dirname, '..', 'init.js')}`
+    it('should initialize the tracer, if no DD_INJECTION_ENABLED', () => {
+      return runTest(cwd, { NODE_OPTIONS }, 'true\n')
+    })
+    it('should not initialize the tracer, if DD_INJECTION_ENABLED', () => {
+      return runTest(cwd, { NODE_OPTIONS, DD_INJECTION_ENABLED }, 'false\n')
+    })
+  })
+  context('when dd-trace in the app dir', () => {
+    const NODE_OPTIONS = '--require dd-trace/init.js'
+    it('should initialize the tracer, if no DD_INJECTION_ENABLED', () => {
+      return runTest(cwd, { NODE_OPTIONS }, 'true\n')
+    })
+    it('should initialize the tracer, if DD_INJECTION_ENABLED', () => {
+      return runTest(cwd, { NODE_OPTIONS, DD_INJECTION_ENABLED }, 'true\n')
+    })
+  })
+})

integration-tests/init/index.js

@@ -0,0 +1,3 @@
+// eslint-disable-next-line no-console
+console.log(!!global._ddtrace)
+process.exit()

package.json

@@ -74,9 +74,9 @@
     "@datadog/native-iast-rewriter": "2.3.1",
     "@datadog/native-iast-taint-tracking": "2.1.0",
     "@datadog/native-metrics": "^2.0.0",
-    "@datadog/pprof": "5.2.0",
+    "@datadog/pprof": "5.3.0",
     "@datadog/sketches-js": "^2.1.0",
-    "@opentelemetry/api": "^1.0.0",
+    "@opentelemetry/api": ">=1.0.0 <1.9.0",
     "@opentelemetry/core": "^1.14.0",
     "crypto-randomuuid": "^1.0.0",
     "dc-polyfill": "^0.1.4",

packages/datadog-esbuild/index.js

@@ -4,6 +4,7 @@
 
 const instrumentations = require('../datadog-instrumentations/src/helpers/instrumentations.js')
 const hooks = require('../datadog-instrumentations/src/helpers/hooks.js')
+const extractPackageAndModulePath = require('../datadog-instrumentations/src/utils/src/extract-package-and-module-path')
 
 for (const hook of Object.values(hooks)) {
   hook()
@@ -21,7 +22,6 @@ for (const instrumentation of Object.values(instrumentations)) {
   }
 }
 
-const NM = 'node_modules/'
 const INSTRUMENTED = Object.keys(instrumentations)
 const RAW_BUILTINS = require('module').builtinModules
 const CHANNEL = 'dd-trace:bundler:load'
@@ -181,33 +181,3 @@ function dotFriendlyResolve (path, directory) {
 
   return require.resolve(path, { paths: [directory] })
 }
-
-/**
- * For a given full path to a module,
- *   return the package name it belongs to and the local path to the module
- *   input: '/foo/node_modules/@co/stuff/foo/bar/baz.js'
- *   output: { pkg: '@co/stuff', path: 'foo/bar/baz.js' }
- */
-function extractPackageAndModulePath (fullPath) {
-  const nm = fullPath.lastIndexOf(NM)
-  if (nm < 0) {
-    return { pkg: null, path: null }
-  }
-
-  const subPath = fullPath.substring(nm + NM.length)
-  const firstSlash = subPath.indexOf('/')
-
-  if (subPath[0] === '@') {
-    const secondSlash = subPath.substring(firstSlash + 1).indexOf('/')
-
-    return {
-      pkg: subPath.substring(0, firstSlash + 1 + secondSlash),
-      path: subPath.substring(firstSlash + 1 + secondSlash + 1)
-    }
-  }
-
-  return {
-    pkg: subPath.substring(0, firstSlash),
-    path: subPath.substring(firstSlash + 1)
-  }
-}