chore: better logging for config state and transaction sampling

CHANGELOG.asciidoc

@@ -37,6 +37,15 @@ for the latest (4.x) releases. The 3.x branch will be maintained until
 2024-03-07 (6 months after the 4.0.0 release).
 
 
+==== Unreleased
+
+[float]
+===== Chores
+
+* Improve trace-level logging to better support debugging central config
+  and transaction sampling issues.
+
+
 [[release-notes-3.51.0]]
 ==== 3.51.0 - 2024/01/12
 

lib/agent.js

@@ -24,6 +24,7 @@ const errors = require('./errors');
 const { InflightEventSet } = require('./InflightEventSet');
 const Instrumentation = require('./instrumentation');
 const { elasticApmAwsLambda } = require('./lambda');
+const logging = require('./logging');
 const Metrics = require('./metrics');
 const parsers = require('./parsers');
 const symbols = require('./symbols');
@@ -324,6 +325,23 @@ Agent.prototype.start = function (opts) {
 
   this.logger.info(preambleData, 'Elastic APM Node.js Agent v%s', version);
 
+  if (!logging.isLoggerCustom(this.logger)) {
+    // Periodically dump the current config (delta from defaults) when logging
+    // at "trace"-level. This allows getting the effective config from a running
+    // agent by setting trace-level logging and getting 1 minute of logs.
+    // (Sometimes getting logs from application *start* is no possible.)
+    setInterval(() => {
+      if (this.logger.isLevelEnabled('trace')) {
+        try {
+          const currConfig = this._conf.getCurrConfig();
+          this.logger.trace({ currConfig }, 'currConfig');
+        } catch (err) {
+          this.logger.trace({ err }, 'error calculating currConfig');
+        }
+      }
+    }, 60 * 1000).unref();
+  }
+
   if (isPreviewVersion) {
     this.logger.warn(
       'Version %s is a pre-release and not intended for use in production environments',

lib/apm-client/apm-client.js

@@ -72,6 +72,8 @@ function createApmClient(config, agent) {
             !logging.isLoggerCustom(agent.logger)
           ) {
             logging.setLogLevel(agent.logger, value);
+            // Hackily also set the HttpApmClient._log level.
+            logging.setLogLevel(client._log, value);
             agent.logger.info(
               `Central config success: updated logger with new logLevel: ${value}`,
             );

lib/config/config.js

@@ -35,6 +35,7 @@ const {
   DEFAULTS,
   CROSS_AGENT_CONFIG_VAR_NAME,
   URL_OPTS,
+  CENTRAL_CONFIG_OPTS,
 } = require('./schema');
 
 const {
@@ -288,6 +289,27 @@ class Config {
     }
     return loggable;
   }
+
+  // Returns an object showing the current config, excluding default values.
+  getCurrConfig() {
+    const currConfig = {};
+
+    // Start with the values from the logging preamble. This selected keys
+    // that were specified, and handles redaction.
+    for (let [k, v] of Object.entries(this.loggingPreambleData.config)) {
+      currConfig[k] = v.value;
+    }
+
+    // Then add the current value of any var possibly set by central config.
+    currConfig.centralConfig = this.centralConfig;
+    if (this.centralConfig) {
+      for (let k of Object.values(CENTRAL_CONFIG_OPTS)) {
+        currConfig[k] = this[k];
+      }
+    }
+
+    return currConfig;
+  }
 }
 
 function readEnv() {

lib/instrumentation/index.js

@@ -542,6 +542,10 @@ Instrumentation.prototype.addEndedTransaction = function (transaction) {
     !transaction.sampled &&
     !agent._apmClient.supportsKeepingUnsampledTransaction()
   ) {
+    agent.logger.debug(
+      { trans: transaction.id, trace: transaction.traceId },
+      'dropping unsampled transaction',
+    );
     return;
   }
 

lib/instrumentation/transaction.js

@@ -92,6 +92,7 @@ function Transaction(agent, name, ...args) {
     trans: this.id,
     parent: this.parentId,
     trace: this.traceId,
+    sampled: this.sampled,
     name: this.name,
     type: this.type,
     subtype: this.subtype,