Add alternative cpu profiler for jvms without jfr

profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java

@@ -43,6 +43,8 @@ public class Configuration implements AutoConfigurationCustomizerProvider {
   private static final boolean DEFAULT_MEMORY_EVENT_RATE_LIMIT_ENABLED = true;
 
   public static final String CONFIG_KEY_ENABLE_PROFILER = PROFILER_ENABLED_PROPERTY;
+  public static final String CONFIG_KEY_PROFILER_JFR = "splunk.profiler.jfr";
+  public static final String CONFIG_KEY_PROFILER_JAVA = "splunk.profiler.java";
   public static final String CONFIG_KEY_PROFILER_DIRECTORY = "splunk.profiler.directory";
   public static final String CONFIG_KEY_RECORDING_DURATION = "splunk.profiler.recording.duration";
   public static final String CONFIG_KEY_KEEP_FILES = "splunk.profiler.keep-files";
@@ -101,6 +103,14 @@ public static String getConfigUrl(ConfigProperties config) {
     return config.getString(CONFIG_KEY_INGEST_URL, ingestUrl);
   }
 
+  public static boolean getProfilerJfrEnabled(ConfigProperties config) {
+    return config.getBoolean(CONFIG_KEY_PROFILER_JFR, true);
+  }
+
+  public static boolean getProfilerJavaEnabled(ConfigProperties config) {
+    return config.getBoolean(CONFIG_KEY_PROFILER_JAVA, false);
+  }
+
   public static boolean getTLABEnabled(ConfigProperties config) {
     boolean memoryEnabled = config.getBoolean(CONFIG_KEY_MEMORY_ENABLED, DEFAULT_MEMORY_ENABLED);
     return config.getBoolean(CONFIG_KEY_TLAB_ENABLED, memoryEnabled);

profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrActivator.java → profiler/src/main/java/com/splunk/opentelemetry/profiler/ProfilerActivator.java

@@ -27,11 +27,12 @@
 import com.splunk.opentelemetry.profiler.allocation.exporter.AllocationEventExporter;
 import com.splunk.opentelemetry.profiler.allocation.exporter.PprofAllocationEventExporter;
 import com.splunk.opentelemetry.profiler.context.SpanContextualizer;
-import com.splunk.opentelemetry.profiler.events.EventPeriods;
+import com.splunk.opentelemetry.profiler.contextstorage.JavaContextStorage;
 import com.splunk.opentelemetry.profiler.exporter.CpuEventExporter;
 import com.splunk.opentelemetry.profiler.exporter.PprofCpuEventExporter;
 import com.splunk.opentelemetry.profiler.util.HelpfulExecutors;
 import io.opentelemetry.api.logs.Logger;
+import io.opentelemetry.api.trace.SpanContext;
 import io.opentelemetry.javaagent.extension.AgentListener;
 import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk;
 import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
@@ -45,43 +46,129 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
+import java.time.Instant;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 
 @AutoService(AgentListener.class)
-public class JfrActivator implements AgentListener {
+public class ProfilerActivator implements AgentListener {
 
   private static final java.util.logging.Logger logger =
-      java.util.logging.Logger.getLogger(JfrActivator.class.getName());
-  private final ExecutorService executor = HelpfulExecutors.newSingleThreadExecutor("JFR Profiler");
+      java.util.logging.Logger.getLogger(ProfilerActivator.class.getName());
   private final ConfigurationLogger configurationLogger = new ConfigurationLogger();
 
   @Override
   public void afterAgent(AutoConfiguredOpenTelemetrySdk autoConfiguredOpenTelemetrySdk) {
     ConfigProperties config = autoConfiguredOpenTelemetrySdk.getConfig();
-    if (notClearForTakeoff(config)) {
+    if (!config.getBoolean(CONFIG_KEY_ENABLE_PROFILER, false)) {
+      logger.fine("Profiler is not enabled.");
+      return;
+    }
+    boolean useJfr = Configuration.getProfilerJfrEnabled(config);
+    boolean javaProfilerEnabled = Configuration.getProfilerJavaEnabled(config);
+    if (useJfr && !JFR.instance.isAvailable()) {
+      // jfr is not available and java profiler is not enabled
+      if (!javaProfilerEnabled) {
+        logger.warning(
+            "JDK Flight Recorder (JFR) is not available in this JVM. Profiling is disabled.");
+        return;
+      }
+
+      // fall back to java profiler
+      logger.fine(
+          "JDK Flight Recorder (JFR) is not available in this JVM, switching to java profiler.");
+      if (Configuration.getTLABEnabled(config)) {
+        logger.warning(
+            "JDK Flight Recorder (JFR) is not available in this JVM. Memory profiling is disabled.");
+      }
+      useJfr = false;
+    }
+    // neither jfr nor java profiler is available
+    if (!useJfr && !javaProfilerEnabled) {
+      logger.fine("Java profiler is disabled.");
       return;
     }
 
     configurationLogger.log(config);
     logger.info("Profiler is active.");
-    executor.submit(
-        logUncaught(
-            () -> activateJfrAndRunForever(config, autoConfiguredOpenTelemetrySdk.getResource())));
-  }
 
-  private boolean notClearForTakeoff(ConfigProperties config) {
-    if (!config.getBoolean(CONFIG_KEY_ENABLE_PROFILER, false)) {
-      logger.fine("Profiler is not enabled.");
-      return true;
+    if (useJfr) {
+      JfrProfiler.run(this, config, autoConfiguredOpenTelemetrySdk.getResource());
+    } else {
+      JavaProfiler.run(this, config, autoConfiguredOpenTelemetrySdk.getResource());
     }
-    if (!JFR.instance.isAvailable()) {
-      logger.warning(
-          "JDK Flight Recorder (JFR) is not available in this JVM. Profiling is disabled.");
-      return true;
+  }
+
+  private static class JfrProfiler {
+    private static final ExecutorService executor =
+        HelpfulExecutors.newSingleThreadExecutor("JFR Profiler");
+
+    static void run(ProfilerActivator activator, ConfigProperties config, Resource resource) {
+      executor.submit(logUncaught(() -> activator.activateJfrAndRunForever(config, resource)));
     }
+  }
+
+  private static class JavaProfiler {
+    private static final ScheduledExecutorService scheduler =
+        HelpfulExecutors.newSingleThreadedScheduledExecutor("Profiler scheduler");
 
-    return false;
+    static void run(ProfilerActivator activator, ConfigProperties config, Resource resource) {
+      int stackDepth = Configuration.getStackDepth(config);
+      LogRecordExporter logsExporter = LogExporterBuilder.fromConfig(config);
+      CpuEventExporter cpuEventExporter =
+          PprofCpuEventExporter.builder()
+              .otelLogger(
+                  activator.buildOtelLogger(
+                      SimpleLogRecordProcessor.create(logsExporter), resource))
+              .period(Configuration.getCallStackInterval(config))
+              .stackDepth(stackDepth)
+              .build();
+      StackTraceFilter stackTraceFilter = activator.buildStackTraceFilter(config, null);
+      boolean onlyTracingSpans = Configuration.getTracingStacksOnly(config);
+
+      Runnable profiler =
+          () -> {
+            Instant now = Instant.now();
+            Map<Thread, StackTraceElement[]> stackTracesMap;
+            Map<Thread, SpanContext> contextMap = new HashMap<>();
+            // disallow context changes while we are taking the thread dump
+            JavaContextStorage.block();
+            try {
+              stackTracesMap = Thread.getAllStackTraces();
+              // copy active context for each thread
+              for (Thread thread : stackTracesMap.keySet()) {
+                SpanContext spanContext = JavaContextStorage.activeContext.get(thread);
+                if (spanContext != null) {
+                  contextMap.put(thread, spanContext);
+                }
+              }
+            } finally {
+              JavaContextStorage.unblock();
+            }
+            for (Map.Entry<Thread, StackTraceElement[]> entry : stackTracesMap.entrySet()) {
+              Thread thread = entry.getKey();
+              StackTraceElement[] stack = entry.getValue();
+
+              if (!stackTraceFilter.test(thread, stack)) {
+                continue;
+              }
+
+              SpanContext spanContext = contextMap.get(thread);
+              if (onlyTracingSpans && !spanContext.isValid()) {
+                continue;
+              }
+
+              cpuEventExporter.export(thread, stack, now, spanContext);
+            }
+            cpuEventExporter.flush();
+          };
+      long period = Configuration.getCallStackInterval(config).toMillis();
+      scheduler.scheduleAtFixedRate(
+          logUncaught(() -> profiler.run()), period, period, TimeUnit.MILLISECONDS);
+    }
   }
 
   private boolean checkOutputDir(Path outputDir) {
@@ -107,7 +194,7 @@ private boolean checkOutputDir(Path outputDir) {
     return true;
   }
 
-  private void outdirWarn(Path dir, String suffix) {
+  private static void outdirWarn(Path dir, String suffix) {
     logger.log(WARNING, "The configured output directory {0} {1}.", new Object[] {dir, suffix});
   }
 
@@ -128,13 +215,12 @@ private void activateJfrAndRunForever(ConfigProperties config, Resource resource
 
     EventReader eventReader = new EventReader();
     SpanContextualizer spanContextualizer = new SpanContextualizer(eventReader);
-    EventPeriods periods = new EventPeriods(jfrSettings::get);
     LogRecordExporter logsExporter = LogExporterBuilder.fromConfig(config);
 
     CpuEventExporter cpuEventExporter =
         PprofCpuEventExporter.builder()
             .otelLogger(buildOtelLogger(SimpleLogRecordProcessor.create(logsExporter), resource))
-            .eventPeriods(periods)
+            .period(Configuration.getCallStackInterval(config))
             .stackDepth(stackDepth)
             .build();
 

profiler/src/main/java/com/splunk/opentelemetry/profiler/SdkCustomizer.java

@@ -17,9 +17,12 @@
 package com.splunk.opentelemetry.profiler;
 
 import static com.splunk.opentelemetry.profiler.Configuration.CONFIG_KEY_ENABLE_PROFILER;
+import static com.splunk.opentelemetry.profiler.Configuration.CONFIG_KEY_PROFILER_JFR;
 import static java.util.Collections.emptyMap;
 
 import com.google.auto.service.AutoService;
+import com.splunk.opentelemetry.profiler.contextstorage.JavaContextStorage;
+import com.splunk.opentelemetry.profiler.contextstorage.JfrContextStorage;
 import io.opentelemetry.context.ContextStorage;
 import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer;
 import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider;
@@ -32,8 +35,12 @@ public class SdkCustomizer implements AutoConfigurationCustomizerProvider {
   public void customize(AutoConfigurationCustomizer autoConfigurationCustomizer) {
     autoConfigurationCustomizer.addPropertiesCustomizer(
         config -> {
-          if (jfrIsAvailable() && jfrIsEnabledInConfig(config)) {
-            ContextStorage.addWrapper(JfrContextStorage::new);
+          if (profilerIsEnabledInConfig(config)) {
+            if (jfrIsAvailable() && jfrIsEnabledInConfig(config)) {
+              ContextStorage.addWrapper(JfrContextStorage::new);
+            } else {
+              ContextStorage.addWrapper(JavaContextStorage::new);
+            }
           }
           return emptyMap();
         });
@@ -43,7 +50,11 @@ private boolean jfrIsAvailable() {
     return JFR.instance.isAvailable();
   }
 
-  private boolean jfrIsEnabledInConfig(ConfigProperties config) {
+  private boolean profilerIsEnabledInConfig(ConfigProperties config) {
     return config.getBoolean(CONFIG_KEY_ENABLE_PROFILER, false);
   }
+
+  private boolean jfrIsEnabledInConfig(ConfigProperties config) {
+    return config.getBoolean(CONFIG_KEY_PROFILER_JFR, true);
+  }
 }

profiler/src/main/java/com/splunk/opentelemetry/profiler/StackTraceFilter.java

@@ -100,17 +100,8 @@ public boolean test(IItem event) {
 
     if (!includeAgentInternalStacks) {
       String threadName = thread.getThreadName();
-      for (String prefix : UNWANTED_PREFIXES) {
-        // if prefix ends with " we expect it to match thread name
-        if (prefix.endsWith("\"")) {
-          // prefix is surrounded by "
-          if (threadName.regionMatches(0, prefix, 1, prefix.length() - 2)) {
-            return false;
-          }
-          // prefix starts with "
-        } else if (threadName.regionMatches(0, prefix, 1, prefix.length() - 1)) {
-          return false;
-        }
+      if (isInternalThread(threadName)) {
+        return false;
       }
     }
     if (!includeJvmInternalStacks) {
@@ -122,6 +113,38 @@ public boolean test(IItem event) {
     return true;
   }
 
+  public boolean test(Thread thread, StackTraceElement[] threadDump) {
+    if (!includeAgentInternalStacks) {
+      String threadName = thread.getName();
+      if (isInternalThread(threadName)) {
+        return false;
+      }
+    }
+    if (!includeJvmInternalStacks) {
+      if (everyFrameIsJvmInternal(threadDump)) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  private static boolean isInternalThread(String threadName) {
+    for (String prefix : UNWANTED_PREFIXES) {
+      // if prefix ends with " we expect it to match thread name
+      if (prefix.endsWith("\"")) {
+        // prefix is surrounded by "
+        if (threadName.regionMatches(0, prefix, 1, prefix.length() - 2)) {
+          return true;
+        }
+        // prefix starts with "
+      } else if (threadName.regionMatches(0, prefix, 1, prefix.length() - 1)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   /**
    * Frames are considered JVM internal if every frame in the stack stars with one of "jdk.", "sun."
    * or "java.".
@@ -167,12 +190,32 @@ private static boolean everyFrameIsJvmInternal(IMCStackTrace stackTrace) {
       if (className == null) {
         continue;
       }
-      if (!className.startsWith("java.")
-          && !className.startsWith("jdk.")
-          && !className.startsWith("sun.")) {
+      if (!isJvmInternalClassName(className)) {
         return false;
       }
     }
     return true;
   }
+
+  private static boolean everyFrameIsJvmInternal(StackTraceElement[] stackTrace) {
+    if (stackTrace == null) {
+      return false;
+    }
+    for (StackTraceElement frame : stackTrace) {
+      String className = frame.getClassName();
+      if (className == null) {
+        continue;
+      }
+      if (!isJvmInternalClassName(className)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private static boolean isJvmInternalClassName(String className) {
+    return className.startsWith("java.")
+        || className.startsWith("jdk.")
+        || className.startsWith("sun.");
+  }
 }