diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java index 7fc9b5e8d..96dfba253 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java @@ -43,6 +43,7 @@ public class Configuration implements AutoConfigurationCustomizerProvider { private static final boolean DEFAULT_MEMORY_EVENT_RATE_LIMIT_ENABLED = true; public static final String CONFIG_KEY_ENABLE_PROFILER = PROFILER_ENABLED_PROPERTY; + public static final String CONFIG_KEY_PROFILER_JFR = "splunk.profiler.jfr"; public static final String CONFIG_KEY_PROFILER_DIRECTORY = "splunk.profiler.directory"; public static final String CONFIG_KEY_RECORDING_DURATION = "splunk.profiler.recording.duration"; public static final String CONFIG_KEY_KEEP_FILES = "splunk.profiler.keep-files"; @@ -101,6 +102,10 @@ public static String getConfigUrl(ConfigProperties config) { return config.getString(CONFIG_KEY_INGEST_URL, ingestUrl); } + public static boolean getProfilerJfrEnabled(ConfigProperties config) { + return config.getBoolean(CONFIG_KEY_PROFILER_JFR, true); + } + public static boolean getTLABEnabled(ConfigProperties config) { boolean memoryEnabled = config.getBoolean(CONFIG_KEY_MEMORY_ENABLED, DEFAULT_MEMORY_ENABLED); return config.getBoolean(CONFIG_KEY_TLAB_ENABLED, memoryEnabled); diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrActivator.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrActivator.java index 37e009519..0737b3f32 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrActivator.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrActivator.java @@ -27,11 +27,12 @@ import com.splunk.opentelemetry.profiler.allocation.exporter.AllocationEventExporter; import com.splunk.opentelemetry.profiler.allocation.exporter.PprofAllocationEventExporter; import com.splunk.opentelemetry.profiler.context.SpanContextualizer; -import com.splunk.opentelemetry.profiler.events.EventPeriods; +import com.splunk.opentelemetry.profiler.contextstorage.JavaContextStorage; import com.splunk.opentelemetry.profiler.exporter.CpuEventExporter; import com.splunk.opentelemetry.profiler.exporter.PprofCpuEventExporter; import com.splunk.opentelemetry.profiler.util.HelpfulExecutors; import io.opentelemetry.api.logs.Logger; +import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.javaagent.extension.AgentListener; import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; @@ -45,43 +46,103 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; @AutoService(AgentListener.class) public class JfrActivator implements AgentListener { private static final java.util.logging.Logger logger = java.util.logging.Logger.getLogger(JfrActivator.class.getName()); - private final ExecutorService executor = HelpfulExecutors.newSingleThreadExecutor("JFR Profiler"); private final ConfigurationLogger configurationLogger = new ConfigurationLogger(); @Override public void afterAgent(AutoConfiguredOpenTelemetrySdk autoConfiguredOpenTelemetrySdk) { ConfigProperties config = autoConfiguredOpenTelemetrySdk.getConfig(); - if (notClearForTakeoff(config)) { + if (!config.getBoolean(CONFIG_KEY_ENABLE_PROFILER, false)) { + logger.fine("Profiler is not enabled."); return; } + boolean useJfr = Configuration.getProfilerJfrEnabled(config); + if (useJfr && !JFR.instance.isAvailable()) { + logger.fine( + "JDK Flight Recorder (JFR) is not available in this JVM, switching to java profiler."); + if (Configuration.getTLABEnabled(config)) { + logger.warning( + "JDK Flight Recorder (JFR) is not available in this JVM. Memory profiling is disabled."); + } + useJfr = false; + } configurationLogger.log(config); logger.info("Profiler is active."); - executor.submit( - logUncaught( - () -> activateJfrAndRunForever(config, autoConfiguredOpenTelemetrySdk.getResource()))); - } - private boolean notClearForTakeoff(ConfigProperties config) { - if (!config.getBoolean(CONFIG_KEY_ENABLE_PROFILER, false)) { - logger.fine("Profiler is not enabled."); - return true; + if (useJfr) { + JfrProfiler.run(this, config, autoConfiguredOpenTelemetrySdk.getResource()); + } else { + JavaProfiler.run(this, config, autoConfiguredOpenTelemetrySdk.getResource()); } - if (!JFR.instance.isAvailable()) { - logger.warning( - "JDK Flight Recorder (JFR) is not available in this JVM. Profiling is disabled."); - return true; + } + + private static class JfrProfiler { + private static final ExecutorService executor = + HelpfulExecutors.newSingleThreadExecutor("JFR Profiler"); + + static void run(JfrActivator activator, ConfigProperties config, Resource resource) { + executor.submit(logUncaught(() -> activator.activateJfrAndRunForever(config, resource))); } + } - return false; + private static class JavaProfiler { + private static final ScheduledExecutorService scheduler = + HelpfulExecutors.newSingleThreadedScheduledExecutor("Profiler scheduler"); + + static void run(JfrActivator activator, ConfigProperties config, Resource resource) { + int stackDepth = Configuration.getStackDepth(config); + LogRecordExporter logsExporter = LogExporterBuilder.fromConfig(config); + CpuEventExporter cpuEventExporter = + PprofCpuEventExporter.builder() + .otelLogger( + activator.buildOtelLogger( + SimpleLogRecordProcessor.create(logsExporter), resource)) + .period(Configuration.getCallStackInterval(config)) + .stackDepth(stackDepth) + .build(); + + Runnable profiler = + () -> { + Instant now = Instant.now(); + Map stackTracesMap; + Map contextMap = new HashMap<>(); + // disallow context changes while we are taking the thread dump + JavaContextStorage.block(); + try { + stackTracesMap = Thread.getAllStackTraces(); + // copy active context for each thread + for (Thread thread : stackTracesMap.keySet()) { + SpanContext spanContext = JavaContextStorage.activeContext.get(thread); + if (spanContext != null) { + contextMap.put(thread, spanContext); + } + } + } finally { + JavaContextStorage.unblock(); + } + for (Map.Entry entry : stackTracesMap.entrySet()) { + Thread thread = entry.getKey(); + SpanContext spanContext = contextMap.get(thread); + cpuEventExporter.export(thread, entry.getValue(), now, spanContext); + } + cpuEventExporter.flush(); + }; + long period = Configuration.getCallStackInterval(config).toMillis(); + scheduler.scheduleAtFixedRate( + logUncaught(() -> profiler.run()), period, period, TimeUnit.MILLISECONDS); + } } private boolean checkOutputDir(Path outputDir) { @@ -107,7 +168,7 @@ private boolean checkOutputDir(Path outputDir) { return true; } - private void outdirWarn(Path dir, String suffix) { + private static void outdirWarn(Path dir, String suffix) { logger.log(WARNING, "The configured output directory {0} {1}.", new Object[] {dir, suffix}); } @@ -128,13 +189,12 @@ private void activateJfrAndRunForever(ConfigProperties config, Resource resource EventReader eventReader = new EventReader(); SpanContextualizer spanContextualizer = new SpanContextualizer(eventReader); - EventPeriods periods = new EventPeriods(jfrSettings::get); LogRecordExporter logsExporter = LogExporterBuilder.fromConfig(config); CpuEventExporter cpuEventExporter = PprofCpuEventExporter.builder() .otelLogger(buildOtelLogger(SimpleLogRecordProcessor.create(logsExporter), resource)) - .eventPeriods(periods) + .period(Configuration.getCallStackInterval(config)) .stackDepth(stackDepth) .build(); diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/SdkCustomizer.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/SdkCustomizer.java index 576102cd9..e74908580 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/SdkCustomizer.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/SdkCustomizer.java @@ -17,9 +17,12 @@ package com.splunk.opentelemetry.profiler; import static com.splunk.opentelemetry.profiler.Configuration.CONFIG_KEY_ENABLE_PROFILER; +import static com.splunk.opentelemetry.profiler.Configuration.CONFIG_KEY_PROFILER_JFR; import static java.util.Collections.emptyMap; import com.google.auto.service.AutoService; +import com.splunk.opentelemetry.profiler.contextstorage.JavaContextStorage; +import com.splunk.opentelemetry.profiler.contextstorage.JfrContextStorage; import io.opentelemetry.context.ContextStorage; import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; @@ -32,8 +35,12 @@ public class SdkCustomizer implements AutoConfigurationCustomizerProvider { public void customize(AutoConfigurationCustomizer autoConfigurationCustomizer) { autoConfigurationCustomizer.addPropertiesCustomizer( config -> { - if (jfrIsAvailable() && jfrIsEnabledInConfig(config)) { - ContextStorage.addWrapper(JfrContextStorage::new); + if (profilerIsEnabledInConfig(config)) { + if (jfrIsAvailable() && jfrIsEnabledInConfig(config)) { + ContextStorage.addWrapper(JfrContextStorage::new); + } else { + ContextStorage.addWrapper(JavaContextStorage::new); + } } return emptyMap(); }); @@ -43,7 +50,11 @@ private boolean jfrIsAvailable() { return JFR.instance.isAvailable(); } - private boolean jfrIsEnabledInConfig(ConfigProperties config) { + private boolean profilerIsEnabledInConfig(ConfigProperties config) { return config.getBoolean(CONFIG_KEY_ENABLE_PROFILER, false); } + + private boolean jfrIsEnabledInConfig(ConfigProperties config) { + return config.getBoolean(CONFIG_KEY_PROFILER_JFR, true); + } } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/contextstorage/AbstractContextStorage.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/contextstorage/AbstractContextStorage.java new file mode 100644 index 000000000..9be0f0838 --- /dev/null +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/contextstorage/AbstractContextStorage.java @@ -0,0 +1,63 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.contextstorage; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.context.Scope; +import javax.annotation.Nullable; + +abstract class AbstractContextStorage implements ContextStorage { + + private final ContextStorage delegate; + private final ThreadLocal activeSpan = ThreadLocal.withInitial(Span::getInvalid); + + AbstractContextStorage(ContextStorage delegate) { + this.delegate = delegate; + } + + @Override + public Scope attach(Context toAttach) { + Scope delegatedScope = delegate.attach(toAttach); + Span span = Span.fromContext(toAttach); + Span current = activeSpan.get(); + // do nothing when active span didn't change + // do nothing if the span isn't sampled + if (span == current || !span.getSpanContext().isSampled()) { + return delegatedScope; + } + + // mark new span as active and generate event + activeSpan.set(span); + activateSpan(span); + return () -> { + // restore previous active span + activeSpan.set(current); + activateSpan(current); + delegatedScope.close(); + }; + } + + protected abstract void activateSpan(Span span); + + @Nullable + @Override + public Context current() { + return delegate.current(); + } +} diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/contextstorage/JavaContextStorage.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/contextstorage/JavaContextStorage.java new file mode 100644 index 000000000..88dac14e7 --- /dev/null +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/contextstorage/JavaContextStorage.java @@ -0,0 +1,79 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.contextstorage; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.instrumentation.api.internal.cache.Cache; + +// active context tracking for java profiler +public class JavaContextStorage extends AbstractContextStorage { + + public static final Cache activeContext = Cache.weak(); + private static final Guard NOP = () -> {}; + private static final BlockingGuard GUARD = new BlockingGuard(); + private static volatile Guard guard = NOP; + + public JavaContextStorage(ContextStorage delegate) { + super(delegate); + } + + public static void block() { + guard = GUARD; + } + + public static void unblock() { + guard = NOP; + GUARD.release(); + } + + @Override + protected void activateSpan(Span span) { + // when taking thread dump we block all thread that attempt to modify the active contexts + guard.stop(); + + SpanContext context = span.getSpanContext(); + if (context.isValid()) { + activeContext.put(Thread.currentThread(), context); + } else { + activeContext.remove(Thread.currentThread()); + } + } + + private interface Guard { + void stop(); + } + + private static class BlockingGuard implements Guard { + + @Override + public synchronized void stop() { + try { + while (guard == GUARD) { + wait(); + } + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + } + } + + synchronized void release() { + notifyAll(); + } + } +} diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrContextStorage.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/contextstorage/JfrContextStorage.java similarity index 59% rename from profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrContextStorage.java rename to profiler/src/main/java/com/splunk/opentelemetry/profiler/contextstorage/JfrContextStorage.java index afd2fba47..0ef3700d2 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrContextStorage.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/contextstorage/JfrContextStorage.java @@ -14,32 +14,28 @@ * limitations under the License. */ -package com.splunk.opentelemetry.profiler; +package com.splunk.opentelemetry.profiler.contextstorage; import com.google.common.annotations.VisibleForTesting; import com.splunk.opentelemetry.profiler.events.ContextAttached; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.TraceFlags; -import io.opentelemetry.context.Context; import io.opentelemetry.context.ContextStorage; -import io.opentelemetry.context.Scope; import java.util.function.Function; -import javax.annotation.Nullable; -class JfrContextStorage implements ContextStorage { +// active context tracking for jfr profiler +public class JfrContextStorage extends AbstractContextStorage { - private final ContextStorage delegate; private final Function newEvent; - private final ThreadLocal activeSpan = ThreadLocal.withInitial(Span::getInvalid); - JfrContextStorage(ContextStorage delegate) { + public JfrContextStorage(ContextStorage delegate) { this(delegate, JfrContextStorage::newEvent); } @VisibleForTesting JfrContextStorage(ContextStorage delegate, Function newEvent) { - this.delegate = delegate; + super(delegate); this.newEvent = newEvent; } @@ -52,28 +48,7 @@ static ContextAttached newEvent(SpanContext spanContext) { } @Override - public Scope attach(Context toAttach) { - Scope delegatedScope = delegate.attach(toAttach); - Span span = Span.fromContext(toAttach); - Span current = activeSpan.get(); - // do nothing when active span didn't change - // do nothing if the span isn't sampled - if (span == current || !span.getSpanContext().isSampled()) { - return delegatedScope; - } - - // mark new span as active and generate event - activeSpan.set(span); - generateEvent(span); - return () -> { - // restore previous active span - activeSpan.set(current); - generateEvent(current); - delegatedScope.close(); - }; - } - - private void generateEvent(Span span) { + protected void activateSpan(Span span) { SpanContext context = span.getSpanContext(); ContextAttached event = newEvent.apply(context); event.begin(); @@ -81,10 +56,4 @@ private void generateEvent(Span span) { event.commit(); } } - - @Nullable - @Override - public Context current() { - return delegate.current(); - } } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/events/EventPeriods.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/events/EventPeriods.java deleted file mode 100644 index ded9b170a..000000000 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/events/EventPeriods.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Splunk Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.splunk.opentelemetry.profiler.events; - -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -public class EventPeriods { - - public static final Duration UNKNOWN = Duration.ZERO; - private final Map cache = new HashMap<>(); - private final Function configFinder; - - public EventPeriods(Function configFinder) { - this.configFinder = configFinder; - } - - public Duration getDuration(String eventName) { - return cache.computeIfAbsent( - eventName, - event -> { - String value = configFinder.apply(event + "#period"); - return parseToDuration(value); - }); - } - - private Duration parseToDuration(String value) { - if (value == null) { - return UNKNOWN; - } - // format is "TTT UUU" where TTT is some numbers and UUU is some units suffix (ms or s) - try { - String[] parts = value.split(" "); - if (parts.length < 2) { - return UNKNOWN; - } - long multiplier = 1; - if ("s".equals(parts[1])) { - multiplier = 1000; - } - return Duration.ofMillis(multiplier * Integer.parseInt(parts[0])); - } catch (NumberFormatException e) { - return UNKNOWN; - } - } -} diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/CpuEventExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/CpuEventExporter.java index 2c239b692..d3bfed59b 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/CpuEventExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/CpuEventExporter.java @@ -17,9 +17,14 @@ package com.splunk.opentelemetry.profiler.exporter; import com.splunk.opentelemetry.profiler.context.StackToSpanLinkage; +import io.opentelemetry.api.trace.SpanContext; +import java.time.Instant; public interface CpuEventExporter { + void export( + Thread thread, StackTraceElement[] stackTrace, Instant eventTime, SpanContext spanContext); + void export(StackToSpanLinkage stackToSpanLinkage); default void flush() {} diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java index 630b23e2a..12cff797f 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java @@ -29,7 +29,6 @@ import com.google.perftools.profiles.ProfileProto.Sample; import com.splunk.opentelemetry.profiler.ProfilingDataType; import com.splunk.opentelemetry.profiler.context.StackToSpanLinkage; -import com.splunk.opentelemetry.profiler.events.EventPeriods; import com.splunk.opentelemetry.profiler.exporter.StackTraceParser.StackTrace; import com.splunk.opentelemetry.profiler.pprof.Pprof; import io.opentelemetry.api.logs.Logger; @@ -38,17 +37,64 @@ import java.time.Instant; public class PprofCpuEventExporter implements CpuEventExporter { - private final EventPeriods eventPeriods; + private Duration period; private final int stackDepth; private final PprofLogDataExporter pprofLogDataExporter; private Pprof pprof = createPprof(); private PprofCpuEventExporter(Builder builder) { - this.eventPeriods = builder.eventPeriods; + this.period = builder.period; this.stackDepth = builder.stackDepth; this.pprofLogDataExporter = new PprofLogDataExporter(builder.otelLogger, ProfilingDataType.CPU); } + @Override + public void export( + Thread thread, StackTraceElement[] stackTrace, Instant eventTime, SpanContext spanContext) { + Sample.Builder sample = Sample.newBuilder(); + + pprof.addLabel(sample, THREAD_ID, thread.getId()); + pprof.addLabel(sample, THREAD_NAME, thread.getName()); + pprof.addLabel(sample, THREAD_STATE, thread.getState().name()); + + if (stackTrace.length > stackDepth) { + pprof.addLabel(sample, THREAD_STACK_TRUNCATED, true); + } + + for (int i = 0; i < Math.min(stackDepth, stackTrace.length); i++) { + StackTraceElement ste = stackTrace[i]; + + String fileName = ste.getFileName(); + if (fileName == null) { + fileName = "unknown"; + } + String className = ste.getClassName(); + if (className == null) { + className = "unknown"; + } + String methodName = ste.getMethodName(); + if (methodName == null) { + methodName = "unknown"; + } + int lineNumber = ste.getLineNumber(); + if (lineNumber < 0) { + lineNumber = 0; + } + sample.addLocationId(pprof.getLocationId(fileName, className, methodName, lineNumber)); + pprof.incFrameCount(); + } + + pprof.addLabel(sample, SOURCE_EVENT_PERIOD, period.toMillis()); + pprof.addLabel(sample, SOURCE_EVENT_TIME, eventTime.toEpochMilli()); + + if (spanContext != null && spanContext.isValid()) { + pprof.addLabel(sample, TRACE_ID, spanContext.getTraceId()); + pprof.addLabel(sample, SPAN_ID, spanContext.getSpanId()); + } + + pprof.getProfileBuilder().addSample(sample); + } + @Override public void export(StackToSpanLinkage stackToSpanLinkage) { StackTrace stackTrace = StackTraceParser.parse(stackToSpanLinkage.getRawStack(), stackDepth); @@ -77,10 +123,7 @@ public void export(StackToSpanLinkage stackToSpanLinkage) { String eventName = stackToSpanLinkage.getSourceEventName(); pprof.addLabel(sample, SOURCE_EVENT_NAME, eventName); - Duration eventPeriod = eventPeriods.getDuration(eventName); - if (!EventPeriods.UNKNOWN.equals(eventPeriod)) { - pprof.addLabel(sample, SOURCE_EVENT_PERIOD, eventPeriod.toMillis()); - } + pprof.addLabel(sample, SOURCE_EVENT_PERIOD, period.toMillis()); Instant time = stackToSpanLinkage.getTime(); pprof.addLabel(sample, SOURCE_EVENT_TIME, time.toEpochMilli()); @@ -120,7 +163,7 @@ public static Builder builder() { public static class Builder { private Logger otelLogger; - private EventPeriods eventPeriods; + private Duration period; private int stackDepth; public PprofCpuEventExporter build() { @@ -132,8 +175,8 @@ public Builder otelLogger(Logger otelLogger) { return this; } - public Builder eventPeriods(EventPeriods eventPeriods) { - this.eventPeriods = eventPeriods; + public Builder period(Duration period) { + this.period = period; return this; } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/ThreadDumpProcessorTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/ThreadDumpProcessorTest.java index 69088b476..a723932ce 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/ThreadDumpProcessorTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/ThreadDumpProcessorTest.java @@ -31,6 +31,7 @@ import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.TraceFlags; import io.opentelemetry.api.trace.TraceState; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -142,7 +143,22 @@ private static List collectResults( SpanContextualizer contextualizer, String threadDump, boolean onlyTracingSpans) { EventReader eventReader = mock(EventReader.class); List results = new ArrayList<>(); - CpuEventExporter profilingEventExporter = results::add; + CpuEventExporter profilingEventExporter = + new CpuEventExporter() { + @Override + public void export( + Thread thread, + StackTraceElement[] stackTrace, + Instant eventTime, + SpanContext spanContext) { + throw new IllegalStateException("should not be called"); + } + + @Override + public void export(StackToSpanLinkage stackToSpanLinkage) { + results.add(stackToSpanLinkage); + } + }; ThreadDumpProcessor processor = ThreadDumpProcessor.builder() .eventReader(eventReader) diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/JfrContextStorageTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/contextstorage/JfrContextStorageTest.java similarity index 98% rename from profiler/src/test/java/com/splunk/opentelemetry/profiler/JfrContextStorageTest.java rename to profiler/src/test/java/com/splunk/opentelemetry/profiler/contextstorage/JfrContextStorageTest.java index 8ce8cf45d..4c0e6f3f2 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/JfrContextStorageTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/contextstorage/JfrContextStorageTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.splunk.opentelemetry.profiler; +package com.splunk.opentelemetry.profiler.contextstorage; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/events/EventPeriodsTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/events/EventPeriodsTest.java deleted file mode 100644 index fa1284405..000000000 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/events/EventPeriodsTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright Splunk Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.splunk.opentelemetry.profiler.events; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.when; - -import java.time.Duration; -import java.util.function.Function; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class EventPeriodsTest { - - @Mock Function configFinder; - - @Test - void testNotCachedFirstParse() { - EventPeriods eventPeriods = new EventPeriods(configFinder); - when(configFinder.apply("jdk.SomeEvent#period")).thenReturn("250 ms"); - Duration result = eventPeriods.getDuration("jdk.SomeEvent"); - assertEquals(Duration.ofMillis(250), result); - } - - @Test - void testCached() { - EventPeriods eventPeriods = new EventPeriods(configFinder); - when(configFinder.apply("jdk.SomeEvent#period")) - .thenReturn("26 s") - .thenThrow(new IllegalStateException()); - Duration result1 = eventPeriods.getDuration("jdk.SomeEvent"); - Duration result2 = eventPeriods.getDuration("jdk.SomeEvent"); - Duration result3 = eventPeriods.getDuration("jdk.SomeEvent"); - assertEquals(Duration.ofSeconds(26), result1); - assertEquals(Duration.ofSeconds(26), result2); - assertEquals(Duration.ofSeconds(26), result3); - } - - @Test - void testNotFoundAlsoCached() { - EventPeriods eventPeriods = new EventPeriods(configFinder); - when(configFinder.apply("jdk.SomeEvent#period")) - .thenReturn(null) - .thenThrow(new IllegalStateException()); - Duration result1 = eventPeriods.getDuration("jdk.SomeEvent"); - Duration result2 = eventPeriods.getDuration("jdk.SomeEvent"); - assertEquals(EventPeriods.UNKNOWN, result1); - assertEquals(EventPeriods.UNKNOWN, result2); - } - - @Test - void testNotParsedAlsoCached() { - EventPeriods eventPeriods = new EventPeriods(configFinder); - when(configFinder.apply("jdk.SomeEvent#period")) - .thenReturn("BLEAK BLOOP") - .thenThrow(new IllegalStateException()); - Duration result1 = eventPeriods.getDuration("jdk.SomeEvent"); - Duration result2 = eventPeriods.getDuration("jdk.SomeEvent"); - assertEquals(EventPeriods.UNKNOWN, result1); - assertEquals(EventPeriods.UNKNOWN, result2); - } - - @Test - void testConfigNotFound() { - EventPeriods eventPeriods = new EventPeriods(configFinder); - when(configFinder.apply("jdk.SomeEvent#period")).thenReturn(null); - Duration result = eventPeriods.getDuration("jdk.SomeEvent"); - assertEquals(EventPeriods.UNKNOWN, result); - } - - @Test - void testEveryChunk() { - // Sometimes the JFR config might have the word "everyChunk" instead of an actual value - EventPeriods eventPeriods = new EventPeriods(configFinder); - when(configFinder.apply("jdk.SomeEvent#period")).thenReturn("everyChunk"); - Duration result = eventPeriods.getDuration("jdk.SomeEvent"); - assertEquals(EventPeriods.UNKNOWN, result); - } -} diff --git a/smoke-tests/src/test/java/com/splunk/opentelemetry/ProfilerSmokeTest.java b/smoke-tests/src/test/java/com/splunk/opentelemetry/ProfilerSmokeTest.java index 6e2cfc2e9..031c9eba6 100644 --- a/smoke-tests/src/test/java/com/splunk/opentelemetry/ProfilerSmokeTest.java +++ b/smoke-tests/src/test/java/com/splunk/opentelemetry/ProfilerSmokeTest.java @@ -24,6 +24,7 @@ import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import com.splunk.opentelemetry.helper.TargetContainerBuilder; import com.splunk.opentelemetry.helper.TargetWaitStrategy; @@ -35,6 +36,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -43,7 +46,6 @@ import jdk.jfr.consumer.RecordingFile; import okhttp3.Request; import okhttp3.Response; -import org.junit.Ignore; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -64,9 +66,15 @@ public abstract class ProfilerSmokeTest { private TestContainerManager containerManager; private TelemetryRetriever telemetryRetriever; private final String jdkVersion; + private final boolean useJfr; ProfilerSmokeTest(String jdkVersion) { + this(jdkVersion, true); + } + + ProfilerSmokeTest(String jdkVersion, boolean useJfr) { this.jdkVersion = jdkVersion; + this.useJfr = useJfr; } public static class TestJdk8 extends ProfilerSmokeTest { @@ -75,14 +83,18 @@ public static class TestJdk8 extends ProfilerSmokeTest { } } - @Ignore - public static class TestJdk11 extends ProfilerSmokeTest { - TestJdk11() { + public static class TestJdk11JfrProfiler extends ProfilerSmokeTest { + TestJdk11JfrProfiler() { super("11"); } } - @Ignore + public static class TestJdk11JavaProfiler extends ProfilerSmokeTest { + TestJdk11JavaProfiler() { + super("11", false); + } + } + public static class TestJdk17 extends ProfilerSmokeTest { TestJdk17() { super("17"); @@ -118,6 +130,8 @@ String getPetclinicImageName() { @Test void ensureJfrFilesContainContextChangeEvents() throws Exception { + assumeTrue(useJfr); + await() .atMost(1, TimeUnit.MINUTES) .pollInterval(1, TimeUnit.SECONDS) @@ -151,10 +165,12 @@ void verifyIngestedLogContent() throws Exception { assertThat(logs.getCpuSamples()).anyMatch(hasThreadName("main")); - assertThat(logs.getMemorySamples()) - .isNotEmpty() - .allMatch(sample -> sample.getAllocated() > 0) - .allMatch(sample -> sample.getThreadName() != null); + if (useJfr) { + assertThat(logs.getMemorySamples()) + .isNotEmpty() + .allMatch(sample -> sample.getAllocated() > 0) + .allMatch(sample -> sample.getThreadName() != null); + } } @Test @@ -257,6 +273,26 @@ private List findJfrFilesInOutputDir() throws Exception { } private void startPetclinic() { + List command = + new ArrayList<>( + Arrays.asList( + "-javaagent:/" + TestContainerManager.TARGET_AGENT_FILENAME, + "-Dotel.resource.attributes=service.name=smoketest,deployment.environment=smokeytown", + "-Dotel.javaagent.debug=true", + "-Dsplunk.profiler.enabled=true", + "-Dsplunk.profiler.tlab.enabled=true", + "-Dsplunk.profiler.directory=/app/jfr", + "-Dsplunk.profiler.keep-files=true", + "-Dsplunk.profiler.call.stack.interval=1001", + "-Dsplunk.profiler.logs-endpoint=http://collector:4317" + // uncomment to enable exporting traces + // "-Dotel.exporter.otlp.endpoint=http://collector:4317" + )); + if (!useJfr) { + command.add("-Dsplunk.profiler.jfr=false"); + } + command.addAll(Arrays.asList("-jar", "/app/spring-petclinic-rest.jar")); + containerManager.startTarget( new TargetContainerBuilder(getPetclinicImageName()) .withTargetPort(PETCLINIC_PORT) @@ -269,20 +305,7 @@ private void startPetclinic() { .withWaitStrategy( new TargetWaitStrategy.Http(Duration.ofMinutes(5), "/petclinic/api/vets")) .withUseDefaultAgentConfiguration(false) - .withCommand( - "-javaagent:/" + TestContainerManager.TARGET_AGENT_FILENAME, - "-Dotel.resource.attributes=service.name=smoketest,deployment.environment=smokeytown", - "-Dotel.javaagent.debug=true", - "-Dsplunk.profiler.enabled=true", - "-Dsplunk.profiler.tlab.enabled=true", - "-Dsplunk.profiler.directory=/app/jfr", - "-Dsplunk.profiler.keep-files=true", - "-Dsplunk.profiler.call.stack.interval=1001", - "-Dsplunk.profiler.logs-endpoint=http://collector:4317", - // uncomment to enable exporting traces - // "-Dotel.exporter.otlp.endpoint=http://collector:4317", - "-jar", - "/app/spring-petclinic-rest.jar")); + .withCommand(command)); logger.info("Petclinic has been started."); diff --git a/testing/jmh-benchmarks/build.gradle.kts b/testing/jmh-benchmarks/build.gradle.kts index 65032b4e6..8320346da 100644 --- a/testing/jmh-benchmarks/build.gradle.kts +++ b/testing/jmh-benchmarks/build.gradle.kts @@ -9,6 +9,7 @@ repositories { dependencies { implementation(project(":profiler")) + implementation("io.opentelemetry:opentelemetry-api") testImplementation("org.junit.jupiter:junit-jupiter-api") testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine") } diff --git a/testing/jmh-benchmarks/src/jmh/java/com/splunk/opentelemetry/profiler/ThreadDumpProcessorBenchmark.java b/testing/jmh-benchmarks/src/jmh/java/com/splunk/opentelemetry/profiler/ThreadDumpProcessorBenchmark.java index 1217bb3f0..0888bd1c1 100644 --- a/testing/jmh-benchmarks/src/jmh/java/com/splunk/opentelemetry/profiler/ThreadDumpProcessorBenchmark.java +++ b/testing/jmh-benchmarks/src/jmh/java/com/splunk/opentelemetry/profiler/ThreadDumpProcessorBenchmark.java @@ -20,8 +20,10 @@ import com.splunk.opentelemetry.profiler.context.StackToSpanLinkage; import com.splunk.opentelemetry.profiler.exporter.CpuEventExporter; import com.splunk.opentelemetry.profiler.old.AgentInternalsFilter; +import io.opentelemetry.api.trace.SpanContext; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; @@ -63,7 +65,18 @@ private void ensureJfrFileExists() throws Exception { private static ThreadDumpProcessor buildNewThreadDumpProcessor() { SpanContextualizer contextualizer = new SpanContextualizer(new EventReader()); - CpuEventExporter cpuEventExporter = x -> {}; + CpuEventExporter cpuEventExporter = + new CpuEventExporter() { + @Override + public void export( + Thread thread, + StackTraceElement[] stackTrace, + Instant eventTime, + SpanContext spanContext) {} + + @Override + public void export(StackToSpanLinkage stackToSpanLinkage) {} + }; return ThreadDumpProcessor.builder() .cpuEventExporter(cpuEventExporter) .spanContextualizer(contextualizer)