From 0374059a0b4a693c5c3993c539318d5557a6051f Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Wed, 7 Jan 2026 10:28:16 +0100 Subject: [PATCH 1/4] Allow running wallclock profiler even without tracer Fix the wallclock context filter logic to allow collecting wallclock profiles even with the tracer being disabled and not providing the tracing context. --- .../ddprof/DatadogProfilerConfig.java | 18 ++ .../JFRBasedProfilingIntegrationTest.java | 163 +++++++++++++----- 2 files changed, 142 insertions(+), 39 deletions(-) diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java index d023d13b219..f9b980f486e 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java @@ -142,7 +142,25 @@ public static boolean getWallCollapsing(ConfigProvider configProvider) { PROFILING_DATADOG_PROFILER_WALL_COLLAPSING_DEFAULT); } + /** + * Checks whether the wall-clock context filter should be used.
+ * The context filter will make wall-clock profiler to pick candidate threads only from threads + * with attached tracing context.
+ * If context filter is not used (this method returns {@literal false}) all threads will be + * considered for wallclock sampling. + * + * @param configProvider the associated config provider + * @return {@literal true} if the wallclock sampler should use context filtering, {@literal false} + * otherwise + */ public static boolean getWallContextFilter(ConfigProvider configProvider) { + // Context filtering requires tracing to be enabled - without tracing, + // there are no span contexts to filter on, so threads would never be added + // to the filter, resulting in no walltime samples. + boolean isTracingEnabled = configProvider.getBoolean(TRACE_ENABLED, true); + if (!isTracingEnabled) { + return false; + } return getBoolean( configProvider, PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER, diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java index 5030247339d..deb8e3af834 100644 --- a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java @@ -491,6 +491,80 @@ void testBogusApiKey(final TestInfo testInfo) throws Exception { 3); } + @Test + @DisplayName("Test wallclock profiling without tracing") + public void testWallclockProfilingWithoutTracing(final TestInfo testInfo) throws Exception { + Assumptions.assumeTrue(OperatingSystem.isLinux()); + testWithRetry( + () -> { + try { + targetProcess = + createProcessBuilder( + profilingServer.getPort(), + tracingServer.getPort(), + VALID_API_KEY, + 0, + PROFILING_START_DELAY_SECONDS, + PROFILING_UPLOAD_PERIOD_SECONDS, + false, + true, + "on", + 0, + logFilePath, + false) + .start(); + + Assumptions.assumeFalse(JavaVirtualMachine.isJ9()); + + final RecordedRequest request = retrieveRequest(); + assertNotNull(request); + + final List items = + FileUpload.parse( + request.getBody().readByteArray(), request.getHeader("Content-Type")); + + FileItem rawJfr = items.get(1); + assertEquals("main.jfr", rawJfr.getName()); + + assertFalse(logHasErrors(logFilePath)); + InputStream eventStream = new ByteArrayInputStream(rawJfr.get()); + eventStream = decompressStream("on", eventStream); + IItemCollection events = JfrLoaderToolkit.loadEvents(eventStream); + assertTrue(events.hasItems()); + + IItemCollection wallclockSamples = + events.apply(ItemFilters.type("datadog.MethodSample")); + assertTrue( + wallclockSamples.hasItems(), "Expected wallclock samples when tracing is disabled"); + + // Verify span context is not present + for (IItemIterable event : wallclockSamples) { + IMemberAccessor rootSpanIdAccessor = + LOCAL_ROOT_SPAN_ID.getAccessor(event.getType()); + IMemberAccessor spanIdAccessor = + SPAN_ID.getAccessor(event.getType()); + for (IItem sample : event) { + assertEquals( + 0, + rootSpanIdAccessor.getMember(sample).longValue(), + "rootSpanId should be 0 when tracing is disabled"); + assertEquals( + 0, + spanIdAccessor.getMember(sample).longValue(), + "spanId should be 0 when tracing is disabled"); + } + } + } finally { + if (targetProcess != null) { + targetProcess.destroyForcibly(); + } + targetProcess = null; + } + }, + testInfo, + 3); + } + @Test @DisplayName("Test shutdown") @Disabled("https://github.com/DataDog/dd-trace-java/pull/5213") @@ -771,7 +845,8 @@ private ProcessBuilder createProcessBuilder( asyncProfilerEnabled, withCompression, exitDelay, - logFilePath); + logFilePath, + true); } private static ProcessBuilder createProcessBuilder( @@ -785,50 +860,60 @@ private static ProcessBuilder createProcessBuilder( final boolean asyncProfilerEnabled, final String withCompression, final int exitDelay, - final Path logFilePath) { + final Path logFilePath, + final boolean tracingEnabled) { final String templateOverride = JFRBasedProfilingIntegrationTest.class .getClassLoader() .getResource("overrides.jfp") .getFile(); - final List command = - Arrays.asList( - javaPath(), - "-Xmx" + System.getProperty("datadog.forkedMaxHeapSize", "1024M"), - "-Xms" + System.getProperty("datadog.forkedMinHeapSize", "64M"), - "-javaagent:" + agentShadowJar(), - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Ddd.trace.agent.port=" + tracerPort, - "-Ddd.service.name=smoke-test-java-app", - "-Ddd.env=smoketest", - "-Ddd.version=99", - "-Ddd.profiling.enabled=true", - "-Ddd.profiling.stackdepth=" + STACK_DEPTH_LIMIT, - "-Ddd.profiling.ddprof.enabled=" + asyncProfilerEnabled, - "-Ddd.profiling.ddprof.alloc.enabled=" + asyncProfilerEnabled, - "-Ddd.profiling.agentless=" + (apiKey != null), - "-Ddd.profiling.start-delay=" + profilingStartDelaySecs, - "-Ddd.profiling.upload.period=" + profilingUploadPeriodSecs, - "-Ddd.profiling.url=http://localhost:" + profilerPort, - "-Ddd.profiling.hotspots.enabled=true", - "-Ddd.profiling.endpoint.collection.enabled=" + endpointCollectionEnabled, - "-Ddd.profiling.upload.timeout=" + PROFILING_UPLOAD_TIMEOUT_SECONDS, - "-Ddd.profiling.debug.dump_path=/tmp/dd-profiler", - "-Ddd.profiling.queueing.time.enabled=true", - "-Ddd.profiling.queueing.time.threshold.millis=0", - "-Ddd.profiling.debug.upload.compression=" + withCompression, - "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug", - "-Ddd.profiling.context.attributes=foo,bar", - "-Dorg.slf4j.simpleLogger.defaultLogLevel=debug", - "-XX:+IgnoreUnrecognizedVMOptions", - "-XX:+UnlockCommercialFeatures", - "-XX:+FlightRecorder", - "-Ddd." + ProfilingConfig.PROFILING_TEMPLATE_OVERRIDE_FILE + "=" + templateOverride, - "-Ddd.jmxfetch.start-delay=" + jmxFetchDelaySecs, - "-jar", - profilingShadowJar(), - Integer.toString(exitDelay)); + final List command = new java.util.ArrayList<>(); + command.add(javaPath()); + command.add("-Xmx" + System.getProperty("datadog.forkedMaxHeapSize", "1024M")); + command.add("-Xms" + System.getProperty("datadog.forkedMinHeapSize", "64M")); + command.add("-javaagent:" + agentShadowJar()); + command.add("-XX:ErrorFile=/tmp/hs_err_pid%p.log"); + command.add("-Ddd.trace.agent.port=" + tracerPort); + if (!tracingEnabled) { + command.add("-Ddd.trace.enabled=false"); + } + command.add("-Ddd.service.name=smoke-test-java-app"); + command.add("-Ddd.env=smoketest"); + command.add("-Ddd.version=99"); + command.add("-Ddd.profiling.enabled=true"); + command.add("-Ddd.profiling.stackdepth=" + STACK_DEPTH_LIMIT); + command.add("-Ddd.profiling.ddprof.enabled=" + asyncProfilerEnabled); + command.add("-Ddd.profiling.ddprof.alloc.enabled=" + asyncProfilerEnabled); + if (!tracingEnabled && asyncProfilerEnabled) { + command.add("-Ddd.profiling.ddprof.wall.enabled=true"); + } + command.add("-Ddd.profiling.agentless=" + (apiKey != null)); + command.add("-Ddd.profiling.start-delay=" + profilingStartDelaySecs); + command.add("-Ddd.profiling.upload.period=" + profilingUploadPeriodSecs); + command.add("-Ddd.profiling.url=http://localhost:" + profilerPort); + command.add("-Ddd.profiling.hotspots.enabled=true"); + command.add("-Ddd.profiling.endpoint.collection.enabled=" + endpointCollectionEnabled); + command.add("-Ddd.profiling.upload.timeout=" + PROFILING_UPLOAD_TIMEOUT_SECONDS); + command.add("-Ddd.profiling.debug.dump_path=/tmp/dd-profiler"); + if (tracingEnabled) { + command.add("-Ddd.profiling.queueing.time.enabled=true"); + command.add("-Ddd.profiling.queueing.time.threshold.millis=0"); + command.add("-Ddd.profiling.context.attributes=foo,bar"); + } + command.add("-Ddd.profiling.debug.upload.compression=" + withCompression); + command.add("-Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug"); + command.add("-Dorg.slf4j.simpleLogger.defaultLogLevel=debug"); + command.add("-XX:+IgnoreUnrecognizedVMOptions"); + command.add("-XX:+UnlockCommercialFeatures"); + command.add("-XX:+FlightRecorder"); + command.add( + "-Ddd." + ProfilingConfig.PROFILING_TEMPLATE_OVERRIDE_FILE + "=" + templateOverride); + command.add("-Ddd.jmxfetch.start-delay=" + jmxFetchDelaySecs); + command.add("-jar"); + command.add(profilingShadowJar()); + command.add(Integer.toString(exitDelay)); + final ProcessBuilder processBuilder = new ProcessBuilder(command); processBuilder.directory(new File(buildDirectory())); From 1dc1321f585f410d386bb718febc55aef6fb1043 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 13 Jan 2026 12:10:33 +0100 Subject: [PATCH 2/4] Simplify changes --- .../datadog/smoketest/JFRBasedProfilingIntegrationTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java index deb8e3af834..7a123aea9d4 100644 --- a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java @@ -875,12 +875,10 @@ private static ProcessBuilder createProcessBuilder( command.add("-javaagent:" + agentShadowJar()); command.add("-XX:ErrorFile=/tmp/hs_err_pid%p.log"); command.add("-Ddd.trace.agent.port=" + tracerPort); - if (!tracingEnabled) { - command.add("-Ddd.trace.enabled=false"); - } command.add("-Ddd.service.name=smoke-test-java-app"); command.add("-Ddd.env=smoketest"); command.add("-Ddd.version=99"); + command.add("-Ddd.trace.enabled=" + tracingEnabled); command.add("-Ddd.profiling.enabled=true"); command.add("-Ddd.profiling.stackdepth=" + STACK_DEPTH_LIMIT); command.add("-Ddd.profiling.ddprof.enabled=" + asyncProfilerEnabled); @@ -913,7 +911,6 @@ private static ProcessBuilder createProcessBuilder( command.add("-jar"); command.add(profilingShadowJar()); command.add(Integer.toString(exitDelay)); - final ProcessBuilder processBuilder = new ProcessBuilder(command); processBuilder.directory(new File(buildDirectory())); From 0e96483939dddd3ab74e482c368a4e6afd7c5608 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 30 Jan 2026 14:34:12 +0100 Subject: [PATCH 3/4] Bump ddprof to 1.37.0 --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b79f12b6f21..a4ee463ff89 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -7,7 +7,7 @@ forbiddenapis = "3.10" spotbugs_annotations = "4.9.8" # DataDog libs and forks -ddprof = "1.36.1" +ddprof = "1.37.0" dogstatsd = "4.4.3" okhttp = "3.12.15" # Datadog fork to support Java 7 From cba4efbac2b0a85d84879ddb215ac96993cd7f5e Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 30 Jan 2026 16:56:20 +0100 Subject: [PATCH 4/4] Fix wallclock profiling without tracing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Explicitly pass filter= to ddprof when context filtering is disabled. Due to ddprof quirk, omitting filter defaults to "0" (enabled), causing zero samples when tracing is off. Empty filter disables it properly. Added unit test to verify filter parameter in all scenarios. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .../profiling/ddprof/DatadogProfiler.java | 6 ++ .../profiling/ddprof/DatadogProfilerTest.java | 58 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java index 0c889108d2d..4ef2c59bfba 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java @@ -304,8 +304,14 @@ String cmdStartProfiling(Path file) throws IllegalStateException { cmd.append('~'); } cmd.append(getWallInterval(configProvider)).append('m'); + // ddprof quirk: if filter parameter is omitted, it defaults to "0" (enabled), + // not empty string (disabled). When enabled without tracing, no threads are added + // to the filter, resulting in zero samples. We must explicitly pass filter= (empty) + // to disable filtering and sample all threads. if (getWallContextFilter(configProvider)) { cmd.append(",filter=0"); + } else { + cmd.append(",filter="); } if (useJvmtiWallclockSampler(configProvider)) { cmd.append(",wallsampler=jvmti"); diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java index c14783629c2..a7b07a8a02e 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java @@ -11,6 +11,7 @@ import datadog.environment.OperatingSystem; import datadog.libs.ddprof.DdprofLibraryLoader; import datadog.trace.api.config.ProfilingConfig; +import datadog.trace.api.config.TraceInstrumentationConfig; import datadog.trace.api.profiling.ProfilingScope; import datadog.trace.api.profiling.RecordingData; import datadog.trace.bootstrap.config.provider.ConfigProvider; @@ -106,6 +107,63 @@ private static Stream profilingModes() { Arguments.of((x & 0x1000) != 0, (x & 0x100) != 0, (x & 0x10) != 0, (x & 0x1) != 0)); } + @ParameterizedTest + @MethodSource("wallContextFilterModes") + void testWallContextFilter(boolean tracingEnabled, boolean contextFilterEnabled) + throws Exception { + // Skip test if profiler native library is not available (e.g., on macOS) + try { + Throwable reason = DdprofLibraryLoader.jvmAccess().getReasonNotLoaded(); + if (reason != null) { + Assumptions.assumeTrue(false, "Profiler not available: " + reason.getMessage()); + } + } catch (Throwable e) { + Assumptions.assumeTrue(false, "Profiler not available: " + e.getMessage()); + } + + Properties props = new Properties(); + props.put(ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_ENABLED, "true"); + props.put(TraceInstrumentationConfig.TRACE_ENABLED, Boolean.toString(tracingEnabled)); + props.put( + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER, + Boolean.toString(contextFilterEnabled)); + + DatadogProfiler profiler = + DatadogProfiler.newInstance(ConfigProvider.withPropertiesOverride(props)); + + Path targetFile = Paths.get("/tmp/target.jfr"); + String cmd = profiler.cmdStartProfiling(targetFile); + + assertTrue(cmd.contains("wall="), "Command should contain wall profiling: " + cmd); + + if (tracingEnabled && contextFilterEnabled) { + assertTrue( + cmd.contains(",filter=0"), + "Command should contain ',filter=0' when tracing and context filter are enabled: " + cmd); + } else { + assertTrue( + cmd.contains(",filter="), + "Command should contain ',filter=' when tracing is disabled or context filter is disabled: " + + cmd); + if (cmd.contains(",filter=0")) { + throw new AssertionError( + "Command should not contain ',filter=0' when tracing is disabled or context filter is disabled: " + + cmd); + } + } + } + + private static Stream wallContextFilterModes() { + return Stream.of( + Arguments.of(true, true), // tracing enabled, context filter enabled -> filter=0 + Arguments.of(true, false), // tracing enabled, context filter disabled -> filter= + Arguments.of( + false, true), // tracing disabled, context filter enabled -> filter= (tracing disabled + // overrides) + Arguments.of(false, false) // tracing disabled, context filter disabled -> filter= + ); + } + @Test public void testContextRegistration() { // warning - the profiler is a process wide singleton and can't be reinitialised