From e7c76c0bc9d1cc1fc1b6b9727aa0250960105a69 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 15 Oct 2026 17:05:56 -0400 Subject: [PATCH 01/30] Log AAS agent supervisor lifecycle at the default level In Azure App Service the tracer starts and supervises the trace agent and DogStatsD. Overlapping worker processes during platform restarts can leave those agents missing or shared, but starts, exits, and stops were only logged at debug level, so default customer logs could not show what happened. - Log each agent start, exit (with exit code and run time), and stop at info, with the tracer PID and, on Java 9+, the agent PID. - Log when an already-running agent not started by this process is used, and when that agent goes away. - After repeated quick failures, move lifecycle lines to debug until the agent stays up, so a crash loop does not flood the log. - Warn when shutdown leaves an agent running. --- .../trace/util/JDK9ProcessPidFunction.java | 15 +++ .../java/datadog/trace/util/PidHelper.java | 26 +++++ .../datadog/trace/util/ProcessSupervisor.java | 107 +++++++++++++++++- 3 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 internal-api/internal-api-9/src/main/java/datadog/trace/util/JDK9ProcessPidFunction.java diff --git a/internal-api/internal-api-9/src/main/java/datadog/trace/util/JDK9ProcessPidFunction.java b/internal-api/internal-api-9/src/main/java/datadog/trace/util/JDK9ProcessPidFunction.java new file mode 100644 index 00000000000..047174b3018 --- /dev/null +++ b/internal-api/internal-api-9/src/main/java/datadog/trace/util/JDK9ProcessPidFunction.java @@ -0,0 +1,15 @@ +package datadog.trace.util; + +import java.util.function.Function; + +/** Use standard API to retrieve the PID of a child process on Java9+. */ +public final class JDK9ProcessPidFunction implements Function { + @Override + public String apply(Process process) { + try { + return Long.toString(process.pid()); + } catch (Throwable e) { + return ""; + } + } +} diff --git a/internal-api/src/main/java/datadog/trace/util/PidHelper.java b/internal-api/src/main/java/datadog/trace/util/PidHelper.java index d69d5726b9f..63a008b8185 100644 --- a/internal-api/src/main/java/datadog/trace/util/PidHelper.java +++ b/internal-api/src/main/java/datadog/trace/util/PidHelper.java @@ -13,6 +13,7 @@ import java.util.Collections; import java.util.Objects; import java.util.Set; +import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -38,6 +39,31 @@ public static long getPidAsLong() { return PID_AS_LONG; } + private static final Function CHILD_PID = findChildPidFunction(); + + /** Returns the PID of a child process, or an empty string when the JVM cannot provide it. */ + public static String getPid(Process process) { + if (null == process || null == CHILD_PID) { + return ""; + } + return CHILD_PID.apply(process); + } + + @SuppressWarnings("unchecked") + private static Function findChildPidFunction() { + if (JavaVirtualMachine.isJavaVersionAtLeast(9)) { + try { + return (Function) + Class.forName("datadog.trace.util.JDK9ProcessPidFunction") + .getDeclaredConstructor() + .newInstance(); + } catch (Throwable e) { + log.debug("JDK9ProcessPidFunction not available", e); + } + } + return null; // Java 8 has no public API for the PID of a child process + } + @SuppressWarnings("unchecked") private static String findPid() { String pid = ""; diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index b232ddc156c..84a20159169 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -6,6 +6,7 @@ import static datadog.trace.util.ProcessSupervisor.Health.HEALTHY; import static datadog.trace.util.ProcessSupervisor.Health.INTERRUPTED; import static datadog.trace.util.ProcessSupervisor.Health.READY_TO_START; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import datadog.trace.api.internal.VisibleForTesting; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; @@ -38,6 +39,14 @@ public interface HealthCheck { private static final long FAULTED_DELAY_MILLIS = 2_000; private static final int MAX_FAULTS = 5; + // Lifecycle events are logged at info level. A process that keeps failing quickly would flood + // the log, so after this many failures in a row the details go to debug level until the + // process stays up for STABLE_RUN_MILLIS. + private static final int QUIET_AFTER_FAILURES = 5; + private static final long STABLE_RUN_MILLIS = 60_000; + + private static final long STOP_WAIT_MILLIS = 200; + private final String imageName; private final ProcessBuilder processBuilder; private final HealthCheck healthCheck; @@ -45,8 +54,10 @@ public interface HealthCheck { private long nextCheckMillis = 0; private Health currentHealth = Health.NEVER_CHECKED; + private Health lastCheckResult = Health.NEVER_CHECKED; private Process currentProcess; private int faults; + private int failuresInARow; private volatile boolean stopping = false; @@ -80,6 +91,7 @@ private void mainLoop() { Thread.sleep(delayMillis); } currentHealth = healthCheck.run(currentHealth); + logHealthCheckChange(currentHealth); if (currentHealth == READY_TO_START) { startProcessAndWait(); } @@ -96,6 +108,28 @@ private void mainLoop() { } } + private void logHealthCheckChange(Health result) { + Health previous = lastCheckResult; + lastCheckResult = result; + if (result == previous) { + return; + } + if (result == HEALTHY) { + // Health checks only run while this supervisor has no running process of its own, + // so a healthy result means another process (usually another worker) is serving it. + failuresInARow = 0; + log.info( + "Process [{}] is already running and was not started by this process (pid {}); using it", + imageName, + PidHelper.getPid()); + } else if (result == READY_TO_START && previous == HEALTHY) { + logLifecycle( + "Process [{}] that this process (pid {}) was using is no longer available", + imageName, + PidHelper.getPid()); + } + } + private void scheduleNextHealthCheck() { long now = System.currentTimeMillis(); if (currentHealth == HEALTHY) { @@ -115,28 +149,86 @@ private void startProcessAndWait() throws Exception { } currentHealth = HEALTHY; faults = 0; + logLifecycle( + "Started process [{}] with pid {} from this process (pid {})", + imageName, + pidOrUnknown(currentProcess), + PidHelper.getPid()); } + long startedMillis = System.currentTimeMillis(); + String childPid = pidOrUnknown(currentProcess); + // Block until the process exits int code = currentProcess.waitFor(); - log.debug("Process [{}] has exited with code {}", imageName, code); currentHealth = code == 0 ? INTERRUPTED : FAULTED; + long ranMillis = System.currentTimeMillis() - startedMillis; + if (ranMillis >= STABLE_RUN_MILLIS) { + failuresInARow = 0; + } + logLifecycle( + "Process [{}] with pid {} exited with code {} after {} ms", + imageName, + childPid, + code, + ranMillis); + if (code != 0 && ++failuresInARow == QUIET_AFTER_FAILURES) { + log.warn( + "Process [{}] has exited with an error {} times in a row; " + + "further starts and exits are logged at debug level until it stays up for {} seconds", + imageName, + failuresInARow, + MILLISECONDS.toSeconds(STABLE_RUN_MILLIS)); + } + // Process is dead, no longer needs to be tracked currentProcess = null; } private void stopProcess() { if (currentProcess != null) { - log.debug("Stopping process: [{}]", imageName); - currentProcess.destroy(); - if (currentProcess.isAlive()) { + String childPid = pidOrUnknown(currentProcess); + log.info("Stopping process [{}] with pid {}", imageName, childPid); + boolean stopped = false; + try { + currentProcess.destroy(); + stopped = currentProcess.waitFor(STOP_WAIT_MILLIS, MILLISECONDS); + if (!stopped) { + currentProcess.destroyForcibly(); + stopped = currentProcess.waitFor(STOP_WAIT_MILLIS, MILLISECONDS); + } + } catch (InterruptedException e) { currentProcess.destroyForcibly(); + Thread.currentThread().interrupt(); + } + if (stopped) { + log.info("Stopped process [{}] with pid {}", imageName, childPid); + } else { + log.warn( + "Process [{}] with pid {} was still running after the stop request", + imageName, + childPid); } currentProcess = null; + } else { + log.debug("No process [{}] started by this process to stop", imageName); } } + private void logLifecycle(String format, Object... arguments) { + if (failuresInARow < QUIET_AFTER_FAILURES) { + log.info(format, arguments); + } else { + log.debug(format, arguments); + } + } + + private static String pidOrUnknown(Process process) { + String pid = PidHelper.getPid(process); + return pid.isEmpty() ? "unknown" : pid; + } + @Override public void close() { stopping = true; @@ -145,6 +237,13 @@ public void close() { supervisorThread.join(THREAD_JOIN_TIMOUT_MS); } catch (Throwable ignored) { } + if (supervisorThread.isAlive()) { + log.warn( + "Supervisor for process [{}] did not finish stopping within {} ms; " + + "the process may be left running", + imageName, + THREAD_JOIN_TIMOUT_MS); + } } @VisibleForTesting From 548da5045c3efcc2586817c24118c69349c239d7 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 15 Oct 2026 17:06:35 -0400 Subject: [PATCH 02/30] Back off AAS agent restarts instead of giving up ProcessSupervisor reset its fault count on every successful start, so an agent that kept exiting was restarted every 2 seconds forever, while a start that kept throwing gave up for good after 5 attempts. - Forget faults only after the process stays up for 60 seconds. - Back off after each fault, doubling from 2 seconds to a cap of 60 seconds, and never stop retrying. - Move repeated start exceptions to debug after the first few, so a permanently failing start does not log a stack trace every minute. --- .../datadog/trace/util/ProcessSupervisor.java | 43 +++++++++++++++---- .../trace/util/ProcessSupervisorTest.java | 31 +++++++++++++ 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index 84a20159169..968277782fa 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -37,7 +37,7 @@ public interface HealthCheck { private static final long HEALTHY_DELAY_MILLIS = 10_000; private static final long FAULTED_DELAY_MILLIS = 2_000; - private static final int MAX_FAULTS = 5; + private static final long MAX_FAULTED_DELAY_MILLIS = 60_000; // Lifecycle events are logged at info level. A process that keeps failing quickly would flood // the log, so after this many failures in a row the details go to debug level until the @@ -56,7 +56,7 @@ public interface HealthCheck { private Health currentHealth = Health.NEVER_CHECKED; private Health lastCheckResult = Health.NEVER_CHECKED; private Process currentProcess; - private int faults; + private final FaultBackoff faultBackoff = new FaultBackoff(); private int failuresInARow; private volatile boolean stopping = false; @@ -81,10 +81,6 @@ public ProcessSupervisor( private void mainLoop() { try { while (!stopping) { - if (currentHealth == FAULTED && ++faults >= MAX_FAULTS) { - log.warn("Failed to start process [{}] after {} attempts", imageName, faults); - break; - } try { long delayMillis = nextCheckMillis - System.currentTimeMillis(); if (delayMillis > 0) { @@ -98,7 +94,11 @@ private void mainLoop() { } catch (InterruptedException e) { currentHealth = INTERRUPTED; } catch (Throwable e) { - log.warn("Exception starting process: [{}]", imageName, e); + if (failuresInARow++ < QUIET_AFTER_FAILURES) { + log.warn("Exception starting process: [{}]", imageName, e); + } else { + log.debug("Exception starting process: [{}]", imageName, e); + } currentHealth = FAULTED; } scheduleNextHealthCheck(); @@ -135,7 +135,7 @@ private void scheduleNextHealthCheck() { if (currentHealth == HEALTHY) { nextCheckMillis = now + HEALTHY_DELAY_MILLIS; } else if (currentHealth == FAULTED) { - nextCheckMillis = now + FAULTED_DELAY_MILLIS; + nextCheckMillis = now + faultBackoff.recordFault(); } else { // interrupted nextCheckMillis = Long.max(nextCheckMillis, now + 100); } @@ -148,7 +148,6 @@ private void startProcessAndWait() throws Exception { currentProcess = processBuilder.start(); } currentHealth = HEALTHY; - faults = 0; logLifecycle( "Started process [{}] with pid {} from this process (pid {})", imageName, @@ -164,6 +163,7 @@ private void startProcessAndWait() throws Exception { currentHealth = code == 0 ? INTERRUPTED : FAULTED; long ranMillis = System.currentTimeMillis() - startedMillis; + faultBackoff.recordUptime(ranMillis); if (ranMillis >= STABLE_RUN_MILLIS) { failuresInARow = 0; } @@ -246,6 +246,31 @@ public void close() { } } + /** + * Counts faults in a row and spaces out the retries that follow them. Faults are forgotten only + * once a process has stayed up for {@link #STABLE_RUN_MILLIS}, so an agent that keeps exiting + * soon after it starts backs off instead of restarting every few seconds. Retries never stop. + */ + static final class FaultBackoff { + private static final int MAX_DOUBLINGS = 5; + + private int faults; + + /** Records a fault and returns how long to wait before the next health check. */ + long recordFault() { + faults++; + int doublings = Math.min(faults - 1, MAX_DOUBLINGS); + return Math.min(FAULTED_DELAY_MILLIS << doublings, MAX_FAULTED_DELAY_MILLIS); + } + + /** Forgets earlier faults once a process has been up for a stable period. */ + void recordUptime(long upMillis) { + if (upMillis >= STABLE_RUN_MILLIS) { + faults = 0; + } + } + } + @VisibleForTesting Process getCurrentProcess() { return currentProcess; diff --git a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java index ff6f6e7143d..06ad1af2355 100644 --- a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java +++ b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java @@ -1,5 +1,6 @@ package datadog.trace.util; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -50,6 +51,36 @@ void processKilledWhenSupervisorClosed() throws InterruptedException { }); } + @Test + void faultBackoffDoublesUpToSixtySecondsAndNeverStops() { + ProcessSupervisor.FaultBackoff backoff = new ProcessSupervisor.FaultBackoff(); + + assertEquals(2_000, backoff.recordFault()); + assertEquals(4_000, backoff.recordFault()); + assertEquals(8_000, backoff.recordFault()); + assertEquals(16_000, backoff.recordFault()); + assertEquals(32_000, backoff.recordFault()); + assertEquals(60_000, backoff.recordFault()); + for (int i = 0; i < 1_000; i++) { + assertEquals(60_000, backoff.recordFault()); + } + } + + @Test + void faultBackoffResetsOnlyAfterStableUptime() { + ProcessSupervisor.FaultBackoff backoff = new ProcessSupervisor.FaultBackoff(); + backoff.recordFault(); + backoff.recordFault(); + + // a start that exits again quickly keeps the earlier faults + backoff.recordUptime(59_999); + assertEquals(8_000, backoff.recordFault()); + + // staying up for the stable period forgets them + backoff.recordUptime(60_000); + assertEquals(2_000, backoff.recordFault()); + } + @Test void processRespawnsWhenKilled() throws InterruptedException { ProcessBuilder processBuilder = createProcessBuilder(); From 949028c7d49f334f19a5eeed172a863c4faac530 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 15 Oct 2026 17:06:36 -0400 Subject: [PATCH 03/30] Health check and restart the AAS agent this tracer started Once ProcessSupervisor started an agent it blocked in waitFor() until the agent exited, so an agent that hung but stayed alive was never checked or replaced. - While its own agent runs, wait in keep-alive steps of 10 seconds and run the health check after each one. - After 3 failed health checks in a row, stop that agent and go back through the normal start path, after the fault backoff. Log the first failed check and the restart once. - Keep tracking an agent that is still running after a stop request, so a restart never starts a second copy next to it. - Supervisors without a health check keep only waiting for the exit. --- .../datadog/trace/util/ProcessSupervisor.java | 81 ++++++++++++++++--- 1 file changed, 70 insertions(+), 11 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index 968277782fa..5918e1d4958 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -47,6 +47,9 @@ public interface HealthCheck { private static final long STOP_WAIT_MILLIS = 200; + // A process this supervisor started is restarted after failing this many health checks in a row + private static final int MAX_FAILED_HEALTH_CHECKS = 3; + private final String imageName; private final ProcessBuilder processBuilder; private final HealthCheck healthCheck; @@ -56,6 +59,8 @@ public interface HealthCheck { private Health currentHealth = Health.NEVER_CHECKED; private Health lastCheckResult = Health.NEVER_CHECKED; private Process currentProcess; + private long currentProcessStartedMillis; + private int failedHealthChecks; private final FaultBackoff faultBackoff = new FaultBackoff(); private int failuresInARow; @@ -86,10 +91,12 @@ private void mainLoop() { if (delayMillis > 0) { Thread.sleep(delayMillis); } - currentHealth = healthCheck.run(currentHealth); - logHealthCheckChange(currentHealth); - if (currentHealth == READY_TO_START) { - startProcessAndWait(); + if (currentProcess == null) { + currentHealth = healthCheck.run(currentHealth); + logHealthCheckChange(currentHealth); + } + if (currentProcess != null || currentHealth == READY_TO_START) { + startProcessAndSupervise(); } } catch (InterruptedException e) { currentHealth = INTERRUPTED; @@ -115,7 +122,7 @@ private void logHealthCheckChange(Health result) { return; } if (result == HEALTHY) { - // Health checks only run while this supervisor has no running process of its own, + // These checks only run while this supervisor has no running process of its own, // so a healthy result means another process (usually another worker) is serving it. failuresInARow = 0; log.info( @@ -141,12 +148,14 @@ private void scheduleNextHealthCheck() { } } - private void startProcessAndWait() throws Exception { + private void startProcessAndSupervise() throws Exception { if (currentProcess == null) { log.debug("Starting process: [{}]", imageName); try (TraceScope ignored = AgentTracer.get().muteTracing()) { currentProcess = processBuilder.start(); } + currentProcessStartedMillis = System.currentTimeMillis(); + failedHealthChecks = 0; currentHealth = HEALTHY; logLifecycle( "Started process [{}] with pid {} from this process (pid {})", @@ -155,14 +164,20 @@ private void startProcessAndWait() throws Exception { PidHelper.getPid()); } - long startedMillis = System.currentTimeMillis(); String childPid = pidOrUnknown(currentProcess); - // Block until the process exits - int code = currentProcess.waitFor(); + // Keep checking on the process until it exits + while (!currentProcess.waitFor(HEALTHY_DELAY_MILLIS, MILLISECONDS)) { + faultBackoff.recordUptime(System.currentTimeMillis() - currentProcessStartedMillis); + if (healthCheck != ALWAYS_READY && !passesHealthCheck(childPid)) { + restartUnhealthyProcess(childPid); + return; + } + } + int code = currentProcess.exitValue(); currentHealth = code == 0 ? INTERRUPTED : FAULTED; - long ranMillis = System.currentTimeMillis() - startedMillis; + long ranMillis = System.currentTimeMillis() - currentProcessStartedMillis; faultBackoff.recordUptime(ranMillis); if (ranMillis >= STABLE_RUN_MILLIS) { failuresInARow = 0; @@ -186,6 +201,49 @@ private void startProcessAndWait() throws Exception { currentProcess = null; } + /** + * Checks on the running process this supervisor started. + * + * @return false once it has failed {@link #MAX_FAILED_HEALTH_CHECKS} health checks in a row + */ + private boolean passesHealthCheck(String childPid) throws InterruptedException { + if (healthCheck.run(HEALTHY) == HEALTHY) { + if (failedHealthChecks > 0) { + logLifecycle( + "Process [{}] with pid {} is passing health checks again", imageName, childPid); + } + failedHealthChecks = 0; + return true; + } + if (++failedHealthChecks == 1) { + logLifecycle( + "Process [{}] with pid {} started by this process (pid {}) failed a health check", + imageName, + childPid, + PidHelper.getPid()); + } + return failedHealthChecks < MAX_FAILED_HEALTH_CHECKS; + } + + private void restartUnhealthyProcess(String childPid) { + if (failuresInARow++ < QUIET_AFTER_FAILURES) { + log.warn( + "Process [{}] with pid {} failed {} health checks in a row; restarting it", + imageName, + childPid, + failedHealthChecks); + } else { + log.debug( + "Process [{}] with pid {} failed {} health checks in a row; restarting it", + imageName, + childPid, + failedHealthChecks); + } + stopProcess(); + // The next health check starts a new process through the normal start path, after a backoff + currentHealth = FAULTED; + } + private void stopProcess() { if (currentProcess != null) { String childPid = pidOrUnknown(currentProcess); @@ -204,13 +262,14 @@ private void stopProcess() { } if (stopped) { log.info("Stopped process [{}] with pid {}", imageName, childPid); + currentProcess = null; } else { + // Keep tracking it, so a restart does not start a second copy while this one runs log.warn( "Process [{}] with pid {} was still running after the stop request", imageName, childPid); } - currentProcess = null; } else { log.debug("No process [{}] started by this process to stop", imageName); } From 3b5b539b2e2c0f47e8c934a24b5e673e140c19de Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 15 Oct 2026 17:06:37 -0400 Subject: [PATCH 04/30] Treat a busy AAS agent pipe as present On Windows, File.exists() can report false for a named pipe whose instances are all busy. The supervisor then took a busy agent for a missing one and started a competing agent, which lost the pipe race and exited. When File.exists() reports false on Windows, open the pipe once and close it again. Only "cannot find the file" means the pipe is absent; "All pipe instances are busy" and other open errors mean a process is serving it. --- .../ddagent/ExternalAgentLauncher.java | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java index 8eea13de224..7cd35fa4aba 100644 --- a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java +++ b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java @@ -10,6 +10,9 @@ import datadog.trace.util.ProcessSupervisor; import java.io.Closeable; import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,7 +94,7 @@ public ProcessSupervisor.Health run(ProcessSupervisor.Health previousHealth) double delayMillis = 50; for (int retries = 0; retries < 7; retries++) { - if (!pipe.exists()) { + if (!pipeExists()) { return READY_TO_START; // no longer bound, start our own external process } @@ -104,11 +107,40 @@ public ProcessSupervisor.Health run(ProcessSupervisor.Health previousHealth) } // otherwise just check that the pipe is still bound - if (pipe.exists()) { + if (pipeExists()) { return HEALTHY; // keep using external process } else { return READY_TO_START; // start our own process } } + + /** + * {@link File#exists()} can report false for a pipe whose instances are all busy, for example + * under load. In that case open the pipe once to tell busy from absent. + */ + private boolean pipeExists() { + if (pipe.exists()) { + return true; + } + if (!OperatingSystem.isWindows()) { + return false; + } + try (RandomAccessFile ignored = new RandomAccessFile(pipe, "rw")) { + return true; + } catch (FileNotFoundException e) { + return !isPipeNotFound(e); + } catch (IOException e) { + return true; // opened, then failed to close + } + } + + /** + * Only "cannot find the file" (ERROR_FILE_NOT_FOUND) means no process is serving the pipe. + * Errors such as "All pipe instances are busy" (ERROR_PIPE_BUSY) mean that one is. + */ + private static boolean isPipeNotFound(FileNotFoundException e) { + String message = e.getMessage(); + return message != null && message.contains("cannot find the file"); + } } } From 85c2e779fd2ac5f4f72e4b5b48f8feb9baf10215 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 15 Oct 2026 17:06:44 -0400 Subject: [PATCH 05/30] Check that the AAS trace agent answers on its pipe The trace agent health check only tested that its pipe existed, so a hung agent that kept its pipe was trusted forever. - After the pipe check passes, send a minimal GET /info over a pipe handle of the check's own, outside the tracer's HTTP clients, with tracing muted. - Treat any HTTP status line as an answer. A busy pipe counts as present. - Pipe reads cannot time out, so send the request on a daemon thread and give up after 2 seconds. While that request is still blocked, later checks fail without sending another one. - DogStatsD never replies, so its check stays "pipe present". --- .../ddagent/ExternalAgentLauncher.java | 93 +++++++++++++++++-- 1 file changed, 86 insertions(+), 7 deletions(-) diff --git a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java index 7cd35fa4aba..e3dfdfff248 100644 --- a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java +++ b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java @@ -1,18 +1,27 @@ package datadog.communication.ddagent; +import static datadog.trace.util.AgentThreadFactory.AgentThread.PROCESS_SUPERVISOR; import static datadog.trace.util.ProcessSupervisor.ALWAYS_READY; import static datadog.trace.util.ProcessSupervisor.Health.HEALTHY; import static datadog.trace.util.ProcessSupervisor.Health.NEVER_CHECKED; import static datadog.trace.util.ProcessSupervisor.Health.READY_TO_START; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import datadog.environment.OperatingSystem; import datadog.trace.api.Config; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.context.TraceScope; +import datadog.trace.util.AgentThreadFactory; import datadog.trace.util.ProcessSupervisor; import java.io.Closeable; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeoutException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,7 +44,7 @@ public ExternalAgentLauncher(Config config) { traceProcessSupervisor = new ProcessSupervisor( - "trace-agent", traceProcessBuilder, healthCheck(config.getAgentNamedPipe())); + "trace-agent", traceProcessBuilder, healthCheck(config.getAgentNamedPipe(), true)); } else { log.warn("Trace agent path not set. Will not start trace agent process"); } @@ -48,7 +57,9 @@ public ExternalAgentLauncher(Config config) { dogStatsDProcessSupervisor = new ProcessSupervisor( - "dogstatsd", dogStatsDProcessBuilder, healthCheck(config.getDogStatsDNamedPipe())); + "dogstatsd", + dogStatsDProcessBuilder, + healthCheck(config.getDogStatsDNamedPipe(), false)); } else { log.warn("DogStatsD path not set. Will not start DogStatsD process"); } @@ -66,18 +77,33 @@ public void close() { } } - private static ProcessSupervisor.HealthCheck healthCheck(String pipeName) { + /** + * @param checkResponse whether the agent answers requests on its pipe. DogStatsD never replies, + * so for it the pipe being present is the whole check. + */ + private static ProcessSupervisor.HealthCheck healthCheck(String pipeName, boolean checkResponse) { return null != pipeName && !pipeName.trim().isEmpty() - ? new NamedPipeHealthCheck(pipeName) + ? new NamedPipeHealthCheck(pipeName, checkResponse) : ALWAYS_READY; } static final class NamedPipeHealthCheck implements ProcessSupervisor.HealthCheck { private static final String NAMED_PIPE_PREFIX = "\\\\.\\pipe\\"; + private static final byte[] INFO_REQUEST = + "GET /info HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" + .getBytes(StandardCharsets.US_ASCII); + private static final int MAX_STATUS_LINE_LENGTH = 64; + private static final long RESPONSE_TIMEOUT_MILLIS = 2_000; + private final File pipe; + private final boolean checkResponse; + + // Only the supervisor thread runs the health check, so this needs no synchronization + private FutureTask pendingInfoRequest; - NamedPipeHealthCheck(String pipeName) { + NamedPipeHealthCheck(String pipeName, boolean checkResponse) { + this.checkResponse = checkResponse; if (pipeName.startsWith(NAMED_PIPE_PREFIX)) { this.pipe = new File(pipeName); } else { @@ -103,17 +129,70 @@ public ProcessSupervisor.Health run(ProcessSupervisor.Health previousHealth) delayMillis = delayMillis * 1.75; } - return HEALTHY; // use existing external process + return respondsOrReadyToStart(); // use existing external process if it responds } // otherwise just check that the pipe is still bound if (pipeExists()) { - return HEALTHY; // keep using external process + return respondsOrReadyToStart(); // keep using external process if it responds } else { return READY_TO_START; // start our own process } } + private ProcessSupervisor.Health respondsOrReadyToStart() throws InterruptedException { + return !checkResponse || agentResponds() ? HEALTHY : READY_TO_START; + } + + /** + * Sends a request on another thread and waits up to {@link #RESPONSE_TIMEOUT_MILLIS} for the + * answer. Reads from a pipe cannot time out, so a request to a hung agent stays blocked until + * the agent exits. Until then later checks fail without sending another request. + */ + private boolean agentResponds() throws InterruptedException { + if (pendingInfoRequest != null && !pendingInfoRequest.isDone()) { + return false; // the last request is still waiting for an answer + } + FutureTask infoRequest = new FutureTask<>(this::sendInfoRequest); + pendingInfoRequest = infoRequest; + AgentThreadFactory.newAgentThread(PROCESS_SUPERVISOR, "-health-check", infoRequest, true) + .start(); + try { + return infoRequest.get(RESPONSE_TIMEOUT_MILLIS, MILLISECONDS); + } catch (ExecutionException | TimeoutException e) { + return false; + } + } + + /** + * Sends a minimal request over a pipe handle of its own, outside the tracer's HTTP clients. + * + * @return true if the agent answers with an HTTP status line, whatever the status + */ + private boolean sendInfoRequest() throws IOException { + try (TraceScope ignored = AgentTracer.get().muteTracing()) { + RandomAccessFile file; + try { + file = new RandomAccessFile(pipe, "rw"); + } catch (FileNotFoundException e) { + // a pipe with all instances busy cannot be asked, but an agent is serving it + return !isPipeNotFound(e); + } + try { + file.write(INFO_REQUEST); + StringBuilder statusLine = new StringBuilder(); + for (int c = file.read(); + c != -1 && c != '\r' && statusLine.length() < MAX_STATUS_LINE_LENGTH; + c = file.read()) { + statusLine.append((char) c); + } + return statusLine.toString().startsWith("HTTP/1."); + } finally { + file.close(); + } + } + } + /** * {@link File#exists()} can report false for a pipe whose instances are all busy, for example * under load. In that case open the pipe once to tell busy from absent. From 6cd7eed3aeba2e33c912d34fa5a9a130115ab5da Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Fri, 16 Oct 2026 12:37:30 -0400 Subject: [PATCH 06/30] Test how ProcessSupervisor supervises the AAS agent The supervisor now health checks its own agent while it runs, restarts it after three failed checks in a row, uses an agent another process started, and never gives up retrying. Only the backoff arithmetic was tested, so coverage of these paths was too low for the build to pass. The new tests start a fake Process from a mocked ProcessBuilder. The fake returns quickly from timed waits, so health checks that normally run ten seconds apart run fast, and each test can choose how the process exits or refuses to stop. They check that the supervisor: - restarts its own process after exactly three failed health checks; - keeps its process when a health check passes before the third failure; - uses a process it did not start, then starts its own once that one goes away (this waits out the real ten second healthy interval); - keeps tracking a process that ignores stop requests instead of starting a second copy; - retries after the process fails to start; - logs the exit code and starts again after the process exits. --- .../trace/util/ProcessSupervisorTest.java | 281 ++++++++++++++++++ 1 file changed, 281 insertions(+) diff --git a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java index 06ad1af2355..7057b1230bb 100644 --- a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java +++ b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java @@ -1,19 +1,61 @@ package datadog.trace.util; +import static datadog.trace.util.ProcessSupervisor.Health.FAULTED; +import static datadog.trace.util.ProcessSupervisor.Health.HEALTHY; +import static datadog.trace.util.ProcessSupervisor.Health.READY_TO_START; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import datadog.trace.test.util.PollingConditions; import datadog.trace.test.util.PortableCommand; +import datadog.trace.util.ProcessSupervisor.HealthCheck; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; // This test looks at the private "currentProcess" variable because the alternative // would be calling "ps -e" repeatedly class ProcessSupervisorTest { + private final Logger supervisorLogger = (Logger) LoggerFactory.getLogger(ProcessSupervisor.class); + private final ListAppender logs = new ListAppender<>(); + + @BeforeEach + void captureLogs() { + logs.start(); + supervisorLogger.addAppender(logs); + } + + @AfterEach + void stopCapturingLogs() { + supervisorLogger.detachAppender(logs); + } + private ProcessBuilder createProcessBuilder() { // Creates a process that never returns on its own return new ProcessBuilder(PortableCommand.runForever()); @@ -122,4 +164,243 @@ void processRespawnsWhenKilled() throws InterruptedException { assertTrue(process == null || !process.isAlive()); }); } + + @Test + void restartsItsOwnProcessAfterThreeFailedHealthChecksInARow() throws IOException { + List started = new CopyOnWriteArrayList<>(); + AtomicInteger checksWhileRunning = new AtomicInteger(); + HealthCheck failsWhileRunning = + previous -> { + if (previous == HEALTHY) { + checksWhileRunning.incrementAndGet(); + return FAULTED; + } + return READY_TO_START; + }; + + try (ProcessSupervisor ignored = + new ProcessSupervisor("test", startsFakeProcesses(started, true), failsWhileRunning)) { + new PollingConditions(5) + .eventually(() -> { + assertEquals(1, started.size()); + assertFalse(started.get(0).isAlive()); + }); + // the replacement only starts after a fault backoff of a few seconds + assertEquals(3, checksWhileRunning.get()); + assertEquals(1, countLogs(Level.INFO, "failed a health check")); + assertEquals(1, countLogs(Level.WARN, "failed 3 health checks in a row; restarting it")); + + // a replacement is started (it keeps failing its health checks, so it is restarted too) + new PollingConditions(5).eventually(() -> assertTrue(started.size() >= 2)); + } + } + + @Test + void keepsItsOwnProcessWhenHealthChecksRecoverBeforeThreeFailures() throws IOException { + List started = new CopyOnWriteArrayList<>(); + AtomicInteger checksWhileRunning = new AtomicInteger(); + // fails two checks in a row, then passes one, over and over + HealthCheck sometimesFailsWhileRunning = + previous -> { + if (previous == HEALTHY) { + return checksWhileRunning.incrementAndGet() % 3 == 0 ? HEALTHY : FAULTED; + } + return READY_TO_START; + }; + + try (ProcessSupervisor ignored = + new ProcessSupervisor( + "test", startsFakeProcesses(started, true), sometimesFailsWhileRunning)) { + new PollingConditions(5).eventually(() -> assertTrue(checksWhileRunning.get() >= 9)); + + assertEquals(1, started.size()); + assertEquals(0, started.get(0).stopRequests.get()); + assertTrue(countLogs(Level.INFO, "is passing health checks again") >= 2); + } + } + + @Test + void usesAProcessItDidNotStartAndStartsItsOwnWhenThatOneGoesAway() throws IOException { + List started = new CopyOnWriteArrayList<>(); + AtomicBoolean otherProcessRunning = new AtomicBoolean(true); + HealthCheck answersWhileAnyProcessRuns = + previous -> otherProcessRunning.get() || !started.isEmpty() ? HEALTHY : READY_TO_START; + + try (ProcessSupervisor ignored = + new ProcessSupervisor( + "test", startsFakeProcesses(started, true), answersWhileAnyProcessRuns)) { + new PollingConditions(5) + .eventually(() -> + assertEquals( + 1, + countLogs(Level.INFO, "is already running and was not started by this process"))); + assertTrue(started.isEmpty()); + + otherProcessRunning.set(false); + + // the next check runs once the ten second healthy interval has passed + new PollingConditions(15) + .eventually(() -> assertEquals(1, countLogs(Level.INFO, "Started process [test]"))); + assertEquals(1, started.size()); + assertEquals(1, countLogs(Level.INFO, "was using is no longer available")); + } + } + + @Test + void keepsTrackingItsProcessWhenItDoesNotStopInsteadOfStartingASecondCopy() throws IOException { + List started = new CopyOnWriteArrayList<>(); + AtomicInteger checksWhileRunning = new AtomicInteger(); + HealthCheck failsWhileRunning = + previous -> { + if (previous == HEALTHY) { + checksWhileRunning.incrementAndGet(); + return FAULTED; + } + return READY_TO_START; + }; + + try (ProcessSupervisor ignored = + new ProcessSupervisor("test", startsFakeProcesses(started, false), failsWhileRunning)) { + // after the failed restart and a fault backoff, it supervises the same process again + new PollingConditions(10).eventually(() -> assertTrue(checksWhileRunning.get() > 3)); + + assertEquals(1, started.size()); + assertTrue(started.get(0).isAlive()); + assertTrue(started.get(0).stopRequests.get() >= 2, "asks to stop, then forcibly"); + assertTrue(countLogs(Level.WARN, "was still running after the stop request") >= 1); + } + } + + @Test + void retriesAfterTheProcessFailsToStart() throws IOException { + List started = new CopyOnWriteArrayList<>(); + ProcessBuilder processBuilder = mock(ProcessBuilder.class); + when(processBuilder.start()) + .thenThrow(new IOException("cannot start")) + .thenAnswer(invocation -> startFakeProcess(started, true)); + + try (ProcessSupervisor ignored = new ProcessSupervisor("test", processBuilder)) { + new PollingConditions(5).eventually(() -> assertEquals(1, started.size())); + + verify(processBuilder, times(2)).start(); + assertEquals(1, countLogs(Level.WARN, "Exception starting process: [test]")); + } + } + + @Test + void logsTheExitCodeAndStartsAgainAfterTheProcessExits() throws IOException { + List started = new CopyOnWriteArrayList<>(); + + try (ProcessSupervisor ignored = + new ProcessSupervisor("test", startsFakeProcesses(started, true))) { + new PollingConditions(5).eventually(() -> assertEquals(1, started.size())); + started.get(0).exit(0); + + new PollingConditions(5).eventually(() -> assertEquals(2, started.size())); + assertEquals(1, countLogs(Level.INFO, "exited with code 0 after")); + } + } + + private static ProcessBuilder startsFakeProcesses( + List started, boolean stopsWhenAsked) throws IOException { + ProcessBuilder processBuilder = mock(ProcessBuilder.class); + when(processBuilder.start()) + .thenAnswer(invocation -> startFakeProcess(started, stopsWhenAsked)); + return processBuilder; + } + + private static FakeProcess startFakeProcess(List started, boolean stopsWhenAsked) { + FakeProcess process = new FakeProcess(stopsWhenAsked); + started.add(process); + return process; + } + + private int countLogs(Level level, String text) { + List events; + synchronized (logs) { + events = new ArrayList<>(logs.list); + } + int count = 0; + for (ILoggingEvent event : events) { + if (event.getLevel() == level && event.getFormattedMessage().contains(text)) { + count++; + } + } + return count; + } + + /** + * Stands in for a child process. Each timed wait returns within {@link #WAIT_MILLIS}, so the + * supervisor's ten second interval between health checks of a running process passes quickly. + */ + private static final class FakeProcess extends Process { + private static final long WAIT_MILLIS = 20; + + private final CountDownLatch exited = new CountDownLatch(1); + private final boolean stopsWhenAsked; + private volatile int exitCode; + final AtomicInteger stopRequests = new AtomicInteger(); + + FakeProcess(boolean stopsWhenAsked) { + this.stopsWhenAsked = stopsWhenAsked; + } + + void exit(int code) { + exitCode = code; + exited.countDown(); + } + + @Override + public boolean waitFor(long timeout, TimeUnit unit) throws InterruptedException { + return exited.await(Math.min(unit.toMillis(timeout), WAIT_MILLIS), MILLISECONDS); + } + + @Override + public int waitFor() throws InterruptedException { + exited.await(); + return exitCode; + } + + @Override + public int exitValue() { + if (isAlive()) { + throw new IllegalThreadStateException("still running"); + } + return exitCode; + } + + @Override + public boolean isAlive() { + return exited.getCount() > 0; + } + + @Override + public void destroy() { + stopRequests.incrementAndGet(); + if (stopsWhenAsked) { + exit(143); + } + } + + @Override + public Process destroyForcibly() { + destroy(); + return this; + } + + @Override + public OutputStream getOutputStream() { + return new ByteArrayOutputStream(); + } + + @Override + public InputStream getInputStream() { + return new ByteArrayInputStream(new byte[0]); + } + + @Override + public InputStream getErrorStream() { + return new ByteArrayInputStream(new byte[0]); + } + } } From 2d02d243b29a62d875102e8b9f17878e2c3e7fc9 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 5 Nov 2026 11:39:48 -0500 Subject: [PATCH 07/30] Name the AAS agent pipe health checks by what they check healthCheck(pipe, true) and healthCheck(pipe, false) didn't say what the flag meant at the call site. Use answersOnPipe for the trace agent, which must answer GET /info, and pipePresent for DogStatsD, which never replies on its pipe. No behavior change. --- .../ddagent/ExternalAgentLauncher.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java index e3dfdfff248..9e805170186 100644 --- a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java +++ b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java @@ -44,7 +44,7 @@ public ExternalAgentLauncher(Config config) { traceProcessSupervisor = new ProcessSupervisor( - "trace-agent", traceProcessBuilder, healthCheck(config.getAgentNamedPipe(), true)); + "trace-agent", traceProcessBuilder, answersOnPipe(config.getAgentNamedPipe())); } else { log.warn("Trace agent path not set. Will not start trace agent process"); } @@ -57,9 +57,7 @@ public ExternalAgentLauncher(Config config) { dogStatsDProcessSupervisor = new ProcessSupervisor( - "dogstatsd", - dogStatsDProcessBuilder, - healthCheck(config.getDogStatsDNamedPipe(), false)); + "dogstatsd", dogStatsDProcessBuilder, pipePresent(config.getDogStatsDNamedPipe())); } else { log.warn("DogStatsD path not set. Will not start DogStatsD process"); } @@ -77,10 +75,16 @@ public void close() { } } - /** - * @param checkResponse whether the agent answers requests on its pipe. DogStatsD never replies, - * so for it the pipe being present is the whole check. - */ + /** Healthy while the agent's pipe is present and the agent answers a request on it. */ + private static ProcessSupervisor.HealthCheck answersOnPipe(String pipeName) { + return healthCheck(pipeName, true); + } + + /** Healthy while the agent's pipe is present. DogStatsD never replies on its pipe. */ + private static ProcessSupervisor.HealthCheck pipePresent(String pipeName) { + return healthCheck(pipeName, false); + } + private static ProcessSupervisor.HealthCheck healthCheck(String pipeName, boolean checkResponse) { return null != pipeName && !pipeName.trim().isEmpty() ? new NamedPipeHealthCheck(pipeName, checkResponse) From 15a9d15dfb3bbe6685b4efaceaa0782f20f8e640 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 5 Nov 2026 11:39:49 -0500 Subject: [PATCH 08/30] Stop the AAS agent with a single TerminateProcess The supervised agents run only on Windows, where Process.destroy() and destroyForcibly() both call TerminateProcess, so the forced second attempt did nothing new. Call destroy() once and wait up to 400 ms, the same total wait as the two 200 ms waits before. A process that is still running after that is kept and tracked as before. --- .../java/datadog/trace/util/ProcessSupervisor.java | 11 ++++------- .../datadog/trace/util/ProcessSupervisorTest.java | 8 +------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index 5918e1d4958..8476b7d3dc8 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -45,7 +45,7 @@ public interface HealthCheck { private static final int QUIET_AFTER_FAILURES = 5; private static final long STABLE_RUN_MILLIS = 60_000; - private static final long STOP_WAIT_MILLIS = 200; + private static final long STOP_WAIT_MILLIS = 400; // A process this supervisor started is restarted after failing this many health checks in a row private static final int MAX_FAILED_HEALTH_CHECKS = 3; @@ -248,16 +248,13 @@ private void stopProcess() { if (currentProcess != null) { String childPid = pidOrUnknown(currentProcess); log.info("Stopping process [{}] with pid {}", imageName, childPid); + // The supervised agents run only on Windows, where destroy() and destroyForcibly() both call + // TerminateProcess: the process stops at once, without a chance to flush. + currentProcess.destroy(); boolean stopped = false; try { - currentProcess.destroy(); stopped = currentProcess.waitFor(STOP_WAIT_MILLIS, MILLISECONDS); - if (!stopped) { - currentProcess.destroyForcibly(); - stopped = currentProcess.waitFor(STOP_WAIT_MILLIS, MILLISECONDS); - } } catch (InterruptedException e) { - currentProcess.destroyForcibly(); Thread.currentThread().interrupt(); } if (stopped) { diff --git a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java index 7057b1230bb..36195666f4d 100644 --- a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java +++ b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java @@ -266,7 +266,7 @@ void keepsTrackingItsProcessWhenItDoesNotStopInsteadOfStartingASecondCopy() thro assertEquals(1, started.size()); assertTrue(started.get(0).isAlive()); - assertTrue(started.get(0).stopRequests.get() >= 2, "asks to stop, then forcibly"); + assertTrue(started.get(0).stopRequests.get() >= 1, "asks it to stop"); assertTrue(countLogs(Level.WARN, "was still running after the stop request") >= 1); } } @@ -382,12 +382,6 @@ public void destroy() { } } - @Override - public Process destroyForcibly() { - destroy(); - return this; - } - @Override public OutputStream getOutputStream() { return new ByteArrayOutputStream(); From 0bcc245f496cddb9537920c829ecba794a40e359 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 5 Nov 2026 12:23:23 -0500 Subject: [PATCH 09/30] Fix AAS agent supervisor review findings - An exit before the process stayed up for 60 s is now a fault whatever its exit code, so it backs off and counts toward quiet logging. The trace agent exits with 0 at once when APM is disabled, and that used to restart about every 100 ms, now with start and exit lines at info. - Lifecycle logs return to info once the running process has stayed up for 60 s, not only after it exits. - A trace agent /info answer that arrives after the 2 s timeout counts at the next check, instead of being dropped by a new request. An agent that always answers in just over 2 s was failing every check. --- .../ddagent/ExternalAgentLauncher.java | 35 +++++++++++++++---- .../datadog/trace/util/ProcessSupervisor.java | 20 +++++++---- .../trace/util/ProcessSupervisorTest.java | 20 +++++++++++ 3 files changed, 62 insertions(+), 13 deletions(-) diff --git a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java index 9e805170186..08592f83935 100644 --- a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java +++ b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java @@ -103,8 +103,9 @@ static final class NamedPipeHealthCheck implements ProcessSupervisor.HealthCheck private final File pipe; private final boolean checkResponse; - // Only the supervisor thread runs the health check, so this needs no synchronization - private FutureTask pendingInfoRequest; + // A request that got no answer in time. Only the supervisor thread runs the health check, so + // this needs no synchronization. + private FutureTask timedOutInfoRequest; NamedPipeHealthCheck(String pipeName, boolean checkResponse) { this.checkResponse = checkResponse; @@ -151,19 +152,39 @@ private ProcessSupervisor.Health respondsOrReadyToStart() throws InterruptedExce /** * Sends a request on another thread and waits up to {@link #RESPONSE_TIMEOUT_MILLIS} for the * answer. Reads from a pipe cannot time out, so a request to a hung agent stays blocked until - * the agent exits. Until then later checks fail without sending another request. + * the agent exits. Until then later checks fail without sending another request. An answer that + * arrives after the timeout still counts at the next check, so a slow agent is not restarted. */ private boolean agentResponds() throws InterruptedException { - if (pendingInfoRequest != null && !pendingInfoRequest.isDone()) { - return false; // the last request is still waiting for an answer + FutureTask lateRequest = timedOutInfoRequest; + if (lateRequest != null) { + if (!lateRequest.isDone()) { + return false; // still waiting for an answer + } + timedOutInfoRequest = null; + if (answered(lateRequest)) { + return true; + } } FutureTask infoRequest = new FutureTask<>(this::sendInfoRequest); - pendingInfoRequest = infoRequest; AgentThreadFactory.newAgentThread(PROCESS_SUPERVISOR, "-health-check", infoRequest, true) .start(); try { return infoRequest.get(RESPONSE_TIMEOUT_MILLIS, MILLISECONDS); - } catch (ExecutionException | TimeoutException e) { + } catch (TimeoutException e) { + timedOutInfoRequest = infoRequest; + return false; + } catch (ExecutionException e) { + return false; + } + } + + /** Returns the result of a finished request, without waiting. */ + private static boolean answered(FutureTask finishedRequest) + throws InterruptedException { + try { + return finishedRequest.get(); + } catch (ExecutionException e) { return false; } } diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index 8476b7d3dc8..54c3a925424 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -168,18 +168,26 @@ private void startProcessAndSupervise() throws Exception { // Keep checking on the process until it exits while (!currentProcess.waitFor(HEALTHY_DELAY_MILLIS, MILLISECONDS)) { - faultBackoff.recordUptime(System.currentTimeMillis() - currentProcessStartedMillis); + long upMillis = System.currentTimeMillis() - currentProcessStartedMillis; + faultBackoff.recordUptime(upMillis); + if (upMillis >= STABLE_RUN_MILLIS) { + failuresInARow = 0; // it stayed up, so log its lifecycle at info level again + } if (healthCheck != ALWAYS_READY && !passesHealthCheck(childPid)) { restartUnhealthyProcess(childPid); return; } } int code = currentProcess.exitValue(); - currentHealth = code == 0 ? INTERRUPTED : FAULTED; - long ranMillis = System.currentTimeMillis() - currentProcessStartedMillis; + boolean stayedUp = ranMillis >= STABLE_RUN_MILLIS; + // An exit before the process stayed up is a fault whatever its code. The trace agent exits + // with 0 when APM is disabled, for example, and restarting that every 100 ms never ends. + boolean failed = code != 0 || !stayedUp; + currentHealth = failed ? FAULTED : INTERRUPTED; + faultBackoff.recordUptime(ranMillis); - if (ranMillis >= STABLE_RUN_MILLIS) { + if (stayedUp) { failuresInARow = 0; } logLifecycle( @@ -188,9 +196,9 @@ private void startProcessAndSupervise() throws Exception { childPid, code, ranMillis); - if (code != 0 && ++failuresInARow == QUIET_AFTER_FAILURES) { + if (failed && ++failuresInARow == QUIET_AFTER_FAILURES) { log.warn( - "Process [{}] has exited with an error {} times in a row; " + "Process [{}] has failed {} times in a row; " + "further starts and exits are logged at debug level until it stays up for {} seconds", imageName, failuresInARow, diff --git a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java index 36195666f4d..e57e75b260e 100644 --- a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java +++ b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java @@ -301,6 +301,26 @@ void logsTheExitCodeAndStartsAgainAfterTheProcessExits() throws IOException { } } + @Test + void backsOffWhenTheProcessKeepsExitingWithCodeZero() throws Exception { + // the trace agent exits with 0 at once when APM is disabled + List started = new CopyOnWriteArrayList<>(); + ProcessBuilder processBuilder = mock(ProcessBuilder.class); + when(processBuilder.start()) + .thenAnswer(invocation -> { + FakeProcess process = startFakeProcess(started, true); + process.exit(0); + return process; + }); + + try (ProcessSupervisor ignored = new ProcessSupervisor("test", processBuilder)) { + new PollingConditions(5).eventually(() -> assertEquals(1, started.size())); + // the first fault waits 2 s; restarting after a clean exit would take about 100 ms + Thread.sleep(1_000); + assertEquals(1, started.size()); + } + } + private static ProcessBuilder startsFakeProcesses( List started, boolean stopsWhenAsked) throws IOException { ProcessBuilder processBuilder = mock(ProcessBuilder.class); From 10502c0ff083f4f70bc9ceefb8e9c6b2d72cf874 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Tue, 17 Nov 2026 10:22:28 -0500 Subject: [PATCH 10/30] Test the Java 9+ child process PID lookup master now checks coverage of internal-api-9, and JDK9ProcessPidFunction had no test. Check that it returns the child's PID, and an empty string when the Process can't tell its PID, which the supervisor logs as "unknown". --- .../util/JDK9ProcessPidFunctionTest.java | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 internal-api/internal-api-9/src/test/java/datadog/trace/util/JDK9ProcessPidFunctionTest.java diff --git a/internal-api/internal-api-9/src/test/java/datadog/trace/util/JDK9ProcessPidFunctionTest.java b/internal-api/internal-api-9/src/test/java/datadog/trace/util/JDK9ProcessPidFunctionTest.java new file mode 100644 index 00000000000..1efc78d9caf --- /dev/null +++ b/internal-api/internal-api-9/src/test/java/datadog/trace/util/JDK9ProcessPidFunctionTest.java @@ -0,0 +1,61 @@ +package datadog.trace.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.InputStream; +import java.io.OutputStream; +import org.junit.jupiter.api.Test; + +class JDK9ProcessPidFunctionTest { + + private final JDK9ProcessPidFunction pidFunction = new JDK9ProcessPidFunction(); + + @Test + void returnsThePidOfTheProcess() { + Process process = + new FakeProcess() { + @Override + public long pid() { + return 42; + } + }; + + assertEquals("42", pidFunction.apply(process)); + } + + @Test + void returnsEmptyWhenTheProcessCannotTellItsPid() { + // Process.pid() throws UnsupportedOperationException unless a subclass supports it + assertEquals("", pidFunction.apply(new FakeProcess())); + } + + private static class FakeProcess extends Process { + @Override + public OutputStream getOutputStream() { + return null; + } + + @Override + public InputStream getInputStream() { + return null; + } + + @Override + public InputStream getErrorStream() { + return null; + } + + @Override + public int waitFor() { + return 0; + } + + @Override + public int exitValue() { + return 0; + } + + @Override + public void destroy() {} + } +} From e81efaa0fabf567b55369b3ebb5b6942ba8ffa18 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Wed, 18 Nov 2026 11:11:55 -0500 Subject: [PATCH 11/30] Update internal-api/src/main/java/datadog/trace/util/PidHelper.java Co-authored-by: Brice Dutheil --- .../java/datadog/trace/util/PidHelper.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/PidHelper.java b/internal-api/src/main/java/datadog/trace/util/PidHelper.java index 63a008b8185..9f7ca016e9c 100644 --- a/internal-api/src/main/java/datadog/trace/util/PidHelper.java +++ b/internal-api/src/main/java/datadog/trace/util/PidHelper.java @@ -39,29 +39,30 @@ public static long getPidAsLong() { return PID_AS_LONG; } - private static final Function CHILD_PID = findChildPidFunction(); + private static final MethodHandle PROCESS_PID = findProcessPid(); /** Returns the PID of a child process, or an empty string when the JVM cannot provide it. */ public static String getPid(Process process) { - if (null == process || null == CHILD_PID) { - return ""; + if (process != null && PROCESS_PID != null) { + try { + return Long.toString((long) PROCESS_PID.invokeExact(process)); + } catch (Throwable e) { + log.debug("Cannot get child PID through JVM API", e); + } } - return CHILD_PID.apply(process); + return ""; } - @SuppressWarnings("unchecked") - private static Function findChildPidFunction() { + private static MethodHandle findProcessPid() { if (JavaVirtualMachine.isJavaVersionAtLeast(9)) { try { - return (Function) - Class.forName("datadog.trace.util.JDK9ProcessPidFunction") - .getDeclaredConstructor() - .newInstance(); + return MethodHandles.publicLookup() + .findVirtual(Process.class, "pid", MethodType.methodType(long.class)); } catch (Throwable e) { - log.debug("JDK9ProcessPidFunction not available", e); + log.debug("Process.pid() not available", e); } } - return null; // Java 8 has no public API for the PID of a child process + return null; } @SuppressWarnings("unchecked") From 66ea5ef70e9bccc93c382398a930b782d5dc97be Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Wed, 18 Nov 2026 11:12:23 -0500 Subject: [PATCH 12/30] Update internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java Co-authored-by: Brice Dutheil --- .../main/java/datadog/trace/util/ProcessSupervisor.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index 54c3a925424..a507d0547a4 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -259,11 +259,16 @@ private void stopProcess() { // The supervised agents run only on Windows, where destroy() and destroyForcibly() both call // TerminateProcess: the process stops at once, without a chance to flush. currentProcess.destroy(); + boolean interrupted = Thread.interrupted(); boolean stopped = false; try { stopped = currentProcess.waitFor(STOP_WAIT_MILLIS, MILLISECONDS); } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + interrupted = true; + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } } if (stopped) { log.info("Stopped process [{}] with pid {}", imageName, childPid); From 3a66484462d035ed73e595b48726cb262b95defc Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Wed, 18 Nov 2026 11:21:15 -0500 Subject: [PATCH 13/30] Fix the PidHelper imports and remove the unused PID function PidHelper now calls Process.pid() through a MethodHandle, but the java.lang.invoke imports were missing, so internal-api didn't compile. Add them, drop the unused Function import, and delete JDK9ProcessPidFunction and its test, which nothing uses anymore. --- .../trace/util/JDK9ProcessPidFunction.java | 15 ----- .../util/JDK9ProcessPidFunctionTest.java | 61 ------------------- .../java/datadog/trace/util/PidHelper.java | 4 +- 3 files changed, 3 insertions(+), 77 deletions(-) delete mode 100644 internal-api/internal-api-9/src/main/java/datadog/trace/util/JDK9ProcessPidFunction.java delete mode 100644 internal-api/internal-api-9/src/test/java/datadog/trace/util/JDK9ProcessPidFunctionTest.java diff --git a/internal-api/internal-api-9/src/main/java/datadog/trace/util/JDK9ProcessPidFunction.java b/internal-api/internal-api-9/src/main/java/datadog/trace/util/JDK9ProcessPidFunction.java deleted file mode 100644 index 047174b3018..00000000000 --- a/internal-api/internal-api-9/src/main/java/datadog/trace/util/JDK9ProcessPidFunction.java +++ /dev/null @@ -1,15 +0,0 @@ -package datadog.trace.util; - -import java.util.function.Function; - -/** Use standard API to retrieve the PID of a child process on Java9+. */ -public final class JDK9ProcessPidFunction implements Function { - @Override - public String apply(Process process) { - try { - return Long.toString(process.pid()); - } catch (Throwable e) { - return ""; - } - } -} diff --git a/internal-api/internal-api-9/src/test/java/datadog/trace/util/JDK9ProcessPidFunctionTest.java b/internal-api/internal-api-9/src/test/java/datadog/trace/util/JDK9ProcessPidFunctionTest.java deleted file mode 100644 index 1efc78d9caf..00000000000 --- a/internal-api/internal-api-9/src/test/java/datadog/trace/util/JDK9ProcessPidFunctionTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package datadog.trace.util; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.io.InputStream; -import java.io.OutputStream; -import org.junit.jupiter.api.Test; - -class JDK9ProcessPidFunctionTest { - - private final JDK9ProcessPidFunction pidFunction = new JDK9ProcessPidFunction(); - - @Test - void returnsThePidOfTheProcess() { - Process process = - new FakeProcess() { - @Override - public long pid() { - return 42; - } - }; - - assertEquals("42", pidFunction.apply(process)); - } - - @Test - void returnsEmptyWhenTheProcessCannotTellItsPid() { - // Process.pid() throws UnsupportedOperationException unless a subclass supports it - assertEquals("", pidFunction.apply(new FakeProcess())); - } - - private static class FakeProcess extends Process { - @Override - public OutputStream getOutputStream() { - return null; - } - - @Override - public InputStream getInputStream() { - return null; - } - - @Override - public InputStream getErrorStream() { - return null; - } - - @Override - public int waitFor() { - return 0; - } - - @Override - public int exitValue() { - return 0; - } - - @Override - public void destroy() {} - } -} diff --git a/internal-api/src/main/java/datadog/trace/util/PidHelper.java b/internal-api/src/main/java/datadog/trace/util/PidHelper.java index 9f7ca016e9c..530999b1c8b 100644 --- a/internal-api/src/main/java/datadog/trace/util/PidHelper.java +++ b/internal-api/src/main/java/datadog/trace/util/PidHelper.java @@ -6,6 +6,9 @@ import datadog.trace.config.inversion.ConfigHelper; import de.thetaphi.forbiddenapis.SuppressForbidden; import java.io.IOException; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; import java.lang.management.ManagementFactory; import java.nio.file.Files; import java.nio.file.Path; @@ -13,7 +16,6 @@ import java.util.Collections; import java.util.Objects; import java.util.Set; -import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; From d4604f1b03fa0c655f86800c971f33548591e100 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Wed, 18 Nov 2026 11:24:35 -0500 Subject: [PATCH 14/30] Count AAS agent failures in one place The start-exception, restart, and exit paths each counted failures in a row and chose the log level themselves, and only the exit path said when the log went quiet. logFailure() and recordFailure() now do both, so the "logged at debug level" notice is written whichever path reaches the fifth failure. --- .../datadog/trace/util/ProcessSupervisor.java | 61 +++++++++++-------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index a507d0547a4..6670b916118 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -101,11 +101,7 @@ private void mainLoop() { } catch (InterruptedException e) { currentHealth = INTERRUPTED; } catch (Throwable e) { - if (failuresInARow++ < QUIET_AFTER_FAILURES) { - log.warn("Exception starting process: [{}]", imageName, e); - } else { - log.debug("Exception starting process: [{}]", imageName, e); - } + logFailure("Exception starting process: [{}]", imageName, e); currentHealth = FAULTED; } scheduleNextHealthCheck(); @@ -196,13 +192,8 @@ private void startProcessAndSupervise() throws Exception { childPid, code, ranMillis); - if (failed && ++failuresInARow == QUIET_AFTER_FAILURES) { - log.warn( - "Process [{}] has failed {} times in a row; " - + "further starts and exits are logged at debug level until it stays up for {} seconds", - imageName, - failuresInARow, - MILLISECONDS.toSeconds(STABLE_RUN_MILLIS)); + if (failed) { + recordFailure(); } // Process is dead, no longer needs to be tracked @@ -234,19 +225,11 @@ private boolean passesHealthCheck(String childPid) throws InterruptedException { } private void restartUnhealthyProcess(String childPid) { - if (failuresInARow++ < QUIET_AFTER_FAILURES) { - log.warn( - "Process [{}] with pid {} failed {} health checks in a row; restarting it", - imageName, - childPid, - failedHealthChecks); - } else { - log.debug( - "Process [{}] with pid {} failed {} health checks in a row; restarting it", - imageName, - childPid, - failedHealthChecks); - } + logFailure( + "Process [{}] with pid {} failed {} health checks in a row; restarting it", + imageName, + childPid, + failedHealthChecks); stopProcess(); // The next health check starts a new process through the normal start path, after a backoff currentHealth = FAULTED; @@ -293,6 +276,34 @@ private void logLifecycle(String format, Object... arguments) { } } + /** + * Counts a failure in a row and logs it at warn level, or at debug level once the log has gone + * quiet. + */ + private void logFailure(String format, Object... arguments) { + if (failuresInARow < QUIET_AFTER_FAILURES) { + log.warn(format, arguments); + } else { + log.debug(format, arguments); + } + recordFailure(); + } + + /** + * Counts a failure in a row. When the count reaches {@link #QUIET_AFTER_FAILURES}, says that + * lifecycle lines go to debug level until the process stays up. + */ + private void recordFailure() { + if (++failuresInARow == QUIET_AFTER_FAILURES) { + log.warn( + "Process [{}] has failed {} times in a row; " + + "further starts and exits are logged at debug level until it stays up for {} seconds", + imageName, + failuresInARow, + MILLISECONDS.toSeconds(STABLE_RUN_MILLIS)); + } + } + private static String pidOrUnknown(Process process) { String pid = PidHelper.getPid(process); return pid.isEmpty() ? "unknown" : pid; From d1e37be3a6b01368d8a15fc384335fd02a43355e Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Wed, 18 Nov 2026 12:29:57 -0500 Subject: [PATCH 15/30] Test the child PID lookup through PidHelper Removing JDK9ProcessPidFunctionTest dropped the child PID cases, so a broken Process.pid() handle would only show up as "unknown" in the logs. Check PidHelper.getPid(Process) on Java 9+: the PID of a process that reports one, and an empty string for one that can't. --- .../trace/util/PidHelperChildPidTest.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 internal-api/internal-api-9/src/test/java/datadog/trace/util/PidHelperChildPidTest.java diff --git a/internal-api/internal-api-9/src/test/java/datadog/trace/util/PidHelperChildPidTest.java b/internal-api/internal-api-9/src/test/java/datadog/trace/util/PidHelperChildPidTest.java new file mode 100644 index 00000000000..64340de6ce2 --- /dev/null +++ b/internal-api/internal-api-9/src/test/java/datadog/trace/util/PidHelperChildPidTest.java @@ -0,0 +1,60 @@ +package datadog.trace.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.InputStream; +import java.io.OutputStream; +import org.junit.jupiter.api.Test; + +/** Covers {@link PidHelper#getPid(Process)} and its cached {@code Process.pid()} handle. */ +class PidHelperChildPidTest { + + @Test + void returnsThePidOfTheProcess() { + Process process = + new FakeProcess() { + @Override + public long pid() { + return 42; + } + }; + + assertEquals("42", PidHelper.getPid(process)); + } + + @Test + void returnsEmptyWhenTheProcessCannotTellItsPid() { + // Process.pid() throws UnsupportedOperationException unless a subclass supports it + assertEquals("", PidHelper.getPid(new FakeProcess())); + } + + private static class FakeProcess extends Process { + @Override + public OutputStream getOutputStream() { + return null; + } + + @Override + public InputStream getInputStream() { + return null; + } + + @Override + public InputStream getErrorStream() { + return null; + } + + @Override + public int waitFor() { + return 0; + } + + @Override + public int exitValue() { + return 0; + } + + @Override + public void destroy() {} + } +} From 3be4b0a7af4902dec79e40fc372fce87f5a742d3 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 19 Nov 2026 12:07:09 -0500 Subject: [PATCH 16/30] Fail the AAS trace agent check on a pipe that stays busy The /info health check counted a busy pipe as an answer, so an agent that stopped accepting connections passed every check and was never restarted. The check now retries opening the pipe for up to 500 ms, like .NET's NamedPipeClientStream.Connect and the tracer's own pipe socket, and fails if the pipe is still busy. The pipe-present check still counts a busy pipe as present. A failed check next to an agent this tracer did not start used to start a duplicate at once. Like dd-trace-dotnet, the supervisor now waits for 3 failed checks in a row first, and counts again after each start. --- .../ddagent/ExternalAgentLauncher.java | 46 ++++++++++++++----- .../datadog/trace/util/ProcessSupervisor.java | 46 +++++++++++++++---- .../trace/util/ProcessSupervisorTest.java | 25 ++++++++++ 3 files changed, 96 insertions(+), 21 deletions(-) diff --git a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java index 08592f83935..58496daf28f 100644 --- a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java +++ b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java @@ -5,6 +5,7 @@ import static datadog.trace.util.ProcessSupervisor.Health.HEALTHY; import static datadog.trace.util.ProcessSupervisor.Health.NEVER_CHECKED; import static datadog.trace.util.ProcessSupervisor.Health.READY_TO_START; +import static datadog.trace.util.ProcessSupervisor.Health.UNHEALTHY; import static java.util.concurrent.TimeUnit.MILLISECONDS; import datadog.environment.OperatingSystem; @@ -99,6 +100,8 @@ static final class NamedPipeHealthCheck implements ProcessSupervisor.HealthCheck .getBytes(StandardCharsets.US_ASCII); private static final int MAX_STATUS_LINE_LENGTH = 64; private static final long RESPONSE_TIMEOUT_MILLIS = 2_000; + // The .NET tracer's default named pipe connect timeout, DD_TRACE_PIPE_TIMEOUT_MS + private static final long OPEN_TIMEOUT_MILLIS = 500; private final File pipe; private final boolean checkResponse; @@ -134,19 +137,20 @@ public ProcessSupervisor.Health run(ProcessSupervisor.Health previousHealth) delayMillis = delayMillis * 1.75; } - return respondsOrReadyToStart(); // use existing external process if it responds + return healthOfBoundProcess(); // use existing external process if it responds } // otherwise just check that the pipe is still bound if (pipeExists()) { - return respondsOrReadyToStart(); // keep using external process if it responds + return healthOfBoundProcess(); // keep using external process if it responds } else { return READY_TO_START; // start our own process } } - private ProcessSupervisor.Health respondsOrReadyToStart() throws InterruptedException { - return !checkResponse || agentResponds() ? HEALTHY : READY_TO_START; + /** Checks the process that has the pipe, which may or may not be the supervisor's own. */ + private ProcessSupervisor.Health healthOfBoundProcess() throws InterruptedException { + return !checkResponse || agentResponds() ? HEALTHY : UNHEALTHY; } /** @@ -194,14 +198,11 @@ private static boolean answered(FutureTask finishedRequest) * * @return true if the agent answers with an HTTP status line, whatever the status */ - private boolean sendInfoRequest() throws IOException { + private boolean sendInfoRequest() throws IOException, InterruptedException { try (TraceScope ignored = AgentTracer.get().muteTracing()) { - RandomAccessFile file; - try { - file = new RandomAccessFile(pipe, "rw"); - } catch (FileNotFoundException e) { - // a pipe with all instances busy cannot be asked, but an agent is serving it - return !isPipeNotFound(e); + RandomAccessFile file = openPipe(); + if (file == null) { + return false; } try { file.write(INFO_REQUEST); @@ -218,6 +219,29 @@ private boolean sendInfoRequest() throws IOException { } } + /** + * Opens the pipe, retrying for up to {@link #OPEN_TIMEOUT_MILLIS} like .NET's {@code + * NamedPipeClientStream.Connect}. A pipe server creates its next pipe instance only after a + * client connects, so an open in between fails with "All pipe instances are busy". An agent + * that stops accepting connections keeps its pipe busy, so a pipe still busy after the retries + * fails the check. + * + * @return the opened pipe, or null if it could not be opened in time + */ + private RandomAccessFile openPipe() throws InterruptedException { + long deadlineNanos = System.nanoTime() + MILLISECONDS.toNanos(OPEN_TIMEOUT_MILLIS); + while (true) { + try { + return new RandomAccessFile(pipe, "rw"); + } catch (FileNotFoundException e) { + if (System.nanoTime() - deadlineNanos >= 0) { + return null; + } + Thread.sleep(1); + } + } + } + /** * {@link File#exists()} can report false for a pipe whose instances are all busy, for example * under load. In that case open the pipe once to tell busy from absent. diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index 6670b916118..cd77814f557 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -6,6 +6,7 @@ import static datadog.trace.util.ProcessSupervisor.Health.HEALTHY; import static datadog.trace.util.ProcessSupervisor.Health.INTERRUPTED; import static datadog.trace.util.ProcessSupervisor.Health.READY_TO_START; +import static datadog.trace.util.ProcessSupervisor.Health.UNHEALTHY; import static java.util.concurrent.TimeUnit.MILLISECONDS; import datadog.trace.api.internal.VisibleForTesting; @@ -23,7 +24,9 @@ public enum Health { READY_TO_START, INTERRUPTED, FAULTED, - HEALTHY + HEALTHY, + /** A process is serving, but did not pass the health check. */ + UNHEALTHY } @FunctionalInterface @@ -47,7 +50,8 @@ public interface HealthCheck { private static final long STOP_WAIT_MILLIS = 400; - // A process this supervisor started is restarted after failing this many health checks in a row + // A process this supervisor started is restarted after failing this many health checks in a row. + // When a process it did not start fails this many, it starts its own. private static final int MAX_FAILED_HEALTH_CHECKS = 3; private final String imageName; @@ -61,6 +65,7 @@ public interface HealthCheck { private Process currentProcess; private long currentProcessStartedMillis; private int failedHealthChecks; + private int otherProcessFailedHealthChecks; private final FaultBackoff faultBackoff = new FaultBackoff(); private int failuresInARow; @@ -94,8 +99,14 @@ private void mainLoop() { if (currentProcess == null) { currentHealth = healthCheck.run(currentHealth); logHealthCheckChange(currentHealth); + otherProcessFailedHealthChecks = + currentHealth == UNHEALTHY ? otherProcessFailedHealthChecks + 1 : 0; } - if (currentProcess != null || currentHealth == READY_TO_START) { + // A copy started next to a process that is still serving cannot take over until that one + // exits, so it is only started once that process has failed several checks in a row. + if (currentProcess != null + || currentHealth == READY_TO_START + || otherProcessFailedHealthChecks >= MAX_FAILED_HEALTH_CHECKS) { startProcessAndSupervise(); } } catch (InterruptedException e) { @@ -117,15 +128,29 @@ private void logHealthCheckChange(Health result) { if (result == previous) { return; } + // These checks only run while this supervisor has no running process of its own, + // so a healthy or unhealthy result means another process (usually another worker) is serving. if (result == HEALTHY) { - // These checks only run while this supervisor has no running process of its own, - // so a healthy result means another process (usually another worker) is serving it. failuresInARow = 0; - log.info( - "Process [{}] is already running and was not started by this process (pid {}); using it", + if (previous == UNHEALTHY) { + log.info( + "Process [{}] that this process (pid {}) is using is passing health checks again", + imageName, + PidHelper.getPid()); + } else { + log.info( + "Process [{}] is already running and was not started by this process (pid {}); using it", + imageName, + PidHelper.getPid()); + } + } else if (result == UNHEALTHY) { + logLifecycle( + "Process [{}] is running but was not started by this process (pid {}) and failed a health" + + " check; starting a new one if it fails {} checks in a row", imageName, - PidHelper.getPid()); - } else if (result == READY_TO_START && previous == HEALTHY) { + PidHelper.getPid(), + MAX_FAILED_HEALTH_CHECKS); + } else if (result == READY_TO_START && (previous == HEALTHY || previous == UNHEALTHY)) { logLifecycle( "Process [{}] that this process (pid {}) was using is no longer available", imageName, @@ -135,7 +160,7 @@ private void logHealthCheckChange(Health result) { private void scheduleNextHealthCheck() { long now = System.currentTimeMillis(); - if (currentHealth == HEALTHY) { + if (currentHealth == HEALTHY || currentHealth == UNHEALTHY) { nextCheckMillis = now + HEALTHY_DELAY_MILLIS; } else if (currentHealth == FAULTED) { nextCheckMillis = now + faultBackoff.recordFault(); @@ -146,6 +171,7 @@ private void scheduleNextHealthCheck() { private void startProcessAndSupervise() throws Exception { if (currentProcess == null) { + otherProcessFailedHealthChecks = 0; log.debug("Starting process: [{}]", imageName); try (TraceScope ignored = AgentTracer.get().muteTracing()) { currentProcess = processBuilder.start(); diff --git a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java index e57e75b260e..6e346654d4f 100644 --- a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java +++ b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java @@ -3,6 +3,7 @@ import static datadog.trace.util.ProcessSupervisor.Health.FAULTED; import static datadog.trace.util.ProcessSupervisor.Health.HEALTHY; import static datadog.trace.util.ProcessSupervisor.Health.READY_TO_START; +import static datadog.trace.util.ProcessSupervisor.Health.UNHEALTHY; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -246,6 +247,30 @@ void usesAProcessItDidNotStartAndStartsItsOwnWhenThatOneGoesAway() throws IOExce } } + @Test + void startsItsOwnProcessOnlyAfterAProcessItDidNotStartFailsThreeChecksInARow() + throws IOException { + List started = new CopyOnWriteArrayList<>(); + AtomicInteger checksBeforeStart = new AtomicInteger(); + HealthCheck otherProcessNeverAnswers = + previous -> { + if (started.isEmpty()) { + checksBeforeStart.incrementAndGet(); + return UNHEALTHY; + } + return HEALTHY; + }; + + try (ProcessSupervisor ignored = + new ProcessSupervisor( + "test", startsFakeProcesses(started, true), otherProcessNeverAnswers)) { + // checks of a process it did not start run ten seconds apart + new PollingConditions(30).eventually(() -> assertEquals(1, started.size())); + assertEquals(3, checksBeforeStart.get()); + assertEquals(1, countLogs(Level.INFO, "starting a new one if it fails 3 checks in a row")); + } + } + @Test void keepsTrackingItsProcessWhenItDoesNotStopInsteadOfStartingASecondCopy() throws IOException { List started = new CopyOnWriteArrayList<>(); From a1cc0283a4818f0f20a0422076e0263d33a1f1c9 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 19 Nov 2026 12:43:43 -0500 Subject: [PATCH 17/30] Name the AAS supervisor's catch-all log line for what it covers It also catches health-check exceptions from a process that is already running, so "Exception starting process" was misleading. --- .../src/main/java/datadog/trace/util/ProcessSupervisor.java | 2 +- .../src/test/java/datadog/trace/util/ProcessSupervisorTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index cd77814f557..9b9e06438eb 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -112,7 +112,7 @@ private void mainLoop() { } catch (InterruptedException e) { currentHealth = INTERRUPTED; } catch (Throwable e) { - logFailure("Exception starting process: [{}]", imageName, e); + logFailure("Exception supervising process: [{}]", imageName, e); currentHealth = FAULTED; } scheduleNextHealthCheck(); diff --git a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java index 6e346654d4f..c6d0c1ad21e 100644 --- a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java +++ b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java @@ -308,7 +308,7 @@ void retriesAfterTheProcessFailsToStart() throws IOException { new PollingConditions(5).eventually(() -> assertEquals(1, started.size())); verify(processBuilder, times(2)).start(); - assertEquals(1, countLogs(Level.WARN, "Exception starting process: [test]")); + assertEquals(1, countLogs(Level.WARN, "Exception supervising process: [test]")); } } From 2783ac5619b4b6dfaa3ebef35d758f391a8b66ce Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 19 Nov 2026 12:43:58 -0500 Subject: [PATCH 18/30] Log why an AAS trace agent health check request failed An unexpected IOException, such as a broken pipe or access denied, looked the same as a hung agent. --- .../datadog/communication/ddagent/ExternalAgentLauncher.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java index 58496daf28f..a868371e87d 100644 --- a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java +++ b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java @@ -179,6 +179,7 @@ private boolean agentResponds() throws InterruptedException { timedOutInfoRequest = infoRequest; return false; } catch (ExecutionException e) { + log.debug("Health check request failed", e.getCause()); return false; } } @@ -189,6 +190,7 @@ private static boolean answered(FutureTask finishedRequest) try { return finishedRequest.get(); } catch (ExecutionException e) { + log.debug("Health check request failed", e.getCause()); return false; } } From 73e3e55a49d3e5375d636e8339db28ddf39e97bb Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 19 Nov 2026 12:46:18 -0500 Subject: [PATCH 19/30] Force-stop a supervised process that is still running after destroy() ProcessSupervisor is a general internal-api class. On Windows, where the AAS agents run, destroy() and destroyForcibly() both call TerminateProcess, but elsewhere destroy() may only ask the process to exit. The total wait stays 400 ms. --- .../datadog/trace/util/ProcessSupervisor.java | 16 ++++++++++++---- .../trace/util/ProcessSupervisorTest.java | 11 +++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index 9b9e06438eb..b63446a953e 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -48,7 +48,8 @@ public interface HealthCheck { private static final int QUIET_AFTER_FAILURES = 5; private static final long STABLE_RUN_MILLIS = 60_000; - private static final long STOP_WAIT_MILLIS = 400; + // How long to wait after each stop request + private static final long STOP_WAIT_MILLIS = 200; // A process this supervisor started is restarted after failing this many health checks in a row. // When a process it did not start fails this many, it starts its own. @@ -265,15 +266,22 @@ private void stopProcess() { if (currentProcess != null) { String childPid = pidOrUnknown(currentProcess); log.info("Stopping process [{}] with pid {}", imageName, childPid); - // The supervised agents run only on Windows, where destroy() and destroyForcibly() both call - // TerminateProcess: the process stops at once, without a chance to flush. - currentProcess.destroy(); + // On Windows, where the AAS agents run, destroy() and destroyForcibly() both call + // TerminateProcess: the process stops at once, without a chance to flush. Elsewhere + // destroy() may ask the process to exit, so destroyForcibly() follows if it is still running. boolean interrupted = Thread.interrupted(); boolean stopped = false; try { + currentProcess.destroy(); stopped = currentProcess.waitFor(STOP_WAIT_MILLIS, MILLISECONDS); + if (!stopped) { + currentProcess.destroyForcibly(); + stopped = currentProcess.waitFor(STOP_WAIT_MILLIS, MILLISECONDS); + } } catch (InterruptedException e) { interrupted = true; + currentProcess.destroyForcibly(); + stopped = !currentProcess.isAlive(); } finally { if (interrupted) { Thread.currentThread().interrupt(); diff --git a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java index c6d0c1ad21e..6cc124a3a3a 100644 --- a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java +++ b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java @@ -292,6 +292,7 @@ void keepsTrackingItsProcessWhenItDoesNotStopInsteadOfStartingASecondCopy() thro assertEquals(1, started.size()); assertTrue(started.get(0).isAlive()); assertTrue(started.get(0).stopRequests.get() >= 1, "asks it to stop"); + assertTrue(started.get(0).forcedStopRequests.get() >= 1, "then forces it to stop"); assertTrue(countLogs(Level.WARN, "was still running after the stop request") >= 1); } } @@ -385,6 +386,7 @@ private static final class FakeProcess extends Process { private final boolean stopsWhenAsked; private volatile int exitCode; final AtomicInteger stopRequests = new AtomicInteger(); + final AtomicInteger forcedStopRequests = new AtomicInteger(); FakeProcess(boolean stopsWhenAsked) { this.stopsWhenAsked = stopsWhenAsked; @@ -427,6 +429,15 @@ public void destroy() { } } + @Override + public Process destroyForcibly() { + forcedStopRequests.incrementAndGet(); + if (stopsWhenAsked) { + exit(137); + } + return this; + } + @Override public OutputStream getOutputStream() { return new ByteArrayOutputStream(); From 30cdef1adda4981e98cc112046f8f0a76d4df238 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 19 Nov 2026 12:47:08 -0500 Subject: [PATCH 20/30] Send AAS trace agent health check requests on one reused thread A new thread for every /info check, every 10 s, added up to thousands of short-lived threads a day. A new request is sent only once the previous one has finished, so one thread is enough. --- .../ddagent/ExternalAgentLauncher.java | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java index a868371e87d..d45c98252be 100644 --- a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java +++ b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java @@ -21,6 +21,8 @@ import java.io.RandomAccessFile; import java.nio.charset.StandardCharsets; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.FutureTask; import java.util.concurrent.TimeoutException; import org.slf4j.Logger; @@ -110,6 +112,10 @@ static final class NamedPipeHealthCheck implements ProcessSupervisor.HealthCheck // this needs no synchronization. private FutureTask timedOutInfoRequest; + // Created on first use. A new request is sent only once the previous one has finished, so one + // thread is enough. + private ExecutorService requestThread; + NamedPipeHealthCheck(String pipeName, boolean checkResponse) { this.checkResponse = checkResponse; if (pipeName.startsWith(NAMED_PIPE_PREFIX)) { @@ -154,10 +160,11 @@ private ProcessSupervisor.Health healthOfBoundProcess() throws InterruptedExcept } /** - * Sends a request on another thread and waits up to {@link #RESPONSE_TIMEOUT_MILLIS} for the - * answer. Reads from a pipe cannot time out, so a request to a hung agent stays blocked until - * the agent exits. Until then later checks fail without sending another request. An answer that - * arrives after the timeout still counts at the next check, so a slow agent is not restarted. + * Sends a request on the request thread and waits up to {@link #RESPONSE_TIMEOUT_MILLIS} for + * the answer. Reads from a pipe cannot time out, so a request to a hung agent stays blocked + * until the agent exits. Until then later checks fail without sending another request. An + * answer that arrives after the timeout still counts at the next check, so a slow agent is not + * restarted. */ private boolean agentResponds() throws InterruptedException { FutureTask lateRequest = timedOutInfoRequest; @@ -171,8 +178,7 @@ private boolean agentResponds() throws InterruptedException { } } FutureTask infoRequest = new FutureTask<>(this::sendInfoRequest); - AgentThreadFactory.newAgentThread(PROCESS_SUPERVISOR, "-health-check", infoRequest, true) - .start(); + requestThread().execute(infoRequest); try { return infoRequest.get(RESPONSE_TIMEOUT_MILLIS, MILLISECONDS); } catch (TimeoutException e) { @@ -184,6 +190,15 @@ private boolean agentResponds() throws InterruptedException { } } + private ExecutorService requestThread() { + if (requestThread == null) { + requestThread = + Executors.newSingleThreadExecutor(task -> + AgentThreadFactory.newAgentThread(PROCESS_SUPERVISOR, "-health-check", task, true)); + } + return requestThread; + } + /** Returns the result of a finished request, without waiting. */ private static boolean answered(FutureTask finishedRequest) throws InterruptedException { From 0d6687ff1dd85da51556843de7072f6f73b0bd95 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 19 Nov 2026 13:14:26 -0500 Subject: [PATCH 21/30] Test the AAS trace agent check's timeout and late answers on any OS agentResponds now takes the request as a Callable, so its timeout logic runs without a named pipe: an answer within the timeout, a late answer counted at the next check, a late failure followed by a new request, and no new request while one is still waiting. --- .../ddagent/ExternalAgentLauncher.java | 9 +- .../ddagent/NamedPipeHealthCheckTest.java | 84 +++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 communication/src/test/java/datadog/communication/ddagent/NamedPipeHealthCheckTest.java diff --git a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java index d45c98252be..f26c638e9a5 100644 --- a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java +++ b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.nio.charset.StandardCharsets; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -156,7 +157,7 @@ public ProcessSupervisor.Health run(ProcessSupervisor.Health previousHealth) /** Checks the process that has the pipe, which may or may not be the supervisor's own. */ private ProcessSupervisor.Health healthOfBoundProcess() throws InterruptedException { - return !checkResponse || agentResponds() ? HEALTHY : UNHEALTHY; + return !checkResponse || agentResponds(this::sendInfoRequest) ? HEALTHY : UNHEALTHY; } /** @@ -165,8 +166,10 @@ private ProcessSupervisor.Health healthOfBoundProcess() throws InterruptedExcept * until the agent exits. Until then later checks fail without sending another request. An * answer that arrives after the timeout still counts at the next check, so a slow agent is not * restarted. + * + * @param request returns true if the agent answered */ - private boolean agentResponds() throws InterruptedException { + boolean agentResponds(Callable request) throws InterruptedException { FutureTask lateRequest = timedOutInfoRequest; if (lateRequest != null) { if (!lateRequest.isDone()) { @@ -177,7 +180,7 @@ private boolean agentResponds() throws InterruptedException { return true; } } - FutureTask infoRequest = new FutureTask<>(this::sendInfoRequest); + FutureTask infoRequest = new FutureTask<>(request); requestThread().execute(infoRequest); try { return infoRequest.get(RESPONSE_TIMEOUT_MILLIS, MILLISECONDS); diff --git a/communication/src/test/java/datadog/communication/ddagent/NamedPipeHealthCheckTest.java b/communication/src/test/java/datadog/communication/ddagent/NamedPipeHealthCheckTest.java new file mode 100644 index 00000000000..603c23d5542 --- /dev/null +++ b/communication/src/test/java/datadog/communication/ddagent/NamedPipeHealthCheckTest.java @@ -0,0 +1,84 @@ +package datadog.communication.ddagent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.test.util.PollingConditions; +import java.io.IOException; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +/** The timeout and late-answer handling of the trace agent check, without a named pipe. */ +class NamedPipeHealthCheckTest { + + private final ExternalAgentLauncher.NamedPipeHealthCheck healthCheck = + new ExternalAgentLauncher.NamedPipeHealthCheck("test-pipe", true); + private final CountDownLatch release = new CountDownLatch(1); + + @AfterEach + void releaseBlockedRequests() { + release.countDown(); + } + + @Test + void countsTheResultOfARequestThatFinishesInTime() throws InterruptedException { + assertTrue(healthCheck.agentResponds(() -> true)); + assertFalse(healthCheck.agentResponds(() -> false)); + assertFalse( + healthCheck.agentResponds(() -> { + throw new IOException("broken pipe"); + })); + } + + @Test + void countsALateAnswerAtTheNextCheck() throws InterruptedException { + assertFalse(healthCheck.agentResponds(answersOnceReleased(true))); + release.countDown(); + + AtomicInteger newRequests = new AtomicInteger(); + new PollingConditions(5) + .eventually(() -> assertTrue(healthCheck.agentResponds(counting(newRequests, false)))); + assertEquals(0, newRequests.get()); + } + + @Test + void sendsANewRequestAfterALateFailure() throws InterruptedException { + assertFalse(healthCheck.agentResponds(answersOnceReleased(false))); + release.countDown(); + + AtomicInteger newRequests = new AtomicInteger(); + new PollingConditions(5) + .eventually(() -> assertTrue(healthCheck.agentResponds(counting(newRequests, true)))); + assertEquals(1, newRequests.get()); + } + + @Test + @Timeout(10) // waiting on the pending request would block the supervisor thread + void sendsNoNewRequestWhileOneIsStillWaiting() throws InterruptedException { + assertFalse(healthCheck.agentResponds(answersOnceReleased(true))); + + AtomicInteger newRequests = new AtomicInteger(); + assertFalse(healthCheck.agentResponds(counting(newRequests, true))); + assertEquals(0, newRequests.get()); + } + + /** A request that answers only after the test releases it, so the first check times out. */ + private Callable answersOnceReleased(boolean answer) { + return () -> { + release.await(); + return answer; + }; + } + + private static Callable counting(AtomicInteger requests, boolean answer) { + return () -> { + requests.incrementAndGet(); + return answer; + }; + } +} From c30290bdf92d86271fe85b3eb6907fd835b9ca62 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 19 Nov 2026 13:16:30 -0500 Subject: [PATCH 22/30] Forget AAS agent failures only after an adopted agent stays healthy A single healthy check of an agent this tracer did not start reset the failure count, so the logs of an agent that was only sometimes healthy never went quiet. Like a process this tracer started, and like dd-trace-dotnet, it now has to stay healthy for 60 s. That also resets the restart backoff, which an adopted agent never reset before. --- .../datadog/trace/util/ProcessSupervisor.java | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index b63446a953e..0e57d744ab5 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -67,6 +67,7 @@ public interface HealthCheck { private long currentProcessStartedMillis; private int failedHealthChecks; private int otherProcessFailedHealthChecks; + private long otherProcessHealthySinceMillis; private final FaultBackoff faultBackoff = new FaultBackoff(); private int failuresInARow; @@ -100,8 +101,7 @@ private void mainLoop() { if (currentProcess == null) { currentHealth = healthCheck.run(currentHealth); logHealthCheckChange(currentHealth); - otherProcessFailedHealthChecks = - currentHealth == UNHEALTHY ? otherProcessFailedHealthChecks + 1 : 0; + recordOtherProcessCheck(currentHealth); } // A copy started next to a process that is still serving cannot take over until that one // exits, so it is only started once that process has failed several checks in a row. @@ -132,7 +132,6 @@ private void logHealthCheckChange(Health result) { // These checks only run while this supervisor has no running process of its own, // so a healthy or unhealthy result means another process (usually another worker) is serving. if (result == HEALTHY) { - failuresInARow = 0; if (previous == UNHEALTHY) { log.info( "Process [{}] that this process (pid {}) is using is passing health checks again", @@ -159,6 +158,28 @@ private void logHealthCheckChange(Health result) { } } + /** + * Tracks a process this supervisor did not start. As with a process it started, earlier failures + * are forgotten only once it has stayed healthy for {@link #STABLE_RUN_MILLIS}, so restarts next + * to an agent that is only sometimes healthy still back off and their logs stay quiet. + */ + private void recordOtherProcessCheck(Health result) { + otherProcessFailedHealthChecks = result == UNHEALTHY ? otherProcessFailedHealthChecks + 1 : 0; + if (result != HEALTHY) { + otherProcessHealthySinceMillis = 0; + return; + } + long now = System.currentTimeMillis(); + if (otherProcessHealthySinceMillis == 0) { + otherProcessHealthySinceMillis = now; + } + long healthyMillis = now - otherProcessHealthySinceMillis; + faultBackoff.recordUptime(healthyMillis); + if (healthyMillis >= STABLE_RUN_MILLIS) { + failuresInARow = 0; + } + } + private void scheduleNextHealthCheck() { long now = System.currentTimeMillis(); if (currentHealth == HEALTHY || currentHealth == UNHEALTHY) { @@ -173,6 +194,7 @@ private void scheduleNextHealthCheck() { private void startProcessAndSupervise() throws Exception { if (currentProcess == null) { otherProcessFailedHealthChecks = 0; + otherProcessHealthySinceMillis = 0; log.debug("Starting process: [{}]", imageName); try (TraceScope ignored = AgentTracer.get().muteTracing()) { currentProcess = processBuilder.start(); From 45d6be21aba0d1c23d3afce6773de623ffa6dd7f Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 19 Nov 2026 13:43:40 -0500 Subject: [PATCH 23/30] Forget AAS agent failures through one helper The backoff and the quiet-logging count follow the same 60 s rule, which three places applied by hand. One helper now does both. --- .../datadog/trace/util/ProcessSupervisor.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index 0e57d744ab5..1fa51e20c5c 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -173,10 +173,17 @@ private void recordOtherProcessCheck(Health result) { if (otherProcessHealthySinceMillis == 0) { otherProcessHealthySinceMillis = now; } - long healthyMillis = now - otherProcessHealthySinceMillis; - faultBackoff.recordUptime(healthyMillis); - if (healthyMillis >= STABLE_RUN_MILLIS) { - failuresInARow = 0; + recordUptime(now - otherProcessHealthySinceMillis); + } + + /** + * Forgets earlier failures, for both the restart backoff and the quiet logging, once a process + * has stayed up or healthy for {@link #STABLE_RUN_MILLIS}. + */ + private void recordUptime(long upMillis) { + faultBackoff.recordUptime(upMillis); + if (upMillis >= STABLE_RUN_MILLIS) { + failuresInARow = 0; // log its lifecycle at info level again } } @@ -213,11 +220,7 @@ private void startProcessAndSupervise() throws Exception { // Keep checking on the process until it exits while (!currentProcess.waitFor(HEALTHY_DELAY_MILLIS, MILLISECONDS)) { - long upMillis = System.currentTimeMillis() - currentProcessStartedMillis; - faultBackoff.recordUptime(upMillis); - if (upMillis >= STABLE_RUN_MILLIS) { - failuresInARow = 0; // it stayed up, so log its lifecycle at info level again - } + recordUptime(System.currentTimeMillis() - currentProcessStartedMillis); if (healthCheck != ALWAYS_READY && !passesHealthCheck(childPid)) { restartUnhealthyProcess(childPid); return; @@ -231,10 +234,7 @@ private void startProcessAndSupervise() throws Exception { boolean failed = code != 0 || !stayedUp; currentHealth = failed ? FAULTED : INTERRUPTED; - faultBackoff.recordUptime(ranMillis); - if (stayedUp) { - failuresInARow = 0; - } + recordUptime(ranMillis); logLifecycle( "Process [{}] with pid {} exited with code {} after {} ms", imageName, From cbf0f413dea3e3db0d2ddf2ab986736e28739238 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 19 Nov 2026 13:51:47 -0500 Subject: [PATCH 24/30] Count AAS agent uptime only while it passes its health checks An agent that was hung and did not stop when asked stays tracked, and once its start was over 60 s ago every wait reset the backoff and the quiet logging. It was then retried about every 12 s, with two warnings each time, for as long as it stayed hung. --- .../src/main/java/datadog/trace/util/ProcessSupervisor.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index 1fa51e20c5c..f62fb782448 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -220,7 +220,11 @@ private void startProcessAndSupervise() throws Exception { // Keep checking on the process until it exits while (!currentProcess.waitFor(HEALTHY_DELAY_MILLIS, MILLISECONDS)) { - recordUptime(System.currentTimeMillis() - currentProcessStartedMillis); + // Time up counts only while the process passes its checks. One that is failing them, or + // that did not stop when asked, has not stayed up in a useful way. + if (failedHealthChecks == 0) { + recordUptime(System.currentTimeMillis() - currentProcessStartedMillis); + } if (healthCheck != ALWAYS_READY && !passesHealthCheck(childPid)) { restartUnhealthyProcess(childPid); return; From ea73362d4ba3fb25c0c340920c119fec0d291ba2 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 19 Nov 2026 13:54:28 -0500 Subject: [PATCH 25/30] Count AAS agent failures in a row only in FaultBackoff The backoff and the quiet logging kept the same count in two counters, reset by hand in each place that records uptime. FaultBackoff now owns the one count: it spaces out retries and says when logging goes quiet. --- .../datadog/trace/util/ProcessSupervisor.java | 62 +++++++++---------- .../trace/util/ProcessSupervisorTest.java | 18 ++++++ 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index f62fb782448..09387ca45eb 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -69,7 +69,6 @@ public interface HealthCheck { private int otherProcessFailedHealthChecks; private long otherProcessHealthySinceMillis; private final FaultBackoff faultBackoff = new FaultBackoff(); - private int failuresInARow; private volatile boolean stopping = false; @@ -173,18 +172,7 @@ private void recordOtherProcessCheck(Health result) { if (otherProcessHealthySinceMillis == 0) { otherProcessHealthySinceMillis = now; } - recordUptime(now - otherProcessHealthySinceMillis); - } - - /** - * Forgets earlier failures, for both the restart backoff and the quiet logging, once a process - * has stayed up or healthy for {@link #STABLE_RUN_MILLIS}. - */ - private void recordUptime(long upMillis) { - faultBackoff.recordUptime(upMillis); - if (upMillis >= STABLE_RUN_MILLIS) { - failuresInARow = 0; // log its lifecycle at info level again - } + faultBackoff.recordUptime(now - otherProcessHealthySinceMillis); } private void scheduleNextHealthCheck() { @@ -192,7 +180,7 @@ private void scheduleNextHealthCheck() { if (currentHealth == HEALTHY || currentHealth == UNHEALTHY) { nextCheckMillis = now + HEALTHY_DELAY_MILLIS; } else if (currentHealth == FAULTED) { - nextCheckMillis = now + faultBackoff.recordFault(); + nextCheckMillis = now + recordFault(); } else { // interrupted nextCheckMillis = Long.max(nextCheckMillis, now + 100); } @@ -223,7 +211,7 @@ private void startProcessAndSupervise() throws Exception { // Time up counts only while the process passes its checks. One that is failing them, or // that did not stop when asked, has not stayed up in a useful way. if (failedHealthChecks == 0) { - recordUptime(System.currentTimeMillis() - currentProcessStartedMillis); + faultBackoff.recordUptime(System.currentTimeMillis() - currentProcessStartedMillis); } if (healthCheck != ALWAYS_READY && !passesHealthCheck(childPid)) { restartUnhealthyProcess(childPid); @@ -238,16 +226,13 @@ private void startProcessAndSupervise() throws Exception { boolean failed = code != 0 || !stayedUp; currentHealth = failed ? FAULTED : INTERRUPTED; - recordUptime(ranMillis); + faultBackoff.recordUptime(ranMillis); logLifecycle( "Process [{}] with pid {} exited with code {} after {} ms", imageName, childPid, code, ranMillis); - if (failed) { - recordFailure(); - } // Process is dead, no longer needs to be tracked currentProcess = null; @@ -329,39 +314,38 @@ private void stopProcess() { } private void logLifecycle(String format, Object... arguments) { - if (failuresInARow < QUIET_AFTER_FAILURES) { + if (!faultBackoff.isQuiet()) { log.info(format, arguments); } else { log.debug(format, arguments); } } - /** - * Counts a failure in a row and logs it at warn level, or at debug level once the log has gone - * quiet. - */ + /** Logs a failure at warn level, or at debug level once the log has gone quiet. */ private void logFailure(String format, Object... arguments) { - if (failuresInARow < QUIET_AFTER_FAILURES) { + if (!faultBackoff.isQuiet()) { log.warn(format, arguments); } else { log.debug(format, arguments); } - recordFailure(); } /** - * Counts a failure in a row. When the count reaches {@link #QUIET_AFTER_FAILURES}, says that - * lifecycle lines go to debug level until the process stays up. + * Records a fault and returns how long to wait before the next health check. When the faults in a + * row reach {@link #QUIET_AFTER_FAILURES}, says that lifecycle lines go to debug level until the + * process stays up. */ - private void recordFailure() { - if (++failuresInARow == QUIET_AFTER_FAILURES) { + private long recordFault() { + long delayMillis = faultBackoff.recordFault(); + if (faultBackoff.faultsInARow() == QUIET_AFTER_FAILURES) { log.warn( "Process [{}] has failed {} times in a row; " + "further starts and exits are logged at debug level until it stays up for {} seconds", imageName, - failuresInARow, + QUIET_AFTER_FAILURES, MILLISECONDS.toSeconds(STABLE_RUN_MILLIS)); } + return delayMillis; } private static String pidOrUnknown(Process process) { @@ -387,9 +371,10 @@ public void close() { } /** - * Counts faults in a row and spaces out the retries that follow them. Faults are forgotten only - * once a process has stayed up for {@link #STABLE_RUN_MILLIS}, so an agent that keeps exiting - * soon after it starts backs off instead of restarting every few seconds. Retries never stop. + * Counts faults in a row, spaces out the retries that follow them, and decides when lifecycle + * logging goes quiet. Faults are forgotten only once a process has stayed up for {@link + * #STABLE_RUN_MILLIS}, so an agent that keeps exiting soon after it starts backs off instead of + * restarting every few seconds. Retries never stop. */ static final class FaultBackoff { private static final int MAX_DOUBLINGS = 5; @@ -403,6 +388,15 @@ long recordFault() { return Math.min(FAULTED_DELAY_MILLIS << doublings, MAX_FAULTED_DELAY_MILLIS); } + int faultsInARow() { + return faults; + } + + /** Whether lifecycle lines go to debug level, after {@link #QUIET_AFTER_FAILURES} faults. */ + boolean isQuiet() { + return faults >= QUIET_AFTER_FAILURES; + } + /** Forgets earlier faults once a process has been up for a stable period. */ void recordUptime(long upMillis) { if (upMillis >= STABLE_RUN_MILLIS) { diff --git a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java index 6cc124a3a3a..29224e6497e 100644 --- a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java +++ b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java @@ -124,6 +124,24 @@ void faultBackoffResetsOnlyAfterStableUptime() { assertEquals(2_000, backoff.recordFault()); } + @Test + void faultBackoffGoesQuietAfterFiveFaultsUntilStableUptime() { + ProcessSupervisor.FaultBackoff backoff = new ProcessSupervisor.FaultBackoff(); + for (int i = 0; i < 4; i++) { + backoff.recordFault(); + } + assertFalse(backoff.isQuiet()); + + backoff.recordFault(); + assertTrue(backoff.isQuiet()); + + backoff.recordUptime(59_999); + assertTrue(backoff.isQuiet()); + + backoff.recordUptime(60_000); + assertFalse(backoff.isQuiet()); + } + @Test void processRespawnsWhenKilled() throws InterruptedException { ProcessBuilder processBuilder = createProcessBuilder(); From 4a06e8d938210a75f98b558ff7ce1f368ed55400 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 19 Nov 2026 13:54:29 -0500 Subject: [PATCH 26/30] Mark ProcessSupervisor.FaultBackoff as visible for testing --- .../src/main/java/datadog/trace/util/ProcessSupervisor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index 09387ca45eb..68bfed402fe 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -376,6 +376,7 @@ public void close() { * #STABLE_RUN_MILLIS}, so an agent that keeps exiting soon after it starts backs off instead of * restarting every few seconds. Retries never stop. */ + @VisibleForTesting static final class FaultBackoff { private static final int MAX_DOUBLINGS = 5; From b79da991962752c8f356d577ca4f89183dc70632 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 19 Nov 2026 13:55:44 -0500 Subject: [PATCH 27/30] Document the AAS supervisor's new fields with Javadoc The repository's guidance asks for Javadoc on field documentation. --- .../ddagent/ExternalAgentLauncher.java | 15 ++++++++++----- .../datadog/trace/util/ProcessSupervisor.java | 18 ++++++++++++------ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java index f26c638e9a5..7c5d199ddcd 100644 --- a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java +++ b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java @@ -103,18 +103,23 @@ static final class NamedPipeHealthCheck implements ProcessSupervisor.HealthCheck .getBytes(StandardCharsets.US_ASCII); private static final int MAX_STATUS_LINE_LENGTH = 64; private static final long RESPONSE_TIMEOUT_MILLIS = 2_000; - // The .NET tracer's default named pipe connect timeout, DD_TRACE_PIPE_TIMEOUT_MS + + /** The .NET tracer's default named pipe connect timeout, {@code DD_TRACE_PIPE_TIMEOUT_MS}. */ private static final long OPEN_TIMEOUT_MILLIS = 500; private final File pipe; private final boolean checkResponse; - // A request that got no answer in time. Only the supervisor thread runs the health check, so - // this needs no synchronization. + /** + * A request that got no answer in time. Only the supervisor thread runs the health check, so + * this needs no synchronization. + */ private FutureTask timedOutInfoRequest; - // Created on first use. A new request is sent only once the previous one has finished, so one - // thread is enough. + /** + * Created on first use. A new request is sent only once the previous one has finished, so one + * thread is enough. + */ private ExecutorService requestThread; NamedPipeHealthCheck(String pipeName, boolean checkResponse) { diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index 68bfed402fe..ea0a7e4f438 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -42,17 +42,23 @@ public interface HealthCheck { private static final long FAULTED_DELAY_MILLIS = 2_000; private static final long MAX_FAULTED_DELAY_MILLIS = 60_000; - // Lifecycle events are logged at info level. A process that keeps failing quickly would flood - // the log, so after this many failures in a row the details go to debug level until the - // process stays up for STABLE_RUN_MILLIS. + /** + * Lifecycle events are logged at info level. A process that keeps failing quickly would flood the + * log, so after this many failures in a row the details go to debug level until the process stays + * up for {@link #STABLE_RUN_MILLIS}. + */ private static final int QUIET_AFTER_FAILURES = 5; + + /** How long a process must stay up, or healthy, before earlier failures are forgotten. */ private static final long STABLE_RUN_MILLIS = 60_000; - // How long to wait after each stop request + /** How long to wait after each stop request. */ private static final long STOP_WAIT_MILLIS = 200; - // A process this supervisor started is restarted after failing this many health checks in a row. - // When a process it did not start fails this many, it starts its own. + /** + * A process this supervisor started is restarted after failing this many health checks in a row. + * When a process it did not start fails this many, it starts its own. + */ private static final int MAX_FAILED_HEALTH_CHECKS = 3; private final String imageName; From 9ce41d09b93bb282b19432be8e901f47477abbea Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Thu, 19 Nov 2026 14:03:11 -0500 Subject: [PATCH 28/30] Document how ProcessSupervisor uses its health check The health check now also judges a running process, and ALWAYS_READY is recognized by identity, so write both down. --- .../datadog/trace/util/ProcessSupervisor.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index ea0a7e4f438..ff9d57f5b2f 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -29,11 +29,27 @@ public enum Health { UNHEALTHY } + /** + * Checks whether the supervised program is healthy. It runs in two situations: + * + *
    + *
  • While this supervisor has no process of its own, to decide what to do: {@link + * Health#READY_TO_START} starts a process, {@link Health#HEALTHY} uses a process that + * another one started, and {@link Health#UNHEALTHY} waits until that process has failed + * several checks in a row before starting a copy. + *
  • While a process this supervisor started is running, with {@code previousHealth} {@link + * Health#HEALTHY}. Any other result is a failed check, and several in a row restart the + * process. + *
+ * + * A supervisor given {@link #ALWAYS_READY} skips the second kind of check. + */ @FunctionalInterface public interface HealthCheck { Health run(Health previousHealth) throws InterruptedException; } + /** Starts a process whenever none is running, and never checks a running one. */ public static final HealthCheck ALWAYS_READY = health -> READY_TO_START; private static final Logger log = LoggerFactory.getLogger(ProcessSupervisor.class); From 2785408cf9ff26da080e7a3563a467f2260bae9f Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Fri, 20 Nov 2026 09:42:55 -0500 Subject: [PATCH 29/30] Stop the AAS trace agent health check thread when the supervisor stops Nothing shut the /info request thread down. HealthCheck gets a close() that the supervisor thread calls when it stops, and the pipe check shuts its request thread down there. A request still blocked on a read of a hung agent's pipe may stay blocked until that agent exits. --- .../ddagent/ExternalAgentLauncher.java | 11 ++++++++ .../datadog/trace/util/ProcessSupervisor.java | 4 +++ .../trace/util/ProcessSupervisorTest.java | 27 +++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java index 7c5d199ddcd..db825b5d4f3 100644 --- a/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java +++ b/communication/src/main/java/datadog/communication/ddagent/ExternalAgentLauncher.java @@ -207,6 +207,17 @@ private ExecutorService requestThread() { return requestThread; } + /** + * Stops the request thread. A request still blocked on a pipe read may stay blocked until the + * agent exits, since the read does not respond to the interrupt. + */ + @Override + public void close() { + if (requestThread != null) { + requestThread.shutdownNow(); + } + } + /** Returns the result of a finished request, without waiting. */ private static boolean answered(FutureTask finishedRequest) throws InterruptedException { diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index ff9d57f5b2f..266c5654360 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -47,6 +47,9 @@ public enum Health { @FunctionalInterface public interface HealthCheck { Health run(Health previousHealth) throws InterruptedException; + + /** Releases what the check holds. Called once, on the supervisor thread, when it stops. */ + default void close() {} } /** Starts a process whenever none is running, and never checks a running one. */ @@ -141,6 +144,7 @@ private void mainLoop() { } } finally { stopProcess(); + healthCheck.close(); } } diff --git a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java index 29224e6497e..b7731d86dd8 100644 --- a/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java +++ b/internal-api/src/test/java/datadog/trace/util/ProcessSupervisorTest.java @@ -289,6 +289,33 @@ void startsItsOwnProcessOnlyAfterAProcessItDidNotStartFailsThreeChecksInARow() } } + @Test + void closesItsHealthCheckWhenClosed() { + AtomicInteger checks = new AtomicInteger(); + AtomicInteger closes = new AtomicInteger(); + HealthCheck otherProcessAnswers = + new HealthCheck() { + @Override + public ProcessSupervisor.Health run(ProcessSupervisor.Health previous) { + checks.incrementAndGet(); + return HEALTHY; + } + + @Override + public void close() { + closes.incrementAndGet(); + } + }; + + ProcessSupervisor processSupervisor = + new ProcessSupervisor("test", createProcessBuilder(), otherProcessAnswers); + new PollingConditions(5).eventually(() -> assertEquals(1, checks.get())); + assertEquals(0, closes.get()); + + processSupervisor.close(); + new PollingConditions(5).eventually(() -> assertEquals(1, closes.get())); + } + @Test void keepsTrackingItsProcessWhenItDoesNotStopInsteadOfStartingASecondCopy() throws IOException { List started = new CopyOnWriteArrayList<>(); From 5907ed24206c4aaeb39067b511491f236ce3edf7 Mon Sep 17 00:00:00 2001 From: "E. Lewis" Date: Fri, 20 Nov 2026 09:44:40 -0500 Subject: [PATCH 30/30] Quiet the AAS supervisor's stop lines with its other lifecycle lines "Stopping process", "Stopped process" and "was still running after the stop request" bypassed the quiet mode, so an agent that kept hanging and getting restarted still logged them at info or warn each cycle. --- .../src/main/java/datadog/trace/util/ProcessSupervisor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java index 266c5654360..7e798939639 100644 --- a/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java +++ b/internal-api/src/main/java/datadog/trace/util/ProcessSupervisor.java @@ -302,7 +302,7 @@ private void restartUnhealthyProcess(String childPid) { private void stopProcess() { if (currentProcess != null) { String childPid = pidOrUnknown(currentProcess); - log.info("Stopping process [{}] with pid {}", imageName, childPid); + logLifecycle("Stopping process [{}] with pid {}", imageName, childPid); // On Windows, where the AAS agents run, destroy() and destroyForcibly() both call // TerminateProcess: the process stops at once, without a chance to flush. Elsewhere // destroy() may ask the process to exit, so destroyForcibly() follows if it is still running. @@ -325,11 +325,11 @@ private void stopProcess() { } } if (stopped) { - log.info("Stopped process [{}] with pid {}", imageName, childPid); + logLifecycle("Stopped process [{}] with pid {}", imageName, childPid); currentProcess = null; } else { // Keep tracking it, so a restart does not start a second copy while this one runs - log.warn( + logFailure( "Process [{}] with pid {} was still running after the stop request", imageName, childPid);