Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
e7c76c0
Log AAS agent supervisor lifecycle at the default level
Lewis-E Oct 15, 2026
548da50
Back off AAS agent restarts instead of giving up
Lewis-E Oct 15, 2026
949028c
Health check and restart the AAS agent this tracer started
Lewis-E Oct 15, 2026
3b5b539
Treat a busy AAS agent pipe as present
Lewis-E Oct 15, 2026
85c2e77
Check that the AAS trace agent answers on its pipe
Lewis-E Oct 15, 2026
6cd7eed
Test how ProcessSupervisor supervises the AAS agent
Lewis-E Oct 16, 2026
2d02d24
Name the AAS agent pipe health checks by what they check
Lewis-E Nov 5, 2026
15a9d15
Stop the AAS agent with a single TerminateProcess
Lewis-E Nov 5, 2026
0bcc245
Fix AAS agent supervisor review findings
Lewis-E Nov 5, 2026
10502c0
Test the Java 9+ child process PID lookup
Lewis-E Nov 17, 2026
e81efaa
Update internal-api/src/main/java/datadog/trace/util/PidHelper.java
Lewis-E Nov 18, 2026
66ea5ef
Update internal-api/src/main/java/datadog/trace/util/ProcessSuperviso…
Lewis-E Nov 18, 2026
3a66484
Fix the PidHelper imports and remove the unused PID function
Lewis-E Nov 18, 2026
d4604f1
Count AAS agent failures in one place
Lewis-E Nov 18, 2026
d1e37be
Test the child PID lookup through PidHelper
Lewis-E Nov 18, 2026
3be4b0a
Fail the AAS trace agent check on a pipe that stays busy
Lewis-E Nov 19, 2026
a1cc028
Name the AAS supervisor's catch-all log line for what it covers
Lewis-E Nov 19, 2026
2783ac5
Log why an AAS trace agent health check request failed
Lewis-E Nov 19, 2026
73e3e55
Force-stop a supervised process that is still running after destroy()
Lewis-E Nov 19, 2026
30cdef1
Send AAS trace agent health check requests on one reused thread
Lewis-E Nov 19, 2026
0d6687f
Test the AAS trace agent check's timeout and late answers on any OS
Lewis-E Nov 19, 2026
c30290b
Forget AAS agent failures only after an adopted agent stays healthy
Lewis-E Nov 19, 2026
45d6be2
Forget AAS agent failures through one helper
Lewis-E Nov 19, 2026
cbf0f41
Count AAS agent uptime only while it passes its health checks
Lewis-E Nov 19, 2026
ea73362
Count AAS agent failures in a row only in FaultBackoff
Lewis-E Nov 19, 2026
4a06e8d
Mark ProcessSupervisor.FaultBackoff as visible for testing
Lewis-E Nov 19, 2026
b79da99
Document the AAS supervisor's new fields with Javadoc
Lewis-E Nov 19, 2026
9ce41d0
Document how ProcessSupervisor uses its health check
Lewis-E Nov 19, 2026
2785408
Stop the AAS trace agent health check thread when the supervisor stops
Lewis-E Nov 20, 2026
5907ed2
Quiet the AAS supervisor's stop lines with its other lifecycle lines
Lewis-E Nov 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,15 +1,31 @@
package datadog.communication.ddagent;

import static datadog.trace.util.AgentThreadFactory.AgentThread.PROCESS_SUPERVISOR;
import static datadog.trace.util.ProcessSupervisor.ALWAYS_READY;
import static datadog.trace.util.ProcessSupervisor.Health.HEALTHY;
import static datadog.trace.util.ProcessSupervisor.Health.NEVER_CHECKED;
import static datadog.trace.util.ProcessSupervisor.Health.READY_TO_START;
import static datadog.trace.util.ProcessSupervisor.Health.UNHEALTHY;
import static java.util.concurrent.TimeUnit.MILLISECONDS;

import datadog.environment.OperatingSystem;
import datadog.trace.api.Config;
import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
import datadog.trace.context.TraceScope;
import datadog.trace.util.AgentThreadFactory;
import datadog.trace.util.ProcessSupervisor;
import java.io.Closeable;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.FutureTask;
import java.util.concurrent.TimeoutException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -32,7 +48,7 @@ public ExternalAgentLauncher(Config config) {

traceProcessSupervisor =
new ProcessSupervisor(
"trace-agent", traceProcessBuilder, healthCheck(config.getAgentNamedPipe()));
"trace-agent", traceProcessBuilder, answersOnPipe(config.getAgentNamedPipe()));
} else {
log.warn("Trace agent path not set. Will not start trace agent process");
}
Expand All @@ -45,7 +61,7 @@ public ExternalAgentLauncher(Config config) {

dogStatsDProcessSupervisor =
new ProcessSupervisor(
"dogstatsd", dogStatsDProcessBuilder, healthCheck(config.getDogStatsDNamedPipe()));
"dogstatsd", dogStatsDProcessBuilder, pipePresent(config.getDogStatsDNamedPipe()));
} else {
log.warn("DogStatsD path not set. Will not start DogStatsD process");
}
Expand All @@ -63,18 +79,51 @@ public void close() {
}
}

private static ProcessSupervisor.HealthCheck healthCheck(String pipeName) {
/** Healthy while the agent's pipe is present and the agent answers a request on it. */
private static ProcessSupervisor.HealthCheck answersOnPipe(String pipeName) {
return healthCheck(pipeName, true);
}

/** Healthy while the agent's pipe is present. DogStatsD never replies on its pipe. */
private static ProcessSupervisor.HealthCheck pipePresent(String pipeName) {
return healthCheck(pipeName, false);
}

private static ProcessSupervisor.HealthCheck healthCheck(String pipeName, boolean checkResponse) {
return null != pipeName && !pipeName.trim().isEmpty()
? new NamedPipeHealthCheck(pipeName)
? new NamedPipeHealthCheck(pipeName, checkResponse)
: ALWAYS_READY;
}

static final class NamedPipeHealthCheck implements ProcessSupervisor.HealthCheck {
private static final String NAMED_PIPE_PREFIX = "\\\\.\\pipe\\";

private static final byte[] INFO_REQUEST =
"GET /info HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"
.getBytes(StandardCharsets.US_ASCII);
private static final int MAX_STATUS_LINE_LENGTH = 64;
private static final long RESPONSE_TIMEOUT_MILLIS = 2_000;

/** The .NET tracer's default named pipe connect timeout, {@code DD_TRACE_PIPE_TIMEOUT_MS}. */
private static final long OPEN_TIMEOUT_MILLIS = 500;

private final File pipe;
private final boolean checkResponse;

NamedPipeHealthCheck(String pipeName) {
/**
* A request that got no answer in time. Only the supervisor thread runs the health check, so
* this needs no synchronization.
*/
private FutureTask<Boolean> timedOutInfoRequest;

/**
* Created on first use. A new request is sent only once the previous one has finished, so one
* thread is enough.
*/
private ExecutorService requestThread;

NamedPipeHealthCheck(String pipeName, boolean checkResponse) {
this.checkResponse = checkResponse;
if (pipeName.startsWith(NAMED_PIPE_PREFIX)) {
this.pipe = new File(pipeName);
} else {
Expand All @@ -91,7 +140,7 @@ public ProcessSupervisor.Health run(ProcessSupervisor.Health previousHealth)

double delayMillis = 50;
for (int retries = 0; retries < 7; retries++) {
if (!pipe.exists()) {
if (!pipeExists()) {
return READY_TO_START; // no longer bound, start our own external process
}

Expand All @@ -100,15 +149,162 @@ public ProcessSupervisor.Health run(ProcessSupervisor.Health previousHealth)
delayMillis = delayMillis * 1.75;
}

return HEALTHY; // use existing external process
return healthOfBoundProcess(); // use existing external process if it responds
}

// otherwise just check that the pipe is still bound
if (pipe.exists()) {
return HEALTHY; // keep using external process
if (pipeExists()) {
return healthOfBoundProcess(); // keep using external process if it responds
} else {
return READY_TO_START; // start our own process
}
}

/** Checks the process that has the pipe, which may or may not be the supervisor's own. */
private ProcessSupervisor.Health healthOfBoundProcess() throws InterruptedException {
return !checkResponse || agentResponds(this::sendInfoRequest) ? HEALTHY : UNHEALTHY;
}

/**
* Sends a request on the request thread and waits up to {@link #RESPONSE_TIMEOUT_MILLIS} for
* the answer. Reads from a pipe cannot time out, so a request to a hung agent stays blocked
* until the agent exits. Until then later checks fail without sending another request. An
* answer that arrives after the timeout still counts at the next check, so a slow agent is not
* restarted.
*
* @param request returns true if the agent answered
*/
boolean agentResponds(Callable<Boolean> request) throws InterruptedException {
FutureTask<Boolean> lateRequest = timedOutInfoRequest;
if (lateRequest != null) {
if (!lateRequest.isDone()) {
return false; // still waiting for an answer
}
timedOutInfoRequest = null;
if (answered(lateRequest)) {
return true;
}
}
FutureTask<Boolean> infoRequest = new FutureTask<>(request);
requestThread().execute(infoRequest);
try {
return infoRequest.get(RESPONSE_TIMEOUT_MILLIS, MILLISECONDS);
} catch (TimeoutException e) {
timedOutInfoRequest = infoRequest;
return false;
} catch (ExecutionException e) {
Comment thread
Lewis-E marked this conversation as resolved.
Resolved
log.debug("Health check request failed", e.getCause());
return false;
}
}

private ExecutorService requestThread() {
if (requestThread == null) {
requestThread =
Executors.newSingleThreadExecutor(task ->
Comment thread
Lewis-E marked this conversation as resolved.
Resolved
AgentThreadFactory.newAgentThread(PROCESS_SUPERVISOR, "-health-check", task, true));
}
return requestThread;
}

/**
* Stops the request thread. A request still blocked on a pipe read may stay blocked until the
* agent exits, since the read does not respond to the interrupt.
*/
@Override
public void close() {
if (requestThread != null) {
requestThread.shutdownNow();
}
}

/** Returns the result of a finished request, without waiting. */
private static boolean answered(FutureTask<Boolean> finishedRequest)
throws InterruptedException {
try {
return finishedRequest.get();
} catch (ExecutionException e) {
log.debug("Health check request failed", e.getCause());
return false;
}
}

/**
* Sends a minimal request over a pipe handle of its own, outside the tracer's HTTP clients.
*
* @return true if the agent answers with an HTTP status line, whatever the status
*/
private boolean sendInfoRequest() throws IOException, InterruptedException {
try (TraceScope ignored = AgentTracer.get().muteTracing()) {
RandomAccessFile file = openPipe();
if (file == null) {
return false;
}
try {
file.write(INFO_REQUEST);
StringBuilder statusLine = new StringBuilder();
for (int c = file.read();
c != -1 && c != '\r' && statusLine.length() < MAX_STATUS_LINE_LENGTH;
c = file.read()) {
statusLine.append((char) c);
}
return statusLine.toString().startsWith("HTTP/1.");
} finally {
file.close();
}
}
}

/**
* Opens the pipe, retrying for up to {@link #OPEN_TIMEOUT_MILLIS} like .NET's {@code
* NamedPipeClientStream.Connect}. A pipe server creates its next pipe instance only after a
* client connects, so an open in between fails with "All pipe instances are busy". An agent
* that stops accepting connections keeps its pipe busy, so a pipe still busy after the retries
* fails the check.
*
* @return the opened pipe, or null if it could not be opened in time
*/
private RandomAccessFile openPipe() throws InterruptedException {
long deadlineNanos = System.nanoTime() + MILLISECONDS.toNanos(OPEN_TIMEOUT_MILLIS);
while (true) {
try {
return new RandomAccessFile(pipe, "rw");
} catch (FileNotFoundException e) {
if (System.nanoTime() - deadlineNanos >= 0) {
return null;
}
Thread.sleep(1);
}
}
}

/**
* {@link File#exists()} can report false for a pipe whose instances are all busy, for example
* under load. In that case open the pipe once to tell busy from absent.
*/
private boolean pipeExists() {
if (pipe.exists()) {
return true;
}
if (!OperatingSystem.isWindows()) {
return false;
}
try (RandomAccessFile ignored = new RandomAccessFile(pipe, "rw")) {
return true;
} catch (FileNotFoundException e) {
return !isPipeNotFound(e);
} catch (IOException e) {
return true; // opened, then failed to close
}
}

/**
* Only "cannot find the file" (ERROR_FILE_NOT_FOUND) means no process is serving the pipe.
* Errors such as "All pipe instances are busy" (ERROR_PIPE_BUSY) mean that one is.
*/
private static boolean isPipeNotFound(FileNotFoundException e) {
String message = e.getMessage();
return message != null && message.contains("cannot find the file");
}
Comment thread
Lewis-E marked this conversation as resolved.
Resolved
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package datadog.communication.ddagent;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import datadog.trace.test.util.PollingConditions;
import java.io.IOException;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;

/** The timeout and late-answer handling of the trace agent check, without a named pipe. */
class NamedPipeHealthCheckTest {

private final ExternalAgentLauncher.NamedPipeHealthCheck healthCheck =
new ExternalAgentLauncher.NamedPipeHealthCheck("test-pipe", true);
private final CountDownLatch release = new CountDownLatch(1);

@AfterEach
void releaseBlockedRequests() {
release.countDown();
}

@Test
void countsTheResultOfARequestThatFinishesInTime() throws InterruptedException {
assertTrue(healthCheck.agentResponds(() -> true));
assertFalse(healthCheck.agentResponds(() -> false));
assertFalse(
healthCheck.agentResponds(() -> {
throw new IOException("broken pipe");
}));
}

@Test
void countsALateAnswerAtTheNextCheck() throws InterruptedException {
assertFalse(healthCheck.agentResponds(answersOnceReleased(true)));
release.countDown();

AtomicInteger newRequests = new AtomicInteger();
new PollingConditions(5)
.eventually(() -> assertTrue(healthCheck.agentResponds(counting(newRequests, false))));
assertEquals(0, newRequests.get());
}

@Test
void sendsANewRequestAfterALateFailure() throws InterruptedException {
assertFalse(healthCheck.agentResponds(answersOnceReleased(false)));
release.countDown();

AtomicInteger newRequests = new AtomicInteger();
new PollingConditions(5)
.eventually(() -> assertTrue(healthCheck.agentResponds(counting(newRequests, true))));
assertEquals(1, newRequests.get());
}

@Test
@Timeout(10) // waiting on the pending request would block the supervisor thread
void sendsNoNewRequestWhileOneIsStillWaiting() throws InterruptedException {
assertFalse(healthCheck.agentResponds(answersOnceReleased(true)));

AtomicInteger newRequests = new AtomicInteger();
assertFalse(healthCheck.agentResponds(counting(newRequests, true)));
assertEquals(0, newRequests.get());
}

/** A request that answers only after the test releases it, so the first check times out. */
private Callable<Boolean> answersOnceReleased(boolean answer) {
return () -> {
release.await();
return answer;
};
}

private static Callable<Boolean> counting(AtomicInteger requests, boolean answer) {
return () -> {
requests.incrementAndGet();
return answer;
};
}
}
Loading