From c23e25e246f475bfbc4112f814b7f9644cd134a5 Mon Sep 17 00:00:00 2001 From: Nikolay Martynov Date: Thu, 12 Nov 2026 14:25:06 -0500 Subject: [PATCH 1/4] Accept messages that exactly fill the send buffer The UTF-8 encoder rewrite in 3ec7da7 (4.4.4) rejects a message whose encoded size equals maxPacketSizeBytes, although it fits an empty send buffer. The retry into a fresh buffer fails the same way, so the message is dropped and BufferOverflowException goes to the error handler. For a multi-valued message, every remaining chunk is dropped with it, because the chunker fills packets up to exactly maxPacketSizeBytes. Co-Authored-By: Claude Opus 5.5 --- .../com/timgroup/statsd/StatsDProcessor.java | 2 +- .../NonBlockingDirectStatsDClientTest.java | 63 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/timgroup/statsd/StatsDProcessor.java b/src/main/java/com/timgroup/statsd/StatsDProcessor.java index fc34b126..0a60a697 100644 --- a/src/main/java/com/timgroup/statsd/StatsDProcessor.java +++ b/src/main/java/com/timgroup/statsd/StatsDProcessor.java @@ -164,7 +164,7 @@ protected void writeBuilderToSendBuffer(ByteBuffer sendBuffer) { byteBuffer[blen++] = (byte) (128 | (decoded & 63)); } - if (blen >= maxPacketSizeBytes) { + if (blen > maxPacketSizeBytes) { throw new BufferOverflowException(); } } diff --git a/src/test/java/com/timgroup/statsd/NonBlockingDirectStatsDClientTest.java b/src/test/java/com/timgroup/statsd/NonBlockingDirectStatsDClientTest.java index 47bbbe55..68ee7654 100644 --- a/src/test/java/com/timgroup/statsd/NonBlockingDirectStatsDClientTest.java +++ b/src/test/java/com/timgroup/statsd/NonBlockingDirectStatsDClientTest.java @@ -5,7 +5,9 @@ import static org.hamcrest.Matchers.hasItem; import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.junit.After; import org.junit.AfterClass; @@ -150,4 +152,65 @@ public void sends_too_long_multivalued_distribution_to_statsd() { server.messagesReceived(), hasItem(comparesEqualTo("my.prefix.mydistribution:667|d|@0.400000|#baz,foo:bar"))); } + + @Test(timeout = 5000L) + public void sends_multivalued_distribution_chunk_that_exactly_fills_the_packet() + throws Exception { + // The first chunk, including its trailing newline, is exactly MAX_PACKET_SIZE bytes. + String first = "my.prefix.mydistribution:4234:2345:678|d|@0.400000|#baz,foo:bar"; + String second = "my.prefix.mydistribution:1|d|@0.400000|#baz,foo:bar"; + assertThat(first.length() + 1, comparesEqualTo(MAX_PACKET_SIZE)); + + RecordingErrorHandler errors = new RecordingErrorHandler(); + DirectStatsDClient exactFitClient = buildClient(errors); + try { + exactFitClient.recordDistributionValues( + "mydistribution", new long[] {4234L, 2345L, 678L, 1L}, 0.4, "foo:bar", "baz"); + waitForMessages(errors, first, second); + } finally { + exactFitClient.stop(); + } + } + + @Test(timeout = 5000L) + public void sends_message_that_exactly_fills_the_packet() throws Exception { + String prefixAndSuffix = "my.prefix.:1|d\n"; + char[] aspect = new char[MAX_PACKET_SIZE - prefixAndSuffix.length()]; + Arrays.fill(aspect, 'x'); + String expected = "my.prefix." + new String(aspect) + ":1|d"; + + RecordingErrorHandler errors = new RecordingErrorHandler(); + DirectStatsDClient exactFitClient = buildClient(errors); + try { + exactFitClient.distribution(new String(aspect), 1L); + waitForMessages(errors, expected); + } finally { + exactFitClient.stop(); + } + } + + private static DirectStatsDClient buildClient(StatsDClientErrorHandler errorHandler) { + return new NonBlockingStatsDClientBuilder() + .withEnvironmentVariables(CLIENT_ENV) + .prefix("my.prefix") + .hostname("localhost") + .port(STATSD_SERVER_PORT) + .enableTelemetry(false) + .originDetectionEnabled(false) + .maxPacketSizeBytes(MAX_PACKET_SIZE) + .errorHandler(errorHandler) + .buildDirectStatsDClient(); + } + + private static void waitForMessages(RecordingErrorHandler errors, String... expected) + throws InterruptedException { + List expectedMessages = Arrays.asList(expected); + while (!server.messagesReceived().containsAll(expectedMessages)) { + List exceptions = errors.getExceptions(); + if (!exceptions.isEmpty()) { + throw new AssertionError("client reported errors: " + exceptions); + } + Thread.sleep(10L); + } + } } From 5e8e78890de34cb970d17b1e850ffe0ea028c706 Mon Sep 17 00:00:00 2001 From: Nikolay Martynov Date: Fri, 13 Nov 2026 07:10:45 -0500 Subject: [PATCH 2/4] Test exact-fit writes on StatsDProcessor directly Replace the client-level tests with StatsDProcessorTest, which reads the encoded packets from the outbound queue instead of a UDP server. It also covers a multibyte message whose encoded size, not its char count, exactly fills the packet. Co-Authored-By: Claude Opus 5.5 --- .../NonBlockingDirectStatsDClientTest.java | 63 ----------- .../timgroup/statsd/StatsDProcessorTest.java | 104 ++++++++++++++++++ 2 files changed, 104 insertions(+), 63 deletions(-) create mode 100644 src/test/java/com/timgroup/statsd/StatsDProcessorTest.java diff --git a/src/test/java/com/timgroup/statsd/NonBlockingDirectStatsDClientTest.java b/src/test/java/com/timgroup/statsd/NonBlockingDirectStatsDClientTest.java index 68ee7654..47bbbe55 100644 --- a/src/test/java/com/timgroup/statsd/NonBlockingDirectStatsDClientTest.java +++ b/src/test/java/com/timgroup/statsd/NonBlockingDirectStatsDClientTest.java @@ -5,9 +5,7 @@ import static org.hamcrest.Matchers.hasItem; import java.io.IOException; -import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.junit.After; import org.junit.AfterClass; @@ -152,65 +150,4 @@ public void sends_too_long_multivalued_distribution_to_statsd() { server.messagesReceived(), hasItem(comparesEqualTo("my.prefix.mydistribution:667|d|@0.400000|#baz,foo:bar"))); } - - @Test(timeout = 5000L) - public void sends_multivalued_distribution_chunk_that_exactly_fills_the_packet() - throws Exception { - // The first chunk, including its trailing newline, is exactly MAX_PACKET_SIZE bytes. - String first = "my.prefix.mydistribution:4234:2345:678|d|@0.400000|#baz,foo:bar"; - String second = "my.prefix.mydistribution:1|d|@0.400000|#baz,foo:bar"; - assertThat(first.length() + 1, comparesEqualTo(MAX_PACKET_SIZE)); - - RecordingErrorHandler errors = new RecordingErrorHandler(); - DirectStatsDClient exactFitClient = buildClient(errors); - try { - exactFitClient.recordDistributionValues( - "mydistribution", new long[] {4234L, 2345L, 678L, 1L}, 0.4, "foo:bar", "baz"); - waitForMessages(errors, first, second); - } finally { - exactFitClient.stop(); - } - } - - @Test(timeout = 5000L) - public void sends_message_that_exactly_fills_the_packet() throws Exception { - String prefixAndSuffix = "my.prefix.:1|d\n"; - char[] aspect = new char[MAX_PACKET_SIZE - prefixAndSuffix.length()]; - Arrays.fill(aspect, 'x'); - String expected = "my.prefix." + new String(aspect) + ":1|d"; - - RecordingErrorHandler errors = new RecordingErrorHandler(); - DirectStatsDClient exactFitClient = buildClient(errors); - try { - exactFitClient.distribution(new String(aspect), 1L); - waitForMessages(errors, expected); - } finally { - exactFitClient.stop(); - } - } - - private static DirectStatsDClient buildClient(StatsDClientErrorHandler errorHandler) { - return new NonBlockingStatsDClientBuilder() - .withEnvironmentVariables(CLIENT_ENV) - .prefix("my.prefix") - .hostname("localhost") - .port(STATSD_SERVER_PORT) - .enableTelemetry(false) - .originDetectionEnabled(false) - .maxPacketSizeBytes(MAX_PACKET_SIZE) - .errorHandler(errorHandler) - .buildDirectStatsDClient(); - } - - private static void waitForMessages(RecordingErrorHandler errors, String... expected) - throws InterruptedException { - List expectedMessages = Arrays.asList(expected); - while (!server.messagesReceived().containsAll(expectedMessages)) { - List exceptions = errors.getExceptions(); - if (!exceptions.isEmpty()) { - throw new AssertionError("client reported errors: " + exceptions); - } - Thread.sleep(10L); - } - } } diff --git a/src/test/java/com/timgroup/statsd/StatsDProcessorTest.java b/src/test/java/com/timgroup/statsd/StatsDProcessorTest.java new file mode 100644 index 00000000..ee6e83ad --- /dev/null +++ b/src/test/java/com/timgroup/statsd/StatsDProcessorTest.java @@ -0,0 +1,104 @@ +package com.timgroup.statsd; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.junit.After; +import org.junit.Test; + +public class StatsDProcessorTest { + + private static final int MAX_PACKET_SIZE = 64; + + private final RecordingErrorHandler errors = new RecordingErrorHandler(); + private StatsDProcessor processor; + + @After + public void stop() throws InterruptedException { + if (processor != null) { + processor.shutdown(false); + } + } + + @Test(timeout = 5000L) + public void writes_message_that_exactly_fills_the_packet() throws Exception { + String packet = packet("", MAX_PACKET_SIZE); + + assertEquals(Arrays.asList(packet), process(new ChunkedMessage(packet), 1)); + } + + @Test(timeout = 5000L) + public void writes_multibyte_message_that_exactly_fills_the_packet() throws Exception { + // One character short of the limit, but 'é' takes two bytes in UTF-8. + String packet = packet("é", MAX_PACKET_SIZE - 1); + + assertEquals(Arrays.asList(packet), process(new ChunkedMessage(packet), 1)); + } + + @Test(timeout = 5000L) + public void writes_every_chunk_when_one_exactly_fills_the_packet() throws Exception { + String first = packet("", MAX_PACKET_SIZE); + String second = packet("", 10); + + assertEquals(Arrays.asList(first, second), process(new ChunkedMessage(first, second), 2)); + } + + /** A packet of {@code length} chars that starts with {@code head} and ends with a newline. */ + private static String packet(String head, int length) { + char[] filler = new char[length - head.length() - 1]; + Arrays.fill(filler, 'x'); + return head + new String(filler) + "\n"; + } + + private List process(Message message, int expectedPackets) throws Exception { + processor = + new StatsDNonBlockingProcessor( + 16, errors, MAX_PACKET_SIZE, 8, 1, 0, 1, Executors.defaultThreadFactory()); + processor.startWorkers("StatsDProcessorTest-"); + assertTrue(processor.send(message)); + + List packets = new ArrayList<>(); + while (packets.size() < expectedPackets) { + List exceptions = errors.getExceptions(); + if (!exceptions.isEmpty()) { + throw new AssertionError("processor reported errors: " + exceptions); + } + ByteBuffer buffer = processor.getOutboundQueue().poll(10, TimeUnit.MILLISECONDS); + // An exact fit makes the processor flush its current buffer first, even when it is + // still empty. + if (buffer == null || buffer.position() == 0) { + continue; + } + buffer.flip(); + packets.add(StandardCharsets.UTF_8.decode(buffer).toString()); + } + return packets; + } + + /** Writes one prepared chunk per call, like a multi-valued message split across packets. */ + private static final class ChunkedMessage extends Message { + private final String[] chunks; + private int next; + + ChunkedMessage(String... chunks) { + super(Message.Type.DISTRIBUTION, TagsCardinality.DEFAULT); + this.chunks = chunks; + } + + @Override + boolean writeTo(StringBuilder builder, int capacity) { + builder.append(chunks[next++]); + return next < chunks.length; + } + + @Override + public void aggregate(Message message) {} + } +} From 4c3574044e11f1bb7e80f3494ac9aa75326bdfe4 Mon Sep 17 00:00:00 2001 From: Nikolay Martynov Date: Mon, 16 Nov 2026 13:41:30 -0500 Subject: [PATCH 3/4] Extract the UTF-8 encoder and test it directly Move the encoding loop from ProcessingTask into StatsDProcessor.Encoder, unchanged apart from the fix, and replace the processor-level StatsDProcessorTest with StatsDProcessorEncoderTest, which calls the encoder with a plain ByteBuffer. Co-Authored-By: Claude Opus 5.5 --- .../com/timgroup/statsd/StatsDProcessor.java | 31 +++++- .../statsd/StatsDProcessorEncoderTest.java | 84 ++++++++++++++ .../timgroup/statsd/StatsDProcessorTest.java | 104 ------------------ 3 files changed, 112 insertions(+), 107 deletions(-) create mode 100644 src/test/java/com/timgroup/statsd/StatsDProcessorEncoderTest.java delete mode 100644 src/test/java/com/timgroup/statsd/StatsDProcessorTest.java diff --git a/src/main/java/com/timgroup/statsd/StatsDProcessor.java b/src/main/java/com/timgroup/statsd/StatsDProcessor.java index 0a60a697..cf5722f9 100644 --- a/src/main/java/com/timgroup/statsd/StatsDProcessor.java +++ b/src/main/java/com/timgroup/statsd/StatsDProcessor.java @@ -34,9 +34,7 @@ public abstract class StatsDProcessor { protected abstract class ProcessingTask implements Runnable { protected StringBuilder builder = new StringBuilder(); - char[] charBuffer = new char[maxPacketSizeBytes]; - // + 4 so that we can check for buffer overflow without computing encoded length first - final byte[] byteBuffer = new byte[maxPacketSizeBytes + 4]; + final Encoder encoder = new Encoder(maxPacketSizeBytes); public final void run() { try { @@ -133,6 +131,33 @@ protected void processLoop() { abstract Message getMessage() throws InterruptedException; protected void writeBuilderToSendBuffer(ByteBuffer sendBuffer) { + encoder.encode(builder, sendBuffer); + } + } + + /** + * Encodes messages as UTF-8 into send buffers. Each processing task has its own, because it + * reuses its scratch arrays between messages. + */ + static final class Encoder { + private final int maxPacketSizeBytes; + private char[] charBuffer; + // + 4 so that we can check for buffer overflow without computing encoded length first + private final byte[] byteBuffer; + + Encoder(int maxPacketSizeBytes) { + this.maxPacketSizeBytes = maxPacketSizeBytes; + this.charBuffer = new char[maxPacketSizeBytes]; + this.byteBuffer = new byte[maxPacketSizeBytes + 4]; + } + + /** + * Appends the UTF-8 encoding of {@code builder} to {@code sendBuffer}. + * + * @throws BufferOverflowException if the encoding is longer than a packet or than the space + * left in {@code sendBuffer}. Nothing is written in that case. + */ + void encode(StringBuilder builder, ByteBuffer sendBuffer) { int length = builder.length(); if (length > charBuffer.length) { charBuffer = new char[length]; diff --git a/src/test/java/com/timgroup/statsd/StatsDProcessorEncoderTest.java b/src/test/java/com/timgroup/statsd/StatsDProcessorEncoderTest.java new file mode 100644 index 00000000..ca68e31b --- /dev/null +++ b/src/test/java/com/timgroup/statsd/StatsDProcessorEncoderTest.java @@ -0,0 +1,84 @@ +package com.timgroup.statsd; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import org.junit.Test; + +public class StatsDProcessorEncoderTest { + + private static final int MAX_PACKET_SIZE = 64; + + private final StatsDProcessor.Encoder encoder = new StatsDProcessor.Encoder(MAX_PACKET_SIZE); + private final ByteBuffer sendBuffer = ByteBuffer.allocate(MAX_PACKET_SIZE); + + @Test + public void encodes_message_that_exactly_fills_the_packet() { + assertEncodesExactFit(message("", MAX_PACKET_SIZE)); + } + + @Test + public void encodes_two_byte_chars_that_exactly_fill_the_packet() { + assertEncodesExactFit(message("é", MAX_PACKET_SIZE - 1)); + } + + @Test + public void encodes_three_byte_chars_that_exactly_fill_the_packet() { + assertEncodesExactFit(message("€", MAX_PACKET_SIZE - 2)); + } + + @Test + public void encodes_surrogate_pair_that_exactly_fills_the_packet() { + // Two chars, four bytes. + assertEncodesExactFit(message("😀", MAX_PACKET_SIZE - 2)); + } + + @Test + public void rejects_message_one_byte_longer_than_the_packet() { + assertRejects(message("", MAX_PACKET_SIZE + 1)); + } + + @Test + public void rejects_multibyte_message_one_byte_longer_than_the_packet() { + // As many chars as fit in a packet, but 'é' takes two bytes. + assertRejects(message("é", MAX_PACKET_SIZE)); + } + + @Test + public void rejects_message_longer_than_the_space_left() { + sendBuffer.put(new byte[10]); + + assertRejects(message("", MAX_PACKET_SIZE - 9)); + } + + /** {@code length} chars: {@code head}, then ASCII filler. */ + private static String message(String head, int length) { + char[] filler = new char[length - head.length()]; + Arrays.fill(filler, 'x'); + return head + new String(filler); + } + + private void assertEncodesExactFit(String message) { + assertEquals(MAX_PACKET_SIZE, message.getBytes(StandardCharsets.UTF_8).length); + + encoder.encode(new StringBuilder(message), sendBuffer); + + sendBuffer.flip(); + assertEquals(message, StandardCharsets.UTF_8.decode(sendBuffer).toString()); + } + + private void assertRejects(String message) { + int position = sendBuffer.position(); + try { + encoder.encode(new StringBuilder(message), sendBuffer); + fail("expected BufferOverflowException"); + } catch (BufferOverflowException expected) { + // Nothing is written on overflow. + assertEquals(position, sendBuffer.position()); + } + } +} diff --git a/src/test/java/com/timgroup/statsd/StatsDProcessorTest.java b/src/test/java/com/timgroup/statsd/StatsDProcessorTest.java deleted file mode 100644 index ee6e83ad..00000000 --- a/src/test/java/com/timgroup/statsd/StatsDProcessorTest.java +++ /dev/null @@ -1,104 +0,0 @@ -package com.timgroup.statsd; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import org.junit.After; -import org.junit.Test; - -public class StatsDProcessorTest { - - private static final int MAX_PACKET_SIZE = 64; - - private final RecordingErrorHandler errors = new RecordingErrorHandler(); - private StatsDProcessor processor; - - @After - public void stop() throws InterruptedException { - if (processor != null) { - processor.shutdown(false); - } - } - - @Test(timeout = 5000L) - public void writes_message_that_exactly_fills_the_packet() throws Exception { - String packet = packet("", MAX_PACKET_SIZE); - - assertEquals(Arrays.asList(packet), process(new ChunkedMessage(packet), 1)); - } - - @Test(timeout = 5000L) - public void writes_multibyte_message_that_exactly_fills_the_packet() throws Exception { - // One character short of the limit, but 'é' takes two bytes in UTF-8. - String packet = packet("é", MAX_PACKET_SIZE - 1); - - assertEquals(Arrays.asList(packet), process(new ChunkedMessage(packet), 1)); - } - - @Test(timeout = 5000L) - public void writes_every_chunk_when_one_exactly_fills_the_packet() throws Exception { - String first = packet("", MAX_PACKET_SIZE); - String second = packet("", 10); - - assertEquals(Arrays.asList(first, second), process(new ChunkedMessage(first, second), 2)); - } - - /** A packet of {@code length} chars that starts with {@code head} and ends with a newline. */ - private static String packet(String head, int length) { - char[] filler = new char[length - head.length() - 1]; - Arrays.fill(filler, 'x'); - return head + new String(filler) + "\n"; - } - - private List process(Message message, int expectedPackets) throws Exception { - processor = - new StatsDNonBlockingProcessor( - 16, errors, MAX_PACKET_SIZE, 8, 1, 0, 1, Executors.defaultThreadFactory()); - processor.startWorkers("StatsDProcessorTest-"); - assertTrue(processor.send(message)); - - List packets = new ArrayList<>(); - while (packets.size() < expectedPackets) { - List exceptions = errors.getExceptions(); - if (!exceptions.isEmpty()) { - throw new AssertionError("processor reported errors: " + exceptions); - } - ByteBuffer buffer = processor.getOutboundQueue().poll(10, TimeUnit.MILLISECONDS); - // An exact fit makes the processor flush its current buffer first, even when it is - // still empty. - if (buffer == null || buffer.position() == 0) { - continue; - } - buffer.flip(); - packets.add(StandardCharsets.UTF_8.decode(buffer).toString()); - } - return packets; - } - - /** Writes one prepared chunk per call, like a multi-valued message split across packets. */ - private static final class ChunkedMessage extends Message { - private final String[] chunks; - private int next; - - ChunkedMessage(String... chunks) { - super(Message.Type.DISTRIBUTION, TagsCardinality.DEFAULT); - this.chunks = chunks; - } - - @Override - boolean writeTo(StringBuilder builder, int capacity) { - builder.append(chunks[next++]); - return next < chunks.length; - } - - @Override - public void aggregate(Message message) {} - } -} From 4c7094fe64c2be66614dfe3850bc9905bdc598bf Mon Sep 17 00:00:00 2001 From: Nikolay Martynov Date: Tue, 17 Nov 2026 07:16:40 -0500 Subject: [PATCH 4/4] Move the + 4 comment next to the + 4 Co-Authored-By: Claude Opus 5.5 --- src/main/java/com/timgroup/statsd/StatsDProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/timgroup/statsd/StatsDProcessor.java b/src/main/java/com/timgroup/statsd/StatsDProcessor.java index cf5722f9..e238d76f 100644 --- a/src/main/java/com/timgroup/statsd/StatsDProcessor.java +++ b/src/main/java/com/timgroup/statsd/StatsDProcessor.java @@ -142,12 +142,12 @@ protected void writeBuilderToSendBuffer(ByteBuffer sendBuffer) { static final class Encoder { private final int maxPacketSizeBytes; private char[] charBuffer; - // + 4 so that we can check for buffer overflow without computing encoded length first private final byte[] byteBuffer; Encoder(int maxPacketSizeBytes) { this.maxPacketSizeBytes = maxPacketSizeBytes; this.charBuffer = new char[maxPacketSizeBytes]; + // + 4 so that we can check for buffer overflow without computing encoded length first this.byteBuffer = new byte[maxPacketSizeBytes + 4]; }