From f18cb4ff665f6126081f956be978c67cf1244719 Mon Sep 17 00:00:00 2001 From: Nikolay Martynov Date: Tue, 17 Nov 2026 08:48:48 -0500 Subject: [PATCH] Don't flush empty buffers The remaining() check still reserves a byte for the separator the processor wrote between messages before messages carried their own newline. An exact fit therefore flushed the current buffer first, even when it was empty, and a message that exactly filled the rest of a buffer went to a new one. When a multibyte message overflowed an empty buffer, that empty buffer was flushed before the retry failed the same way. Drop the extra byte. When an empty buffer overflows, rethrow the BufferOverflowException without flushing it, so handlers see the same exception whether or not an earlier message is buffered. Co-Authored-By: Claude Opus 5.5 --- .../com/timgroup/statsd/StatsDProcessor.java | 7 +- .../timgroup/statsd/StatsDProcessorTest.java | 153 ++++++++++++++++++ 2 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 src/test/java/com/timgroup/statsd/StatsDProcessorTest.java diff --git a/src/main/java/com/timgroup/statsd/StatsDProcessor.java b/src/main/java/com/timgroup/statsd/StatsDProcessor.java index e238d76f..1b338661 100644 --- a/src/main/java/com/timgroup/statsd/StatsDProcessor.java +++ b/src/main/java/com/timgroup/statsd/StatsDProcessor.java @@ -97,7 +97,7 @@ protected void processLoop() { throw new InvalidMessageException(MESSAGE_TOO_LONG, builder.toString()); } - if (sendBuffer.remaining() < (lowerBoundSize + 1)) { + if (sendBuffer.remaining() < lowerBoundSize) { outboundQueue.put(sendBuffer); sendBuffer = bufferPool.borrow(); } @@ -105,6 +105,11 @@ protected void processLoop() { try { writeBuilderToSendBuffer(sendBuffer); } catch (BufferOverflowException boe) { + if (sendBuffer.position() == 0) { + // Multibyte characters made the message longer than a packet, + // so a fresh buffer would overflow too. + throw boe; + } outboundQueue.put(sendBuffer); sendBuffer = bufferPool.borrow(); writeBuilderToSendBuffer(sendBuffer); diff --git a/src/test/java/com/timgroup/statsd/StatsDProcessorTest.java b/src/test/java/com/timgroup/statsd/StatsDProcessorTest.java new file mode 100644 index 00000000..3a3514d9 --- /dev/null +++ b/src/test/java/com/timgroup/statsd/StatsDProcessorTest.java @@ -0,0 +1,153 @@ +package com.timgroup.statsd; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.junit.After; +import org.junit.Test; + +public class StatsDProcessorTest { + + private static final int MAX_PACKET_SIZE = 64; + + private final RecordingErrorHandler errors = new RecordingErrorHandler(); + private StatsDProcessor processor; + + @After + public void stop() throws InterruptedException { + if (processor != null) { + processor.shutdown(false); + } + } + + @Test(timeout = 5000L) + public void writes_message_that_exactly_fills_the_packet() throws Exception { + String packet = packet("", MAX_PACKET_SIZE); + + assertEquals(Arrays.asList(packet), process(new ChunkedMessage(packet), 1)); + } + + @Test(timeout = 5000L) + public void writes_message_that_exactly_fills_the_rest_of_a_packet() throws Exception { + String first = packet("", 20); + String second = packet("", MAX_PACKET_SIZE - 20); + + assertEquals(Arrays.asList(first + second), process(new ChunkedMessage(first, second), 1)); + } + + @Test(timeout = 5000L) + public void reports_multibyte_overflow_of_an_empty_buffer() throws Exception { + // Fewer chars than fit in a packet, but more bytes: 'é' takes two bytes in UTF-8. + assertEquals( + Arrays.asList(), processOverflow(new ChunkedMessage(multibyte(40)), 0)); + } + + @Test(timeout = 5000L) + public void reports_multibyte_overflow_after_a_buffered_chunk() throws Exception { + String first = packet("", 20); + + assertEquals( + Arrays.asList(first), processOverflow(new ChunkedMessage(first, multibyte(40)), 1)); + } + + /** {@code length} chars, mostly two-byte 'é', ending with a newline. */ + private static String multibyte(int length) { + char[] chars = new char[length - 1]; + Arrays.fill(chars, 'é'); + return new String(chars) + "\n"; + } + + /** + * Processes a message that overflows a packet, and returns the packets written before the + * overflow. + */ + private List processOverflow(Message message, int expectedPackets) throws Exception { + processor = start(); + assertTrue(processor.send(message)); + + while (errors.getExceptions().isEmpty()) { + Thread.sleep(10L); + } + // The client tests rely on this type: InvalidMessageException means the message was + // rejected before it was encoded. + assertEquals(BufferOverflowException.class, errors.getExceptions().get(0).getClass()); + + List packets = new ArrayList<>(); + ByteBuffer buffer; + while ((buffer = processor.getOutboundQueue().poll()) != null) { + if (buffer.position() == 0) { + throw new AssertionError("processor flushed an empty buffer"); + } + buffer.flip(); + packets.add(StandardCharsets.UTF_8.decode(buffer).toString()); + } + assertEquals(expectedPackets, packets.size()); + return packets; + } + + /** A packet of {@code length} chars that starts with {@code head} and ends with a newline. */ + private static String packet(String head, int length) { + char[] filler = new char[length - head.length() - 1]; + Arrays.fill(filler, 'x'); + return head + new String(filler) + "\n"; + } + + private StatsDProcessor start() throws Exception { + StatsDProcessor started = + new StatsDNonBlockingProcessor( + 16, errors, MAX_PACKET_SIZE, 8, 1, 0, 1, Executors.defaultThreadFactory()); + started.startWorkers("StatsDProcessorTest-"); + return started; + } + + private List process(Message message, int expectedPackets) throws Exception { + processor = start(); + assertTrue(processor.send(message)); + + List packets = new ArrayList<>(); + while (packets.size() < expectedPackets) { + List exceptions = errors.getExceptions(); + if (!exceptions.isEmpty()) { + throw new AssertionError("processor reported errors: " + exceptions); + } + ByteBuffer buffer = processor.getOutboundQueue().poll(10, TimeUnit.MILLISECONDS); + if (buffer == null) { + continue; + } + if (buffer.position() == 0) { + throw new AssertionError("processor flushed an empty buffer"); + } + buffer.flip(); + packets.add(StandardCharsets.UTF_8.decode(buffer).toString()); + } + return packets; + } + + /** Writes one prepared chunk per call, like a multi-valued message split across packets. */ + private static final class ChunkedMessage extends Message { + private final String[] chunks; + private int next; + + ChunkedMessage(String... chunks) { + super(Message.Type.DISTRIBUTION, TagsCardinality.DEFAULT); + this.chunks = chunks; + } + + @Override + boolean writeTo(StringBuilder builder, int capacity) { + builder.append(chunks[next++]); + return next < chunks.length; + } + + @Override + public void aggregate(Message message) {} + } +}