diff --git a/src/main/java/com/timgroup/statsd/StatsDProcessor.java b/src/main/java/com/timgroup/statsd/StatsDProcessor.java index e238d76f..1b338661 100644 --- a/src/main/java/com/timgroup/statsd/StatsDProcessor.java +++ b/src/main/java/com/timgroup/statsd/StatsDProcessor.java @@ -97,7 +97,7 @@ protected void processLoop() { throw new InvalidMessageException(MESSAGE_TOO_LONG, builder.toString()); } - if (sendBuffer.remaining() < (lowerBoundSize + 1)) { + if (sendBuffer.remaining() < lowerBoundSize) { outboundQueue.put(sendBuffer); sendBuffer = bufferPool.borrow(); } @@ -105,6 +105,11 @@ protected void processLoop() { try { writeBuilderToSendBuffer(sendBuffer); } catch (BufferOverflowException boe) { + if (sendBuffer.position() == 0) { + // Multibyte characters made the message longer than a packet, + // so a fresh buffer would overflow too. + throw boe; + } outboundQueue.put(sendBuffer); sendBuffer = bufferPool.borrow(); writeBuilderToSendBuffer(sendBuffer); diff --git a/src/test/java/com/timgroup/statsd/StatsDProcessorTest.java b/src/test/java/com/timgroup/statsd/StatsDProcessorTest.java new file mode 100644 index 00000000..3a3514d9 --- /dev/null +++ b/src/test/java/com/timgroup/statsd/StatsDProcessorTest.java @@ -0,0 +1,153 @@ +package com.timgroup.statsd; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.junit.After; +import org.junit.Test; + +public class StatsDProcessorTest { + + private static final int MAX_PACKET_SIZE = 64; + + private final RecordingErrorHandler errors = new RecordingErrorHandler(); + private StatsDProcessor processor; + + @After + public void stop() throws InterruptedException { + if (processor != null) { + processor.shutdown(false); + } + } + + @Test(timeout = 5000L) + public void writes_message_that_exactly_fills_the_packet() throws Exception { + String packet = packet("", MAX_PACKET_SIZE); + + assertEquals(Arrays.asList(packet), process(new ChunkedMessage(packet), 1)); + } + + @Test(timeout = 5000L) + public void writes_message_that_exactly_fills_the_rest_of_a_packet() throws Exception { + String first = packet("", 20); + String second = packet("", MAX_PACKET_SIZE - 20); + + assertEquals(Arrays.asList(first + second), process(new ChunkedMessage(first, second), 1)); + } + + @Test(timeout = 5000L) + public void reports_multibyte_overflow_of_an_empty_buffer() throws Exception { + // Fewer chars than fit in a packet, but more bytes: 'é' takes two bytes in UTF-8. + assertEquals( + Arrays.asList(), processOverflow(new ChunkedMessage(multibyte(40)), 0)); + } + + @Test(timeout = 5000L) + public void reports_multibyte_overflow_after_a_buffered_chunk() throws Exception { + String first = packet("", 20); + + assertEquals( + Arrays.asList(first), processOverflow(new ChunkedMessage(first, multibyte(40)), 1)); + } + + /** {@code length} chars, mostly two-byte 'é', ending with a newline. */ + private static String multibyte(int length) { + char[] chars = new char[length - 1]; + Arrays.fill(chars, 'é'); + return new String(chars) + "\n"; + } + + /** + * Processes a message that overflows a packet, and returns the packets written before the + * overflow. + */ + private List processOverflow(Message message, int expectedPackets) throws Exception { + processor = start(); + assertTrue(processor.send(message)); + + while (errors.getExceptions().isEmpty()) { + Thread.sleep(10L); + } + // The client tests rely on this type: InvalidMessageException means the message was + // rejected before it was encoded. + assertEquals(BufferOverflowException.class, errors.getExceptions().get(0).getClass()); + + List packets = new ArrayList<>(); + ByteBuffer buffer; + while ((buffer = processor.getOutboundQueue().poll()) != null) { + if (buffer.position() == 0) { + throw new AssertionError("processor flushed an empty buffer"); + } + buffer.flip(); + packets.add(StandardCharsets.UTF_8.decode(buffer).toString()); + } + assertEquals(expectedPackets, packets.size()); + return packets; + } + + /** A packet of {@code length} chars that starts with {@code head} and ends with a newline. */ + private static String packet(String head, int length) { + char[] filler = new char[length - head.length() - 1]; + Arrays.fill(filler, 'x'); + return head + new String(filler) + "\n"; + } + + private StatsDProcessor start() throws Exception { + StatsDProcessor started = + new StatsDNonBlockingProcessor( + 16, errors, MAX_PACKET_SIZE, 8, 1, 0, 1, Executors.defaultThreadFactory()); + started.startWorkers("StatsDProcessorTest-"); + return started; + } + + private List process(Message message, int expectedPackets) throws Exception { + processor = start(); + assertTrue(processor.send(message)); + + List packets = new ArrayList<>(); + while (packets.size() < expectedPackets) { + List exceptions = errors.getExceptions(); + if (!exceptions.isEmpty()) { + throw new AssertionError("processor reported errors: " + exceptions); + } + ByteBuffer buffer = processor.getOutboundQueue().poll(10, TimeUnit.MILLISECONDS); + if (buffer == null) { + continue; + } + if (buffer.position() == 0) { + throw new AssertionError("processor flushed an empty buffer"); + } + buffer.flip(); + packets.add(StandardCharsets.UTF_8.decode(buffer).toString()); + } + return packets; + } + + /** Writes one prepared chunk per call, like a multi-valued message split across packets. */ + private static final class ChunkedMessage extends Message { + private final String[] chunks; + private int next; + + ChunkedMessage(String... chunks) { + super(Message.Type.DISTRIBUTION, TagsCardinality.DEFAULT); + this.chunks = chunks; + } + + @Override + boolean writeTo(StringBuilder builder, int capacity) { + builder.append(chunks[next++]); + return next < chunks.length; + } + + @Override + public void aggregate(Message message) {} + } +}