From 0b08fe8761f1ef9d74ecb0190e73927c2206eef0 Mon Sep 17 00:00:00 2001 From: rp872 <348401367+rp872@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:52:35 +0000 Subject: [PATCH 1/6] Skip whitespace without unreading the character that ends it skip_over_whitespace() called skip_whitespace(), which unreads the first character after the whitespace, then read_char() to read it again. The text reader does this before every token. Return that character directly instead. The cases match the old code: a '/' that does not start a comment is returned as '/', EOF is returned as EOF, and read_char() still counts newlines. --- .../ion/impl/IonReaderTextRawTokensX.java | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/amazon/ion/impl/IonReaderTextRawTokensX.java b/src/main/java/com/amazon/ion/impl/IonReaderTextRawTokensX.java index 9a7a445a6..56cde628a 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderTextRawTokensX.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderTextRawTokensX.java @@ -807,8 +807,31 @@ private final int skip_over_whitespace() throws IOException */ private final int skip_over_whitespace(CommentStrategy commentStrategy) throws IOException { - skip_whitespace(commentStrategy); - return read_char(); + // Same as skip_whitespace(commentStrategy) then read_char(), without unreading and + // re-reading the character that ends the whitespace. + int c; + for (;;) { + c = read_char(); + switch (c) { + case ' ': + case '\t': + // new line normalization and counting is handled in read_char + case CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_1: + case CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_2: + case CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_3: + case CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_1: + case CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_2: + case CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_3: + continue; + case '/': + if (commentStrategy.onComment(this)) { + continue; + } + return c; + default: + return c; + } + } } /** From 39d3cad50269f93591c3f39eee32b068a4c806e4 Mon Sep 17 00:00:00 2001 From: rp872 <348401367+rp872@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:52:35 +0000 Subject: [PATCH 2/6] Unread within the current page without a page lookup UnifiedInputStreamX.unread() looked up the current page and its starting offset on every call to check whether the character being stepped back over is still in the current page. Cache that starting offset (_unread_floor) in make_page_current(), the one place a page becomes current. While a page is current its starting offset can only decrease (unreading before its first character lowers it), and a page is only reloaded by refill(), which makes it current again through make_page_current(), so the cached value stays an upper bound of the starting offset: a position above it is inside the page. When _pos is above it, unread() only decrements _pos; every other case takes the existing path. --- .../com/amazon/ion/impl/UnifiedInputStreamX.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/com/amazon/ion/impl/UnifiedInputStreamX.java b/src/main/java/com/amazon/ion/impl/UnifiedInputStreamX.java index 2858ab352..f486fa513 100644 --- a/src/main/java/com/amazon/ion/impl/UnifiedInputStreamX.java +++ b/src/main/java/com/amazon/ion/impl/UnifiedInputStreamX.java @@ -76,6 +76,10 @@ abstract class UnifiedInputStreamX UnifiedSavePointManagerX _save_points; + // An upper bound of the current page's starting offset: unread() can step back without a page + // lookup while _pos is above it. See make_page_current(). + private int _unread_floor = Integer.MAX_VALUE; + // factories to construct an appropriate input stream // based on the input source @@ -119,6 +123,7 @@ public void close() throws IOException { _eof = true; + _unread_floor = Integer.MAX_VALUE; _buffer.clear(); } @@ -176,6 +181,11 @@ protected final void make_page_current(UnifiedDataPageX curr, int idx, int pos, _limit = limit; _pos = pos; _eof = false; + // While a page is current its starting offset can only decrease (unreading before its first + // character lowers it), and a page is only reloaded by refill(), which makes it current again + // through this method. So this value stays an upper bound of the starting offset, and a + // position above it is inside the page, until the next call. + _unread_floor = curr.getStartingOffset(); if (is_byte_data()) { _bytes = curr.getByteBuffer(); } @@ -194,6 +204,12 @@ private final boolean is_byte_data() { } public final void unread(int c) { + // Fast path: stepping back over the character just read, within the current page. + if (_pos > _unread_floor && !_eof && c >= 0 && c <= _max_char_value) { + _pos--; + verify_matched_unread(c); + return; + } if (c == -1) { return; } From 9c8eeb49d618f4330ec41a7d1c7e3584aac4bb60 Mon Sep 17 00:00:00 2001 From: rp872 <348401367+rp872@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:52:35 +0000 Subject: [PATCH 3/6] Copy and skip plain-ASCII runs in bulk in the text reader Strings, symbol identifiers and skipped values were scanned one character at a time through UnifiedInputStreamX.read(). Add helpers that consume a run of characters directly from the current page: appendAsciiRun() copies accepted ASCII characters into a StringBuilder and skipRun() skips them. Neither refills; the existing character-at-a-time loop continues with the character that stopped the run, or with read() at the end of the page. Each table accepts exactly the characters the existing loop passes over without acting on them, so everything else (escapes, quotes, line ends and line counting, comments, brackets, non-ASCII) still goes through the existing code. Spaces and tabs are skipped with the rest when skipping a container, since the container scan ignores them. Used when loading double-quoted strings and symbol identifiers, and when skipping double-quoted strings, containers and blobs. --- .../ion/impl/IonReaderTextRawTokensX.java | 36 ++++++++++ .../amazon/ion/impl/UnifiedInputStreamX.java | 69 ++++++++++++++++++- 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/amazon/ion/impl/IonReaderTextRawTokensX.java b/src/main/java/com/amazon/ion/impl/IonReaderTextRawTokensX.java index 56cde628a..b34833279 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderTextRawTokensX.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderTextRawTokensX.java @@ -100,6 +100,34 @@ public Appendable append(char c) throws IOException // Used for tracking terminator characters when skipping a container private final ArrayList containerSkipTerminatorStack = new ArrayList<>(CONTAINER_STACK_INITIAL_CAPACITY); + // Tables for the UnifiedInputStreamX run helpers, indexed by ASCII character. + /** Symbol identifier characters: a-z, A-Z, 0-9, '$', '_'. */ + private static final boolean[] SYMBOL_CHARS = new boolean[128]; + /** Characters a double-quoted string takes as they are: printable ASCII other than '"' and '\'. */ + private static final boolean[] DOUBLE_QUOTED_PLAIN = new boolean[128]; + /** Characters skipping a double-quoted string must look at: the quote, backslash and line ends. */ + private static final boolean[] DOUBLE_QUOTED_SKIP_STOP = new boolean[128]; + /** + * Characters skip_over_container() must look at: line ends (for line counting), comments, + * quotes and brackets. Spaces and tabs mean nothing to it, so they are skipped with the rest. + */ + private static final boolean[] CONTAINER_SKIP_STOP = new boolean[128]; + /** Characters skip_over_blob() must look at: the closing brace and line ends. */ + private static final boolean[] BLOB_SKIP_STOP = new boolean[128]; + static { + BLOB_SKIP_STOP['}'] = BLOB_SKIP_STOP['\r'] = BLOB_SKIP_STOP['\n'] = true; + for (int c = 0; c < 128; c++) { + SYMBOL_CHARS[c] = IonTokenConstsX.isValidSymbolCharacter(c); + DOUBLE_QUOTED_PLAIN[c] = c >= 0x20 && c < 0x7F && c != '"' && c != '\\'; + } + for (char c : "\"\\\r\n".toCharArray()) { + DOUBLE_QUOTED_SKIP_STOP[c] = true; + } + for (char c : "\r\n/\"'(){}[]".toCharArray()) { + CONTAINER_SKIP_STOP[c] = true; + } + } + /** * IonTokenReader constructor requires a UnifiedInputStream * as the source of bytes/chars that serve as the basic input @@ -1229,6 +1257,8 @@ private void skip_over_container(int terminator) throws IOException int c; for (;;) { + // Characters with no meaning to the scan below. + _stream.skipRun(CONTAINER_SKIP_STOP); c = skip_over_whitespace(); switch (c) { case -1: @@ -1891,6 +1921,7 @@ protected void load_symbol_identifier(StringBuilder sb) throws IOException // Valid symbol chars (a-z, A-Z, 0-9, $, _) are all ASCII and never newlines. int c; for (;;) { + _stream.appendAsciiRun(sb, SYMBOL_CHARS); c = _stream.read(); if (c >= 0 && c <= 0x7F && IonTokenConstsX.isValidSymbolCharacter(c)) { sb.append((char)c); @@ -2062,6 +2093,7 @@ private final void skip_double_quoted_string_helper() throws IOException { int c; for (;;) { + _stream.skipRun(DOUBLE_QUOTED_SKIP_STOP); c = read_string_char(ProhibitedCharacters.NONE); switch (c) { case -1: @@ -2087,6 +2119,9 @@ protected int load_double_quoted_string(StringBuilder sb, boolean is_clob) boolean expectLowSurrogate = false; for (;;) { + // Bulk copy of plain printable ASCII (not '"' or '\'), the same characters the + // per-character fast path below accepts; a clob takes them unchanged too. + _stream.appendAsciiRun(sb, DOUBLE_QUOTED_PLAIN); // Fast path: read directly from stream for common ASCII characters. // ASCII printable chars (0x20-0x7E) excluding '"' (0x22) and '\' (0x5C) // don't need prohibited char checks, UTF-8 decoding, or surrogate handling. @@ -2497,6 +2532,7 @@ private void skip_over_blob(SavePoint sp) throws IOException { int c; for (;;) { + _stream.skipRun(BLOB_SKIP_STOP); c = _stream.read(); // Fast path: base64 chars (> ' ' and not '}') are the common case if (c > ' ' && c != '}') { diff --git a/src/main/java/com/amazon/ion/impl/UnifiedInputStreamX.java b/src/main/java/com/amazon/ion/impl/UnifiedInputStreamX.java index f486fa513..430070047 100644 --- a/src/main/java/com/amazon/ion/impl/UnifiedInputStreamX.java +++ b/src/main/java/com/amazon/ion/impl/UnifiedInputStreamX.java @@ -79,7 +79,8 @@ abstract class UnifiedInputStreamX // An upper bound of the current page's starting offset: unread() can step back without a page // lookup while _pos is above it. See make_page_current(). private int _unread_floor = Integer.MAX_VALUE; - + // staging for appendAsciiRun() + private final char[] _scratch = new char[256]; // factories to construct an appropriate input stream // based on the input source @@ -127,6 +128,72 @@ public void close() _buffer.clear(); } + /** + * Appends to {@code sb} the characters at the read position, up to the end of the + * current page, that are ASCII and accepted by {@code accept} (indexed by character). + * Consumes them. Never refills, so callers continue with {@link #read()}. + */ + final void appendAsciiRun(StringBuilder sb, boolean[] accept) + { + final int start = _pos; + int pos = start; + final int limit = _limit; + if (pos >= limit || _eof) return; + if (_is_byte_data) { + final byte[] bytes = _bytes; + final char[] scratch = _scratch; + int n = 0; + while (pos < limit) { + int c = bytes[pos]; // negative for >= 0x80 + if (c < 0 || !accept[c]) break; + if (n == scratch.length) { sb.append(scratch, 0, n); n = 0; } + scratch[n++] = (char) c; + pos++; + } + if (n > 0) sb.append(scratch, 0, n); + } + else { + final char[] chars = _chars; + while (pos < limit) { + int c = chars[pos]; + if (c >= 128 || !accept[c]) break; + pos++; + } + sb.append(chars, start, pos - start); + } + _pos += pos - start; + } + + /** + * Consumes the characters at the read position, up to the end of the current page, + * that are not in {@code stop} (indexed by character; characters >= 128 never stop). + * Never refills, so callers continue with {@link #read()}. + */ + final void skipRun(boolean[] stop) + { + final int start = _pos; + int pos = start; + final int limit = _limit; + if (pos >= limit || _eof) return; + if (_is_byte_data) { + final byte[] bytes = _bytes; + while (pos < limit) { + int c = bytes[pos]; + if (c >= 0 && stop[c]) break; + pos++; + } + } + else { + final char[] chars = _chars; + while (pos < limit) { + int c = chars[pos]; + if (c < 128 && stop[c]) break; + pos++; + } + } + _pos += pos - start; + } + public final boolean isEOF() { return _eof; // (_state == UIS_STATE.STATE_EOF); } From b04eefcf77704383e944b686c7ca2434c838c28c Mon Sep 17 00:00:00 2001 From: rp872 <348401367+rp872@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:52:35 +0000 Subject: [PATCH 4/6] Decode whole base64 groups straight from the page Blobs were decoded a byte at a time through read_base64_byte(), which reads each base64 character through UnifiedInputStreamX.read(). Decode groups of four alphabet characters directly from the current page into the lob buffer instead, for byte and character sources. Decoding stops at whitespace, '=' (which the alphabet table maps to -1), the closing brace, a non-ASCII or invalid character, or the end of the page, and leaves all of those to read_base64_byte(), so padding, wrapped base64 and errors behave as before. readBytes() goes back to bulk decoding after each such character whenever no decoded bytes are pending, never asks for more than the requested length, and still marks the token finished only when it reached the end of the lob. --- .../ion/impl/IonReaderTextRawTokensX.java | 11 ++++ .../amazon/ion/impl/IonReaderTextSystemX.java | 11 +++- .../amazon/ion/impl/UnifiedInputStreamX.java | 53 +++++++++++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/amazon/ion/impl/IonReaderTextRawTokensX.java b/src/main/java/com/amazon/ion/impl/IonReaderTextRawTokensX.java index b34833279..3eacf2379 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderTextRawTokensX.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderTextRawTokensX.java @@ -2670,6 +2670,17 @@ private final int read_hex_escape_sequence_value(int len) throws IOException return hexchar; } + /** + * Decodes as many whole base64 groups as the input allows without any of the special cases + * read_base64_byte() handles, into {@code dst}; returns the bytes written (a multiple of 3, + * at most {@code len}). Callers continue with read_base64_byte(). + */ + final int read_base64_bytes_bulk(byte[] dst, int off, int len) + { + if (_base64_prefetch_count > 0) return 0; + return 3 * _stream.decodeBase64Groups(dst, off, len / 3, BASE64_CHAR_TO_BIN); + } + public final int read_base64_byte() throws IOException { int b; diff --git a/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java b/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java index 10b82ef71..e2a53cc6e 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java @@ -856,10 +856,19 @@ private int readBytes(byte[] buffer, int offset, int len) switch (_lob_token) { // BLOB case IonTokenConstsX.TOKEN_OPEN_DOUBLE_BRACE: - while (len-- > 0) { + while (len > 0) { + // Whole base64 groups straight from the page, when no decoded bytes are pending. + int bulk = _scanner.read_base64_bytes_bulk(buffer, offset, len); + if (bulk > 0) { + offset += bulk; + len -= bulk; + c = 0; // bytes were read, so the end of the lob was not reached + continue; + } c = _scanner.read_base64_byte(); if (c < 0) break; buffer[offset++] = (byte)c; + len--; } break; // CLOB diff --git a/src/main/java/com/amazon/ion/impl/UnifiedInputStreamX.java b/src/main/java/com/amazon/ion/impl/UnifiedInputStreamX.java index 430070047..3ed34500e 100644 --- a/src/main/java/com/amazon/ion/impl/UnifiedInputStreamX.java +++ b/src/main/java/com/amazon/ion/impl/UnifiedInputStreamX.java @@ -194,6 +194,59 @@ final void skipRun(boolean[] stop) _pos += pos - start; } + /** + * Decodes base64 groups of four alphabet characters (no whitespace, no padding) at the + * read position into {@code dst}, up to {@code maxGroups} groups and the end of the current + * page; consumes them. Returns the number of groups decoded (3 bytes each). Anything else + * (whitespace, '=', a closing brace, an invalid character) is left to the caller's + * character-at-a-time decoding. {@code charToBin} maps a character below 256 to its 6-bit + * value, and every other character, '=' included, to -1. + */ + final int decodeBase64Groups(byte[] dst, int off, int maxGroups, int[] charToBin) + { + if (_eof) return 0; + final int start = _pos; + int pos = start; + final int limit = _limit; + int groups = 0; + if (_is_byte_data) { + final byte[] bytes = _bytes; + while (groups < maxGroups && pos + 4 <= limit) { + int c1 = bytes[pos], c2 = bytes[pos + 1], c3 = bytes[pos + 2], c4 = bytes[pos + 3]; + if ((c1 | c2 | c3 | c4) < 0) break; // non-ASCII + int b1 = charToBin[c1], b2 = charToBin[c2], b3 = charToBin[c3], b4 = charToBin[c4]; + if ((b1 | b2 | b3 | b4) < 0) break; // not in the alphabet + putBase64Group(dst, off, b1, b2, b3, b4); + off += 3; + pos += 4; + groups++; + } + } + else { + final char[] chars = _chars; + while (groups < maxGroups && pos + 4 <= limit) { + int c1 = chars[pos], c2 = chars[pos + 1], c3 = chars[pos + 2], c4 = chars[pos + 3]; + if ((c1 | c2 | c3 | c4) >= 128) break; // non-ASCII + int b1 = charToBin[c1], b2 = charToBin[c2], b3 = charToBin[c3], b4 = charToBin[c4]; + if ((b1 | b2 | b3 | b4) < 0) break; // not in the alphabet + putBase64Group(dst, off, b1, b2, b3, b4); + off += 3; + pos += 4; + groups++; + } + } + _pos += pos - start; + return groups; + } + + /** The 3 bytes four 6-bit base64 values decode to. */ + private static void putBase64Group(byte[] dst, int off, int b1, int b2, int b3, int b4) + { + dst[off] = (byte) ((b1 << 2) | (b2 >> 4)); + dst[off + 1] = (byte) ((b2 << 4) | (b3 >> 2)); + dst[off + 2] = (byte) ((b3 << 6) | b4); + } + public final boolean isEOF() { return _eof; // (_state == UIS_STATE.STATE_EOF); } From 6cd00b250cf6981b5af18f50e9fc06f1dc6decbc Mon Sep 17 00:00:00 2001 From: rp872 <348401367+rp872@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:52:35 +0000 Subject: [PATCH 5/6] Read unquoted field names in one pass For a field name, parse_to_next_value() finished the name token to a save point, which skips over it, and then read the name back by reactivating that save point. Each field name was scanned twice and paid for a save-point push and pop. When the name is an unquoted identifier and nothing is loaded yet, read it directly with token_contents_load(), which records the save point and marks the token finished, after making the same state change finish_value() made; every other case keeps the existing path. After the name the stream is now before the whitespace that follows it, not after it. The next nextToken() skips that whitespace either way, so spans and line numbers are unchanged. One diagnostic moves: a keyword used as a field name (true, false, null, nan) is now reported at the end of the name, where the stream is, not after the spaces and comments that follow it. --- .../amazon/ion/impl/IonReaderTextRawX.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/amazon/ion/impl/IonReaderTextRawX.java b/src/main/java/com/amazon/ion/impl/IonReaderTextRawX.java index 4449ef412..1962cc141 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderTextRawX.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderTextRawX.java @@ -852,9 +852,22 @@ protected final void parse_to_next_value() throws IOException throw new IllegalStateException("field names have to be in structs"); } //finish_value(_current_value_save_point); - finish_and_save_value(); - - sb = token_contents_load(t); + if (t == IonTokenConstsX.TOKEN_SYMBOL_IDENTIFIER + && !_current_value_save_point_loaded + && !_current_value_buffer_loaded + && _scanner.isUnfinishedToken()) { + // Load the name in one pass, instead of skipping it to a save point + // and then reading it back from there (finish_and_save_value() below). + // The state changes are the ones finish_value() makes; token_contents_load() + // records the save point and marks the token finished. + set_state(get_state_after_value()); + _has_next_called = false; + sb = token_contents_load(t); + } + else { + finish_and_save_value(); + sb = token_contents_load(t); + } SymbolToken sym = parseSymbolToken("a field name", sb, t); set_fieldname(sym); From 002e02abd8267de012e991f2e7a434cb3f02d20d Mon Sep 17 00:00:00 2001 From: rp872 <348401367+rp872@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:52:36 +0000 Subject: [PATCH 6/6] Test the text reader fast paths IonReaderTextFastPathsTest has two parts. Reader-level tests read strings, symbols and blobs longer than a page, skipped containers whose contents look like their closing bracket, skipped multi-line values, malformed blobs, prefix reads of cached blobs, field-name spans and errors, and random documents written by the text writer, from byte arrays, streams that return a few bytes per read (so pages end anywhere), strings and readers. They check results, not which path produced them, and pass before and after the fast paths; they would not notice a fast path no longer being called, which is left to benchmarks. Unit tests call the new UnifiedInputStreamX helpers and the bulk base64 read directly, and check that each consumes a whole run or whole groups and stops at the right character. Together the tests fail when any character table, the unread floor, a non-ASCII check, the base64 group decoding or the bulk base64 read is broken. --- .../ion/impl/IonReaderTextFastPathsTest.java | 540 ++++++++++++++++++ 1 file changed, 540 insertions(+) create mode 100644 src/test/java/com/amazon/ion/impl/IonReaderTextFastPathsTest.java diff --git a/src/test/java/com/amazon/ion/impl/IonReaderTextFastPathsTest.java b/src/test/java/com/amazon/ion/impl/IonReaderTextFastPathsTest.java new file mode 100644 index 000000000..04b4a9d09 --- /dev/null +++ b/src/test/java/com/amazon/ion/impl/IonReaderTextFastPathsTest.java @@ -0,0 +1,540 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.impl; + +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonException; +import com.amazon.ion.IonList; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSexp; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonText; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.OffsetSpan; +import com.amazon.ion.SpanProvider; +import com.amazon.ion.TextSpan; +import com.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.system.IonTextWriterBuilder; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The text reader's bulk paths (runs of plain ASCII copied or skipped within a page, base64 groups decoded from + * the page, field names read in one pass, unread without a page lookup) give the same results as the + * character-at-a-time code they shortcut: wherever a page ends, whatever the input source, and when values are + * skipped instead of read. + */ +public class IonReaderTextFastPathsTest { + + private static final IonSystem SYSTEM = IonSystemBuilder.standard().build(); + /** UnifiedInputStreamX's page size for stream input: values longer than this span pages. */ + private static final int PAGE = UnifiedInputStreamX.DEFAULT_PAGE_SIZE; + + /** + * How the reader gets its input. Streams returning a few bytes per read end pages at arbitrary positions. One-byte + * reads are only used with small inputs: each refill takes a new page while a value is being saved, so a value + * longer than a page read a byte at a time would need a page per byte. + */ + enum Source { + BYTES, STREAM, STREAM_1_BYTE_READS, STREAM_4093_BYTE_READS, STRING, READER; + + IonReader open(String text) { + byte[] bytes = text.getBytes(StandardCharsets.UTF_8); + switch (this) { + case BYTES: return SYSTEM.newReader(bytes); + case STREAM: return SYSTEM.newReader(new ByteArrayInputStream(bytes)); + case STREAM_1_BYTE_READS: return SYSTEM.newReader(new LimitedReads(bytes, 1)); + case STREAM_4093_BYTE_READS: return SYSTEM.newReader(new LimitedReads(bytes, 4093)); + case STRING: return SYSTEM.newReader(text); + case READER: return SYSTEM.newReader(new StringReader(text)); + default: throw new IllegalStateException(); + } + } + } + + /** Returns at most {@code max} bytes per read. */ + private static final class LimitedReads extends FilterInputStream { + private final int max; + + LimitedReads(byte[] bytes, int max) { + super(new ByteArrayInputStream(bytes)); + this.max = max; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return super.read(b, off, Math.min(len, max)); + } + } + + private static IonDatagram load(Source source, String text) { + IonDatagram datagram = SYSTEM.newDatagram(); + try (IonReader reader = source.open(text)) { + IonWriter writer = SYSTEM.newWriter(datagram); + writer.writeValues(reader); + writer.close(); + } catch (IOException e) { + throw new IonException(e); + } + return datagram; + } + + @ParameterizedTest + @EnumSource(value = Source.class, names = "STREAM_1_BYTE_READS", mode = EnumSource.Mode.EXCLUDE) + public void stringsAndSymbolsLongerThanAPage(Source source) { + StringBuilder text = new StringBuilder(); + String plain = repeat("abcdefghijklmnopqrstuvwxyz0123456789 !#$%&()*+,-./:;<=>?@[]^_`{|}~", PAGE / 40); + // Plain runs broken by every kind of character the bulk copy must stop at. + text.append('"').append(plain).append("\\n\\\"\\\\\\u00e9é\\U0001F600😀").append(plain).append("\" "); + String identifier = "_$" + repeat("abcXYZ019", PAGE / 6); + text.append(identifier).append(' '); + text.append("{").append(identifier).append(":\"").append(plain).append("\"}"); + + IonDatagram values = load(source, text.toString()); + + assertEquals(3, values.size()); + assertEquals(plain + "\n\"\\\u00e9\u00e9\uD83D\uDE00\uD83D\uDE00" + plain, stringOf(values.get(0))); + assertEquals(identifier, stringOf(values.get(1))); + assertEquals(plain, stringOf(((IonStruct) values.get(2)).get(identifier))); + } + + @ParameterizedTest + @EnumSource(value = Source.class, names = "STREAM_1_BYTE_READS", mode = EnumSource.Mode.EXCLUDE) + public void blobsDecodeTheSameWithWhitespaceAndPadding(Source source) { + Random random = new Random(1); + List blobs = new ArrayList<>(); + StringBuilder text = new StringBuilder(); + for (int length : new int[] {0, 1, 2, 3, 4, 5, 57, 58, 59, PAGE, PAGE + 1, 3 * PAGE + 2}) { + byte[] bytes = new byte[length]; + random.nextBytes(bytes); + blobs.add(bytes); + String base64 = Base64.getEncoder().encodeToString(bytes); + // Plain, wrapped every 76 characters, and with whitespace inside a 4-character group. + text.append("{{").append(base64).append("}} "); + text.append("{{ ").append(Base64.getMimeEncoder().encodeToString(bytes)).append("\n}} "); + text.append("{{").append(base64.length() > 2 ? base64.substring(0, 2) + " \t" + base64.substring(2) : base64) + .append("}} "); + } + + try (IonReader reader = source.open(text.toString())) { + for (byte[] expected : blobs) { + for (int form = 0; form < 3; form++) { + assertEquals(IonType.BLOB, reader.next()); + assertEquals(expected.length, reader.byteSize()); + assertArrayEquals(expected, reader.newBytes()); + } + } + assertNull(reader.next()); + } catch (IOException e) { + throw new IonException(e); + } + } + + @ParameterizedTest + @EnumSource(value = Source.class, names = "STREAM_1_BYTE_READS", mode = EnumSource.Mode.EXCLUDE) + public void skippedValuesLeaveTheReaderOnTheNextValue(Source source) { + String big = repeat("x", PAGE + 3); + // Each skipped value holds quotes, comments, escapes and lobs whose contents look like its closing bracket. + String text = "{a:\"" + big + "\", 'b': {{" + Base64.getEncoder().encodeToString(big.getBytes(StandardCharsets.US_ASCII)) + + "}}, c:[\"]\", ']', '\\']', {{\"]}}\"}}, /* ] */ (\")\" ']') // ]\n], d:{e:" + big + ", x:'}', y:\"}\"/*}*/," + + "z:{{\"}\"}} // }\n}, f:\"\\\"}\", g:(')' \")\" /*)*/ // )\n), h:end} 42"; + + try (IonReader reader = source.open(text)) { + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + List names = new ArrayList<>(); + while (reader.next() != null) { + names.add(reader.getFieldName()); // values skipped, never read + } + reader.stepOut(); + assertEquals(Arrays.asList("a", "b", "c", "d", "f", "g", "h"), names); + assertEquals(IonType.INT, reader.next()); + assertEquals(42, reader.intValue()); + assertNull(reader.next()); + } catch (IOException e) { + throw new IonException(e); + } + } + + /** Field names read in one pass leave value spans, line numbers and columns as they were. */ + @ParameterizedTest + @EnumSource(Source.class) + public void valueSpansAfterFieldNames(Source source) { + String text = "{abc:1,\n de : \"x\",\r\n f/*c*/:{{AQID}},'q':true,\n\tgh\n:sym}"; + // What the reader reported before the fast paths, pinned as-is: the point is that they stay the same. + // (The line and column are those of where the reader started looking for the field, not of the value.) + long[][] expected = { // offset, line, column of each field's span + {1, 1, 2}, + {10, 1, 7}, + {24, 2, 13}, + {40, 3, 17}, + {51, 3, 26}, + }; + try (IonReader reader = source.open(text)) { + reader.next(); + reader.stepIn(); + SpanProvider spans = reader.asFacet(SpanProvider.class); + for (long[] position : expected) { + reader.next(); + OffsetSpan offset = spans.currentSpan().asFacet(OffsetSpan.class); + TextSpan line = spans.currentSpan().asFacet(TextSpan.class); + if (offset != null) { + assertEquals(position[0], offset.getStartOffset(), "offset of " + reader.getFieldName()); + } + assertEquals(position[1], line.getStartLine(), "line of " + reader.getFieldName()); + assertEquals(position[2], line.getStartColumn(), "column of " + reader.getFieldName()); + } + assertNull(reader.next()); + } catch (IOException e) { + throw new IonException(e); + } + } + + /** The bulk skips stop at line ends, so line numbers after skipped multi-line values are unchanged. */ + @ParameterizedTest + @EnumSource(Source.class) + public void lineNumbersAfterSkippedMultiLineValues(Source source) throws IOException { + for (String nl : new String[] {"\n", "\r\n", "\r"}) { + // A list, a struct holding a blob and an escaped line end, a blob and a string, each over two lines. + String text = "[1," + nl + "2]" + nl + + "{a:\"x\"," + nl + " b:{{" + nl + "AQID" + nl + "}}, c:\"y\\" + nl + "z\"}" + nl + + "{{ AQID" + nl + "BAUG }}" + nl + + "\"p\\" + nl + "q\"" + nl + + "end"; + IonType[] types = {IonType.LIST, IonType.STRUCT, IonType.BLOB, IonType.STRING, IonType.SYMBOL}; + long[] lines = {1, 3, 8, 10, 12}; + try (IonReader reader = source.open(text)) { + SpanProvider spans = reader.asFacet(SpanProvider.class); + for (int i = 0; i < types.length; i++) { + assertEquals(types[i], reader.next()); + TextSpan span = spans.currentSpan().asFacet(TextSpan.class); + assertEquals(lines[i], span.getStartLine(), types[i] + " after line ends " + escape(nl)); + assertEquals(1, span.getStartColumn(), types[i] + " after line ends " + escape(nl)); + } + assertNull(reader.next()); + } + } + } + + /** A raw line end is still an error in a double-quoted string that is skipped, not read. */ + @ParameterizedTest + @EnumSource(Source.class) + public void skippedStringsWithRawLineEndsAreErrors(Source source) { + for (String text : new String[] {"\"x\ny\" 1", "\"x\ry\" 1", "{a:\"x\ny\", b:1}", "[\"x\ny\"] 1"}) { + IonException e = assertThrows(IonException.class, () -> { + try (IonReader reader = source.open(text)) { + while (reader.next() != null) { + if (reader.getType() == IonType.STRUCT) { + reader.stepIn(); + while (reader.next() != null) { + // field values skipped + } + reader.stepOut(); + } + } + } + }, escape(text)); + assertTrue(e.getMessage().contains("at line 2 offset 1"), escape(text) + ": " + e.getMessage()); + } + } + + private static String escape(String s) { + return s.replace("\r", "\\r").replace("\n", "\\n"); + } + + @ParameterizedTest + @EnumSource(Source.class) + public void identifiersStopAtOperatorCharacters(Source source) { + IonDatagram values = load(source, "(abc.def a.b x...y z+w)"); + assertEquals(SYSTEM.singleValue("(abc '.' def a '.' b x '...' y z '+' w)"), values.get(0)); + } + + /** + * Field-name errors keep their exception and message. A keyword used as a field name is reported at the end of + * the name. (Before the name was read in one pass, the position also included the spaces and comments after it.) + */ + @Test + public void fieldNameErrors() { + String parsing = "com.amazon.ion.impl.IonReaderTextRawX$IonReaderTextParsingException"; + String token = "com.amazon.ion.impl.IonReaderTextRawTokensX$IonReaderTextTokenException"; + String[][] cases = { // input, exception class, message (or its start, for the last one) + {"{abc 1}", parsing, "Syntax error at line 1 offset 6: field name must be followed by a colon, not a TOKEN_UNKNOWN_NUMERIC"}, + {"{abc /*c*/ 1}", parsing, "Syntax error at line 1 offset 12: field name must be followed by a colon, not a TOKEN_UNKNOWN_NUMERIC"}, + {"{abc", parsing, "Syntax error at line 1 offset 5: field name must be followed by a colon, not a TOKEN_EOF"}, + {"{true:1}", parsing, "Syntax error at line 1 offset 6: Cannot use unquoted keyword true as a field name"}, + {"{null:1}", parsing, "Syntax error at line 1 offset 6: Cannot use unquoted keyword null as a field name"}, + {"{true :1}", parsing, "Syntax error at line 1 offset 6: Cannot use unquoted keyword true as a field name"}, + {"{nan :1}", parsing, "Syntax error at line 1 offset 5: Cannot use unquoted keyword nan as a field name"}, + {"{x:{nan /*c*/ : 1}}", parsing, "Syntax error at line 1 offset 8: Cannot use unquoted keyword nan as a field name"}, + {"{true\n:1}", parsing, "Syntax error at line 1 offset 6: Cannot use unquoted keyword true as a field name"}, + {"{false/*c*/:1}", parsing, "Syntax error at line 1 offset 7: Cannot use unquoted keyword false as a field name"}, + {"{nan\r\n :1}", parsing, "Syntax error at line 1 offset 5: Cannot use unquoted keyword nan as a field name"}, + {"{abc\u00e9:1}", token, "bad character ["}, + }; + for (String[] c : cases) { + for (Source source : Source.values()) { + IonException e = assertThrows(IonException.class, () -> load(source, c[0]), source + ": " + escape(c[0])); + assertEquals(c[1], e.getClass().getName(), source + ": " + escape(c[0])); + if (c[1].equals(token)) { + assertTrue(e.getMessage().startsWith(c[2]) && e.getMessage().endsWith("at line 1 offset 6"), + source + ": " + escape(c[0]) + ": " + e.getMessage()); + } else { + assertEquals(c[2], e.getMessage(), source + ": " + escape(c[0])); + } + } + } + } + + /** Non-ASCII characters in a blob are left to the existing decoder, which rejects them as before. */ + @ParameterizedTest + @EnumSource(Source.class) + public void nonAsciiInBase64IsRejectedAsBefore(Source source) { + for (String text : new String[] {"{{AAAA\u00e9AAA}}", "{{AAAA\u4e2dAAA}}", "{{\u00e9AAA}}", "{{AAAAAAAA\u4e2d}}"}) { + IonException e = assertThrows(IonException.class, () -> { + try (IonReader reader = source.open(text)) { + reader.next(); + reader.newBytes(); + } + }, source + ": " + text); + assertEquals("com.amazon.ion.impl.IonReaderTextRawTokensX$IonReaderTextTokenException", e.getClass().getName(), + source + ": " + text + ": " + e); + assertTrue(e.getMessage().startsWith("invalid character ") && e.getMessage().contains(" encountered in base64 value"), + source + ": " + text + ": " + e.getMessage()); + } + } + + /** + * After byteSize(), which decodes and caches the whole blob, one getBytes() shorter than the blob returns its + * first bytes from that cache, and the reader then moves on to the next value. This does not reach the point + * where bulk decoding hands over to the existing decoder with decoded bytes pending: that is covered by + * bulkBase64ReturnsWholeGroupsWithinTheRequestedLength. (Reading an uncached text blob in pieces is a separate, + * pre-existing problem.) + */ + @ParameterizedTest + @EnumSource(Source.class) + public void cachedPrefixReads(Source source) throws IOException { + byte[] bytes = new byte[200]; + new Random(5).nextBytes(bytes); + for (String base64 : new String[] {Base64.getEncoder().encodeToString(bytes), Base64.getMimeEncoder().encodeToString(bytes)}) { + for (int length = 1; length <= 8; length++) { + try (IonReader reader = source.open("{{" + base64 + "}} 7")) { + assertEquals(IonType.BLOB, reader.next()); + assertEquals(bytes.length, reader.byteSize()); + byte[] buffer = new byte[length]; + assertEquals(length, reader.getBytes(buffer, 0, length)); + assertArrayEquals(Arrays.copyOf(bytes, length), buffer, "length " + length); + assertEquals(IonType.INT, reader.next()); + assertEquals(7, reader.intValue()); + } + } + } + } + + // The stream helpers, called directly: each consumes a whole run, stops where it should, and never refills. + + private static final boolean[] SYMBOL = new boolean[128]; + private static final boolean[] QUOTE = new boolean[128]; + static { + for (int c = 0; c < 128; c++) { + SYMBOL[c] = IonTokenConstsX.isValidSymbolCharacter(c); + } + QUOTE['"'] = true; + } + + private static UnifiedInputStreamX[] streams(String text) throws IOException { + byte[] bytes = text.getBytes(StandardCharsets.UTF_8); + return new UnifiedInputStreamX[] { + UnifiedInputStreamX.makeStream(bytes), UnifiedInputStreamX.makeStream(new ByteArrayInputStream(bytes)), + UnifiedInputStreamX.makeStream(text), UnifiedInputStreamX.makeStream(new StringReader(text)), + }; + } + + @Test + public void appendAsciiRunCopiesTheAcceptedRun() throws IOException { + for (UnifiedInputStreamX in : streams("abc123_$-rest")) { + StringBuilder sb = new StringBuilder("x"); + in.appendAsciiRun(sb, SYMBOL); + assertEquals("xabc123_$", sb.toString()); + assertEquals('-', in.read()); + } + for (UnifiedInputStreamX in : streams("ab\u00e9")) { + StringBuilder sb = new StringBuilder(); + in.appendAsciiRun(sb, SYMBOL); + assertEquals("ab", sb.toString()); + int next = in.read(); + assertEquals(true, next == 0xC3 || next == 0xE9, "the non-ASCII character is left: " + next); + } + StringBuilder sb = new StringBuilder(); + UnifiedInputStreamX.makeStream(new LimitedReads("abcdefghij".getBytes(StandardCharsets.US_ASCII), 4)).appendAsciiRun(sb, SYMBOL); + assertEquals("abcd", sb.toString(), "stops at the end of the page"); + } + + @Test + public void skipRunIgnoresNonAscii() throws IOException { + for (UnifiedInputStreamX in : streams("ab \u00e9{\"rest")) { + in.skipRun(QUOTE); // non-ASCII never stops a skipRun + assertEquals('"', in.read()); + } + } + + @Test + public void decodeBase64GroupsDecodesWholeGroupsOnly() throws IOException { + int[] table = IonReaderTextRawTokensX.BASE64_CHAR_TO_BIN; + String[][] cases = { // input, max groups, decoded text, next character + {"QUJDREVG}", "9", "ABCDEF", "}"}, + {"QUJDREVG}", "1", "ABC", "R"}, + {"QUJDR0g=}", "9", "ABC", "R"}, // '=' is left to the existing decoder + {"QUJD RE}", "9", "ABC", " "}, // so is whitespace + {"QUJDRE", "9", "ABC", "R"}, // and a group the page doesn't hold in full + {"QUJD\u00e9AAA", "9", "ABC", null}, // and non-ASCII + {"QUJD\u4e2dAAA", "9", "ABC", null}, + }; + for (String[] c : cases) { + for (UnifiedInputStreamX in : streams(c[0])) { + byte[] dst = new byte[30]; + int groups = in.decodeBase64Groups(dst, 0, Integer.parseInt(c[1]), table); + assertEquals(c[2], new String(dst, 0, 3 * groups, StandardCharsets.US_ASCII), escape(c[0])); + int next = in.read(); + if (c[3] != null) { + assertEquals(c[3].charAt(0), next, escape(c[0])); + } else { + assertEquals(true, next >= 0x80, escape(c[0]) + ": the non-ASCII character is left: " + next); + } + } + } + } + + @Test + public void bulkBase64ReturnsWholeGroupsWithinTheRequestedLength() throws IOException { + for (UnifiedInputStreamX in : streams("QUJDREVGR0hJ}}")) { + IonReaderTextRawTokensX scanner = new IonReaderTextRawTokensX(in); + byte[] dst = new byte[9]; + assertEquals(6, scanner.read_base64_bytes_bulk(dst, 0, 8)); + assertEquals("ABCDEF", new String(dst, 0, 6, StandardCharsets.US_ASCII)); + assertEquals('G', scanner.read_base64_byte()); + } + } + + /** Random documents written by the text writer read back equal to what was written, from every source. */ + @ParameterizedTest + @EnumSource(value = Source.class, names = "STREAM_1_BYTE_READS", mode = EnumSource.Mode.EXCLUDE) + public void randomDocumentsRoundTrip(Source source) throws IOException { + Random random = new Random(42); + for (int i = 0; i < 300; i++) { + IonDatagram written = SYSTEM.newDatagram(); + for (int n = random.nextInt(5); n >= 0; n--) { + written.add(randomValue(random, 0)); + } + for (IonTextWriterBuilder builder : new IonTextWriterBuilder[] { + IonTextWriterBuilder.standard(), IonTextWriterBuilder.pretty()}) { + StringBuilder text = new StringBuilder(); + try (IonWriter writer = builder.build(text)) { + written.writeTo(writer); + } + assertEquals(written, load(source, text.toString()), text.toString()); + } + } + } + + private static IonValue randomValue(Random random, int depth) { + switch (random.nextInt(depth > 2 ? 6 : 9)) { + case 0: return SYSTEM.newString(randomText(random)); + case 1: return SYSTEM.newSymbol(random.nextBoolean() ? randomIdentifier(random) : randomText(random)); + case 2: return SYSTEM.newDecimal(new java.math.BigDecimal(random.nextLong()).movePointLeft(random.nextInt(8))); + case 3: { + byte[] bytes = new byte[random.nextInt(50) == 0 ? PAGE + random.nextInt(100) : random.nextInt(80)]; + random.nextBytes(bytes); + return SYSTEM.newBlob(bytes); + } + case 4: return SYSTEM.newInt(random.nextLong()); + case 5: return SYSTEM.newBool(random.nextBoolean()); + case 6: { + IonStruct struct = SYSTEM.newEmptyStruct(); + for (int n = random.nextInt(5); n > 0; n--) { + struct.add(random.nextBoolean() ? randomIdentifier(random) : randomText(random), randomValue(random, depth + 1)); + } + return struct; + } + case 7: { + IonList list = SYSTEM.newEmptyList(); + for (int n = random.nextInt(5); n > 0; n--) { + list.add(randomValue(random, depth + 1)); + } + return list; + } + default: { + IonSexp sexp = SYSTEM.newEmptySexp(); + for (int n = random.nextInt(5); n > 0; n--) { + sexp.add(randomValue(random, depth + 1)); + } + return sexp; + } + } + } + + private static String randomIdentifier(Random random) { + String start = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_$"; + String rest = start + "0123456789"; + StringBuilder s = new StringBuilder().append(start.charAt(random.nextInt(start.length()))); + for (int n = random.nextInt(random.nextInt(50) == 0 ? PAGE + 100 : 20); n > 0; n--) { + s.append(rest.charAt(random.nextInt(rest.length()))); + } + String identifier = s.toString(); + return identifier.equals("true") || identifier.equals("false") || identifier.equals("null") + || identifier.equals("nan") || identifier.startsWith("$") ? "_" + identifier : identifier; + } + + private static String randomText(Random random) { + StringBuilder s = new StringBuilder(); + for (int n = random.nextInt(random.nextInt(50) == 0 ? PAGE + 100 : 60); n > 0; n--) { + int kind = random.nextInt(30); + if (kind < 24) { + s.append((char) (0x20 + random.nextInt(0x5F))); + } else { + switch (kind) { + case 24: s.append('\n'); break; + case 25: s.append('\t'); break; + case 26: s.append('é'); break; + case 27: s.append("日本"); break; + case 28: s.appendCodePoint(0x1F600); break; + default: s.append((char) random.nextInt(0x20)); break; + } + } + } + return s.toString(); + } + + private static String stringOf(IonValue value) { + return value == null ? null : ((IonText) value).stringValue(); + } + + private static String repeat(String s, int times) { + StringBuilder out = new StringBuilder(s.length() * times); + for (int i = 0; i < times; i++) { + out.append(s); + } + return out.toString(); + } +}