From 1f31641d0a600c1b89b6cc71d4dacc190eaac29d Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Tue, 13 Oct 2026 08:21:21 +0200 Subject: [PATCH 1/3] refactor: carry typed ScalarValue in encoder stats instead of serialized byte[] (#531) EncodeResult, CascadeStep, SegmentStats and the min/max helpers now hold ProtoScalarValue; serialization happens once at the FlatBuffer boundary (VortexWriter#buildArrayFlatBuffer) and zone-map pruning reads the typed value directly. No wire change. Co-Authored-By: Claude Sonnet 5 --- .../dfa1/vortex/writer/DictColumnState.java | 13 ++--- .../dfa1/vortex/writer/VortexWriter.java | 18 +++---- .../dfa1/vortex/writer/ZoneMapStatCodec.java | 28 +++++------ .../writer/encode/AlpEncodingEncoder.java | 18 +++---- .../writer/encode/AlpRdEncodingEncoder.java | 10 ++-- .../encode/BitpackedEncodingEncoder.java | 14 +++--- .../writer/encode/BoolEncodingEncoder.java | 8 +-- .../vortex/writer/encode/CascadeStep.java | 12 ++--- .../encode/ConstantEncodingEncoder.java | 8 +-- .../encode/DateTimePartsEncodingEncoder.java | 5 +- .../writer/encode/DeltaEncodingEncoder.java | 12 ++--- .../writer/encode/DictEncodingEncoder.java | 12 ++--- .../vortex/writer/encode/EncodeResult.java | 14 +++--- .../writer/encode/ExtEncodingEncoder.java | 5 +- .../FrameOfReferenceEncodingEncoder.java | 4 +- .../writer/encode/FsstEncodingEncoder.java | 7 +-- .../writer/encode/MaskedEncodingEncoder.java | 9 ++-- .../writer/encode/OnPairEncodingEncoder.java | 7 +-- .../encode/PrimitiveEncodingEncoder.java | 48 +++++++++--------- .../writer/encode/RleEncodingEncoder.java | 5 +- .../writer/encode/RunEndEncodingEncoder.java | 12 ++--- .../encode/SequenceEncodingEncoder.java | 10 ++-- .../writer/encode/SparseEncodingEncoder.java | 2 +- .../writer/encode/VarBinEncodingEncoder.java | 10 ++-- .../writer/encode/ZigZagEncodingEncoder.java | 6 +-- .../vortex/writer/encode/ZoneMapStats.java | 24 ++++----- .../writer/encode/ZstdEncodingEncoder.java | 3 +- .../vortex/writer/ZoneMapStatCodecTest.java | 50 +++++++++---------- .../encode/AlpRdEncodingEncoderTest.java | 8 +-- .../encode/ConstantEncodingEncoderTest.java | 12 ++--- .../encode/DeltaEncodingEncoderTest.java | 8 +-- .../encode/DictEncodingEncoderTest.java | 12 ++--- .../writer/encode/ExtEncodingEncoderTest.java | 8 +-- .../FrameOfReferenceEncodingEncoderTest.java | 12 ++--- .../encode/FsstEncodingEncoderTest.java | 16 +++--- .../encode/MaskedEncodingEncoderTest.java | 4 +- .../encode/PrimitiveEncodingEncoderTest.java | 14 +++--- .../encode/RunEndEncodingEncoderTest.java | 8 +-- .../encode/SequenceEncodingEncoderTest.java | 8 +-- .../encode/ZigZagEncodingEncoderTest.java | 12 ++--- 40 files changed, 253 insertions(+), 243 deletions(-) diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java b/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java index a7589e7c1..d89ab347d 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.writer.encode.LongIntMap; @@ -70,8 +71,8 @@ final class DictColumnState { private final List chunkValidity = new ArrayList<>(); private final List chunkRowCounts = new ArrayList<>(); private final List chunkNullCounts = new ArrayList<>(); - private final List chunkStatsMin = new ArrayList<>(); - private final List chunkStatsMax = new ArrayList<>(); + private final List chunkStatsMin = new ArrayList<>(); + private final List chunkStatsMax = new ArrayList<>(); private final List chunkStatsSum = new ArrayList<>(); private long codeArrayBytes; @@ -138,11 +139,11 @@ List chunkNullCounts() { return chunkNullCounts; } - List chunkStatsMin() { + List chunkStatsMin() { return chunkStatsMin; } - List chunkStatsMax() { + List chunkStatsMax() { return chunkStatsMax; } @@ -288,12 +289,12 @@ boolean ingestDictChunk(Object data) { chunkStatsMax.add(null); chunkStatsSum.add(null); } else if (varBin) { - byte[][] mm = VarBinEncodingEncoder.minMaxStats((String[]) values); + ProtoScalarValue[] mm = VarBinEncodingEncoder.minMaxStats((String[]) values); chunkStatsMin.add(mm != null ? mm[0] : null); chunkStatsMax.add(mm != null ? mm[1] : null); chunkStatsSum.add(null); } else { - byte[][] mm = PrimitiveEncodingEncoder.minMaxStats(ptype, values); + ProtoScalarValue[] mm = PrimitiveEncodingEncoder.minMaxStats(ptype, values); chunkStatsMin.add(mm != null ? mm[0] : null); chunkStatsMax.add(mm != null ? mm[1] : null); chunkStatsSum.add(PrimitiveEncodingEncoder.sumStat(ptype, values)); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index 616a9ce98..e657b914a 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.fbs.FbsBuilder; import io.github.dfa1.vortex.writer.encode.DateTimePartsData; import io.github.dfa1.vortex.writer.encode.FixedSizeListData; @@ -853,13 +854,13 @@ private EncodedSegment encodeSegment(DType dtype, Object data, EncodingEncoder e // fallback computes them from the untouched input, independent of which encoder ran. // This is what makes stats coverage an encoder-independent guarantee rather than a // per-encoder convention every new encoder has to remember (ADR 0025). - byte[] min; - byte[] max; + ProtoScalarValue min; + ProtoScalarValue max; if (result.hasStats()) { min = result.statsMin(); max = result.statsMax(); } else { - byte[][] fallback = ZoneMapStatCodec.columnMinMax(dtype, data); + ProtoScalarValue[] fallback = ZoneMapStatCodec.columnMinMax(dtype, data); min = fallback != null ? fallback[0] : null; max = fallback != null ? fallback[1] : null; } @@ -1005,9 +1006,9 @@ private ByteBuffer buildArrayFlatBuffer(EncodeResult result, int[] paddings, lon // (Rust's flat writer doesn't either — flat/writer.rs retains only pre-computed stats); the // per-zone sum lives in the vortex.stats zone-map table emitted by flushZoneMaps(). int minVec = result.hasStats() - ? io.github.dfa1.vortex.core.fbs.FbsArrayStats.createMinVector(fbb, result.statsMin()) : 0; + ? io.github.dfa1.vortex.core.fbs.FbsArrayStats.createMinVector(fbb, result.statsMin().encode()) : 0; int maxVec = result.hasStats() - ? io.github.dfa1.vortex.core.fbs.FbsArrayStats.createMaxVector(fbb, result.statsMax()) : 0; + ? io.github.dfa1.vortex.core.fbs.FbsArrayStats.createMaxVector(fbb, result.statsMax().encode()) : 0; // forceDefaults only while building ArrayStats, so null_count = 0 is serialized (flatbuffers // omits a scalar equal to its default otherwise) — matching the Rust writer and letting the // reader prune IS NULL on zero-null chunks. Reset immediately so the Array/ArrayNode tables @@ -1142,7 +1143,7 @@ private DType columnDtype(ColumnName colName) { /// min/max) rather than dropping the stat for the whole column — MIN/MAX/SUM are nullable per /// zone, matching Rust. Field/bit order follows ZonedStatsSchema: MAX(3), MIN(4), SUM(5), /// NULL_COUNT(6). - private void emitZoneMap(ColumnName colName, DType minMaxDtype, List minBytes, List maxBytes, + private void emitZoneMap(ColumnName colName, DType minMaxDtype, List minBytes, List maxBytes, DType sumDtype, List sumBytes, long[] nullCounts, long[] rowCounts) throws IOException { int nZones = nullCounts.length; @@ -1548,8 +1549,7 @@ private record SegRef(long offset, long len, SegmentStats stats) { // S6218: the byte[] stat components are never value-compared — SegmentStats instances are only // read positionally, so the default identity equals is fine. - @SuppressWarnings("java:S6218") - private record SegmentStats(byte[] min, byte[] max, byte[] sum, long nullCount) { + private record SegmentStats(ProtoScalarValue min, ProtoScalarValue max, byte[] sum, long nullCount) { } private record EncodedSegment(EncodeResult result, Arena arena, SegmentStats stats) { @@ -1568,7 +1568,7 @@ private record ZoneMapRef(int zonesSegIdx, long nZones, byte[] metadata) { /// @param nullableCodes whether the codes child carries validity (Rust's `is_nullable_codes`) private record DictColRef(int valuesSegIdx, long valuesLen, List codesSegIdxes, List codesRowCounts, List chunkRowCounts, List chunkNullCounts, - List chunkStatsMin, List chunkStatsMax, List chunkStatsSum, + List chunkStatsMin, List chunkStatsMax, List chunkStatsSum, boolean nullableCodes) { long totalRows() { return chunkRowCounts.stream().mapToLong(Long::longValue).sum(); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java index 98676721c..d553fae16 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java @@ -17,7 +17,7 @@ /// Pure helpers for `vortex.stats` zone-map stats (ADR): deriving the per-zone min/max/sum dtype /// for a column, encoding a zone-map layout's `vortex.stats` metadata bytes, and decoding each -/// zone's serialized [ProtoScalarValue] stat back into a typed array the zone-map's struct +/// zone's serialized [ProtoScalarValue] stat back into a typed array the zone-map's struc /// encoder expects. Matches [io.github.dfa1.vortex.reader]'s `ZonedStatsSchema`. final class ZoneMapStatCodec { @@ -31,7 +31,7 @@ private ZoneMapStatCodec() { } /// The zone length to declare for a one-zone-per-chunk zone map over chunks of `rowCounts` - /// rows: the shared chunk length when every chunk but the last has it (and the last is not + /// rows: the shared chunk length when every chunk but the last has it (and the last is no /// empty and no longer), else `0`. /// /// Rust reads the declared length as a uniform stride - row `r` belongs to zone @@ -120,7 +120,7 @@ static byte[] columnSum(DType dtype, Object data) { /// The serialized `{min, max}` pair for `data` of logical type `dtype`, or `null` when the /// column has no recordable min/max ([#zoneMinMaxDtype] returns `null` for the same `dtype` /// shapes). This is the generic fallback every column gets regardless of which - /// [io.github.dfa1.vortex.writer.encode.EncodingEncoder] wrote it — mirroring the Rust + /// [io.github.dfa1.vortex.writer.encode.EncodingEncoder] wrote it — mirroring the Rus /// reference, which computes stats through one generic reduction for every encoding rather /// than trusting each encoding to report its own (see [ComparableValues]'s Rust-parity note). /// `VortexWriter#writeSegment` (private) only calls this when the winning encoder's own @@ -138,7 +138,7 @@ static byte[] columnSum(DType dtype, Object data) { /// @param dtype the segment's logical type /// @param data the segment's input data, possibly [NullableData]- or [ComparableValues]-wrapped /// @return a `{min, max}` pair, or `null` when `dtype` has no recordable min/max - static byte[][] columnMinMax(DType dtype, Object data) { + static ProtoScalarValue[] columnMinMax(DType dtype, Object data) { if (data instanceof ComparableValues cv) { return PrimitiveEncodingEncoder.minMaxStats(cv.ptype(), cv.values()); } @@ -167,7 +167,7 @@ private static Object compactIfNullable(PType ptype, Object data) { /// or an encoder that does not compute one) marks that zone invalid in `valid` rather than /// failing the whole column's zone-map — matching Rust, which stores min/max as a nullable /// per-zone stat. - static Object zoneStatValues(DType minMaxDtype, List statBytes, boolean[] valid) throws IOException { + static Object zoneStatValues(DType minMaxDtype, List statBytes, boolean[] valid) throws IOException { return switch (minMaxDtype) { case DType.Primitive p -> statColumn(p.ptype(), statBytes, valid); case DType.Utf8 _ -> statStringColumn(statBytes, valid); @@ -185,14 +185,14 @@ static Object sumColumn(DType sumDtype, List sumBytes, boolean[] valid) double[] a = new double[n]; for (int i = 0; i < n; i++) { valid[i] = sumBytes.get(i) != null; - a[i] = valid[i] ? scalarDouble(sumBytes.get(i)) : 0.0; + a[i] = valid[i] ? scalarDouble(decodeScalar(sumBytes.get(i))) : 0.0; } return a; } long[] a = new long[n]; for (int i = 0; i < n; i++) { valid[i] = sumBytes.get(i) != null; - a[i] = valid[i] ? scalarLong(sumBytes.get(i)) : 0L; + a[i] = valid[i] ? scalarLong(decodeScalar(sumBytes.get(i))) : 0L; } return a; } @@ -202,11 +202,11 @@ static Object sumColumn(DType sumDtype, List sumBytes, boolean[] valid) /// A `null` entry in `statBytes` sets `valid[i]` to `false` and fills the slot with `""` (a /// sum-neutral-style placeholder never read back, since [NullableData] carries validity /// separately from values). - private static String[] statStringColumn(List statBytes, boolean[] valid) throws IOException { + private static String[] statStringColumn(List statBytes, boolean[] valid) throws IOException { String[] out = new String[statBytes.size()]; for (int i = 0; i < out.length; i++) { valid[i] = statBytes.get(i) != null; - out[i] = valid[i] ? decodeScalar(statBytes.get(i)).string_value() : ""; + out[i] = valid[i] ? statBytes.get(i).string_value() : ""; } return out; } @@ -214,7 +214,7 @@ private static String[] statStringColumn(List statBytes, boolean[] valid /// Builds the per-zone values array in the storage shape the primitive encoder expects, decoding /// each zone's serialized [ProtoScalarValue] stat. A `null` entry in `statBytes` sets `valid[i]` /// to `false` and fills the slot with a zero placeholder, never read back. - private static Object statColumn(PType ptype, List statBytes, boolean[] valid) throws IOException { + private static Object statColumn(PType ptype, List statBytes, boolean[] valid) throws IOException { int n = statBytes.size(); return switch (ptype) { case I8, U8 -> { @@ -277,16 +277,16 @@ private static Object statColumn(PType ptype, List statBytes, boolean[] }; } - private static long scalarLong(byte[] bytes) throws IOException { + private static long scalarLong(ProtoScalarValue sv) { // Integer columns serialize min/max as int64 (signed) or uint64 (unsigned). - ProtoScalarValue sv = decodeScalar(bytes); + return sv.int64_value() != null ? sv.int64_value() : sv.uint64_value(); } - private static double scalarDouble(byte[] bytes) throws IOException { + private static double scalarDouble(ProtoScalarValue sv) { // Float columns serialize min/max as f64 (F64), f32 (F32) or f16 (F16, the half bits). // Branch rather than use a ternary so each path widens explicitly instead of mixing boxed types. - ProtoScalarValue sv = decodeScalar(bytes); + if (sv.f64_value() != null) { return sv.f64_value(); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoder.java index f3ef8285f..b62dbced8 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoder.java @@ -216,8 +216,8 @@ private static AlpF64Data computeF64(double[] values) { } } - byte[] statsMin = n > 0 ? scalarF64(min) : null; - byte[] statsMax = n > 0 ? scalarF64(max) : null; + ProtoScalarValue statsMin = n > 0 ? scalarF64(min) : null; + ProtoScalarValue statsMax = n > 0 ? scalarF64(max) : null; return new AlpF64Data(expE, expF, encodedArr, patchIndices, patchValues, statsMin, statsMax); } @@ -401,8 +401,8 @@ private static EncodeResult encodeF32(float[] values, EncodeContext ctx) { } } - byte[] statsMin = n > 0 ? scalarF32(min) : null; - byte[] statsMax = n > 0 ? scalarF32(max) : null; + ProtoScalarValue statsMin = n > 0 ? scalarF32(min) : null; + ProtoScalarValue statsMax = n > 0 ? scalarF32(max) : null; MemorySegment encodedBuf = ctx.arena().allocate((long) n * 4, 4); for (int i = 0; i < n; i++) { @@ -440,17 +440,17 @@ private static EncodeResult encodeF32(float[] values, EncodeContext ctx) { EncodedBuffer.of(patchBuffers.offsets(), patchBuffers.offsetType())), statsMin, statsMax); } - private static byte[] scalarF64(double v) { - return ProtoScalarValue.ofF64Value(v).encode(); + private static ProtoScalarValue scalarF64(double v) { + return ProtoScalarValue.ofF64Value(v); } - private static byte[] scalarF32(float v) { - return ProtoScalarValue.ofF32Value(v).encode(); + private static ProtoScalarValue scalarF32(float v) { + return ProtoScalarValue.ofF32Value(v); } @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. private record AlpF64Data(int expE, int expF, long[] encodedArr, List patchIndices, List patchValues, - byte[] statsMin, byte[] statsMax) { + ProtoScalarValue statsMin, ProtoScalarValue statsMax) { } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoder.java index d52bbd6d8..e988029e1 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoder.java @@ -295,7 +295,7 @@ private static Map buildLookup(short[] dict) { private static EncodeResult buildEncodeResult( short[] dict, int rightBitWidth, short[] leftCodes, Object rightPartsData, DType rightDtype, - List excPos, List excVals, byte[] statsMin, byte[] statsMax, EncodeContext ctx) { + List excPos, List excVals, ProtoScalarValue statsMin, ProtoScalarValue statsMax, EncodeContext ctx) { EncodingEncoder bp = ctx.lookupEncoder(EncodingId.FASTLANES_BITPACKED); EncodeResult leftResult = bp.encode(DType.U16, leftCodes, ctx); @@ -366,12 +366,12 @@ private static EncodeResult buildEncodeResult( return new EncodeResult(root, List.copyOf(allBuffers), statsMin, statsMax); } - private static byte[] scalarF64(double v) { - return ProtoScalarValue.ofF64Value(v).encode(); + private static ProtoScalarValue scalarF64(double v) { + return ProtoScalarValue.ofF64Value(v); } - private static byte[] scalarF32(float v) { - return ProtoScalarValue.ofF32Value(v).encode(); + private static ProtoScalarValue scalarF32(float v) { + return ProtoScalarValue.ofF32Value(v); } private static EncodeResult emptyResult(DType rightDtype, EncodeContext ctx) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java index 4bb59ed8b..0d570b8d1 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java @@ -81,8 +81,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { if (hasNegative) { MemorySegment widePacked = packFastLanes(longs, n, typeBits, typeBits, ctx.arena()); byte[] wideMeta = new ProtoBitPackedMetadata(typeBits, 0, null).encode(); - byte[] wideMin = statsBytes(ptype, signedMin); - byte[] wideMax = statsBytes(ptype, signedMax); + ProtoScalarValue wideMin = statsBytes(ptype, signedMin); + ProtoScalarValue wideMax = statsBytes(ptype, signedMax); EncodeNode wideRoot = new EncodeNode(EncodingId.FASTLANES_BITPACKED, MemorySegment.ofArray(wideMeta), new EncodeNode[0], new int[]{0}); return new EncodeResult(wideRoot, List.of(EncodedBuffer.of(widePacked, ptype)), wideMin, wideMax); @@ -92,8 +92,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment packed = packFastLanes(longs, n, bitWidth, typeBits, ctx.arena()); - byte[] statsMin = n > 0 ? statsBytes(ptype, signedMin) : null; - byte[] statsMax = n > 0 ? statsBytes(ptype, signedMax) : null; + ProtoScalarValue statsMin = n > 0 ? statsBytes(ptype, signedMin) : null; + ProtoScalarValue statsMax = n > 0 ? statsBytes(ptype, signedMax) : null; if (bitWidth >= typeBits) { // Packed width covers full type range — no overflow possible, no patches needed. @@ -243,10 +243,10 @@ private static void storeWords(MemorySegment seg, long off, long[] words, int ty } } - private static byte[] statsBytes(PType ptype, long value) { + private static ProtoScalarValue statsBytes(PType ptype, long value) { if (ptype.isUnsigned()) { - return ProtoScalarValue.ofUint64Value(value).encode(); + return ProtoScalarValue.ofUint64Value(value); } - return ProtoScalarValue.ofInt64Value(value).encode(); + return ProtoScalarValue.ofInt64Value(value); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BoolEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BoolEncodingEncoder.java index 260ab394c..a81b77bab 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BoolEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BoolEncodingEncoder.java @@ -54,11 +54,11 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { break; } } - byte[] statsMin = bools.length > 0 - ? ProtoScalarValue.ofBoolValue(!hasFalse).encode() + ProtoScalarValue statsMin = bools.length > 0 + ? ProtoScalarValue.ofBoolValue(!hasFalse) : null; - byte[] statsMax = bools.length > 0 - ? ProtoScalarValue.ofBoolValue(hasTrue).encode() + ProtoScalarValue statsMax = bools.length > 0 + ? ProtoScalarValue.ofBoolValue(hasTrue) : null; return EncodeResult.simple(encodingId(), EncodedBuffer.bytes(encodeBool(bools, ctx.arena())), statsMin, statsMax); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadeStep.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadeStep.java index c68ab4601..746588c60 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadeStep.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadeStep.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer.encode; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import java.util.List; /// One step in cascade-aware encoding: a partially-assembled node tree plus open child slots. @@ -17,16 +18,15 @@ /// @param partialRoot partially-assembled root encode node (may be `null` when not applicable) /// @param ownedBuffers buffers owned directly by the root node, before child buffers are appended /// @param openChildren child slots to be filled recursively by the cascading compressor -/// @param statsMin serialized minimum stat bytes, or `null` -/// @param statsMax serialized maximum stat bytes, or `null` +/// @param statsMin minimum stat scalar, or `null` +/// @param statsMax maximum stat scalar, or `null` /// @param applicable `false` if this encoding cannot handle the input data -@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. public record CascadeStep( EncodeNode partialRoot, List ownedBuffers, List openChildren, - byte[] statsMin, - byte[] statsMax, + ProtoScalarValue statsMin, + ProtoScalarValue statsMax, boolean applicable ) { /// This step with zone-map bounds attached, for an encoder whose entry point knows the @@ -36,7 +36,7 @@ public record CascadeStep( /// @param stats a `{min, max}` pair from [ZoneMapStats#of(io.github.dfa1.vortex.core.model.DType, Object)], /// or `null` when the input carried nothing to bound /// @return a copy of this step carrying `stats` - public CascadeStep withStats(byte[][] stats) { + public CascadeStep withStats(ProtoScalarValue[] stats) { return stats == null || !applicable ? this : new CascadeStep(partialRoot, ownedBuffers, openChildren, stats[0], stats[1], applicable); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java index 1e56368a4..d6e841380 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java @@ -70,9 +70,10 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { .orElseThrow(() -> new VortexException(EncodingId.VORTEX_CONSTANT, "not a constant array")); ProtoScalarValue scalar = buildScalar(ptype, firstRaw); byte[] scalarBytes = scalar.encode(); + ProtoScalarValue statsScalar = scalar; // A constant array's min and max are both the one repeated value, by construction -- no // scan needed. Empty arrays report no stats, matching every other encoder's convention. - byte[] stats = Array.getLength(data) > 0 ? scalarBytes : null; + ProtoScalarValue stats = Array.getLength(data) > 0 ? statsScalar : null; return EncodeResult.simple(EncodingId.VORTEX_CONSTANT, EncodedBuffer.bytes(MemorySegment.ofArray(scalarBytes)), stats, stats); } @@ -101,10 +102,11 @@ private static EncodeResult encodeUtf8(String[] data) { if (data.length == 0 || !isConstantStrings(data)) { throw new VortexException(EncodingId.VORTEX_CONSTANT, "not a constant array"); } - byte[] scalarBytes = ProtoScalarValue.ofStringValue(data[0]).encode(); + ProtoScalarValue scalar = ProtoScalarValue.ofStringValue(data[0]); + byte[] scalarBytes = scalar.encode(); // Both extremes are the one repeated value, by construction return EncodeResult.simple(EncodingId.VORTEX_CONSTANT, EncodedBuffer.bytes(MemorySegment.ofArray(scalarBytes)), - scalarBytes, scalarBytes); + scalar, scalar); } private static boolean isConstantStrings(String[] data) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java index a52e9b3bc..5e47d5b70 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer.encode; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; @@ -81,7 +82,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { // Bounds come from the undivided timestamps, in storage space: the split children are // days/seconds/subseconds and none of them orders the column. Without this, choosing // datetimeparts over vortex.ext would trade the column's zone map for its size. - byte[][] stats = PrimitiveEncodingEncoder.minMaxStats(PType.I64, d.timestamps()); + ProtoScalarValue[] stats = PrimitiveEncodingEncoder.minMaxStats(PType.I64, d.timestamps()); return new EncodeResult(root, List.copyOf(allBuffers), PrimitiveEncodingEncoder.minOf(stats), PrimitiveEncodingEncoder.maxOf(stats)); } @@ -112,7 +113,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext encodeC new ChildSlot(DType.I64, seconds, 1, SELF), new ChildSlot(DType.I64, subseconds, 2, SELF)); - byte[][] stats = PrimitiveEncodingEncoder.minMaxStats(PType.I64, d.timestamps()); + ProtoScalarValue[] stats = PrimitiveEncodingEncoder.minMaxStats(PType.I64, d.timestamps()); return new CascadeStep(partialRoot, List.of(), children, PrimitiveEncodingEncoder.minOf(stats), PrimitiveEncodingEncoder.maxOf(stats), true); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoder.java index b86e52dfb..510effd59 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoder.java @@ -80,7 +80,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { /// @param statsMin zone-map minimum, `null` when empty /// @param statsMax zone-map maximum, `null` when empty @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. - private record Deltas(long[] bases, long[] deltas, long paddedLen, byte[] statsMin, byte[] statsMax) { + private record Deltas(long[] bases, long[] deltas, long paddedLen, ProtoScalarValue statsMin, ProtoScalarValue statsMax) { byte[] metadata() { return new ProtoDeltaMetadata(paddedLen, 0).encode(); @@ -140,16 +140,16 @@ private static Deltas deltas(long[] longs, PType ptype) { System.arraycopy(chunkDelta, 0, deltasAll, chunk * FastLanes.CHUNK, FastLanes.CHUNK); } - byte[] statsMin = n > 0 ? statsBytes(ptype, minVal) : null; - byte[] statsMax = n > 0 ? statsBytes(ptype, maxVal) : null; + ProtoScalarValue statsMin = n > 0 ? statsBytes(ptype, minVal) : null; + ProtoScalarValue statsMax = n > 0 ? statsBytes(ptype, maxVal) : null; return new Deltas(basesAll, deltasAll, paddedLen, statsMin, statsMax); } - private static byte[] statsBytes(PType ptype, long value) { + private static ProtoScalarValue statsBytes(PType ptype, long value) { if (ptype.isUnsigned()) { - return ProtoScalarValue.ofUint64Value(value).encode(); + return ProtoScalarValue.ofUint64Value(value); } - return ProtoScalarValue.ofInt64Value(value).encode(); + return ProtoScalarValue.ofInt64Value(value); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java index 9ca424f4c..32c72af49 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java @@ -136,7 +136,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { new EncodeNode[]{codesNode, valuesNode}, new int[0]); - byte[][] stats = PrimitiveEncodingEncoder.minMaxStats(((DType.Primitive) dtype).ptype(), data); + ProtoScalarValue[] stats = PrimitiveEncodingEncoder.minMaxStats(((DType.Primitive) dtype).ptype(), data); return new EncodeResult(rootNode, List.of(EncodedBuffer.of(d.valuesBuf(), ((DType.Primitive) dtype).ptype()), EncodedBuffer.of(codesBuf, codePType)), PrimitiveEncodingEncoder.minOf(stats), PrimitiveEncodingEncoder.maxOf(stats)); } @@ -160,7 +160,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { new int[0]); DType codesDtype = new DType.Primitive(codePType, false); - byte[][] stats = PrimitiveEncodingEncoder.minMaxStats(((DType.Primitive) dtype).ptype(), data); + ProtoScalarValue[] stats = PrimitiveEncodingEncoder.minMaxStats(((DType.Primitive) dtype).ptype(), data); return new CascadeStep(partialRoot, List.of(), List.of(new ChildSlot(codesDtype, d.codesArr(), 0, CODES_EXCLUDED), new ChildSlot(dtype, d.valuesArr(), 1, Set.of(EncodingId.VORTEX_DICT))), @@ -218,8 +218,8 @@ private static CascadeStep encodeUtf8Cascade(String[] strings, EncodeContext ctx String minStr = valueMap.keySet().stream().min(String::compareTo).orElse(null); String maxStr = valueMap.keySet().stream().max(String::compareTo).orElse(null); - byte[] statsMin = minStr != null ? ProtoScalarValue.ofStringValue(minStr).encode() : null; - byte[] statsMax = maxStr != null ? ProtoScalarValue.ofStringValue(maxStr).encode() : null; + ProtoScalarValue statsMin = minStr != null ? ProtoScalarValue.ofStringValue(minStr) : null; + ProtoScalarValue statsMax = maxStr != null ? ProtoScalarValue.ofStringValue(maxStr) : null; return new CascadeStep(partialRoot, List.of(), List.of(new ChildSlot(new DType.Primitive(codePType, false), codesArr, 0, CODES_EXCLUDED), @@ -357,8 +357,8 @@ private static EncodeResult encodeUtf8(String[] strings, EncodeContext ctx) { String minStr = valueMap.keySet().stream().min(String::compareTo).orElse(null); String maxStr = valueMap.keySet().stream().max(String::compareTo).orElse(null); - byte[] statsMin = minStr != null ? ProtoScalarValue.ofStringValue(minStr).encode() : null; - byte[] statsMax = maxStr != null ? ProtoScalarValue.ofStringValue(maxStr).encode() : null; + ProtoScalarValue statsMin = minStr != null ? ProtoScalarValue.ofStringValue(minStr) : null; + ProtoScalarValue statsMax = maxStr != null ? ProtoScalarValue.ofStringValue(maxStr) : null; return new EncodeResult(root, List.of(EncodedBuffer.bytes(dictBytesBuf), EncodedBuffer.of(dictOffsetsBuf, PType.I64), EncodedBuffer.of(codesBuf, codePType)), statsMin, statsMax); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java index 44f90cd45..ca7a195de 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer.encode; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.model.EncodingId; import java.lang.foreign.MemorySegment; @@ -9,14 +10,13 @@ /// /// @param rootNode the root encode node describing the encoding tree structure /// @param encodedBuffers flat list of data buffers in the order referenced by `rootNode` -/// @param statsMin serialized minimum value bytes for zone-map pruning, or `null` -/// @param statsMax serialized maximum value bytes for zone-map pruning, or `null` -@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. +/// @param statsMin minimum value for zone-map pruning, or `null` +/// @param statsMax maximum value for zone-map pruning, or `null` public record EncodeResult( EncodeNode rootNode, List encodedBuffers, - byte[] statsMin, - byte[] statsMax + ProtoScalarValue statsMin, + ProtoScalarValue statsMax ) { /// The raw bytes of [#encodedBuffers()], in the same order. /// @@ -32,7 +32,7 @@ public List buffers() { /// @param min serialized minimum stat bytes, or `null` /// @param max serialized maximum stat bytes, or `null` /// @return an [EncodeResult] backed by a single-buffer leaf node - public static EncodeResult simple(EncodingId encodingId, EncodedBuffer data, byte[] min, byte[] max) { + public static EncodeResult simple(EncodingId encodingId, EncodedBuffer data, ProtoScalarValue min, ProtoScalarValue max) { return new EncodeResult(EncodeNode.leaf(encodingId, 0), List.of(data), min, max); } @@ -53,7 +53,7 @@ public static EncodeResult simple(EncodingId encodingId, EncodedBuffer data) { /// @param stats a `{min, max}` pair as returned by the encoders' `minMaxStats` helpers, or /// `null` when the input carried nothing to bound /// @return a copy of this result carrying `stats` - public EncodeResult withStats(byte[][] stats) { + public EncodeResult withStats(ProtoScalarValue[] stats) { return stats == null ? this : new EncodeResult(rootNode, encodedBuffers, stats[0], stats[1]); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoder.java index d8889af11..31901d08b 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoder.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer.encode; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; @@ -67,7 +68,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { ChildSlot slot = new ChildSlot(ext.storageDType(), data, 0, Set.of(EncodingId.VORTEX_EXT)); // The cascade reads stats off the step, not off the children it later resolves, so the // bounds have to be computed here rather than inherited from the storage slot. - byte[][] stats = storageStats(ext.storageDType(), data); + ProtoScalarValue[] stats = storageStats(ext.storageDType(), data); return new CascadeStep(partialRoot, List.of(), List.of(slot), PrimitiveEncodingEncoder.minOf(stats), PrimitiveEncodingEncoder.maxOf(stats), true); } @@ -79,7 +80,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { /// @param storage the extension's storage dtype /// @param data the storage-shaped values being encoded /// @return a two-element `{min, max}` array of encoded scalars, or `null` - private static byte[][] storageStats(DType storage, Object data) { + private static ProtoScalarValue[] storageStats(DType storage, Object data) { return storage instanceof DType.Primitive(PType ptype, boolean ignored) ? PrimitiveEncodingEncoder.minMaxStats(ptype, data) : null; diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java index ab5bddc57..b422aa953 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java @@ -64,7 +64,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { EncodeNode child = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); EncodeNode root = new EncodeNode(EncodingId.FASTLANES_FOR, meta, new EncodeNode[]{child}, new int[0]); - byte[][] stats = PrimitiveEncodingEncoder.minMaxStats(ptype, data); + ProtoScalarValue[] stats = PrimitiveEncodingEncoder.minMaxStats(ptype, data); return new EncodeResult(root, List.of(EncodedBuffer.of(residuals, ptype)), PrimitiveEncodingEncoder.minOf(stats), PrimitiveEncodingEncoder.maxOf(stats)); } @@ -90,7 +90,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext encodeC EncodeNode partialRoot = new EncodeNode(EncodingId.FASTLANES_FOR, meta, new EncodeNode[1], new int[0]); ChildSlot slot = new ChildSlot(dtype, residualsAsNativeArray(longs, ref, ptype), 0, Set.of(EncodingId.FASTLANES_FOR)); - byte[][] stats = PrimitiveEncodingEncoder.minMaxStats(ptype, data); + ProtoScalarValue[] stats = PrimitiveEncodingEncoder.minMaxStats(ptype, data); return new CascadeStep(partialRoot, List.of(), List.of(slot), PrimitiveEncodingEncoder.minOf(stats), PrimitiveEncodingEncoder.maxOf(stats), true); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java index 6ae5748c2..e3781f8b5 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer.encode; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.compute.PrimitiveArrays; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; @@ -70,7 +71,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { /// /// @param data the Utf8 (`String[]`) or Binary (`byte[][]`) values being encoded /// @return a two-element `{min, max}` array of encoded scalars, or `null` - private static byte[][] zoneMapStats(Object data) { + private static ProtoScalarValue[] zoneMapStats(Object data) { return data instanceof String[] strings ? VarBinEncodingEncoder.minMaxStats(strings) : null; } @@ -87,7 +88,7 @@ private static byte[][] zoneMapStats(Object data) { /// @param arena arena backing the length/offset buffers /// @param stats the `{min, max}` zone-map scalars from [#zoneMapStats(Object)], or `null` /// @return the terminal `vortex.fsst` encode result - static EncodeResult toEncodeResult(Fsst c, Arena arena, byte[][] stats) { + static EncodeResult toEncodeResult(Fsst c, Arena arena, ProtoScalarValue[] stats) { long uncompLenBytes = c.uncompLenPType().byteSize(); MemorySegment uncompLenBuf = arena.allocate(Math.max(c.n() * uncompLenBytes, 1)); for (int i = 0; i < c.n(); i++) { @@ -132,7 +133,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { return CascadeStep.terminal(encode(dtype, data, ctx)); } Fsst c = compress(data, ctx.arena()); - byte[][] stats = zoneMapStats(data); + ProtoScalarValue[] stats = zoneMapStats(data); Object uncompLens = PrimitiveArrays.fromIntsArray(c.uncompLens(), c.uncompLenPType(), EncodingId.VORTEX_FSST); Object codesOffsets = PrimitiveArrays.fromIntsArray(c.codesOffsets(), c.codesOffPType(), EncodingId.VORTEX_FSST); EncodeNode partialRoot = new EncodeNode( diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java index a09f735a6..82d395304 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer.encode; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; @@ -61,9 +62,9 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { null, new EncodeNode[]{valuesResult.rootNode(), validityNode}, new int[0]); - byte[][] stats = maskedMinMaxStats(nonNullable, values, validity); - byte[] statsMin = stats == null ? null : stats[0]; - byte[] statsMax = stats == null ? null : stats[1]; + ProtoScalarValue[] stats = maskedMinMaxStats(nonNullable, values, validity); + ProtoScalarValue statsMin = stats == null ? null : stats[0]; + ProtoScalarValue statsMax = stats == null ? null : stats[1]; EncodeResult masked = new EncodeResult(root, buffers, statsMin, statsMax); // Rust never wraps a dict-encoded nullable string column in a row mask: its dict builder @@ -94,7 +95,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { /// @param values the dense values array from the [NullableData] carrier /// @param validity per-row validity, aligned with `values` /// @return a two-element `{min, max}` array of encoded scalars, or `null` when no row is valid - private static byte[][] maskedMinMaxStats(DType nonNullable, Object values, boolean[] validity) { + private static ProtoScalarValue[] maskedMinMaxStats(DType nonNullable, Object values, boolean[] validity) { return ZoneMapStats.of(nonNullable, new NullableData(values, validity)); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/OnPairEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/OnPairEncodingEncoder.java index d1a05816a..83946deae 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/OnPairEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/OnPairEncodingEncoder.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer.encode; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.compute.PrimitiveArrays; import io.github.dfa1.vortex.core.io.PTypeIO; import io.github.dfa1.vortex.core.model.DType; @@ -66,7 +67,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { } EncodeNode root = new EncodeNode(EncodingId.VORTEX_ONPAIR, MemorySegment.ofArray(c.metadata()), children, new int[]{0}); - byte[][] stats = zoneMapStats(data); + ProtoScalarValue[] stats = zoneMapStats(data); return new EncodeResult(root, buffers, stats != null ? stats[0] : null, stats != null ? stats[1] : null); } @@ -87,11 +88,11 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { } EncodeNode partialRoot = new EncodeNode(EncodingId.VORTEX_ONPAIR, MemorySegment.ofArray(c.metadata()), new EncodeNode[4], new int[]{0}); - byte[][] stats = zoneMapStats(data); + ProtoScalarValue[] stats = zoneMapStats(data); return new CascadeStep(partialRoot, List.of(EncodedBuffer.bytes(c.dictBytes(ctx.arena()))), slots, stats != null ? stats[0] : null, stats != null ? stats[1] : null, true); } - private static byte[][] zoneMapStats(Object data) { + private static ProtoScalarValue[] zoneMapStats(Object data) { return data instanceof String[] strings ? VarBinEncodingEncoder.minMaxStats(strings) : null; } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java index 6d1ee0bc9..e20b80141 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java @@ -32,9 +32,9 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment seg = data instanceof byte[] bytes ? MemorySegment.ofArray(bytes) : PrimitiveArrays.toSegment(data, ptype, ctx.arena()); - byte[] min = null; - byte[] max = null; - byte[][] stats = minMaxStats(ptype, data); + ProtoScalarValue min = null; + ProtoScalarValue max = null; + ProtoScalarValue[] stats = minMaxStats(ptype, data); if (stats != null) { min = stats[0]; max = stats[1]; @@ -50,7 +50,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { /// @param ptype the primitive type of `data` /// @param data the raw primitive array (e.g. `long[]`, `int[]`, `String`-free) /// @return a two-element `{min, max}` array of encoded scalars, or `null` if `data` is empty - public static byte[][] minMaxStats(PType ptype, Object data) { + public static ProtoScalarValue[] minMaxStats(PType ptype, Object data) { if (Array.getLength(data) == 0) { return null; } @@ -60,13 +60,13 @@ public static byte[][] minMaxStats(PType ptype, Object data) { return null; } return switch (ptype) { - case I8, I16, I32, I64 -> new byte[][]{scalarI64(minMax[0]), scalarI64(minMax[1])}; - case U8, U16, U32, U64 -> new byte[][]{scalarU64(minMax[0]), scalarU64(minMax[1])}; - case F16 -> new byte[][]{scalarF16(Float.float16ToFloat((short) minMax[0])), + case I8, I16, I32, I64 -> new ProtoScalarValue[]{scalarI64(minMax[0]), scalarI64(minMax[1])}; + case U8, U16, U32, U64 -> new ProtoScalarValue[]{scalarU64(minMax[0]), scalarU64(minMax[1])}; + case F16 -> new ProtoScalarValue[]{scalarF16(Float.float16ToFloat((short) minMax[0])), scalarF16(Float.float16ToFloat((short) minMax[1]))}; - case F32 -> new byte[][]{scalarF32(Float.intBitsToFloat((int) minMax[0])), + case F32 -> new ProtoScalarValue[]{scalarF32(Float.intBitsToFloat((int) minMax[0])), scalarF32(Float.intBitsToFloat((int) minMax[1]))}; - case F64 -> new byte[][]{scalarF64(Double.longBitsToDouble(minMax[0])), + case F64 -> new ProtoScalarValue[]{scalarF64(Double.longBitsToDouble(minMax[0])), scalarF64(Double.longBitsToDouble(minMax[1]))}; }; } @@ -76,7 +76,7 @@ public static byte[][] minMaxStats(PType ptype, Object data) { /// /// @param stats a [#minMaxStats] result, possibly `null` /// @return the serialized min scalar, or `null` - public static byte[] minOf(byte[][] stats) { + public static ProtoScalarValue minOf(ProtoScalarValue[] stats) { return stats == null ? null : stats[0]; } @@ -85,7 +85,7 @@ public static byte[] minOf(byte[][] stats) { /// /// @param stats a [#minMaxStats] result, possibly `null` /// @return the serialized max scalar, or `null` - public static byte[] maxOf(byte[][] stats) { + public static ProtoScalarValue maxOf(ProtoScalarValue[] stats) { return stats == null ? null : stats[1]; } @@ -111,35 +111,35 @@ public static byte[] sumStat(PType ptype, Object data) { return switch (ptype) { case I8, I16, I32, I64 -> { OptionalLong sum = ops.sum(data, ptype); - yield sum.isEmpty() ? null : scalarI64(sum.getAsLong()); + yield sum.isEmpty() ? null : scalarI64(sum.getAsLong()).encode(); } case U8, U16, U32, U64 -> { OptionalLong sum = ops.sum(data, ptype); - yield sum.isEmpty() ? null : scalarU64(sum.getAsLong()); + yield sum.isEmpty() ? null : scalarU64(sum.getAsLong()).encode(); } - case F16, F32, F64 -> scalarF64(ops.sumFloating(data, ptype)); + case F16, F32, F64 -> scalarF64(ops.sumFloating(data, ptype)).encode(); }; } - private static byte[] scalarI64(long v) { - return ProtoScalarValue.ofInt64Value(v).encode(); + private static ProtoScalarValue scalarI64(long v) { + return ProtoScalarValue.ofInt64Value(v); } - private static byte[] scalarU64(long v) { - return ProtoScalarValue.ofUint64Value(v).encode(); + private static ProtoScalarValue scalarU64(long v) { + return ProtoScalarValue.ofUint64Value(v); } // Rust types an F16 column's min/max as F16 scalars (`f16_value`, the half bits as a u64); an // f32 scalar on an F16 column is rejected on read ("expected F32 dtype for F32Value, got f16"). - private static byte[] scalarF16(float v) { - return ProtoScalarValue.ofF16Value(Short.toUnsignedLong(Float.floatToFloat16(v))).encode(); + private static ProtoScalarValue scalarF16(float v) { + return ProtoScalarValue.ofF16Value(Short.toUnsignedLong(Float.floatToFloat16(v))); } - private static byte[] scalarF32(float v) { - return ProtoScalarValue.ofF32Value(v).encode(); + private static ProtoScalarValue scalarF32(float v) { + return ProtoScalarValue.ofF32Value(v); } - private static byte[] scalarF64(double v) { - return ProtoScalarValue.ofF64Value(v).encode(); + private static ProtoScalarValue scalarF64(double v) { + return ProtoScalarValue.ofF64Value(v); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java index a3cd60ae9..efb66cc98 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer.encode; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; @@ -105,7 +106,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { } // Run-length encoding preserves the value set, so the runs bound the column exactly as the // raw values do. - byte[][] stats = ZoneMapStats.of(dtype, data); + ProtoScalarValue[] stats = ZoneMapStats.of(dtype, data); Runs runs = runs(longs); MemorySegment valuesSeg = fromLongs(runs.values(), runs.valuesCount(), ptype, ctx.arena()); @@ -156,7 +157,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { new ChildSlot(dtype, PrimitiveArrays.fromBitsArray(values, ptype, EncodingId.FASTLANES_RLE), 0, VALUES_EXCLUDED), new ChildSlot(new DType.Primitive(PType.U16, false), runs.indices(), 1, POSITIONS_EXCLUDED), new ChildSlot(new DType.Primitive(PType.U64, false), runs.offsets(), 2, POSITIONS_EXCLUDED)); - byte[][] stats = ZoneMapStats.of(dtype, data); + ProtoScalarValue[] stats = ZoneMapStats.of(dtype, data); return new CascadeStep(partialRoot, List.of(), slots, ZoneMapStats.minOf(stats), ZoneMapStats.maxOf(stats), true); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java index 03248b637..4959180b3 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java @@ -189,8 +189,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { EncodeNode valuesNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); EncodeNode root = new EncodeNode(EncodingId.VORTEX_RUNEND, MemorySegment.ofArray(metaBytes), new EncodeNode[]{endsNode, valuesNode}, new int[0]); - byte[] statsMin = n > 0 ? statsBytes(ptype, minVal) : null; - byte[] statsMax = n > 0 ? statsBytes(ptype, maxVal) : null; + ProtoScalarValue statsMin = n > 0 ? statsBytes(ptype, minVal) : null; + ProtoScalarValue statsMax = n > 0 ? statsBytes(ptype, maxVal) : null; return new EncodeResult(root, List.of(EncodedBuffer.of(endsBuf, PType.U32), EncodedBuffer.of(valuesBuf, ptype)), statsMin, statsMax); } @@ -243,15 +243,15 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { new EncodeNode[]{null, null}, new int[0]); ChildSlot endsSlot = new ChildSlot(new DType.Primitive(PType.U32, false), ends, 0, ENDS_EXCLUDED); ChildSlot valuesSlot = new ChildSlot(dtype, PrimitiveArrays.fromLongsArray(values, ptype, EncodingId.VORTEX_RUNEND), 1, VALUES_EXCLUDED); - byte[][] stats = ZoneMapStats.of(dtype, data); + ProtoScalarValue[] stats = ZoneMapStats.of(dtype, data); return new CascadeStep(partialRoot, List.of(), List.of(endsSlot, valuesSlot), ZoneMapStats.minOf(stats), ZoneMapStats.maxOf(stats), true); } - private static byte[] statsBytes(PType ptype, long value) { + private static ProtoScalarValue statsBytes(PType ptype, long value) { if (ptype.isUnsigned()) { - return ProtoScalarValue.ofUint64Value(value).encode(); + return ProtoScalarValue.ofUint64Value(value); } - return ProtoScalarValue.ofInt64Value(value).encode(); + return ProtoScalarValue.ofInt64Value(value); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java index 6797c4730..e9b7131ad 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java @@ -86,18 +86,18 @@ private static EncodeResult encodeInteger(PType pt, Object data) { // A perfect arithmetic sequence is monotonic (or constant) end to end, so its extremes are // always the first and last element -- no separate scan needed. boolean unsign = pt.isUnsigned(); - byte[] statsMin = null; - byte[] statsMax = null; + ProtoScalarValue statsMin = null; + ProtoScalarValue statsMax = null; if (n > 0) { long last = base + (n - 1) * multiplier; boolean baseIsMin = unsign ? Long.compareUnsigned(base, last) <= 0 : base <= last; - statsMin = buildIntScalar(pt, baseIsMin ? base : last).encode(); - statsMax = buildIntScalar(pt, baseIsMin ? last : base).encode(); + statsMin = buildIntScalar(pt, baseIsMin ? base : last); + statsMax = buildIntScalar(pt, baseIsMin ? last : base); } return buildResult(baseScalar, mulScalar, statsMin, statsMax); } - private static EncodeResult buildResult(ProtoScalarValue base, ProtoScalarValue mul, byte[] statsMin, byte[] statsMax) { + private static EncodeResult buildResult(ProtoScalarValue base, ProtoScalarValue mul, ProtoScalarValue statsMin, ProtoScalarValue statsMax) { ProtoSequenceMetadata meta = new ProtoSequenceMetadata(base, mul); MemorySegment metaBuf = MemorySegment.ofArray(meta.encode()); EncodeNode node = new EncodeNode(EncodingId.VORTEX_SEQUENCE, metaBuf, new EncodeNode[0], new int[]{}); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java index 47d510ec8..29a347b0b 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java @@ -81,7 +81,7 @@ public Estimate expectedRatio(DType dtype, ArrayAndStats data, EncodeContext ctx public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { // `data` is the whole logical column -- the fill value and patch positions are derived // from it below -- so the column's bounds are just its bounds, patches included. - byte[][] stats = ZoneMapStats.of(dtype, data); + ProtoScalarValue[] stats = ZoneMapStats.of(dtype, data); if (!(dtype instanceof DType.Primitive p)) { return CascadeStep.notApplicable(); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java index 11ad705bd..237421719 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java @@ -56,9 +56,9 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { // Zone-map min/max stats are lexicographic-string-only (minMaxStats(String[])); binary // blobs (e.g. audio bytes) aren't usefully zone-mapped, so skip stats for that path. - byte[][] stats = strings != null ? minMaxStats(strings) : null; - byte[] statsMin = stats != null ? stats[0] : null; - byte[] statsMax = stats != null ? stats[1] : null; + ProtoScalarValue[] stats = strings != null ? minMaxStats(strings) : null; + ProtoScalarValue statsMin = stats != null ? stats[0] : null; + ProtoScalarValue statsMax = stats != null ? stats[1] : null; EncodeNode offsetsNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); EncodeNode root = new EncodeNode(EncodingId.VORTEX_VARBIN, MemorySegment.ofArray(metaBytes), @@ -73,7 +73,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { /// /// @param strings the string values (may contain `null`) /// @return a two-element `{min, max}` array of encoded scalars, or `null` if all entries are `null` - public static byte[][] minMaxStats(String[] strings) { + public static ProtoScalarValue[] minMaxStats(String[] strings) { String minStr = null; String maxStr = null; for (String s : strings) { @@ -90,6 +90,6 @@ public static byte[][] minMaxStats(String[] strings) { if (minStr == null) { return null; } - return new byte[][]{ProtoScalarValue.ofStringValue(minStr).encode(), ProtoScalarValue.ofStringValue(maxStr).encode()}; + return new ProtoScalarValue[]{ProtoScalarValue.ofStringValue(minStr), ProtoScalarValue.ofStringValue(maxStr)}; } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java index 7bdd31431..ed96bff4c 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java @@ -131,8 +131,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { }; EncodeNode child = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); EncodeNode root = new EncodeNode(EncodingId.VORTEX_ZIGZAG, null, new EncodeNode[]{child}, new int[0]); - byte[] statsMin = n > 0 ? ProtoScalarValue.ofInt64Value(minMax[0]).encode() : null; - byte[] statsMax = n > 0 ? ProtoScalarValue.ofInt64Value(minMax[1]).encode() : null; + ProtoScalarValue statsMin = n > 0 ? ProtoScalarValue.ofInt64Value(minMax[0]) : null; + ProtoScalarValue statsMax = n > 0 ? ProtoScalarValue.ofInt64Value(minMax[1]) : null; return new EncodeResult(root, List.of(EncodedBuffer.of(seg, signed)), statsMin, statsMax); } @@ -173,7 +173,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { ChildSlot slot = new ChildSlot(new DType.Primitive(unsigned, false), PrimitiveArrays.fromLongsArray(encoded, unsigned, EncodingId.VORTEX_ZIGZAG), 0, ENCODED_EXCLUDED); // zigzag is not order-preserving, so the bounds come from the original signed values - byte[][] stats = ZoneMapStats.of(dtype, data); + ProtoScalarValue[] stats = ZoneMapStats.of(dtype, data); return new CascadeStep(partialRoot, List.of(), List.of(slot), ZoneMapStats.minOf(stats), ZoneMapStats.maxOf(stats), true); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZoneMapStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZoneMapStats.java index cd636a62d..e8e288ed6 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZoneMapStats.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZoneMapStats.java @@ -38,7 +38,7 @@ private ZoneMapStats() { /// @param dtype the column's dtype; a nullable one is read through to its non-nullable form /// @param data the values being encoded /// @return a two-element `{min, max}` array of encoded scalars, or `null` - static byte[][] of(DType dtype, Object data) { + static ProtoScalarValue[] of(DType dtype, Object data) { if (data instanceof NullableData(var values, var validity)) { DType nonNullable = dtype == null ? null : dtype.withNullable(false); if (nonNullable instanceof DType.Primitive p) { @@ -68,7 +68,7 @@ static byte[][] of(DType dtype, Object data) { /// @param dtype the decimal dtype (precision picks the width) /// @param values the decimals, `null` at masked-out rows /// @return a two-element `{min, max}` array of encoded scalars, or `null` when none is valid - static byte[][] ofDecimal(DType.Decimal dtype, BigDecimal[] values) { + static ProtoScalarValue[] ofDecimal(DType.Decimal dtype, BigDecimal[] values) { BigDecimal[] valid = Arrays.stream(values).filter(Objects::nonNull).toArray(BigDecimal[]::new); int width = DecimalEncodingEncoder.storageWidth(dtype.precision()); return ofDecimal(DecimalEncodingEncoder.pack(valid, width, Arena.ofAuto()), width); @@ -87,7 +87,7 @@ static byte[][] ofDecimal(DType.Decimal dtype, BigDecimal[] values) { /// @param values the packed values /// @param byteWidth bytes per value, 1/2/4/8/16/32 /// @return a two-element `{min, max}` array of encoded scalars, or `null` when empty - static byte[][] ofDecimal(MemorySegment values, int byteWidth) { + static ProtoScalarValue[] ofDecimal(MemorySegment values, int byteWidth) { long n = values.byteSize() / byteWidth; if (n == 0) { return null; @@ -103,7 +103,7 @@ static byte[][] ofDecimal(MemorySegment values, int byteWidth) { maxOff = off; } } - return new byte[][]{decimalScalar(values, minOff, byteWidth), decimalScalar(values, maxOff, byteWidth)}; + return new ProtoScalarValue[]{decimalScalar(values, minOff, byteWidth), decimalScalar(values, maxOff, byteWidth)}; } /// Bounds for a decimal column already widened into `long` values, encoded at the same @@ -111,12 +111,12 @@ static byte[][] ofDecimal(MemorySegment values, int byteWidth) { /// /// @param values the unscaled values /// @return a two-element `{min, max}` array of encoded scalars, or `null` when empty - static byte[][] ofDecimal(long[] values) { + static ProtoScalarValue[] ofDecimal(long[] values) { if (values.length == 0) { return null; } long[] minMax = SimdOperationsSupport.preferred().minMax(values, PType.I64); - return new byte[][]{decimalScalar(minMax[0]), decimalScalar(minMax[1])}; + return new ProtoScalarValue[]{decimalScalar(minMax[0]), decimalScalar(minMax[1])}; } /// Compares two equal-width little-endian two's-complement values in place: the most @@ -137,24 +137,24 @@ private static int compareTwosComplement(MemorySegment values, long a, long b, i return 0; } - private static byte[] decimalScalar(MemorySegment values, long offset, int byteWidth) { + private static ProtoScalarValue decimalScalar(MemorySegment values, long offset, int byteWidth) { return ProtoScalarValue.ofBytesValue( - values.asSlice(offset, byteWidth).toArray(ValueLayout.JAVA_BYTE)).encode(); + values.asSlice(offset, byteWidth).toArray(ValueLayout.JAVA_BYTE)); } - private static byte[] decimalScalar(long value) { + private static ProtoScalarValue decimalScalar(long value) { byte[] le = new byte[Long.BYTES]; for (int i = 0; i < Long.BYTES; i++) { le[i] = (byte) (value >>> (8 * i)); } - return ProtoScalarValue.ofBytesValue(le).encode(); + return ProtoScalarValue.ofBytesValue(le); } /// The min half of an [#of(DType, Object)] result, or `null`. /// /// @param stats an [#of(DType, Object)] result, possibly `null` /// @return the serialized min scalar, or `null` - static byte[] minOf(byte[][] stats) { + static ProtoScalarValue minOf(ProtoScalarValue[] stats) { return stats == null ? null : stats[0]; } @@ -162,7 +162,7 @@ static byte[] minOf(byte[][] stats) { /// /// @param stats an [#of(DType, Object)] result, possibly `null` /// @return the serialized max scalar, or `null` - static byte[] maxOf(byte[][] stats) { + static ProtoScalarValue maxOf(ProtoScalarValue[] stats) { return stats == null ? null : stats[1]; } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java index 11385caba..a80c88d0c 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer.encode; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.zstd.ZstdCompressContext; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; @@ -113,7 +114,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { requireBinding(ZSTD_BINDING_PRESENT); // Compression does not reorder anything, so the column's bounds are the input's bounds. // Computed once here rather than in each of the four shape-specific paths below. - byte[][] stats = ZoneMapStats.of(dtype, data); + ProtoScalarValue[] stats = ZoneMapStats.of(dtype, data); if (data instanceof NullableData nd) { if (dtype instanceof DType.Primitive dt) { return encodeNullablePrimitive(dt, nd, ctx).withStats(stats); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java index d6c98d8ad..e0423091b 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java @@ -67,11 +67,11 @@ void signed_reportsRealMinMax() throws IOException { long[] data = {10L, -5L, 3L, 7L}; // When - byte[][] stats = ZoneMapStatCodec.columnMinMax(DTypes.I64, data); + ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(DTypes.I64, data); // Then - assertThat(scalar(stats[0]).int64_value()).isEqualTo(-5L); - assertThat(scalar(stats[1]).int64_value()).isEqualTo(10L); + assertThat(stats[0].int64_value()).isEqualTo(-5L); + assertThat(stats[1].int64_value()).isEqualTo(10L); } @Test @@ -80,11 +80,11 @@ void unsigned_usesUnsignedComparison() throws IOException { int[] data = {1, -1, 5}; // When - byte[][] stats = ZoneMapStatCodec.columnMinMax(DTypes.U32, data); + ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(DTypes.U32, data); // Then - assertThat(scalar(stats[0]).uint64_value()).isEqualTo(1L); - assertThat(scalar(stats[1]).uint64_value()).isEqualTo(4294967295L); + assertThat(stats[0].uint64_value()).isEqualTo(1L); + assertThat(stats[1].uint64_value()).isEqualTo(4294967295L); } @Test @@ -96,11 +96,11 @@ void nullable_compactsToValidElementsOnly() throws IOException { NullableData data = new NullableData(values, validity); // When - byte[][] stats = ZoneMapStatCodec.columnMinMax(DTypes.I64.asNullable(), data); + ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(DTypes.I64.asNullable(), data); // Then - assertThat(scalar(stats[0]).int64_value()).isEqualTo(-50L); - assertThat(scalar(stats[1]).int64_value()).isEqualTo(100L); + assertThat(stats[0].int64_value()).isEqualTo(-50L); + assertThat(stats[1].int64_value()).isEqualTo(100L); } @Test @@ -109,7 +109,7 @@ void empty_returnsNull() { long[] data = {}; // When - byte[][] stats = ZoneMapStatCodec.columnMinMax(DTypes.I64, data); + ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(DTypes.I64, data); // Then assertThat(stats).isNull(); @@ -128,11 +128,11 @@ void plainPrimitiveStorage_unwrapsToStoragePrimitive() throws IOException { long[] data = {100L, 200L, 50L}; // When - byte[][] stats = ZoneMapStatCodec.columnMinMax(TIMESTAMP_MS, data); + ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(TIMESTAMP_MS, data); // Then - assertThat(scalar(stats[0]).int64_value()).isEqualTo(50L); - assertThat(scalar(stats[1]).int64_value()).isEqualTo(200L); + assertThat(stats[0].int64_value()).isEqualTo(50L); + assertThat(stats[1].int64_value()).isEqualTo(200L); } @Test @@ -154,11 +154,11 @@ public Object values() { FakeCarrier carrier = new FakeCarrier(new long[]{9L, 1L, 5L}); // When - byte[][] stats = ZoneMapStatCodec.columnMinMax(TIMESTAMP_MS, carrier); + ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(TIMESTAMP_MS, carrier); // Then - assertThat(scalar(stats[0]).int64_value()).isEqualTo(1L); - assertThat(scalar(stats[1]).int64_value()).isEqualTo(9L); + assertThat(stats[0].int64_value()).isEqualTo(1L); + assertThat(stats[1].int64_value()).isEqualTo(9L); } @Test @@ -169,11 +169,11 @@ void nullablePlainStorage_compactsToValidElementsOnly() throws IOException { NullableData data = new NullableData(values, validity); // When - byte[][] stats = ZoneMapStatCodec.columnMinMax(TIMESTAMP_MS, data); + ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(TIMESTAMP_MS, data); // Then - assertThat(scalar(stats[0]).int64_value()).isEqualTo(300L); - assertThat(scalar(stats[1]).int64_value()).isEqualTo(300L); + assertThat(stats[0].int64_value()).isEqualTo(300L); + assertThat(stats[1].int64_value()).isEqualTo(300L); } } @@ -186,11 +186,11 @@ void reportsLexicographicMinMax() throws IOException { String[] data = {"banana", "apple", "cherry"}; // When - byte[][] stats = ZoneMapStatCodec.columnMinMax(DTypes.UTF8, data); + ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(DTypes.UTF8, data); // Then - assertThat(scalar(stats[0]).string_value()).isEqualTo("apple"); - assertThat(scalar(stats[1]).string_value()).isEqualTo("cherry"); + assertThat(stats[0].string_value()).isEqualTo("apple"); + assertThat(stats[1].string_value()).isEqualTo("cherry"); } @Test @@ -201,11 +201,11 @@ void nullable_skipsNullEntriesDirectly() throws IOException { NullableData data = new NullableData(values, new boolean[]{false, true, false, true}); // When - byte[][] stats = ZoneMapStatCodec.columnMinMax(DTypes.UTF8.asNullable(), data); + ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(DTypes.UTF8.asNullable(), data); // Then - assertThat(scalar(stats[0]).string_value()).isEqualTo("apple"); - assertThat(scalar(stats[1]).string_value()).isEqualTo("banana"); + assertThat(stats[0].string_value()).isEqualTo("apple"); + assertThat(stats[1].string_value()).isEqualTo("banana"); } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoderTest.java index 56c75accb..1dfd77216 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoderTest.java @@ -237,8 +237,8 @@ void encode_f64_reportsMinMaxStats() throws IOException { EncodeResult result = new AlpRdEncodingEncoder().encode(DTypes.F64, values, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).f64_value()).isEqualTo(-3.25); - assertThat(scalar(result.statsMax()).f64_value()).isEqualTo(10.0); + assertThat(result.statsMin().f64_value()).isEqualTo(-3.25); + assertThat(result.statsMax().f64_value()).isEqualTo(10.0); } @Test @@ -250,8 +250,8 @@ void encode_f32_reportsMinMaxStats() throws IOException { EncodeResult result = new AlpRdEncodingEncoder().encode(DTypes.F32, values, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).f32_value()).isEqualTo(-3.25f); - assertThat(scalar(result.statsMax()).f32_value()).isEqualTo(10.0f); + assertThat(result.statsMin().f32_value()).isEqualTo(-3.25f); + assertThat(result.statsMax().f32_value()).isEqualTo(10.0f); } @Test diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java index 04787d81b..92fbe891c 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java @@ -399,8 +399,8 @@ void encode_i64_reportsValueAsMinAndMax() throws java.io.IOException { EncodeResult result = ENCODER.encode(DTypes.I64, new long[]{constant, constant}, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(constant); - assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(constant); + assertThat(result.statsMin().int64_value()).isEqualTo(constant); + assertThat(result.statsMax().int64_value()).isEqualTo(constant); } @Test @@ -412,8 +412,8 @@ void encode_f64_reportsValueAsMinAndMax() throws java.io.IOException { EncodeResult result = ENCODER.encode(DTypes.F64, new double[]{constant, constant, constant}, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).f64_value()).isEqualTo(constant); - assertThat(scalar(result.statsMax()).f64_value()).isEqualTo(constant); + assertThat(result.statsMin().f64_value()).isEqualTo(constant); + assertThat(result.statsMax().f64_value()).isEqualTo(constant); } @Test @@ -425,8 +425,8 @@ void encode_f16_reportsValueAsF16MinAndMax() throws java.io.IOException { EncodeResult result = ENCODER.encode(DTypes.F16, data, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).f16_value()).isEqualTo(0x4100L); - assertThat(scalar(result.statsMax()).f16_value()).isEqualTo(0x4100L); + assertThat(result.statsMin().f16_value()).isEqualTo(0x4100L); + assertThat(result.statsMax().f16_value()).isEqualTo(0x4100L); } @Test diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoderTest.java index a69ec1048..237687e6a 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoderTest.java @@ -183,8 +183,8 @@ void encode_signedI64_statsCarryMinAndMax() throws Exception { // Then — signed stats use the int64 scalar field, min/max by signed ordering assertThat(result.hasStats()).isTrue(); - assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(-10L); - assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(50L); + assertThat(result.statsMin().int64_value()).isEqualTo(-10L); + assertThat(result.statsMax().int64_value()).isEqualTo(50L); } @Test @@ -197,8 +197,8 @@ void encode_unsignedU64_statsUseUnsignedOrderingAndField() throws Exception { EncodeResult result = ENCODER.encode(DTypes.U64, data, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).uint64_value()).isEqualTo(1L); - assertThat(scalar(result.statsMax()).uint64_value()).isEqualTo(-1L); + assertThat(result.statsMin().uint64_value()).isEqualTo(1L); + assertThat(result.statsMax().uint64_value()).isEqualTo(-1L); } @Test diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoderTest.java index 16276eab8..fa9977b24 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoderTest.java @@ -219,8 +219,8 @@ void encode_primitive_statsCarryMinAndMaxOfLogicalValues() throws IOException { // Then — min/max reflect the logical column values (10..30), not the dict codes (0..2) assertThat(result.hasStats()).isTrue(); - assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(10L); - assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(30L); + assertThat(result.statsMin().int64_value()).isEqualTo(10L); + assertThat(result.statsMax().int64_value()).isEqualTo(30L); } @Test @@ -234,8 +234,8 @@ void encodeCascade_primitive_statsCarryMinAndMaxOfLogicalValues() throws IOExcep CascadeStep result = ENCODER.encodeCascade(DTypes.I64, data, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(10L); - assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(30L); + assertThat(result.statsMin().int64_value()).isEqualTo(10L); + assertThat(result.statsMax().int64_value()).isEqualTo(30L); } @Test @@ -247,8 +247,8 @@ void encode_primitive_unsignedU32_statsUseUnsignedField() throws IOException { EncodeResult result = ENCODER.encode(new DType.Primitive(PType.U32, false), data, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).uint64_value()).isEqualTo(100L); - assertThat(scalar(result.statsMax()).uint64_value()).isEqualTo(300L); + assertThat(result.statsMin().uint64_value()).isEqualTo(100L); + assertThat(result.statsMax().uint64_value()).isEqualTo(300L); } @Test diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java index 33f5c148e..4b345f4e8 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java @@ -48,8 +48,8 @@ void encode_extensionColumn_reportsStorageSpaceBounds() throws Exception { // Then — the reader resolves an extension column's MIN/MAX against its storage dtype assertThat(result.hasStats()).isTrue(); - assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(1_600_000_000_000L); - assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(1_800_000_000_000L); + assertThat(result.statsMin().int64_value()).isEqualTo(1_600_000_000_000L); + assertThat(result.statsMax().int64_value()).isEqualTo(1_800_000_000_000L); } @Test @@ -65,8 +65,8 @@ void encodeCascade_extensionColumn_reportsTheSameBounds() throws Exception { // Then assertThat(result.openChildren()).isNotEmpty(); - assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(1_600_000_000_000L); - assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(1_800_000_000_000L); + assertThat(result.statsMin().int64_value()).isEqualTo(1_600_000_000_000L); + assertThat(result.statsMax().int64_value()).isEqualTo(1_800_000_000_000L); } @Test diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoderTest.java index c5111c5be..cc5667e6f 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoderTest.java @@ -248,8 +248,8 @@ void encode_signedI64_statsCarryMinAndMax() throws IOException { // Then assertThat(result.hasStats()).isTrue(); - assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(990L); - assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(1050L); + assertThat(result.statsMin().int64_value()).isEqualTo(990L); + assertThat(result.statsMax().int64_value()).isEqualTo(1050L); } @Test @@ -261,8 +261,8 @@ void encode_unsignedU32_statsUseUnsignedField() throws IOException { EncodeResult result = ENCODER.encode(new DType.Primitive(PType.U32, false), data, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).uint64_value()).isEqualTo(100L); - assertThat(scalar(result.statsMax()).uint64_value()).isEqualTo(300L); + assertThat(result.statsMin().uint64_value()).isEqualTo(100L); + assertThat(result.statsMax().uint64_value()).isEqualTo(300L); } @Test @@ -277,8 +277,8 @@ void encodeCascade_signedI64_statsCarryMinAndMax() throws IOException { // Then assertThat(result.applicable()).isTrue(); - assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(990L); - assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(1050L); + assertThat(result.statsMin().int64_value()).isEqualTo(990L); + assertThat(result.statsMax().int64_value()).isEqualTo(1050L); } private static ProtoScalarValue scalar(byte[] bytes) throws IOException { diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java index be7a52584..c75f4a77a 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java @@ -56,8 +56,8 @@ void encode_utf8_reportsLexicographicMinMax() throws Exception { // Then assertThat(result.hasStats()).isTrue(); - assertThat(scalar(result.statsMin()).string_value()).isEqualTo("apple"); - assertThat(scalar(result.statsMax()).string_value()).isEqualTo("quince"); + assertThat(result.statsMin().string_value()).isEqualTo("apple"); + assertThat(result.statsMax().string_value()).isEqualTo("quince"); } @Test @@ -72,10 +72,10 @@ void encode_utf8_matchesWhatVarBinWouldReport() throws Exception { .encode(DTypes.UTF8, data, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(fsst.statsMin()).string_value()) - .isEqualTo(scalar(varbin.statsMin()).string_value()); - assertThat(scalar(fsst.statsMax()).string_value()) - .isEqualTo(scalar(varbin.statsMax()).string_value()); + assertThat(fsst.statsMin().string_value()) + .isEqualTo(varbin.statsMin().string_value()); + assertThat(fsst.statsMax().string_value()) + .isEqualTo(varbin.statsMax().string_value()); } @Test @@ -91,8 +91,8 @@ void encodeCascade_utf8_carriesTheSameStats() throws Exception { // Then assertThat(result.openChildren()).isNotEmpty(); - assertThat(scalar(result.statsMin()).string_value()).isEqualTo("apple"); - assertThat(scalar(result.statsMax()).string_value()).isEqualTo("quince"); + assertThat(result.statsMin().string_value()).isEqualTo("apple"); + assertThat(result.statsMax().string_value()).isEqualTo("quince"); } @Test diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoderTest.java index baf1b0734..9c201c2f8 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoderTest.java @@ -261,8 +261,8 @@ void mixedValidity_statsIgnoreThePlaceholderAtInvalidSlots() throws java.io.IOEx // Then assertThat(result.hasStats()).isTrue(); - assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(500L); - assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(900L); + assertThat(result.statsMin().int64_value()).isEqualTo(500L); + assertThat(result.statsMax().int64_value()).isEqualTo(900L); } @Test diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java index 8e465831c..6b7f420f4 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java @@ -145,11 +145,11 @@ void leadingNanIsSkipped() { double[] values = {Double.NaN, 3.0, -1.0}; // When - byte[][] result = PrimitiveEncodingEncoder.minMaxStats(PType.F64, values); + ProtoScalarValue[] result = PrimitiveEncodingEncoder.minMaxStats(PType.F64, values); // Then - assertThat(result[0]).isEqualTo(ProtoScalarValue.ofF64Value(-1.0).encode()); - assertThat(result[1]).isEqualTo(ProtoScalarValue.ofF64Value(3.0).encode()); + assertThat(result[0]).isEqualTo(ProtoScalarValue.ofF64Value(-1.0)); + assertThat(result[1]).isEqualTo(ProtoScalarValue.ofF64Value(3.0)); } @Test @@ -158,7 +158,7 @@ void allNanHasNoMinMax() { float[] values = {Float.NaN, Float.NaN}; // When - byte[][] result = PrimitiveEncodingEncoder.minMaxStats(PType.F32, values); + ProtoScalarValue[] result = PrimitiveEncodingEncoder.minMaxStats(PType.F32, values); // Then assertThat(result).isNull(); @@ -171,11 +171,11 @@ void infinitiesAreValidExtremes() { double[] values = {Double.POSITIVE_INFINITY}; // When - byte[][] result = PrimitiveEncodingEncoder.minMaxStats(PType.F64, values); + ProtoScalarValue[] result = PrimitiveEncodingEncoder.minMaxStats(PType.F64, values); // Then - assertThat(result[0]).isEqualTo(ProtoScalarValue.ofF64Value(Double.POSITIVE_INFINITY).encode()); - assertThat(result[1]).isEqualTo(ProtoScalarValue.ofF64Value(Double.POSITIVE_INFINITY).encode()); + assertThat(result[0]).isEqualTo(ProtoScalarValue.ofF64Value(Double.POSITIVE_INFINITY)); + assertThat(result[1]).isEqualTo(ProtoScalarValue.ofF64Value(Double.POSITIVE_INFINITY)); } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java index 107e70441..6db1f7233 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java @@ -190,8 +190,8 @@ void encode_i64_reportsMinMaxAcrossRuns() throws java.io.IOException { EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(-5L); - assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(10L); + assertThat(result.statsMin().int64_value()).isEqualTo(-5L); + assertThat(result.statsMax().int64_value()).isEqualTo(10L); } @Test @@ -204,8 +204,8 @@ void encode_u32_usesUnsignedComparison() throws java.io.IOException { EncodeResult result = ENCODER.encode(DTypes.U32, data, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).uint64_value()).isEqualTo(1L); - assertThat(scalar(result.statsMax()).uint64_value()).isEqualTo(4294967295L); + assertThat(result.statsMin().uint64_value()).isEqualTo(1L); + assertThat(result.statsMax().uint64_value()).isEqualTo(4294967295L); } @Test diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java index 5e49a2de8..9acce2a66 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java @@ -410,8 +410,8 @@ void encode_i64_increasing_reportsFirstAsMinLastAsMax() throws java.io.IOExcepti EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(10L); - assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(40L); + assertThat(result.statsMin().int64_value()).isEqualTo(10L); + assertThat(result.statsMax().int64_value()).isEqualTo(40L); } @Test @@ -423,8 +423,8 @@ void encode_i64_decreasing_reportsLastAsMinFirstAsMax() throws java.io.IOExcepti EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(10L); - assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(40L); + assertThat(result.statsMin().int64_value()).isEqualTo(10L); + assertThat(result.statsMax().int64_value()).isEqualTo(40L); } @Test diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoderTest.java index 924c4f99e..6f82355ce 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoderTest.java @@ -246,8 +246,8 @@ void encode_i32_reportsMinMaxOfOriginalValues() throws java.io.IOException { EncodeResult result = ENCODER.encode(DTypes.I32, data, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(-1L); - assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(5L); + assertThat(result.statsMin().int64_value()).isEqualTo(-1L); + assertThat(result.statsMax().int64_value()).isEqualTo(5L); } @Test @@ -259,8 +259,8 @@ void encode_i64_reportsMinMaxOfOriginalValues() throws java.io.IOException { EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(Long.MIN_VALUE); - assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(Long.MAX_VALUE); + assertThat(result.statsMin().int64_value()).isEqualTo(Long.MIN_VALUE); + assertThat(result.statsMax().int64_value()).isEqualTo(Long.MAX_VALUE); } @Test @@ -317,8 +317,8 @@ void encodeCascade_boundsComeFromSignedValues() { CascadeStep result = sut.encodeCascade(DTypes.I64, data, EncodeTestHelper.testCtx()); // Then - assertThat(result.statsMin()).isEqualTo(ProtoScalarValue.ofInt64Value(-100L).encode()); - assertThat(result.statsMax()).isEqualTo(ProtoScalarValue.ofInt64Value(50L).encode()); + assertThat(result.statsMin()).isEqualTo(ProtoScalarValue.ofInt64Value(-100L)); + assertThat(result.statsMax()).isEqualTo(ProtoScalarValue.ofInt64Value(50L)); } @Test From bf5d930028b89067d6e3799f0572ace8fe03daa5 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Tue, 13 Oct 2026 08:45:32 +0200 Subject: [PATCH 2/3] refactor: type the zone-map sum stat as ProtoScalarValue too (#531) Co-Authored-By: Claude Sonnet 5 --- .../github/dfa1/vortex/writer/DictColumnState.java | 4 ++-- .../io/github/dfa1/vortex/writer/VortexWriter.java | 6 +++--- .../dfa1/vortex/writer/ZoneMapStatCodec.java | 14 ++++---------- .../writer/encode/PrimitiveEncodingEncoder.java | 8 ++++---- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java b/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java index d89ab347d..247a4e1fb 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java @@ -73,7 +73,7 @@ final class DictColumnState { private final List chunkNullCounts = new ArrayList<>(); private final List chunkStatsMin = new ArrayList<>(); private final List chunkStatsMax = new ArrayList<>(); - private final List chunkStatsSum = new ArrayList<>(); + private final List chunkStatsSum = new ArrayList<>(); private long codeArrayBytes; DictColumnState(DType dtype) { @@ -147,7 +147,7 @@ List chunkStatsMax() { return chunkStatsMax; } - List chunkStatsSum() { + List chunkStatsSum() { return chunkStatsSum; } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index e657b914a..fc48a23e8 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -1144,7 +1144,7 @@ private DType columnDtype(ColumnName colName) { /// zone, matching Rust. Field/bit order follows ZonedStatsSchema: MAX(3), MIN(4), SUM(5), /// NULL_COUNT(6). private void emitZoneMap(ColumnName colName, DType minMaxDtype, List minBytes, List maxBytes, - DType sumDtype, List sumBytes, long[] nullCounts, + DType sumDtype, List sumBytes, long[] nullCounts, long[] rowCounts) throws IOException { int nZones = nullCounts.length; boolean[] allValid = new boolean[nZones]; @@ -1549,7 +1549,7 @@ private record SegRef(long offset, long len, SegmentStats stats) { // S6218: the byte[] stat components are never value-compared — SegmentStats instances are only // read positionally, so the default identity equals is fine. - private record SegmentStats(ProtoScalarValue min, ProtoScalarValue max, byte[] sum, long nullCount) { + private record SegmentStats(ProtoScalarValue min, ProtoScalarValue max, ProtoScalarValue sum, long nullCount) { } private record EncodedSegment(EncodeResult result, Arena arena, SegmentStats stats) { @@ -1568,7 +1568,7 @@ private record ZoneMapRef(int zonesSegIdx, long nZones, byte[] metadata) { /// @param nullableCodes whether the codes child carries validity (Rust's `is_nullable_codes`) private record DictColRef(int valuesSegIdx, long valuesLen, List codesSegIdxes, List codesRowCounts, List chunkRowCounts, List chunkNullCounts, - List chunkStatsMin, List chunkStatsMax, List chunkStatsSum, + List chunkStatsMin, List chunkStatsMax, List chunkStatsSum, boolean nullableCodes) { long totalRows() { return chunkRowCounts.stream().mapToLong(Long::longValue).sum(); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java index d553fae16..1e174fe7b 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java @@ -10,7 +10,6 @@ import io.github.dfa1.vortex.writer.encode.VarBinEncodingEncoder; import java.io.IOException; -import java.lang.foreign.MemorySegment; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.List; @@ -109,7 +108,7 @@ static DType zoneSumDtype(DType dtype) { /// The serialized per-chunk SUM scalar for `data` of logical type `dtype`, or `null` when the /// column is not summable (non-primitive) or the sum overflowed. Validity placeholders are zero /// and therefore sum-neutral, so a nullable carrier sums correctly via its values. - static byte[] columnSum(DType dtype, Object data) { + static ProtoScalarValue columnSum(DType dtype, Object data) { if (!(dtype instanceof DType.Primitive p)) { return null; } @@ -178,21 +177,21 @@ static Object zoneStatValues(DType minMaxDtype, List statBytes /// Builds the per-zone SUM array for `sumDtype` (i64/u64 → `long[]`, f64 → `double[]`), decoding /// each zone's serialized scalar. Zones whose sum overflowed carry a `null` entry in `sumBytes`; /// `valid[i]` is set accordingly so the stat field reports them as null. - static Object sumColumn(DType sumDtype, List sumBytes, boolean[] valid) throws IOException { + static Object sumColumn(DType sumDtype, List sumBytes, boolean[] valid) throws IOException { PType ptype = ((DType.Primitive) sumDtype).ptype(); int n = sumBytes.size(); if (ptype == PType.F64) { double[] a = new double[n]; for (int i = 0; i < n; i++) { valid[i] = sumBytes.get(i) != null; - a[i] = valid[i] ? scalarDouble(decodeScalar(sumBytes.get(i))) : 0.0; + a[i] = valid[i] ? scalarDouble(sumBytes.get(i)) : 0.0; } return a; } long[] a = new long[n]; for (int i = 0; i < n; i++) { valid[i] = sumBytes.get(i) != null; - a[i] = valid[i] ? scalarLong(decodeScalar(sumBytes.get(i))) : 0L; + a[i] = valid[i] ? scalarLong(sumBytes.get(i)) : 0L; } return a; } @@ -295,9 +294,4 @@ private static double scalarDouble(ProtoScalarValue sv) { } return sv.f32_value(); } - - private static ProtoScalarValue decodeScalar(byte[] bytes) throws IOException { - MemorySegment seg = MemorySegment.ofArray(bytes); - return ProtoScalarValue.decode(seg, 0, seg.byteSize()); - } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java index e20b80141..d61a5d273 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java @@ -103,7 +103,7 @@ public static ProtoScalarValue maxOf(ProtoScalarValue[] stats) { // Most likely dead weight: Rust's default vortex.zoned layout writes no zone sum (#447) and vortex-jni // cannot read one, so only the legacy vortex.stats layout and segment/dict-chunk stats carry this value, // observable through our own columnZoneStats. Kept NaN-skipping for parity (#523); candidate for removal. - public static byte[] sumStat(PType ptype, Object data) { + public static ProtoScalarValue sumStat(PType ptype, Object data) { if (Array.getLength(data) == 0) { return null; } @@ -111,13 +111,13 @@ public static byte[] sumStat(PType ptype, Object data) { return switch (ptype) { case I8, I16, I32, I64 -> { OptionalLong sum = ops.sum(data, ptype); - yield sum.isEmpty() ? null : scalarI64(sum.getAsLong()).encode(); + yield sum.isEmpty() ? null : scalarI64(sum.getAsLong()); } case U8, U16, U32, U64 -> { OptionalLong sum = ops.sum(data, ptype); - yield sum.isEmpty() ? null : scalarU64(sum.getAsLong()).encode(); + yield sum.isEmpty() ? null : scalarU64(sum.getAsLong()); } - case F16, F32, F64 -> scalarF64(ops.sumFloating(data, ptype)).encode(); + case F16, F32, F64 -> scalarF64(ops.sumFloating(data, ptype)); }; } From 194ffebe07f33b99e5c49091235963e65d374069 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Tue, 13 Oct 2026 08:48:35 +0200 Subject: [PATCH 3/3] refactor: MinMax record replaces the two-element stats array (#531) Co-Authored-By: Claude Sonnet 5 --- .../dfa1/vortex/writer/DictColumnState.java | 13 ++--- .../dfa1/vortex/writer/VortexWriter.java | 7 +-- .../dfa1/vortex/writer/ZoneMapStatCodec.java | 3 +- .../vortex/writer/encode/CascadeStep.java | 4 +- .../encode/DateTimePartsEncodingEncoder.java | 5 +- .../writer/encode/DictEncodingEncoder.java | 4 +- .../vortex/writer/encode/EncodeResult.java | 4 +- .../writer/encode/ExtEncodingEncoder.java | 5 +- .../FrameOfReferenceEncodingEncoder.java | 4 +- .../writer/encode/FsstEncodingEncoder.java | 11 ++-- .../writer/encode/MaskedEncodingEncoder.java | 8 +-- .../dfa1/vortex/writer/encode/MinMax.java | 10 ++++ .../writer/encode/OnPairEncodingEncoder.java | 11 ++-- .../encode/PrimitiveEncodingEncoder.java | 36 ++++++------- .../writer/encode/RleEncodingEncoder.java | 5 +- .../writer/encode/RunEndEncodingEncoder.java | 2 +- .../writer/encode/SparseEncodingEncoder.java | 2 +- .../writer/encode/VarBinEncodingEncoder.java | 10 ++-- .../writer/encode/ZigZagEncodingEncoder.java | 2 +- .../vortex/writer/encode/ZoneMapStats.java | 20 ++++---- .../writer/encode/ZstdEncodingEncoder.java | 3 +- .../vortex/writer/ZoneMapStatCodecTest.java | 51 ++++++++++--------- .../encode/PrimitiveEncodingEncoderTest.java | 14 ++--- 23 files changed, 121 insertions(+), 113 deletions(-) create mode 100644 writer/src/main/java/io/github/dfa1/vortex/writer/encode/MinMax.java diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java b/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java index 247a4e1fb..82b3559a9 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer; +import io.github.dfa1.vortex.writer.encode.MinMax; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; @@ -289,14 +290,14 @@ boolean ingestDictChunk(Object data) { chunkStatsMax.add(null); chunkStatsSum.add(null); } else if (varBin) { - ProtoScalarValue[] mm = VarBinEncodingEncoder.minMaxStats((String[]) values); - chunkStatsMin.add(mm != null ? mm[0] : null); - chunkStatsMax.add(mm != null ? mm[1] : null); + MinMax mm = VarBinEncodingEncoder.minMaxStats((String[]) values); + chunkStatsMin.add(mm != null ? mm.min() : null); + chunkStatsMax.add(mm != null ? mm.max() : null); chunkStatsSum.add(null); } else { - ProtoScalarValue[] mm = PrimitiveEncodingEncoder.minMaxStats(ptype, values); - chunkStatsMin.add(mm != null ? mm[0] : null); - chunkStatsMax.add(mm != null ? mm[1] : null); + MinMax mm = PrimitiveEncodingEncoder.minMaxStats(ptype, values); + chunkStatsMin.add(mm != null ? mm.min() : null); + chunkStatsMax.add(mm != null ? mm.max() : null); chunkStatsSum.add(PrimitiveEncodingEncoder.sumStat(ptype, values)); } codeArrayBytes += 2L * len; diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index fc48a23e8..f9325fb60 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer; +import io.github.dfa1.vortex.writer.encode.MinMax; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.fbs.FbsBuilder; import io.github.dfa1.vortex.writer.encode.DateTimePartsData; @@ -860,9 +861,9 @@ private EncodedSegment encodeSegment(DType dtype, Object data, EncodingEncoder e min = result.statsMin(); max = result.statsMax(); } else { - ProtoScalarValue[] fallback = ZoneMapStatCodec.columnMinMax(dtype, data); - min = fallback != null ? fallback[0] : null; - max = fallback != null ? fallback[1] : null; + MinMax fallback = ZoneMapStatCodec.columnMinMax(dtype, data); + min = fallback != null ? fallback.min() : null; + max = fallback != null ? fallback.max() : null; } SegmentStats stats = new SegmentStats(min, max, ZoneMapStatCodec.columnSum(dtype, data), nullCount); return new EncodedSegment(result, arena, stats); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java index 1e174fe7b..271c9aa33 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer; +import io.github.dfa1.vortex.writer.encode.MinMax; import io.github.dfa1.vortex.core.compute.PrimitiveArrays; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; @@ -137,7 +138,7 @@ static ProtoScalarValue columnSum(DType dtype, Object data) { /// @param dtype the segment's logical type /// @param data the segment's input data, possibly [NullableData]- or [ComparableValues]-wrapped /// @return a `{min, max}` pair, or `null` when `dtype` has no recordable min/max - static ProtoScalarValue[] columnMinMax(DType dtype, Object data) { + static MinMax columnMinMax(DType dtype, Object data) { if (data instanceof ComparableValues cv) { return PrimitiveEncodingEncoder.minMaxStats(cv.ptype(), cv.values()); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadeStep.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadeStep.java index 746588c60..09c44d5db 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadeStep.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadeStep.java @@ -36,10 +36,10 @@ public record CascadeStep( /// @param stats a `{min, max}` pair from [ZoneMapStats#of(io.github.dfa1.vortex.core.model.DType, Object)], /// or `null` when the input carried nothing to bound /// @return a copy of this step carrying `stats` - public CascadeStep withStats(ProtoScalarValue[] stats) { + public CascadeStep withStats(MinMax stats) { return stats == null || !applicable ? this - : new CascadeStep(partialRoot, ownedBuffers, openChildren, stats[0], stats[1], applicable); + : new CascadeStep(partialRoot, ownedBuffers, openChildren, stats.min(), stats.max(), applicable); } /// Convenience: terminal step — no open children, result is final. diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java index 5e47d5b70..d5950120f 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java @@ -1,6 +1,5 @@ package io.github.dfa1.vortex.writer.encode; -import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; @@ -82,7 +81,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { // Bounds come from the undivided timestamps, in storage space: the split children are // days/seconds/subseconds and none of them orders the column. Without this, choosing // datetimeparts over vortex.ext would trade the column's zone map for its size. - ProtoScalarValue[] stats = PrimitiveEncodingEncoder.minMaxStats(PType.I64, d.timestamps()); + MinMax stats = PrimitiveEncodingEncoder.minMaxStats(PType.I64, d.timestamps()); return new EncodeResult(root, List.copyOf(allBuffers), PrimitiveEncodingEncoder.minOf(stats), PrimitiveEncodingEncoder.maxOf(stats)); } @@ -113,7 +112,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext encodeC new ChildSlot(DType.I64, seconds, 1, SELF), new ChildSlot(DType.I64, subseconds, 2, SELF)); - ProtoScalarValue[] stats = PrimitiveEncodingEncoder.minMaxStats(PType.I64, d.timestamps()); + MinMax stats = PrimitiveEncodingEncoder.minMaxStats(PType.I64, d.timestamps()); return new CascadeStep(partialRoot, List.of(), children, PrimitiveEncodingEncoder.minOf(stats), PrimitiveEncodingEncoder.maxOf(stats), true); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java index 32c72af49..4c9c26df8 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java @@ -136,7 +136,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { new EncodeNode[]{codesNode, valuesNode}, new int[0]); - ProtoScalarValue[] stats = PrimitiveEncodingEncoder.minMaxStats(((DType.Primitive) dtype).ptype(), data); + MinMax stats = PrimitiveEncodingEncoder.minMaxStats(((DType.Primitive) dtype).ptype(), data); return new EncodeResult(rootNode, List.of(EncodedBuffer.of(d.valuesBuf(), ((DType.Primitive) dtype).ptype()), EncodedBuffer.of(codesBuf, codePType)), PrimitiveEncodingEncoder.minOf(stats), PrimitiveEncodingEncoder.maxOf(stats)); } @@ -160,7 +160,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { new int[0]); DType codesDtype = new DType.Primitive(codePType, false); - ProtoScalarValue[] stats = PrimitiveEncodingEncoder.minMaxStats(((DType.Primitive) dtype).ptype(), data); + MinMax stats = PrimitiveEncodingEncoder.minMaxStats(((DType.Primitive) dtype).ptype(), data); return new CascadeStep(partialRoot, List.of(), List.of(new ChildSlot(codesDtype, d.codesArr(), 0, CODES_EXCLUDED), new ChildSlot(dtype, d.valuesArr(), 1, Set.of(EncodingId.VORTEX_DICT))), diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java index ca7a195de..241a09e13 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java @@ -53,10 +53,10 @@ public static EncodeResult simple(EncodingId encodingId, EncodedBuffer data) { /// @param stats a `{min, max}` pair as returned by the encoders' `minMaxStats` helpers, or /// `null` when the input carried nothing to bound /// @return a copy of this result carrying `stats` - public EncodeResult withStats(ProtoScalarValue[] stats) { + public EncodeResult withStats(MinMax stats) { return stats == null ? this - : new EncodeResult(rootNode, encodedBuffers, stats[0], stats[1]); + : new EncodeResult(rootNode, encodedBuffers, stats.min(), stats.max()); } /// Returns `true` if both `statsMin` and `statsMax` are present. diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoder.java index 31901d08b..b91d2fd78 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoder.java @@ -1,6 +1,5 @@ package io.github.dfa1.vortex.writer.encode; -import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; @@ -68,7 +67,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { ChildSlot slot = new ChildSlot(ext.storageDType(), data, 0, Set.of(EncodingId.VORTEX_EXT)); // The cascade reads stats off the step, not off the children it later resolves, so the // bounds have to be computed here rather than inherited from the storage slot. - ProtoScalarValue[] stats = storageStats(ext.storageDType(), data); + MinMax stats = storageStats(ext.storageDType(), data); return new CascadeStep(partialRoot, List.of(), List.of(slot), PrimitiveEncodingEncoder.minOf(stats), PrimitiveEncodingEncoder.maxOf(stats), true); } @@ -80,7 +79,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { /// @param storage the extension's storage dtype /// @param data the storage-shaped values being encoded /// @return a two-element `{min, max}` array of encoded scalars, or `null` - private static ProtoScalarValue[] storageStats(DType storage, Object data) { + private static MinMax storageStats(DType storage, Object data) { return storage instanceof DType.Primitive(PType ptype, boolean ignored) ? PrimitiveEncodingEncoder.minMaxStats(ptype, data) : null; diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java index b422aa953..00e391bb2 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java @@ -64,7 +64,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { EncodeNode child = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); EncodeNode root = new EncodeNode(EncodingId.FASTLANES_FOR, meta, new EncodeNode[]{child}, new int[0]); - ProtoScalarValue[] stats = PrimitiveEncodingEncoder.minMaxStats(ptype, data); + MinMax stats = PrimitiveEncodingEncoder.minMaxStats(ptype, data); return new EncodeResult(root, List.of(EncodedBuffer.of(residuals, ptype)), PrimitiveEncodingEncoder.minOf(stats), PrimitiveEncodingEncoder.maxOf(stats)); } @@ -90,7 +90,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext encodeC EncodeNode partialRoot = new EncodeNode(EncodingId.FASTLANES_FOR, meta, new EncodeNode[1], new int[0]); ChildSlot slot = new ChildSlot(dtype, residualsAsNativeArray(longs, ref, ptype), 0, Set.of(EncodingId.FASTLANES_FOR)); - ProtoScalarValue[] stats = PrimitiveEncodingEncoder.minMaxStats(ptype, data); + MinMax stats = PrimitiveEncodingEncoder.minMaxStats(ptype, data); return new CascadeStep(partialRoot, List.of(), List.of(slot), PrimitiveEncodingEncoder.minOf(stats), PrimitiveEncodingEncoder.maxOf(stats), true); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java index e3781f8b5..4c12340e3 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java @@ -1,6 +1,5 @@ package io.github.dfa1.vortex.writer.encode; -import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.compute.PrimitiveArrays; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; @@ -71,7 +70,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { /// /// @param data the Utf8 (`String[]`) or Binary (`byte[][]`) values being encoded /// @return a two-element `{min, max}` array of encoded scalars, or `null` - private static ProtoScalarValue[] zoneMapStats(Object data) { + private static MinMax zoneMapStats(Object data) { return data instanceof String[] strings ? VarBinEncodingEncoder.minMaxStats(strings) : null; } @@ -88,7 +87,7 @@ private static ProtoScalarValue[] zoneMapStats(Object data) { /// @param arena arena backing the length/offset buffers /// @param stats the `{min, max}` zone-map scalars from [#zoneMapStats(Object)], or `null` /// @return the terminal `vortex.fsst` encode result - static EncodeResult toEncodeResult(Fsst c, Arena arena, ProtoScalarValue[] stats) { + static EncodeResult toEncodeResult(Fsst c, Arena arena, MinMax stats) { long uncompLenBytes = c.uncompLenPType().byteSize(); MemorySegment uncompLenBuf = arena.allocate(Math.max(c.n() * uncompLenBytes, 1)); for (int i = 0; i < c.n(); i++) { @@ -112,7 +111,7 @@ static EncodeResult toEncodeResult(Fsst c, Arena arena, ProtoScalarValue[] stats List.of(EncodedBuffer.of(c.symBuf(), PType.U64), EncodedBuffer.bytes(c.symLenBuf()), EncodedBuffer.bytes(c.compBuf()), EncodedBuffer.of(uncompLenBuf, c.uncompLenPType()), EncodedBuffer.of(codesOffBuf, c.codesOffPType())), - stats != null ? stats[0] : null, stats != null ? stats[1] : null); + stats != null ? stats.min() : null, stats != null ? stats.max() : null); } /// Cascading FSST: expose the per-row uncompressed-length and code-offset children as open @@ -133,7 +132,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { return CascadeStep.terminal(encode(dtype, data, ctx)); } Fsst c = compress(data, ctx.arena()); - ProtoScalarValue[] stats = zoneMapStats(data); + MinMax stats = zoneMapStats(data); Object uncompLens = PrimitiveArrays.fromIntsArray(c.uncompLens(), c.uncompLenPType(), EncodingId.VORTEX_FSST); Object codesOffsets = PrimitiveArrays.fromIntsArray(c.codesOffsets(), c.codesOffPType(), EncodingId.VORTEX_FSST); EncodeNode partialRoot = new EncodeNode( @@ -146,7 +145,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { EncodedBuffer.bytes(c.compBuf())), List.of(new ChildSlot(new DType.Primitive(c.uncompLenPType(), false), uncompLens, 0, SELF), new ChildSlot(new DType.Primitive(c.codesOffPType(), false), codesOffsets, 1, SELF)), - stats != null ? stats[0] : null, stats != null ? stats[1] : null, true); + stats != null ? stats.min() : null, stats != null ? stats.max() : null, true); } /// The FSST-specific product of compression: the symbol-table buffers, the wire code stream, the diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java index 82d395304..4231ac12f 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java @@ -62,9 +62,9 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { null, new EncodeNode[]{valuesResult.rootNode(), validityNode}, new int[0]); - ProtoScalarValue[] stats = maskedMinMaxStats(nonNullable, values, validity); - ProtoScalarValue statsMin = stats == null ? null : stats[0]; - ProtoScalarValue statsMax = stats == null ? null : stats[1]; + MinMax stats = maskedMinMaxStats(nonNullable, values, validity); + ProtoScalarValue statsMin = stats == null ? null : stats.min(); + ProtoScalarValue statsMax = stats == null ? null : stats.max(); EncodeResult masked = new EncodeResult(root, buffers, statsMin, statsMax); // Rust never wraps a dict-encoded nullable string column in a row mask: its dict builder @@ -95,7 +95,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { /// @param values the dense values array from the [NullableData] carrier /// @param validity per-row validity, aligned with `values` /// @return a two-element `{min, max}` array of encoded scalars, or `null` when no row is valid - private static ProtoScalarValue[] maskedMinMaxStats(DType nonNullable, Object values, boolean[] validity) { + private static MinMax maskedMinMaxStats(DType nonNullable, Object values, boolean[] validity) { return ZoneMapStats.of(nonNullable, new NullableData(values, validity)); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MinMax.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MinMax.java new file mode 100644 index 000000000..dbfd1cc68 --- /dev/null +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MinMax.java @@ -0,0 +1,10 @@ +package io.github.dfa1.vortex.writer.encode; + +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; + +/// A column's zone-map minimum and maximum, kept typed until the writer serializes them. +/// +/// @param min the smallest value +/// @param max the largest value +public record MinMax(ProtoScalarValue min, ProtoScalarValue max) { +} diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/OnPairEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/OnPairEncodingEncoder.java index 83946deae..1ff7ca22d 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/OnPairEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/OnPairEncodingEncoder.java @@ -1,6 +1,5 @@ package io.github.dfa1.vortex.writer.encode; -import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.compute.PrimitiveArrays; import io.github.dfa1.vortex.core.io.PTypeIO; import io.github.dfa1.vortex.core.model.DType; @@ -67,8 +66,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { } EncodeNode root = new EncodeNode(EncodingId.VORTEX_ONPAIR, MemorySegment.ofArray(c.metadata()), children, new int[]{0}); - ProtoScalarValue[] stats = zoneMapStats(data); - return new EncodeResult(root, buffers, stats != null ? stats[0] : null, stats != null ? stats[1] : null); + MinMax stats = zoneMapStats(data); + return new EncodeResult(root, buffers, stats != null ? stats.min() : null, stats != null ? stats.max() : null); } /// Leaves the four integer children open, so the cascade can bit-pack, frame-of-reference or @@ -88,11 +87,11 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { } EncodeNode partialRoot = new EncodeNode(EncodingId.VORTEX_ONPAIR, MemorySegment.ofArray(c.metadata()), new EncodeNode[4], new int[]{0}); - ProtoScalarValue[] stats = zoneMapStats(data); + MinMax stats = zoneMapStats(data); return new CascadeStep(partialRoot, List.of(EncodedBuffer.bytes(c.dictBytes(ctx.arena()))), slots, - stats != null ? stats[0] : null, stats != null ? stats[1] : null, true); + stats != null ? stats.min() : null, stats != null ? stats.max() : null, true); } - private static ProtoScalarValue[] zoneMapStats(Object data) { + private static MinMax zoneMapStats(Object data) { return data instanceof String[] strings ? VarBinEncodingEncoder.minMaxStats(strings) : null; } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java index d61a5d273..8f58616e8 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java @@ -34,10 +34,10 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { : PrimitiveArrays.toSegment(data, ptype, ctx.arena()); ProtoScalarValue min = null; ProtoScalarValue max = null; - ProtoScalarValue[] stats = minMaxStats(ptype, data); + MinMax stats = minMaxStats(ptype, data); if (stats != null) { - min = stats[0]; - max = stats[1]; + min = stats.min(); + max = stats.max(); } return EncodeResult.simple(EncodingId.VORTEX_PRIMITIVE, EncodedBuffer.of(seg, ptype), min, max); } @@ -50,7 +50,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { /// @param ptype the primitive type of `data` /// @param data the raw primitive array (e.g. `long[]`, `int[]`, `String`-free) /// @return a two-element `{min, max}` array of encoded scalars, or `null` if `data` is empty - public static ProtoScalarValue[] minMaxStats(PType ptype, Object data) { + public static MinMax minMaxStats(PType ptype, Object data) { if (Array.getLength(data) == 0) { return null; } @@ -60,33 +60,33 @@ public static ProtoScalarValue[] minMaxStats(PType ptype, Object data) { return null; } return switch (ptype) { - case I8, I16, I32, I64 -> new ProtoScalarValue[]{scalarI64(minMax[0]), scalarI64(minMax[1])}; - case U8, U16, U32, U64 -> new ProtoScalarValue[]{scalarU64(minMax[0]), scalarU64(minMax[1])}; - case F16 -> new ProtoScalarValue[]{scalarF16(Float.float16ToFloat((short) minMax[0])), - scalarF16(Float.float16ToFloat((short) minMax[1]))}; - case F32 -> new ProtoScalarValue[]{scalarF32(Float.intBitsToFloat((int) minMax[0])), - scalarF32(Float.intBitsToFloat((int) minMax[1]))}; - case F64 -> new ProtoScalarValue[]{scalarF64(Double.longBitsToDouble(minMax[0])), - scalarF64(Double.longBitsToDouble(minMax[1]))}; + case I8, I16, I32, I64 -> new MinMax(scalarI64(minMax[0]), scalarI64(minMax[1])); + case U8, U16, U32, U64 -> new MinMax(scalarU64(minMax[0]), scalarU64(minMax[1])); + case F16 -> new MinMax(scalarF16(Float.float16ToFloat((short) minMax[0])), + scalarF16(Float.float16ToFloat((short) minMax[1]))); + case F32 -> new MinMax(scalarF32(Float.intBitsToFloat((int) minMax[0])), + scalarF32(Float.intBitsToFloat((int) minMax[1]))); + case F64 -> new MinMax(scalarF64(Double.longBitsToDouble(minMax[0])), + scalarF64(Double.longBitsToDouble(minMax[1]))); }; } /// The min half of a [#minMaxStats] result, or `null` when `stats` itself is `null` (empty - /// data). Saves callers the repeated `stats == null ? null : stats[0]` idiom. + /// data). Saves callers the repeated `stats == null ? null : stats.min()` idiom. /// /// @param stats a [#minMaxStats] result, possibly `null` /// @return the serialized min scalar, or `null` - public static ProtoScalarValue minOf(ProtoScalarValue[] stats) { - return stats == null ? null : stats[0]; + public static ProtoScalarValue minOf(MinMax stats) { + return stats == null ? null : stats.min(); } /// The max half of a [#minMaxStats] result, or `null` when `stats` itself is `null` (empty - /// data). Saves callers the repeated `stats == null ? null : stats[1]` idiom. + /// data). Saves callers the repeated `stats == null ? null : stats.max()` idiom. /// /// @param stats a [#minMaxStats] result, possibly `null` /// @return the serialized max scalar, or `null` - public static ProtoScalarValue maxOf(ProtoScalarValue[] stats) { - return stats == null ? null : stats[1]; + public static ProtoScalarValue maxOf(MinMax stats) { + return stats == null ? null : stats.max(); } /// Computes the serialized SUM [io.github.dfa1.vortex.core.proto.ProtoScalarValue] for a raw primitive diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java index efb66cc98..ec070535c 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java @@ -1,6 +1,5 @@ package io.github.dfa1.vortex.writer.encode; -import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; @@ -106,7 +105,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { } // Run-length encoding preserves the value set, so the runs bound the column exactly as the // raw values do. - ProtoScalarValue[] stats = ZoneMapStats.of(dtype, data); + MinMax stats = ZoneMapStats.of(dtype, data); Runs runs = runs(longs); MemorySegment valuesSeg = fromLongs(runs.values(), runs.valuesCount(), ptype, ctx.arena()); @@ -157,7 +156,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { new ChildSlot(dtype, PrimitiveArrays.fromBitsArray(values, ptype, EncodingId.FASTLANES_RLE), 0, VALUES_EXCLUDED), new ChildSlot(new DType.Primitive(PType.U16, false), runs.indices(), 1, POSITIONS_EXCLUDED), new ChildSlot(new DType.Primitive(PType.U64, false), runs.offsets(), 2, POSITIONS_EXCLUDED)); - ProtoScalarValue[] stats = ZoneMapStats.of(dtype, data); + MinMax stats = ZoneMapStats.of(dtype, data); return new CascadeStep(partialRoot, List.of(), slots, ZoneMapStats.minOf(stats), ZoneMapStats.maxOf(stats), true); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java index 4959180b3..ab8ab7413 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java @@ -243,7 +243,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { new EncodeNode[]{null, null}, new int[0]); ChildSlot endsSlot = new ChildSlot(new DType.Primitive(PType.U32, false), ends, 0, ENDS_EXCLUDED); ChildSlot valuesSlot = new ChildSlot(dtype, PrimitiveArrays.fromLongsArray(values, ptype, EncodingId.VORTEX_RUNEND), 1, VALUES_EXCLUDED); - ProtoScalarValue[] stats = ZoneMapStats.of(dtype, data); + MinMax stats = ZoneMapStats.of(dtype, data); return new CascadeStep(partialRoot, List.of(), List.of(endsSlot, valuesSlot), ZoneMapStats.minOf(stats), ZoneMapStats.maxOf(stats), true); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java index 29a347b0b..0c86b621a 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java @@ -81,7 +81,7 @@ public Estimate expectedRatio(DType dtype, ArrayAndStats data, EncodeContext ctx public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { // `data` is the whole logical column -- the fill value and patch positions are derived // from it below -- so the column's bounds are just its bounds, patches included. - ProtoScalarValue[] stats = ZoneMapStats.of(dtype, data); + MinMax stats = ZoneMapStats.of(dtype, data); if (!(dtype instanceof DType.Primitive p)) { return CascadeStep.notApplicable(); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java index 237421719..614419772 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java @@ -56,9 +56,9 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { // Zone-map min/max stats are lexicographic-string-only (minMaxStats(String[])); binary // blobs (e.g. audio bytes) aren't usefully zone-mapped, so skip stats for that path. - ProtoScalarValue[] stats = strings != null ? minMaxStats(strings) : null; - ProtoScalarValue statsMin = stats != null ? stats[0] : null; - ProtoScalarValue statsMax = stats != null ? stats[1] : null; + MinMax stats = strings != null ? minMaxStats(strings) : null; + ProtoScalarValue statsMin = stats != null ? stats.min() : null; + ProtoScalarValue statsMax = stats != null ? stats.max() : null; EncodeNode offsetsNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); EncodeNode root = new EncodeNode(EncodingId.VORTEX_VARBIN, MemorySegment.ofArray(metaBytes), @@ -73,7 +73,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { /// /// @param strings the string values (may contain `null`) /// @return a two-element `{min, max}` array of encoded scalars, or `null` if all entries are `null` - public static ProtoScalarValue[] minMaxStats(String[] strings) { + public static MinMax minMaxStats(String[] strings) { String minStr = null; String maxStr = null; for (String s : strings) { @@ -90,6 +90,6 @@ public static ProtoScalarValue[] minMaxStats(String[] strings) { if (minStr == null) { return null; } - return new ProtoScalarValue[]{ProtoScalarValue.ofStringValue(minStr), ProtoScalarValue.ofStringValue(maxStr)}; + return new MinMax(ProtoScalarValue.ofStringValue(minStr), ProtoScalarValue.ofStringValue(maxStr)); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java index ed96bff4c..7ec823767 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java @@ -173,7 +173,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { ChildSlot slot = new ChildSlot(new DType.Primitive(unsigned, false), PrimitiveArrays.fromLongsArray(encoded, unsigned, EncodingId.VORTEX_ZIGZAG), 0, ENCODED_EXCLUDED); // zigzag is not order-preserving, so the bounds come from the original signed values - ProtoScalarValue[] stats = ZoneMapStats.of(dtype, data); + MinMax stats = ZoneMapStats.of(dtype, data); return new CascadeStep(partialRoot, List.of(), List.of(slot), ZoneMapStats.minOf(stats), ZoneMapStats.maxOf(stats), true); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZoneMapStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZoneMapStats.java index e8e288ed6..bc0b4d1fb 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZoneMapStats.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZoneMapStats.java @@ -38,7 +38,7 @@ private ZoneMapStats() { /// @param dtype the column's dtype; a nullable one is read through to its non-nullable form /// @param data the values being encoded /// @return a two-element `{min, max}` array of encoded scalars, or `null` - static ProtoScalarValue[] of(DType dtype, Object data) { + static MinMax of(DType dtype, Object data) { if (data instanceof NullableData(var values, var validity)) { DType nonNullable = dtype == null ? null : dtype.withNullable(false); if (nonNullable instanceof DType.Primitive p) { @@ -68,7 +68,7 @@ static ProtoScalarValue[] of(DType dtype, Object data) { /// @param dtype the decimal dtype (precision picks the width) /// @param values the decimals, `null` at masked-out rows /// @return a two-element `{min, max}` array of encoded scalars, or `null` when none is valid - static ProtoScalarValue[] ofDecimal(DType.Decimal dtype, BigDecimal[] values) { + static MinMax ofDecimal(DType.Decimal dtype, BigDecimal[] values) { BigDecimal[] valid = Arrays.stream(values).filter(Objects::nonNull).toArray(BigDecimal[]::new); int width = DecimalEncodingEncoder.storageWidth(dtype.precision()); return ofDecimal(DecimalEncodingEncoder.pack(valid, width, Arena.ofAuto()), width); @@ -87,7 +87,7 @@ static ProtoScalarValue[] ofDecimal(DType.Decimal dtype, BigDecimal[] values) { /// @param values the packed values /// @param byteWidth bytes per value, 1/2/4/8/16/32 /// @return a two-element `{min, max}` array of encoded scalars, or `null` when empty - static ProtoScalarValue[] ofDecimal(MemorySegment values, int byteWidth) { + static MinMax ofDecimal(MemorySegment values, int byteWidth) { long n = values.byteSize() / byteWidth; if (n == 0) { return null; @@ -103,7 +103,7 @@ static ProtoScalarValue[] ofDecimal(MemorySegment values, int byteWidth) { maxOff = off; } } - return new ProtoScalarValue[]{decimalScalar(values, minOff, byteWidth), decimalScalar(values, maxOff, byteWidth)}; + return new MinMax(decimalScalar(values, minOff, byteWidth), decimalScalar(values, maxOff, byteWidth)); } /// Bounds for a decimal column already widened into `long` values, encoded at the same @@ -111,12 +111,12 @@ static ProtoScalarValue[] ofDecimal(MemorySegment values, int byteWidth) { /// /// @param values the unscaled values /// @return a two-element `{min, max}` array of encoded scalars, or `null` when empty - static ProtoScalarValue[] ofDecimal(long[] values) { + static MinMax ofDecimal(long[] values) { if (values.length == 0) { return null; } long[] minMax = SimdOperationsSupport.preferred().minMax(values, PType.I64); - return new ProtoScalarValue[]{decimalScalar(minMax[0]), decimalScalar(minMax[1])}; + return new MinMax(decimalScalar(minMax[0]), decimalScalar(minMax[1])); } /// Compares two equal-width little-endian two's-complement values in place: the most @@ -154,15 +154,15 @@ private static ProtoScalarValue decimalScalar(long value) { /// /// @param stats an [#of(DType, Object)] result, possibly `null` /// @return the serialized min scalar, or `null` - static ProtoScalarValue minOf(ProtoScalarValue[] stats) { - return stats == null ? null : stats[0]; + static ProtoScalarValue minOf(MinMax stats) { + return stats == null ? null : stats.min(); } /// The max half of an [#of(DType, Object)] result, or `null`. /// /// @param stats an [#of(DType, Object)] result, possibly `null` /// @return the serialized max scalar, or `null` - static ProtoScalarValue maxOf(ProtoScalarValue[] stats) { - return stats == null ? null : stats[1]; + static ProtoScalarValue maxOf(MinMax stats) { + return stats == null ? null : stats.max(); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java index a80c88d0c..8eadac46c 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java @@ -1,6 +1,5 @@ package io.github.dfa1.vortex.writer.encode; -import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.zstd.ZstdCompressContext; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; @@ -114,7 +113,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { requireBinding(ZSTD_BINDING_PRESENT); // Compression does not reorder anything, so the column's bounds are the input's bounds. // Computed once here rather than in each of the four shape-specific paths below. - ProtoScalarValue[] stats = ZoneMapStats.of(dtype, data); + MinMax stats = ZoneMapStats.of(dtype, data); if (data instanceof NullableData nd) { if (dtype instanceof DType.Primitive dt) { return encodeNullablePrimitive(dt, nd, ctx).withStats(stats); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java index e0423091b..187d93d9a 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer; +import io.github.dfa1.vortex.writer.encode.MinMax; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; @@ -67,11 +68,11 @@ void signed_reportsRealMinMax() throws IOException { long[] data = {10L, -5L, 3L, 7L}; // When - ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(DTypes.I64, data); + MinMax stats = ZoneMapStatCodec.columnMinMax(DTypes.I64, data); // Then - assertThat(stats[0].int64_value()).isEqualTo(-5L); - assertThat(stats[1].int64_value()).isEqualTo(10L); + assertThat(stats.min().int64_value()).isEqualTo(-5L); + assertThat(stats.max().int64_value()).isEqualTo(10L); } @Test @@ -80,11 +81,11 @@ void unsigned_usesUnsignedComparison() throws IOException { int[] data = {1, -1, 5}; // When - ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(DTypes.U32, data); + MinMax stats = ZoneMapStatCodec.columnMinMax(DTypes.U32, data); // Then - assertThat(stats[0].uint64_value()).isEqualTo(1L); - assertThat(stats[1].uint64_value()).isEqualTo(4294967295L); + assertThat(stats.min().uint64_value()).isEqualTo(1L); + assertThat(stats.max().uint64_value()).isEqualTo(4294967295L); } @Test @@ -96,11 +97,11 @@ void nullable_compactsToValidElementsOnly() throws IOException { NullableData data = new NullableData(values, validity); // When - ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(DTypes.I64.asNullable(), data); + MinMax stats = ZoneMapStatCodec.columnMinMax(DTypes.I64.asNullable(), data); // Then - assertThat(stats[0].int64_value()).isEqualTo(-50L); - assertThat(stats[1].int64_value()).isEqualTo(100L); + assertThat(stats.min().int64_value()).isEqualTo(-50L); + assertThat(stats.max().int64_value()).isEqualTo(100L); } @Test @@ -109,7 +110,7 @@ void empty_returnsNull() { long[] data = {}; // When - ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(DTypes.I64, data); + MinMax stats = ZoneMapStatCodec.columnMinMax(DTypes.I64, data); // Then assertThat(stats).isNull(); @@ -128,11 +129,11 @@ void plainPrimitiveStorage_unwrapsToStoragePrimitive() throws IOException { long[] data = {100L, 200L, 50L}; // When - ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(TIMESTAMP_MS, data); + MinMax stats = ZoneMapStatCodec.columnMinMax(TIMESTAMP_MS, data); // Then - assertThat(stats[0].int64_value()).isEqualTo(50L); - assertThat(stats[1].int64_value()).isEqualTo(200L); + assertThat(stats.min().int64_value()).isEqualTo(50L); + assertThat(stats.max().int64_value()).isEqualTo(200L); } @Test @@ -154,11 +155,11 @@ public Object values() { FakeCarrier carrier = new FakeCarrier(new long[]{9L, 1L, 5L}); // When - ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(TIMESTAMP_MS, carrier); + MinMax stats = ZoneMapStatCodec.columnMinMax(TIMESTAMP_MS, carrier); // Then - assertThat(stats[0].int64_value()).isEqualTo(1L); - assertThat(stats[1].int64_value()).isEqualTo(9L); + assertThat(stats.min().int64_value()).isEqualTo(1L); + assertThat(stats.max().int64_value()).isEqualTo(9L); } @Test @@ -169,11 +170,11 @@ void nullablePlainStorage_compactsToValidElementsOnly() throws IOException { NullableData data = new NullableData(values, validity); // When - ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(TIMESTAMP_MS, data); + MinMax stats = ZoneMapStatCodec.columnMinMax(TIMESTAMP_MS, data); // Then - assertThat(stats[0].int64_value()).isEqualTo(300L); - assertThat(stats[1].int64_value()).isEqualTo(300L); + assertThat(stats.min().int64_value()).isEqualTo(300L); + assertThat(stats.max().int64_value()).isEqualTo(300L); } } @@ -186,11 +187,11 @@ void reportsLexicographicMinMax() throws IOException { String[] data = {"banana", "apple", "cherry"}; // When - ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(DTypes.UTF8, data); + MinMax stats = ZoneMapStatCodec.columnMinMax(DTypes.UTF8, data); // Then - assertThat(stats[0].string_value()).isEqualTo("apple"); - assertThat(stats[1].string_value()).isEqualTo("cherry"); + assertThat(stats.min().string_value()).isEqualTo("apple"); + assertThat(stats.max().string_value()).isEqualTo("cherry"); } @Test @@ -201,11 +202,11 @@ void nullable_skipsNullEntriesDirectly() throws IOException { NullableData data = new NullableData(values, new boolean[]{false, true, false, true}); // When - ProtoScalarValue[] stats = ZoneMapStatCodec.columnMinMax(DTypes.UTF8.asNullable(), data); + MinMax stats = ZoneMapStatCodec.columnMinMax(DTypes.UTF8.asNullable(), data); // Then - assertThat(stats[0].string_value()).isEqualTo("apple"); - assertThat(stats[1].string_value()).isEqualTo("banana"); + assertThat(stats.min().string_value()).isEqualTo("apple"); + assertThat(stats.max().string_value()).isEqualTo("banana"); } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java index 6b7f420f4..9fe98613a 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java @@ -145,11 +145,11 @@ void leadingNanIsSkipped() { double[] values = {Double.NaN, 3.0, -1.0}; // When - ProtoScalarValue[] result = PrimitiveEncodingEncoder.minMaxStats(PType.F64, values); + MinMax result = PrimitiveEncodingEncoder.minMaxStats(PType.F64, values); // Then - assertThat(result[0]).isEqualTo(ProtoScalarValue.ofF64Value(-1.0)); - assertThat(result[1]).isEqualTo(ProtoScalarValue.ofF64Value(3.0)); + assertThat(result.min()).isEqualTo(ProtoScalarValue.ofF64Value(-1.0)); + assertThat(result.max()).isEqualTo(ProtoScalarValue.ofF64Value(3.0)); } @Test @@ -158,7 +158,7 @@ void allNanHasNoMinMax() { float[] values = {Float.NaN, Float.NaN}; // When - ProtoScalarValue[] result = PrimitiveEncodingEncoder.minMaxStats(PType.F32, values); + MinMax result = PrimitiveEncodingEncoder.minMaxStats(PType.F32, values); // Then assertThat(result).isNull(); @@ -171,11 +171,11 @@ void infinitiesAreValidExtremes() { double[] values = {Double.POSITIVE_INFINITY}; // When - ProtoScalarValue[] result = PrimitiveEncodingEncoder.minMaxStats(PType.F64, values); + MinMax result = PrimitiveEncodingEncoder.minMaxStats(PType.F64, values); // Then - assertThat(result[0]).isEqualTo(ProtoScalarValue.ofF64Value(Double.POSITIVE_INFINITY)); - assertThat(result[1]).isEqualTo(ProtoScalarValue.ofF64Value(Double.POSITIVE_INFINITY)); + assertThat(result.min()).isEqualTo(ProtoScalarValue.ofF64Value(Double.POSITIVE_INFINITY)); + assertThat(result.max()).isEqualTo(ProtoScalarValue.ofF64Value(Double.POSITIVE_INFINITY)); } }