diff --git a/java/vortex-spark/common/src/test/java/dev/vortex/spark/VortexListElementPruningIoTest.java b/java/vortex-spark/common/src/test/java/dev/vortex/spark/VortexListElementPruningIoTest.java
new file mode 100644
index 00000000000..17e80b0b5e2
--- /dev/null
+++ b/java/vortex-spark/common/src/test/java/dev/vortex/spark/VortexListElementPruningIoTest.java
@@ -0,0 +1,295 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+package dev.vortex.spark;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.Function;
+import java.util.stream.Stream;
+import org.apache.spark.sql.Column;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.functions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+/**
+ * Tests that reading a leaf inside lists of structs nested in structs reads only that leaf from storage, through both
+ * Spark read paths, and returns what Parquet returns.
+ *
+ * The table nests lists the way wide event schemas do, one to four lists deep under structs:
+ *
+ *
+ * id
+ * r: struct
+ * q: struct
+ * imp: array<struct
+ * id, w0..w7
+ * banner: struct
+ * w, h, b0..b7
+ * format: array<struct<w, h, f0..f7>>
+ * tags: array<string>
+ * seats: array<struct
+ * name, s0..s7
+ * deals: array<struct
+ * price, d0..d7
+ * segs: array<struct<v, g0..g7>>
+ *
+ *
+ * Every struct carries eight wide string siblings, so a read that pulls in a leaf's siblings shows up in the bytes
+ * read. Each list level has null lists, null elements and empty lists. The table is written with
+ * {@code vortex.listLayout}, without it, and as Parquet to compare against.
+ */
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public final class VortexListElementPruningIoTest {
+ private static final int ROWS = 8192;
+ private static final String PRUNING = "spark.sql.optimizer.nestedSchemaPruning.enabled";
+ private static final String USE_V1 = "spark.sql.sources.useV1SourceList";
+
+ private SparkSession spark;
+ private String listLayoutPath;
+ private String flatPath;
+ private String parquetPath;
+
+ @BeforeAll
+ public void setUp(@TempDir Path tempDir) {
+ spark = SparkSession.builder()
+ .appName("VortexListElementPruningIoTest")
+ .master("local[2]")
+ .config("spark.driver.host", "127.0.0.1")
+ .config("spark.sql.shuffle.partitions", "2")
+ .config("spark.sql.adaptive.enabled", "false")
+ .config(PRUNING, "true")
+ // Spark prunes below two arrays only through an index, which ANSI mode fails on an empty array.
+ .config("spark.sql.ansi.enabled", "false")
+ .config("spark.ui.enabled", "false")
+ .config(
+ "spark.hadoop.fs.file.impl",
+ VortexNestedSchemaPruningIoTest.CountingLocalFileSystem.class.getName())
+ .config("spark.hadoop.fs.file.impl.disable.cache", "true")
+ .getOrCreate();
+
+ Dataset rows = spark.range(0, ROWS, 1, 2)
+ .selectExpr("id", "named_struct('q', named_struct('imp', " + imps() + ")) AS r");
+ listLayoutPath = tempDir.resolve("list_layout").toString();
+ rows.write().format("vortex").option("vortex.listLayout", "true").save(listLayoutPath);
+ flatPath = tempDir.resolve("flat").toString();
+ rows.write().format("vortex").save(flatPath);
+ parquetPath = tempDir.resolve("parquet").toString();
+ rows.write().parquet(parquetPath);
+ }
+
+ @AfterAll
+ public void tearDown() {
+ if (spark != null) {
+ spark.stop();
+ SparkSession.clearActiveSession();
+ SparkSession.clearDefaultSession();
+ }
+ }
+
+ /** Eight wide string fields named {@code prefix0..7}, distinct per row and position. */
+ private static String wide(String prefix, String key) {
+ StringBuilder out = new StringBuilder();
+ for (int k = 0; k < 8; k++) {
+ out.append(String.format(
+ Locale.ROOT, ", '%s%d', sha2(concat_ws(':', '%s%d', %s), 256)", prefix, k, prefix, k, key));
+ }
+ return out.toString();
+ }
+
+ /**
+ * A list of {@code element} over {@code sequence(0, max - 1)}, null when {@code seed % nullEvery == 0}, with a
+ * length of {@code seed % (max + 1)}, so some lists are empty, and a null element when {@code (seed + i) % 7 == 0}.
+ */
+ private static String list(String var, String seed, int max, int nullEvery, String element) {
+ return String.format(
+ Locale.ROOT,
+ "CASE WHEN (%s) %% %d = 0 THEN NULL ELSE transform(filter(sequence(0, %d), %s -> %s < (%s) %% %d),"
+ + " %s -> CASE WHEN (%s + %s) %% 7 = 0 THEN NULL ELSE %s END) END",
+ seed,
+ nullEvery,
+ max - 1,
+ var,
+ var,
+ seed,
+ max + 1,
+ var,
+ seed,
+ var,
+ element);
+ }
+
+ private static String imps() {
+ String format = list(
+ "k",
+ "id + i + 3",
+ 3,
+ 5,
+ "named_struct('w', id + i * 10 + k, 'h', id + k" + wide("f", "id, i, k") + ")");
+ String banner = "CASE WHEN (id + i) % 6 = 0 THEN NULL ELSE named_struct('w', id * 3 + i, 'h', id + i"
+ + wide("b", "id, i") + ", 'format', " + format + ") END";
+ String tags = list("t", "id + i + 1", 4, 9, "concat('tag', id % 13, '-', t)");
+ String segs = list(
+ "g",
+ "id + i + c + d + 2",
+ 3,
+ 6,
+ "named_struct('v', id + i + c + d + g" + wide("g", "id, i, c, d, g") + ")");
+ String deals = list(
+ "d",
+ "id + i + c + 1",
+ 2,
+ 4,
+ "named_struct('price', id * 7 + i + c + d" + wide("d", "id, i, c, d") + ", 'segs', " + segs + ")");
+ String seats = list(
+ "c",
+ "id + i",
+ 3,
+ 8,
+ "named_struct('name', concat('seat', c)" + wide("s", "id, i, c") + ", 'deals', " + deals + ")");
+ String imp = "named_struct('id', id * 10 + i" + wide("w", "id, i") + ", 'banner', " + banner + ", 'tags', "
+ + tags + ", 'seats', " + seats + ")";
+ return list("i", "id", 4, 11, imp);
+ }
+
+ private static final Column IMP = functions.col("r.q.imp");
+
+ /**
+ * Leaves at one to four lists deep, each with the column read whole for comparison: the leaf's list, or the
+ * innermost list around it.
+ */
+ static Stream leaves() {
+ return Stream.of(
+ Arguments.of("r.q.imp.id (1 list)", IMP.getField("id"), IMP),
+ Arguments.of(
+ "r.q.imp.banner.w (1 list, struct below)",
+ IMP.getField("banner").getField("w"),
+ IMP),
+ Arguments.of(
+ "r.q.imp[0].banner.format.w (2 lists)",
+ IMP.getItem(0).getField("banner").getField("format").getField("w"),
+ IMP.getItem(0).getField("banner").getField("format")),
+ Arguments.of("r.q.imp.tags (list of strings in a list)", IMP.getField("tags"), IMP),
+ Arguments.of(
+ "r.q.imp[0].seats.deals[0].price (3 lists)",
+ IMP.getItem(0)
+ .getField("seats")
+ .getField("deals")
+ .getItem(0)
+ .getField("price"),
+ IMP.getItem(0).getField("seats")),
+ Arguments.of(
+ "r.q.imp[0].seats[0].deals[0].segs.v (4 lists)",
+ IMP.getItem(0)
+ .getField("seats")
+ .getItem(0)
+ .getField("deals")
+ .getItem(0)
+ .getField("segs")
+ .getField("v"),
+ IMP.getItem(0)
+ .getField("seats")
+ .getItem(0)
+ .getField("deals")
+ .getItem(0)
+ .getField("segs")));
+ }
+
+ static Stream leavesByPath() {
+ return leaves().flatMap(
+ a -> Stream.of(true, false).map(v1 -> Arguments.of(a.get()[0], a.get()[1], a.get()[2], v1)));
+ }
+
+ @ParameterizedTest(name = "{0}, V1={3}")
+ @MethodSource("leavesByPath")
+ public void leafInsideListsReadsOnlyThatLeaf(String name, Column leaf, Column whole, boolean v1) {
+ withReadPath(v1, () -> {
+ Dataset table = spark.read().format("vortex").load(listLayoutPath);
+ long leafBytes = bytesRead(table.select(leaf));
+ long wholeBytes = bytesRead(table.select(whole));
+ long flatLeaf =
+ bytesRead(spark.read().format("vortex").load(flatPath).select(leaf));
+ System.out.printf(
+ Locale.ROOT,
+ "bytes read, V1=%s, %s: list layout %d (around it %d), flat %d%n",
+ v1,
+ name,
+ leafBytes,
+ wholeBytes,
+ flatLeaf);
+ assertTrue(
+ leafBytes * 5 < wholeBytes,
+ name + " read " + leafBytes + " bytes; the list around it read " + wholeBytes);
+ assertMatchesParquet(t -> t.select(functions.col("id"), leaf));
+ });
+ }
+
+ @ParameterizedTest(name = "V1={0}")
+ @MethodSource("readPaths")
+ public void severalLeavesAndFiltersMatchParquet(boolean v1) {
+ withReadPath(v1, () -> {
+ assertMatchesParquet(t -> t.select(
+ functions.col("id"),
+ IMP.getField("id"),
+ IMP.getField("banner").getField("h"),
+ IMP.getItem(1).getField("seats").getField("name")));
+ assertMatchesParquet(t -> t.where(functions.size(IMP).gt(2))
+ .select(
+ functions.col("id"),
+ IMP.getItem(0).getField("banner").getField("format").getField("h")));
+ assertMatchesParquet(t -> t.where(IMP.isNull()).select(functions.col("id"), IMP.getField("id")));
+ assertMatchesParquet(t -> t.where(functions.array_contains(
+ IMP.getField("id"), functions.col("id").multiply(10)))
+ .select(functions.col("id"), IMP.getField("tags")));
+ });
+ }
+
+ static Stream readPaths() {
+ return Stream.of(true, false);
+ }
+
+ private void withReadPath(boolean v1, Runnable body) {
+ String previous = spark.conf().get(USE_V1, "");
+ spark.conf().set(USE_V1, v1 ? "vortex" : "");
+ try {
+ body.run();
+ } finally {
+ spark.conf().set(USE_V1, previous);
+ }
+ }
+
+ /** Asserts {@code query} returns Parquet's rows from both Vortex tables, with and without pruning. */
+ private void assertMatchesParquet(Function, Dataset> query) {
+ List expected =
+ query.apply(spark.read().parquet(parquetPath)).orderBy("id").collectAsList();
+ for (String path : List.of(listLayoutPath, flatPath)) {
+ Dataset read = query.apply(spark.read().format("vortex").load(path));
+ assertEquals(expected, read.orderBy("id").collectAsList(), path);
+ spark.conf().set(PRUNING, "false");
+ try {
+ assertEquals(expected, read.orderBy("id").collectAsList(), path + ", unpruned");
+ } finally {
+ spark.conf().set(PRUNING, "true");
+ }
+ }
+ }
+
+ /** Hashes every selected value so the whole result is decoded, and returns the bytes read from storage. */
+ private static long bytesRead(Dataset query) {
+ VortexNestedSchemaPruningIoTest.BYTES_READ.set(0);
+ query.selectExpr("bit_xor(xxhash64(*))").collectAsList();
+ return VortexNestedSchemaPruningIoTest.BYTES_READ.get();
+ }
+}
diff --git a/vortex-jni/goldenfiles/list_layout/event_shape.vortex b/vortex-jni/goldenfiles/list_layout/event_shape.vortex
new file mode 100644
index 00000000000..bb49deb7cde
Binary files /dev/null and b/vortex-jni/goldenfiles/list_layout/event_shape.vortex differ
diff --git a/vortex-jni/goldenfiles/list_layout/nullable_struct_elements.vortex b/vortex-jni/goldenfiles/list_layout/nullable_struct_elements.vortex
new file mode 100644
index 00000000000..9195dfd4248
Binary files /dev/null and b/vortex-jni/goldenfiles/list_layout/nullable_struct_elements.vortex differ
diff --git a/vortex-jni/goldenfiles/list_layout/zero_row_chunks.vortex b/vortex-jni/goldenfiles/list_layout/zero_row_chunks.vortex
new file mode 100644
index 00000000000..20d4b6bcd1f
Binary files /dev/null and b/vortex-jni/goldenfiles/list_layout/zero_row_chunks.vortex differ
diff --git a/vortex-jni/src/list_layout_golden.rs b/vortex-jni/src/list_layout_golden.rs
index 62921cb5fad..f38ba78d9b2 100644
--- a/vortex-jni/src/list_layout_golden.rs
+++ b/vortex-jni/src/list_layout_golden.rs
@@ -30,6 +30,8 @@ use vortex::array::arrays::ListArray;
use vortex::array::arrays::PrimitiveArray;
use vortex::array::arrays::StructArray;
use vortex::array::arrays::VarBinViewArray;
+use vortex::array::dtype::DType;
+use vortex::array::stream::ArrayStreamAdapter;
use vortex::array::stream::ArrayStreamExt;
use vortex::array::validity::Validity;
use vortex::buffer::Buffer;
@@ -88,6 +90,13 @@ enum Case {
InStruct,
/// Lists that are all empty.
AllEmpty,
+ /// A wide event schema's shape: a nullable struct holding nullable lists of nullable structs, which hold a nullable
+ /// struct, a list of strings and lists nested four deep.
+ EventShape,
+ /// Nullable lists of nullable structs.
+ NullableStructElements,
+ /// Nullable lists written as chunks with empty chunks before, between and after them.
+ ZeroRowChunks,
}
impl Case {
@@ -100,6 +109,9 @@ impl Case {
Case::Nested => "nested",
Case::InStruct => "in_struct",
Case::AllEmpty => "all_empty",
+ Case::EventShape => "event_shape",
+ Case::NullableStructElements => "nullable_struct_elements",
+ Case::ZeroRowChunks => "zero_row_chunks",
}
}
@@ -196,6 +208,34 @@ impl Case {
)?;
StructArray::from_fields(&[("l", l)])?.into_array()
}
+ Case::EventShape => event_shape(session, rows)?,
+ Case::NullableStructElements => {
+ let offsets = offsets(rows, |row| (row * 5 + 2) % 5);
+ let n = usize::try_from(last(&offsets))?;
+ let elements = StructArray::try_new(
+ ["a", "b"].into(),
+ [
+ PrimitiveArray::from_option_iter(
+ (0..n).map(|i| (i % 4 != 0).then_some(i as i64)),
+ )
+ .into_array(),
+ VarBinViewArray::from_iter_nullable_str(
+ (0..n).map(|i| (i % 6 != 0).then(|| format!("b{i}"))),
+ )
+ .into_array(),
+ ],
+ n,
+ Validity::from_iter((0..n).map(|i| i % 9 != 0)),
+ )?;
+ let validity = Validity::from_iter((0..rows).map(|row| row % 7 != 0));
+ list(session, elements.into_array(), offsets, validity)?
+ }
+ Case::ZeroRowChunks => {
+ let offsets = offsets(rows, |row| (row * 3 + 1) % 4);
+ let elements = Buffer::from_iter((0..last(&offsets)).map(|i| i64::from(i) * 7));
+ let validity = Validity::from_iter((0..rows).map(|row| row % 10 != 0));
+ list(session, elements.into_array(), offsets, validity)?
+ }
Case::AllEmpty => list(
session,
Buffer::::empty().into_array(),
@@ -203,15 +243,178 @@ impl Case {
Validity::NonNullable,
)?,
};
- let name = if matches!(self, Case::InStruct) {
- "s"
- } else {
- "l"
+ let name = match self {
+ Case::InStruct => "s",
+ Case::EventShape => "r",
+ _ => "l",
};
Ok(StructArray::from_fields(&[("id", id), (name, lists)])?.into_array())
}
}
+/// Nullable structs of `fields`, each struct valid unless `null(row)`.
+fn nullable_struct(
+ fields: &[(&str, ArrayRef)],
+ len: usize,
+ null: impl Fn(usize) -> bool,
+) -> VortexResult {
+ Ok(StructArray::try_new(
+ fields.iter().map(|(n, _)| *n).collect::>().into(),
+ fields.iter().map(|(_, a)| a.clone()),
+ len,
+ Validity::from_iter((0..len).map(|row| !null(row))),
+ )?
+ .into_array())
+}
+
+fn nullable_i64(len: usize, f: impl Fn(usize) -> Option) -> ArrayRef {
+ PrimitiveArray::from_option_iter((0..len).map(f)).into_array()
+}
+
+/// A sibling string field, so a leaf's siblings are real columns. Values repeat, to keep the file small.
+fn wide_strings(len: usize, tag: &str) -> ArrayRef {
+ VarBinViewArray::from_iter_nullable_str(
+ (0..len).map(|i| (i % 5 != 0).then(|| format!("{tag}{}", i % 50))),
+ )
+ .into_array()
+}
+
+/// A nullable list over `elements`, each of `rows` lists holding `len(row)` and null when `null(row)`.
+fn nullable_list(
+ session: &VortexSession,
+ rows: usize,
+ elements: impl FnOnce(usize) -> VortexResult,
+ len: impl Fn(usize) -> usize,
+ null: impl Fn(usize) -> bool,
+) -> VortexResult {
+ let offsets = offsets(rows, |row| if null(row) { 0 } else { len(row) });
+ let n = usize::try_from(last(&offsets))?;
+ list(
+ session,
+ elements(n)?,
+ offsets,
+ Validity::from_iter((0..rows).map(|row| !null(row))),
+ )
+}
+
+/// [`Case::EventShape`]'s `r`: `{q: {imp: [{id, w, banner: {w, h, b, format: [{w, h, f}]}, tags: [utf8],
+/// seats: [{name, s, deals: [{price, d, segs: [{v, g}]}]}]}]}}`, nullable at every level.
+fn event_shape(session: &VortexSession, rows: usize) -> VortexResult {
+ let imp = nullable_list(
+ session,
+ rows,
+ |n| {
+ let format = nullable_list(
+ session,
+ n,
+ |m| {
+ nullable_struct(
+ &[
+ ("w", nullable_i64(m, |i| Some(i as i64 * 3))),
+ ("h", nullable_i64(m, |i| (i % 4 != 0).then_some(i as i64))),
+ ("f", wide_strings(m, "f")),
+ ],
+ m,
+ |i| i % 8 == 0,
+ )
+ },
+ |i| i % 3,
+ |i| i % 5 == 0,
+ )?;
+ let banner = nullable_struct(
+ &[
+ ("w", nullable_i64(n, |i| Some(i as i64 * 2))),
+ ("h", nullable_i64(n, |i| (i % 3 != 0).then_some(i as i64))),
+ ("b", wide_strings(n, "b")),
+ ("format", format),
+ ],
+ n,
+ |i| i % 6 == 0,
+ )?;
+ let tags = nullable_list(
+ session,
+ n,
+ |m| {
+ Ok(VarBinViewArray::from_iter_nullable_str(
+ (0..m).map(|i| (i % 7 != 0).then(|| format!("t{}", i % 13))),
+ )
+ .into_array())
+ },
+ |i| i % 4,
+ |i| i % 9 == 0,
+ )?;
+ let seats = nullable_list(
+ session,
+ n,
+ |m| {
+ let deals = nullable_list(
+ session,
+ m,
+ |k| {
+ let segs = nullable_list(
+ session,
+ k,
+ |j| {
+ nullable_struct(
+ &[
+ ("v", nullable_i64(j, |i| Some(i as i64))),
+ ("g", wide_strings(j, "g")),
+ ],
+ j,
+ |i| i % 10 == 0,
+ )
+ },
+ |i| i % 3,
+ |i| i % 6 == 0,
+ )?;
+ nullable_struct(
+ &[
+ (
+ "price",
+ nullable_i64(k, |i| (i % 5 != 0).then_some(i as i64 * 11)),
+ ),
+ ("d", wide_strings(k, "d")),
+ ("segs", segs),
+ ],
+ k,
+ |i| i % 7 == 0,
+ )
+ },
+ |i| i % 3,
+ |i| i % 4 == 0,
+ )?;
+ nullable_struct(
+ &[
+ ("name", wide_strings(m, "seat")),
+ ("s", wide_strings(m, "s")),
+ ("deals", deals),
+ ],
+ m,
+ |i| i % 8 == 0,
+ )
+ },
+ |i| i % 4,
+ |i| i % 8 == 0,
+ )?;
+ nullable_struct(
+ &[
+ ("id", nullable_i64(n, |i| Some(i as i64 * 10))),
+ ("w", wide_strings(n, "w")),
+ ("banner", banner),
+ ("tags", tags),
+ ("seats", seats),
+ ],
+ n,
+ |i| i % 7 == 0,
+ )
+ },
+ |row| row % 5,
+ |row| row % 11 == 0,
+ )?;
+ let q = nullable_struct(&[("imp", imp)], rows, |row| row % 13 == 0)?;
+ nullable_struct(&[("q", q)], rows, |row| row % 17 == 0)
+}
+
/// Offsets of `rows` lists, the list of row `row` holding `len(row)` elements.
fn offsets(rows: usize, len: impl Fn(usize) -> usize) -> Buffer {
let mut end = 0u32;
@@ -244,16 +447,39 @@ fn list(
/// Writes `array` as a `vortex.listLayout` write does.
fn write(array: ArrayRef) -> VortexResult {
+ write_chunks(array.dtype().clone(), vec![array])
+}
+
+/// Writes `chunks` as one stream, as a `vortex.listLayout` write of several batches does.
+fn write_chunks(dtype: DType, chunks: Vec) -> VortexResult {
let session = write_session(&new_session(), true)?;
let mut bytes = ByteBufferMut::empty();
- RUNTIME.block_on(
- session
- .write_options()
- .write(&mut bytes, array.to_array_stream()),
- )?;
+ let stream = ArrayStreamAdapter::new(dtype, futures::stream::iter(chunks.into_iter().map(Ok)));
+ RUNTIME.block_on(session.write_options().write(&mut bytes, stream))?;
Ok(bytes.freeze())
}
+/// Writes `case` the way its golden file was written: in one stream, and for [`Case::ZeroRowChunks`] as several
+/// populated chunks with empty ones before, between and after them.
+fn write_case(session: &VortexSession, case: Case) -> VortexResult {
+ let array = case.array(session)?;
+ if !matches!(case, Case::ZeroRowChunks) {
+ return write(array);
+ }
+ let empty = || array.slice(0..0);
+ let chunks = vec![
+ empty()?,
+ array.slice(0..7_000)?,
+ empty()?,
+ empty()?,
+ array.slice(7_000..15_000)?,
+ empty()?,
+ array.slice(15_000..ROWS)?,
+ empty()?,
+ ];
+ write_chunks(array.dtype().clone(), chunks)
+}
+
/// Opens `bytes` as the JNI reader does.
fn open(session: &VortexSession, bytes: ByteBuffer) -> VortexResult {
file_open_options(session.open_options()).open_buffer(bytes)
@@ -336,7 +562,7 @@ fn filtered(
array.filter(Mask::from_iter(keep.into_bit_buffer().iter()))
}
-const CASES: [Case; 7] = [
+const CASES: [Case; 10] = [
Case::Primitive,
Case::Nullable,
Case::Utf8,
@@ -344,6 +570,9 @@ const CASES: [Case; 7] = [
Case::Nested,
Case::InStruct,
Case::AllEmpty,
+ Case::EventShape,
+ Case::NullableStructElements,
+ Case::ZeroRowChunks,
];
#[template]
@@ -355,6 +584,9 @@ const CASES: [Case; 7] = [
#[case::nested(Case::Nested)]
#[case::in_struct(Case::InStruct)]
#[case::all_empty(Case::AllEmpty)]
+#[case::event_shape(Case::EventShape)]
+#[case::nullable_struct_elements(Case::NullableStructElements)]
+#[case::zero_row_chunks(Case::ZeroRowChunks)]
fn cases(#[case] case: Case) {}
#[apply(cases)]
@@ -373,7 +605,7 @@ fn golden_file_has_list_layouts(#[case] case: Case) -> VortexResult<()> {
fn writer_writes_the_golden_list_layouts(#[case] case: Case) -> VortexResult<()> {
let session = new_session();
let golden = list_layouts(golden(&session, case)?.footer().layout())?;
- let written = open(&session, write(case.array(&session)?)?)?;
+ let written = open(&session, write_case(&session, case)?)?;
assert_eq!(
list_layouts(written.footer().layout())?,
golden,
@@ -441,6 +673,28 @@ fn golden_file_reads_row_indices(#[case] case: Case) -> VortexResult<()> {
#[case::element_field(Case::StructElements, list_transform(col("l"), element("a")))]
#[case::nullable_element_field(Case::StructElements, list_transform(col("l"), element("b")))]
#[case::utf8_elements(Case::Utf8, col("l"))]
+#[case::nullable_struct_element_field(
+ Case::NullableStructElements,
+ list_transform(col("l"), element("b"))
+)]
+#[case::zero_row_chunks(Case::ZeroRowChunks, col("l"))]
+#[case::event_leaf_one_list(Case::EventShape, list_transform(imp(), element("id")))]
+#[case::event_leaf_below_a_struct(Case::EventShape, list_transform(imp(), path(&["banner", "w"])))]
+#[case::event_strings_in_a_list(Case::EventShape, list_transform(imp(), element("tags")))]
+#[case::event_leaf_two_lists(
+ Case::EventShape,
+ list_transform(
+ imp(),
+ lambda(
+ "x",
+ list_transform(get_item("format", get_item("banner", var("x"))), element("w"))
+ )
+ )
+)]
+#[case::event_leaf_four_lists(
+ Case::EventShape,
+ list_transform(imp(), lambda("x", seats_deals_segs_v(var("x"))))
+)]
fn golden_file_reads_projections(
#[case] case: Case,
#[case] projection: Expression,
@@ -470,6 +724,30 @@ fn golden_file_reads_filters(#[case] case: Case, #[case] filter: Expression) ->
Ok(())
}
+/// `r.q.imp` of [`Case::EventShape`].
+fn imp() -> Expression {
+ get_item("imp", get_item("q", col("r")))
+}
+
+/// `x -> x.....`, for `list_transform`.
+fn path(fields: &[&str]) -> Lambda {
+ let body = fields.iter().fold(var("x"), |e, f| get_item(*f, e));
+ Lambda::new(["x"], body).vortex_expect("lambda binds one parameter")
+}
+
+/// ` -> body`, for `list_transform`.
+fn lambda(name: &str, body: Expression) -> Lambda {
+ Lambda::new([name], body).vortex_expect("lambda binds one parameter")
+}
+
+/// `imp.seats[].deals[].segs[].v` below one `imp` element: three more lists, four in all.
+fn seats_deals_segs_v(imp: Expression) -> Expression {
+ let segs = |deal: Expression| list_transform(get_item("segs", deal), element("v"));
+ let deals =
+ |seat: Expression| list_transform(get_item("deals", seat), lambda("d", segs(var("d"))));
+ list_transform(get_item("seats", imp), lambda("s", deals(var("s"))))
+}
+
/// `x -> x.`, for `list_transform`.
fn element(field: &str) -> Lambda {
Lambda::new(["x"], get_item(field, var("x"))).vortex_expect("lambda binds one parameter")
@@ -487,7 +765,7 @@ fn write_missing_golden_files() -> VortexResult<()> {
continue;
}
fs::create_dir_all(path.parent().vortex_expect("golden files have a directory"))?;
- fs::write(&path, write(case.array(&session)?)?.as_slice())?;
+ fs::write(&path, write_case(&session, case)?.as_slice())?;
}
Ok(())
}