From b8d7042ccefc6b12e58a1cb3c258ccc54c6f3eb5 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sat, 24 Oct 2026 03:04:25 +1100 Subject: [PATCH 1/3] test(eql): hold json_tree containment to jsonb over generated documents The live json_tree oracle checked a fixed corpus and a fixed list of needles. The only randomized containment check compared the projection with an in-memory model of jsonb containment, never with PostgreSQL, so nothing generated was ever answered by the database itself. postgres_containment_over_generated_documents_is_jsonb_containment stores 300 generated documents encrypted beside their plaintexts and asks 500 needles: random values, sub-documents pruned from stored documents, and those sub-documents with one edit. For every needle the rows the encrypted @> and <@ select must be the rows jsonb @> and <@ select over the plaintexts, as PostgreSQL computes them. It runs once with the planner held to the GIN index and once with every index path off, and checks each @> plan with EXPLAIN. <@ may differ only where the SQL says it does (a stored root scalar inside a needle root array), and the test asserts that is the whole difference. It also checks -> at 20 generated key paths: the entry exists exactly where jsonb #> finds a node, and a leaf opens to the value jsonb holds there. The seed is fixed (1336), logged on every run, and replaced by EQL_JSON_TREE_SEED. A failure prints the seed, the needle and every stored document the two answers disagree on. test:encryption:postgres passes --show-output so the seed line reaches the CI log. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- packages/eql/mise.toml | 5 +- .../eql/tests/encryption/tests/json_tree.rs | 772 +++++++++++++++++- 2 files changed, 775 insertions(+), 2 deletions(-) diff --git a/packages/eql/mise.toml b/packages/eql/mise.toml index 9f1995443..756626154 100644 --- a/packages/eql/mise.toml +++ b/packages/eql/mise.toml @@ -343,7 +343,10 @@ psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -f release/cipherstash-encrypt- for suite in text_eq v4_types v4_order v4_date_time_numeric v4_range v4_text_search v4_ore_block json_tree; do listing=$(cargo test --locked -p eql-encryption-tests --test "$suite" -- --ignored --list | sed -n 's/: test$//p') [ -n "$listing" ] || { echo "error: '--ignored' selects no test in $suite; the PostgreSQL test has gone missing." >&2; exit 1; } - cargo test --locked -p eql-encryption-tests --test "$suite" -- --ignored + # `--show-output`: a passing test's output is otherwise captured, and the + # generated containment oracle in json_tree logs the seed it ran under + # (EQL_JSON_TREE_SEED replays it) and what it checked. + cargo test --locked -p eql-encryption-tests --test "$suite" -- --ignored --show-output done """ diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index 202129658..f905789fb 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -6,7 +6,8 @@ //! `@>` and `<@` over the encrypted column against PostgreSQL's own //! `jsonb @>` over the plaintexts, for the design's corpus and random //! documents, with a GIN index on the projection that the planner is shown -//! to use. +//! to use, and again for a few hundred generated documents and needles under +//! a logged seed (`EQL_JSON_TREE_SEED` replays one). mod common; @@ -435,6 +436,775 @@ fn postgres_containment_over_the_encrypted_column_is_jsonb_containment() { assert!(nulls.get::<_, bool>(0)); } +// ============================================================================= +// PostgreSQL: generated documents +// ============================================================================= + +/// The environment variable that replaces the generated oracle's seed. +const SEED_ENV: &str = "EQL_JSON_TREE_SEED"; + +/// The seed the generated oracle runs under when [`SEED_ENV`] is unset. +/// Fixed, so every CI run checks the same documents; logged, so a failure +/// replays from the log. +const DEFAULT_SEED: u64 = 1336; + +/// How many documents are stored, and how many needles of each kind ask. +const STORED_DOCUMENTS: usize = 300; +const RANDOM_NEEDLES: usize = 100; +const EXTRACTED_NEEDLES: usize = 200; +const NEAR_MISS_NEEDLES: usize = 200; + +/// Numbers, several of them one value to `jsonb` in different +/// representations: `1`, `1.0` and `1e0`; `100` and `1e2` (an integer and +/// a float); `0`, `0.0` and `-0.0`; `0.5` and `5e-1`. A value is stored as +/// `serde_json` parsed it and PostgreSQL is sent `serde_json`'s spelling of +/// the same value, so each side sees one plaintext. +const NUMBERS: &[&str] = &[ + "0", + "0.0", + "-0.0", + "1", + "1.0", + "1e0", + "2", + "2.00", + "-1", + "-1.0", + "10", + "1e1", + "100", + "1e2", + "0.5", + "5e-1", + "1.5", + "123456789012", + "1e20", +]; + +/// Strings, some spelled like a number, a boolean or null, so that a kind +/// confused with another shows as a wrong answer. +const STRINGS: &[&str] = &["x", "y", "", "1", "1.0", "2", "true", "null", "a b", "ü"]; + +/// Object keys: few, so they repeat across levels and siblings. The paths +/// the `->` check asks at are built from the first four. +const KEYS: &[&str] = &["a", "b", "c", "x", "1", ""]; +const PATH_KEYS: &[&str] = &["a", "b", "c", "x"]; + +/// SplitMix64: a small, well-mixed deterministic generator, so a seed +/// names one run exactly on every platform. +struct Gen(u64); + +impl Gen { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } + + fn chance(&mut self, percent: usize) -> bool { + self.below(100) < percent + } + + fn pick<'a>(&mut self, items: &[&'a str]) -> &'a str { + items[self.below(items.len())] + } + + fn scalar(&mut self) -> Value { + match self.below(10) { + 0..=3 => serde_json::from_str(self.pick(NUMBERS)).unwrap(), + 4..=6 => Value::String(self.pick(STRINGS).to_owned()), + 7 | 8 => Value::Bool(self.chance(50)), + _ => Value::Null, + } + } + + /// A value with at most `depth` levels of containers below it. + fn value(&mut self, depth: u32) -> Value { + if depth == 0 { + return self.scalar(); + } + match self.below(10) { + 0..=3 => self.scalar(), + 4..=6 => self.object(depth), + _ => self.array(depth), + } + } + + /// An object of up to four members, empty at depth 0. A key drawn + /// twice keeps the later value, as `jsonb` does. + fn object(&mut self, depth: u32) -> Value { + let mut map = Map::new(); + if depth > 0 { + for _ in 0..self.below(5) { + let key = self.pick(KEYS).to_owned(); + let _ = map.insert(key, self.value(depth - 1)); + } + } + Value::Object(map) + } + + /// An array of up to four elements, empty at depth 0: sometimes all + /// objects, sometimes all arrays, otherwise mixed, and an element is + /// sometimes a copy of an earlier one. + fn array(&mut self, depth: u32) -> Value { + let mut items: Vec = Vec::new(); + if depth > 0 { + let shape = self.below(4); + for _ in 0..self.below(5) { + let item = if !items.is_empty() && self.chance(25) { + items[self.below(items.len())].clone() + } else { + match shape { + 0 => self.object(depth - 1), + 1 => self.array(depth - 1), + _ => self.value(depth - 1), + } + }; + items.push(item); + } + } + Value::Array(items) + } + + /// A stored document: mostly an object, sometimes a root array, and + /// sometimes a root scalar. + fn document(&mut self) -> Value { + match self.below(20) { + 0..=11 => self.object(4), + 12..=16 => self.array(3), + _ => self.scalar(), + } + } + + /// A sub-document `v` contains by construction: an object keeps some of + /// its members, an array some of its elements (in any order, and + /// repeated or not, since `jsonb` array containment ignores both), each + /// pruned in turn; a scalar is itself. + fn prune(&mut self, v: &Value) -> Value { + match v { + Value::Object(map) => { + let mut out = Map::new(); + for (key, member) in map { + if self.chance(60) { + let _ = out.insert(key.clone(), self.prune(member)); + } + } + Value::Object(out) + } + Value::Array(items) => { + let mut out = Vec::new(); + if !items.is_empty() { + for _ in 0..self.below(items.len() + 1) { + let item = &items[self.below(items.len())]; + out.push(self.prune(item)); + } + } + Value::Array(out) + } + scalar => scalar.clone(), + } + } + + /// A node somewhere inside `v`, `v` itself included. + fn node<'v>(&mut self, v: &'v Value) -> &'v Value { + if self.chance(40) { + return v; + } + match v { + Value::Object(map) if !map.is_empty() => { + let member = map.values().nth(self.below(map.len())).unwrap(); + self.node(member) + } + Value::Array(items) if !items.is_empty() => { + let item = &items[self.below(items.len())]; + self.node(item) + } + _ => v, + } + } + + /// A needle drawn from the stored document `doc`. Mostly a pruning of + /// the whole document, which `doc` contains by construction (`root`); + /// for a root array, sometimes one of its scalar elements bare, which + /// `jsonb` also counts as contained; otherwise a pruning of a node + /// inside it, which other rows may hold. + fn extract(&mut self, doc: &Value) -> (Value, bool) { + if let Value::Array(items) = doc { + let scalars: Vec<&Value> = items + .iter() + .filter(|v| !v.is_object() && !v.is_array()) + .collect(); + if !scalars.is_empty() && self.chance(25) { + return (scalars[self.below(scalars.len())].clone(), true); + } + } + if self.chance(70) { + (self.prune(doc), true) + } else { + let node = self.node(doc); + (self.prune(node), false) + } + } + + /// One edit at a node inside `v`: change a leaf, add a key, add an + /// element, or change a node's kind. + fn mutate(&mut self, v: &mut Value) { + if self.chance(60) { + match v { + Value::Object(map) if !map.is_empty() => { + let i = self.below(map.len()); + let member = map.values_mut().nth(i).unwrap(); + return self.mutate(member); + } + Value::Array(items) if !items.is_empty() => { + let i = self.below(items.len()); + return self.mutate(&mut items[i]); + } + _ => {} + } + } + match (self.below(4), &mut *v) { + (0, Value::Object(map)) => { + let key = self.pick(KEYS).to_owned(); + let _ = map.insert(key, self.value(1)); + } + (1, Value::Array(items)) => items.push(self.value(1)), + (2, scalar) if !scalar.is_object() && !scalar.is_array() => { + let mut other = self.scalar(); + while other == *scalar { + other = self.scalar(); + } + *scalar = other; + } + (_, Value::Object(map)) => *v = Value::Array(map.values().cloned().collect()), + (_, Value::Array(items)) => { + let key = self.pick(KEYS).to_owned(); + *v = json!({ key: items.first().cloned().unwrap_or(Value::Null) }); + } + (_, scalar) => { + *v = if self.chance(50) { + json!([scalar.clone()]) + } else { + json!({ self.pick(KEYS): scalar.clone() }) + }; + } + } + } +} + +/// The features the generator is meant to cover, counted over the stored +/// documents, so a change to it that stops producing one fails here +/// instead of quietly testing less. +#[derive(Debug, Default)] +struct Census { + root_objects: usize, + root_arrays: usize, + root_scalars: usize, + empty_objects: usize, + empty_arrays: usize, + arrays_of_objects: usize, + arrays_of_arrays: usize, + duplicate_elements: usize, + keys_on_two_levels: usize, + float_spelled_integers: usize, + numeric_strings: usize, + booleans: usize, + nulls: usize, + max_depth: usize, +} + +impl Census { + fn of(docs: &[Value]) -> Self { + let mut census = Census::default(); + for doc in docs { + match doc { + Value::Object(_) => census.root_objects += 1, + Value::Array(_) => census.root_arrays += 1, + _ => census.root_scalars += 1, + } + let mut keys = + std::collections::BTreeMap::<&str, std::collections::BTreeSet>::new(); + census.walk(doc, 0, &mut keys); + if keys.values().any(|depths| depths.len() > 1) { + census.keys_on_two_levels += 1; + } + } + census + } + + fn walk<'v>( + &mut self, + v: &'v Value, + depth: usize, + keys: &mut std::collections::BTreeMap<&'v str, std::collections::BTreeSet>, + ) { + self.max_depth = self.max_depth.max(depth); + match v { + Value::Object(map) => { + if map.is_empty() { + self.empty_objects += 1; + } + for (key, member) in map { + let _ = keys.entry(key).or_default().insert(depth); + self.walk(member, depth + 1, keys); + } + } + Value::Array(items) => { + if items.is_empty() { + self.empty_arrays += 1; + } + if items.iter().any(Value::is_object) { + self.arrays_of_objects += 1; + } + if items.iter().any(Value::is_array) { + self.arrays_of_arrays += 1; + } + if items + .iter() + .enumerate() + .any(|(i, a)| items[i + 1..].contains(a)) + { + self.duplicate_elements += 1; + } + for item in items { + self.walk(item, depth + 1, keys); + } + } + Value::Number(n) => { + if n.is_f64() && n.as_f64().is_some_and(|f| f.fract() == 0.0) { + self.float_spelled_integers += 1; + } + } + Value::String(s) => { + if s.parse::().is_ok() { + self.numeric_strings += 1; + } + } + Value::Bool(_) => self.booleans += 1, + Value::Null => self.nulls += 1, + } + } + + fn assert_covers_the_generator(&self) { + let counts = [ + ("root objects", self.root_objects), + ("root arrays", self.root_arrays), + ("root scalars", self.root_scalars), + ("empty objects", self.empty_objects), + ("empty arrays", self.empty_arrays), + ("arrays of objects", self.arrays_of_objects), + ("arrays of arrays", self.arrays_of_arrays), + ("duplicate array elements", self.duplicate_elements), + ("a key on two levels", self.keys_on_two_levels), + ("integers spelled as floats", self.float_spelled_integers), + ("strings spelled as numbers", self.numeric_strings), + ("booleans", self.booleans), + ("nulls", self.nulls), + ]; + for (what, count) in counts { + assert!( + count > 0, + "the generated documents hold no {what}: {self:?}" + ); + } + assert!(self.max_depth >= 4, "nested a few levels: {self:?}"); + } +} + +/// Where a needle came from, for the failure message. +#[derive(Clone, Copy)] +enum Origin { + Random, + /// A pruning of the stored document's root, so that row holds it. + Contained(usize), + /// A pruning of a node inside the stored document. + Inside(usize), + /// A contained needle with one edit. + NearMiss(usize), +} + +impl std::fmt::Display for Origin { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Origin::Random => write!(f, "random"), + Origin::Contained(row) => write!(f, "pruned from row {row}"), + Origin::Inside(row) => write!(f, "pruned from a node of row {row}"), + Origin::NearMiss(row) => write!(f, "near miss of row {row}"), + } + } +} + +struct Needle { + origin: Origin, + value: Value, + contains: String, + contained_by: String, +} + +/// Needs an EQL v4 installation. The property behind the fixed corpus +/// above, over generated documents: a few hundred stored encrypted beside +/// their plaintexts, and several hundred needles (random values, +/// sub-documents of stored documents, and those sub-documents with one +/// edit). For every needle, the rows the encrypted `@>` and `<@` select are +/// the rows PostgreSQL's own `jsonb` operators select over the plaintexts, +/// once with the planner held to the GIN index and once with every index +/// path turned off, each plan confirmed by `EXPLAIN`. `<@` differs from +/// `jsonb` only where the SQL documents it does, a stored root scalar +/// inside a needle root array, and the test asserts that is the whole +/// difference. Then `->` at generated key paths: the entry is there exactly +/// where `jsonb #>` finds a node, and a leaf opens to the value `jsonb` +/// holds there. +/// +/// The seed is fixed and logged; `EQL_JSON_TREE_SEED` replaces it. A failure +/// names the seed, the needle and each stored document the two answers +/// disagree on. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_containment_over_generated_documents_is_jsonb_containment() { + let seed = match std::env::var(SEED_ENV) { + Ok(text) => text + .trim() + .parse::() + .unwrap_or_else(|e| panic!("{SEED_ENV}={text:?}: {e}")), + Err(_) => DEFAULT_SEED, + }; + let replay = format!("seed {seed}; replay with {SEED_ENV}={seed}"); + eprintln!("json_tree generated containment: {replay}"); + let started = std::time::Instant::now(); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = plan(); + let mut db = connect(); + let mut rng = Gen(seed); + + // The stored documents, sealed, then written in one statement. + let docs: Vec = (0..STORED_DOCUMENTS).map(|_| rng.document()).collect(); + let census = Census::of(&docs); + census.assert_covers_the_generator(); + let ids: Vec = (0..docs.len() as i32).collect(); + let plains: Vec = docs.iter().map(Value::to_string).collect(); + let sealed: Vec = docs + .iter() + .map(|doc| serde_json::to_string(&wait(&runtime, seal(&cipher, doc))).unwrap()) + .collect(); + db.batch_execute( + "CREATE TEMP TABLE json_tree_generated (id int PRIMARY KEY, plain jsonb NOT NULL, doc public.eql_v4_json_tree NOT NULL)", + ) + .unwrap(); + db.execute( + "INSERT INTO json_tree_generated + SELECT id, plain::jsonb, doc::jsonb::public.eql_v4_json_tree + FROM unnest($1::int[], $2::text[], $3::text[]) AS u(id, plain, doc)", + &[&ids, &plains, &sealed], + ) + .unwrap(); + db.batch_execute( + "CREATE INDEX json_tree_generated_doc ON json_tree_generated USING gin ((eql_v4.json_tree_query(doc)) jsonb_path_ops); + ANALYZE json_tree_generated;", + ) + .unwrap(); + let stored_at = started.elapsed(); + + // The needles, each with both operands. + let mut drawn: Vec<(Origin, Value)> = Vec::new(); + for _ in 0..RANDOM_NEEDLES { + drawn.push((Origin::Random, rng.value(2))); + } + for _ in 0..EXTRACTED_NEEDLES { + let from = rng.below(docs.len()); + let (value, root) = rng.extract(&docs[from]); + let origin = if root { + Origin::Contained(from) + } else { + Origin::Inside(from) + }; + drawn.push((origin, value)); + } + for _ in 0..NEAR_MISS_NEEDLES { + let from = rng.below(docs.len()); + let mut value = rng.prune(&docs[from]); + rng.mutate(&mut value); + drawn.push((Origin::NearMiss(from), value)); + } + let operand = |query| serde_json::to_string(&JsonTreeQuery::try_from(&query).unwrap()).unwrap(); + let needles: Vec = drawn + .into_iter() + .map(|(origin, value)| Needle { + contains: operand( + wait(&runtime, cipher.query(op::Contains, &value).on(&plan)).unwrap(), + ), + contained_by: operand( + wait(&runtime, cipher.query(op::ContainedBy, &value).on(&plan)).unwrap(), + ), + origin, + value, + }) + .collect(); + + // The oracle: PostgreSQL's `jsonb @>` and `<@` over the plaintexts, for + // every needle, in one statement. + let plain_needles: Vec = needles.iter().map(|n| n.value.to_string()).collect(); + let oracle: Vec<(Vec, Vec)> = db + .query( + "SELECT coalesce(array_agg(g.id ORDER BY g.id) FILTER (WHERE g.plain @> n.v), '{}'), + coalesce(array_agg(g.id ORDER BY g.id) FILTER (WHERE g.plain <@ n.v), '{}') + FROM (SELECT i, t::jsonb AS v FROM unnest($1::text[]) WITH ORDINALITY AS u(t, i)) AS n + CROSS JOIN json_tree_generated AS g + GROUP BY n.i ORDER BY n.i", + &[&plain_needles], + ) + .unwrap() + .iter() + .map(|r| (r.get(0), r.get(1))) + .collect(); + assert_eq!(oracle.len(), needles.len()); + + // The generator's own claims, checked by the oracle: a pruned root is + // contained by the row it came from, and the edits miss often enough + // to be near misses rather than more hits. + let mut near_misses_missing = 0; + for (needle, (contains, _)) in needles.iter().zip(&oracle) { + match needle.origin { + Origin::Contained(from) => assert!( + contains.contains(&(from as i32)), + "{replay}: row {from} should contain its own sub-document {}\nstored: {}", + needle.value, + docs[from] + ), + Origin::NearMiss(from) if !contains.contains(&(from as i32)) => { + near_misses_missing += 1; + } + _ => {} + } + } + assert!( + near_misses_missing * 4 >= NEAR_MISS_NEEDLES, + "{replay}: only {near_misses_missing} of {NEAR_MISS_NEEDLES} near misses miss their source row" + ); + let matched: usize = oracle.iter().map(|(c, _)| c.len()).sum(); + let hit = oracle.iter().filter(|(c, _)| !c.is_empty()).count(); + + let show = |ids: &[i32]| -> String { + ids.iter() + .map(|id| format!("\n row {id}: {}", docs[*id as usize])) + .collect() + }; + let mut failures: Vec = Vec::new(); + let compare = |failures: &mut Vec, + what: &str, + needle: &Needle, + index: usize, + expected: &[i32], + encrypted: &[i32]| { + if expected != encrypted { + let only_jsonb: Vec = expected + .iter() + .copied() + .filter(|id| !encrypted.contains(id)) + .collect(); + let only_encrypted: Vec = encrypted + .iter() + .copied() + .filter(|id| !expected.contains(id)) + .collect(); + failures.push(format!( + "{what}, needle {index} ({}): {}\n jsonb: {expected:?}\n encrypted: {encrypted:?}\n only jsonb:{}\n only encrypted:{}", + needle.origin, + needle.value, + show(&only_jsonb), + show(&only_encrypted), + )); + } + }; + + let ids_of = |db: &mut postgres::Client, sql: &str, param: &str| -> Vec { + db.query(sql, &[¶m]) + .unwrap_or_else(|e| panic!("{replay}: {sql}: {e}")) + .iter() + .map(|r| r.get(0)) + .collect() + }; + let explain = |db: &mut postgres::Client, sql: &str, param: &str| -> String { + db.query(&format!("EXPLAIN {sql}"), &[¶m]) + .unwrap_or_else(|e| panic!("{replay}: EXPLAIN {sql}: {e}")) + .iter() + .map(|r| r.get::<_, String>(0)) + .collect::>() + .join("\n") + }; + const CONTAINS: &str = + "SELECT id FROM json_tree_generated WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree ORDER BY id"; + const CONTAINED_BY: &str = + "SELECT id FROM json_tree_generated WHERE doc <@ $1::text::jsonb::eql_v4.query_json_tree ORDER BY id"; + + // Both execution paths. The GIN index on the projection serves `@>` + // (`jsonb_path_ops` has no `<@` strategy, so `<@` filters a scan on both + // paths); every `@>` plan is checked to be the one the path names. + let mut excluded = 0; + for (path, settings) in [ + ("index", "SET enable_seqscan = off"), + ( + "seqscan", + "SET enable_seqscan = on; SET enable_bitmapscan = off; SET enable_indexscan = off; SET enable_indexonlyscan = off", + ), + ] { + db.batch_execute(settings).unwrap(); + for (index, (needle, (contains, contained_by))) in needles.iter().zip(&oracle).enumerate() { + let text = explain(&mut db, CONTAINS, &needle.contains); + let planned = match path { + "index" => text.contains("Bitmap Index Scan on json_tree_generated_doc"), + _ => text.contains("Seq Scan on json_tree_generated") && !text.contains("Index"), + }; + assert!( + planned, + "{replay}: `@>` for needle {index} on the {path} path planned otherwise:\n{text}" + ); + let encrypted = ids_of(&mut db, CONTAINS, &needle.contains); + compare(&mut failures, &format!("@> ({path})"), needle, index, contains, &encrypted); + + // `<@`: everything `jsonb` selects, less a stored root scalar + // when the needle is a root array, the one case the SQL + // documents it cannot answer. + let expected: Vec = contained_by + .iter() + .copied() + .filter(|id| { + let stored = &docs[*id as usize]; + !(needle.value.is_array() && !stored.is_object() && !stored.is_array()) + }) + .collect(); + if path == "index" { + excluded += contained_by.len() - expected.len(); + } + let encrypted = ids_of(&mut db, CONTAINED_BY, &needle.contained_by); + compare(&mut failures, &format!("<@ ({path})"), needle, index, &expected, &encrypted); + } + db.batch_execute("RESET ALL").unwrap(); + } + let compared_at = started.elapsed(); + + // `->` at generated key paths: the entry is there exactly where `jsonb` + // finds a node at the path (`#>` reads a key of an array as no node, as + // a strict path does), and a leaf opens to the value `jsonb` holds there, + // compared by PostgreSQL's `jsonb =`, which compares numbers by value. + let mut paths: Vec> = PATH_KEYS.iter().map(|k| vec![*k]).collect(); + for a in PATH_KEYS { + for b in PATH_KEYS { + paths.push(vec![*a, *b]); + } + } + let mut opened: Vec = Vec::new(); + let mut held: Vec = Vec::new(); + let mut at: Vec<(String, i32)> = Vec::new(); + let mut containers = 0; + for steps in &paths { + let path = format!("$.{}", steps.join(".")); + let selector = wait( + &runtime, + keyset.json_selector(&JsonPath::parse(&path).unwrap(), column()), + ) + .unwrap() + .to_string(); + let keys: Vec = steps.iter().map(|s| (*s).to_owned()).collect(); + for row in db + .query( + "SELECT id, (plain #> $2::text[])::text, jsonb_typeof(plain #> $2::text[]), (doc -> $1::text)::jsonb::text + FROM json_tree_generated ORDER BY id", + &[&selector, &keys], + ) + .unwrap() + { + let id: i32 = row.get(0); + let want: Option = row.get(1); + let kind: Option = row.get(2); + let got: Option = row.get(3); + match (want, got) { + (None, None) => {} + (Some(_), Some(_)) if matches!(kind.as_deref(), Some("object" | "array")) => { + containers += 1; + } + (Some(want), Some(got)) => { + let entry: JsonTreeEntry = serde_json::from_str(&got).unwrap(); + match wait(&runtime, cipher.decrypt_as::(entry, column())) { + Ok(value) => { + opened.push(value.to_string()); + held.push(want); + at.push((path.clone(), id)); + } + Err(error) => failures.push(format!( + "-> {path}, row {id}: the entry did not open: {error}\n stored: {}", + docs[id as usize] + )), + } + } + (want, got) => failures.push(format!( + "-> {path}, row {id}: jsonb has {want:?}, the encrypted column {}\n stored: {}", + if got.is_some() { "an entry" } else { "none" }, + docs[id as usize] + )), + } + } + } + let differing: Vec = db + .query( + "SELECT i FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS u(opened, held, i) + WHERE opened::jsonb IS DISTINCT FROM held::jsonb ORDER BY i", + &[&opened, &held], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + for i in differing { + let i = i as usize - 1; + let (path, id) = &at[i]; + failures.push(format!( + "-> {path}, row {id}: opened {}, jsonb holds {}\n stored: {}", + opened[i], held[i], docs[*id as usize] + )); + } + eprintln!( + "json_tree generated containment: {replay}; {} documents, {} needles ({RANDOM_NEEDLES} random, {EXTRACTED_NEEDLES} extracted, {NEAR_MISS_NEEDLES} near misses, {near_misses_missing} of them missing their source row); {hit} needles match some row, {matched} matches in all; {excluded} root scalars excluded from `<@`; `->` opened {} leaves and found {containers} containers at {} paths; stored in {stored_at:?}, containment compared in {:?}, total {:?}", + docs.len(), + needles.len(), + opened.len(), + paths.len(), + compared_at - stored_at, + started.elapsed(), + ); + assert!( + failures.is_empty(), + "{replay}: {} disagreements with jsonb; the first ten:\n{}", + failures.len(), + failures + .iter() + .take(10) + .cloned() + .collect::>() + .join("\n") + ); + assert!( + excluded > 0, + "{replay}: no stored root scalar sat inside a needle root array, so the `<@` exception went unexercised" + ); + assert!( + opened.len() > 50 && containers > 10, + "{replay}: the paths should reach many leaves and containers: {} leaves, {containers} containers", + opened.len() + ); +} + /// Needs an EQL v4 installation. Extraction by selector opens to the leaf; /// the subtree walk by a client-derived token returns exactly the nodes /// under it and reassembles; the array functions count and list elements; From dd9bc51d2512a26a3fccad783cdc0215a7068afc Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sat, 24 Oct 2026 11:53:49 +1100 Subject: [PATCH 2/3] test(eql): generate the json_tree containment cases with proptest The generated containment oracle drew its documents and needles from a hand-rolled SplitMix64 generator, one fixed batch per run, and could not shrink a failure. It now draws them from proptest strategies: a prop_recursive JSON value with the same coverage (nested objects and arrays, repeated keys, arrays of objects and of arrays, duplicate elements, numbers equal by value in different representations, numeric strings, booleans, null, empty containers, root scalars and arrays), and needles that are random values, prunings of a stored document, or such a pruning with one edit. A derived needle names its document by an Index and its pruning by a keep mask, so it follows the documents as they shrink and shrinks toward a smaller needle itself. A fault injected into the <@ exception shrinks to one stored 0 and the needle [0]. A case is a batch of documents and needles, checked in a few set-based statements: one correlated subquery per needle, which the planner serves from the GIN index when sequential scans are off. The table has no primary key, so the GIN index is the only one there is to take. Every case asserts with EXPLAIN that each plan is the one it asked for. The three properties (@> on both plans, <@ on both plans less its documented exception, and -> at generated key paths) are separate tests under given_generated_documents_in_postgres, and a database-free test holds the document strategy to every feature the census counts. The module lives in tests/json_tree/generated.rs. The RNG is ChaCha with seed 1336 and 64 cases by default. PROPTEST_RNG_SEED and PROPTEST_CASES override them, as for any proptest run, and both are logged. Failures persist to the crate's proptest-regressions/json_tree.txt through FileFailurePersistence::Direct, because SourceParallel resolves file!() from the working directory, which is not where this nested workspace member's paths are rooted. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- packages/eql/Cargo.lock | 1 + packages/eql/mise.toml | 5 +- packages/eql/tests/encryption/Cargo.toml | 4 + .../proptest-regressions/json_tree.txt | 6 + .../eql/tests/encryption/tests/json_tree.rs | 775 +----------- .../encryption/tests/json_tree/generated.rs | 1126 +++++++++++++++++ 6 files changed, 1149 insertions(+), 768 deletions(-) create mode 100644 packages/eql/tests/encryption/proptest-regressions/json_tree.txt create mode 100644 packages/eql/tests/encryption/tests/json_tree/generated.rs diff --git a/packages/eql/Cargo.lock b/packages/eql/Cargo.lock index 24b13d5f4..138a68c37 100644 --- a/packages/eql/Cargo.lock +++ b/packages/eql/Cargo.lock @@ -1251,6 +1251,7 @@ dependencies = [ "eql-bindings", "eql-domains", "postgres", + "proptest", "rust_decimal", "serde", "serde_json", diff --git a/packages/eql/mise.toml b/packages/eql/mise.toml index 756626154..79bea8ab0 100644 --- a/packages/eql/mise.toml +++ b/packages/eql/mise.toml @@ -344,8 +344,9 @@ for suite in text_eq v4_types v4_order v4_date_time_numeric v4_range v4_text_sea listing=$(cargo test --locked -p eql-encryption-tests --test "$suite" -- --ignored --list | sed -n 's/: test$//p') [ -n "$listing" ] || { echo "error: '--ignored' selects no test in $suite; the PostgreSQL test has gone missing." >&2; exit 1; } # `--show-output`: a passing test's output is otherwise captured, and the - # generated containment oracle in json_tree logs the seed it ran under - # (EQL_JSON_TREE_SEED replays it) and what it checked. + # generated containment properties in json_tree log the proptest seed and + # case count they ran under (PROPTEST_RNG_SEED and PROPTEST_CASES replay + # them) and what they checked. cargo test --locked -p eql-encryption-tests --test "$suite" -- --ignored --show-output done """ diff --git a/packages/eql/tests/encryption/Cargo.toml b/packages/eql/tests/encryption/Cargo.toml index 6ee6ec13c..c73a1c5ae 100644 --- a/packages/eql/tests/encryption/Cargo.toml +++ b/packages/eql/tests/encryption/Cargo.toml @@ -54,6 +54,10 @@ postgres = "0.19" # compared. # cipherstash-client = "=0.42.3" +# The generated containment oracle (tests/json_tree/generated.rs). The same +# requirement eql-domains and tests/sqlx carry, so the lockfile gains no +# second copy. +proptest = "1" # The tampering test decodes a stored body, changes one byte and re-encodes # it. The same line eql-bindings uses, so the lockfile gains no second copy. base64 = "0.22" diff --git a/packages/eql/tests/encryption/proptest-regressions/json_tree.txt b/packages/eql/tests/encryption/proptest-regressions/json_tree.txt new file mode 100644 index 000000000..341ba3ff3 --- /dev/null +++ b/packages/eql/tests/encryption/proptest-regressions/json_tree.txt @@ -0,0 +1,6 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index f905789fb..9b00f89b6 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -6,8 +6,9 @@ //! `@>` and `<@` over the encrypted column against PostgreSQL's own //! `jsonb @>` over the plaintexts, for the design's corpus and random //! documents, with a GIN index on the projection that the planner is shown -//! to use, and again for a few hundred generated documents and needles under -//! a logged seed (`EQL_JSON_TREE_SEED` replays one). +//! to use, and again as proptest properties over generated batches of +//! documents and needles (`json_tree/generated.rs`; `PROPTEST_RNG_SEED` +//! replays a run). mod common; @@ -440,770 +441,12 @@ fn postgres_containment_over_the_encrypted_column_is_jsonb_containment() { // PostgreSQL: generated documents // ============================================================================= -/// The environment variable that replaces the generated oracle's seed. -const SEED_ENV: &str = "EQL_JSON_TREE_SEED"; - -/// The seed the generated oracle runs under when [`SEED_ENV`] is unset. -/// Fixed, so every CI run checks the same documents; logged, so a failure -/// replays from the log. -const DEFAULT_SEED: u64 = 1336; - -/// How many documents are stored, and how many needles of each kind ask. -const STORED_DOCUMENTS: usize = 300; -const RANDOM_NEEDLES: usize = 100; -const EXTRACTED_NEEDLES: usize = 200; -const NEAR_MISS_NEEDLES: usize = 200; - -/// Numbers, several of them one value to `jsonb` in different -/// representations: `1`, `1.0` and `1e0`; `100` and `1e2` (an integer and -/// a float); `0`, `0.0` and `-0.0`; `0.5` and `5e-1`. A value is stored as -/// `serde_json` parsed it and PostgreSQL is sent `serde_json`'s spelling of -/// the same value, so each side sees one plaintext. -const NUMBERS: &[&str] = &[ - "0", - "0.0", - "-0.0", - "1", - "1.0", - "1e0", - "2", - "2.00", - "-1", - "-1.0", - "10", - "1e1", - "100", - "1e2", - "0.5", - "5e-1", - "1.5", - "123456789012", - "1e20", -]; - -/// Strings, some spelled like a number, a boolean or null, so that a kind -/// confused with another shows as a wrong answer. -const STRINGS: &[&str] = &["x", "y", "", "1", "1.0", "2", "true", "null", "a b", "ü"]; - -/// Object keys: few, so they repeat across levels and siblings. The paths -/// the `->` check asks at are built from the first four. -const KEYS: &[&str] = &["a", "b", "c", "x", "1", ""]; -const PATH_KEYS: &[&str] = &["a", "b", "c", "x"]; - -/// SplitMix64: a small, well-mixed deterministic generator, so a seed -/// names one run exactly on every platform. -struct Gen(u64); - -impl Gen { - fn next(&mut self) -> u64 { - self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); - let mut z = self.0; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); - z ^ (z >> 31) - } - - fn below(&mut self, n: usize) -> usize { - (self.next() % n as u64) as usize - } - - fn chance(&mut self, percent: usize) -> bool { - self.below(100) < percent - } - - fn pick<'a>(&mut self, items: &[&'a str]) -> &'a str { - items[self.below(items.len())] - } - - fn scalar(&mut self) -> Value { - match self.below(10) { - 0..=3 => serde_json::from_str(self.pick(NUMBERS)).unwrap(), - 4..=6 => Value::String(self.pick(STRINGS).to_owned()), - 7 | 8 => Value::Bool(self.chance(50)), - _ => Value::Null, - } - } - - /// A value with at most `depth` levels of containers below it. - fn value(&mut self, depth: u32) -> Value { - if depth == 0 { - return self.scalar(); - } - match self.below(10) { - 0..=3 => self.scalar(), - 4..=6 => self.object(depth), - _ => self.array(depth), - } - } - - /// An object of up to four members, empty at depth 0. A key drawn - /// twice keeps the later value, as `jsonb` does. - fn object(&mut self, depth: u32) -> Value { - let mut map = Map::new(); - if depth > 0 { - for _ in 0..self.below(5) { - let key = self.pick(KEYS).to_owned(); - let _ = map.insert(key, self.value(depth - 1)); - } - } - Value::Object(map) - } - - /// An array of up to four elements, empty at depth 0: sometimes all - /// objects, sometimes all arrays, otherwise mixed, and an element is - /// sometimes a copy of an earlier one. - fn array(&mut self, depth: u32) -> Value { - let mut items: Vec = Vec::new(); - if depth > 0 { - let shape = self.below(4); - for _ in 0..self.below(5) { - let item = if !items.is_empty() && self.chance(25) { - items[self.below(items.len())].clone() - } else { - match shape { - 0 => self.object(depth - 1), - 1 => self.array(depth - 1), - _ => self.value(depth - 1), - } - }; - items.push(item); - } - } - Value::Array(items) - } - - /// A stored document: mostly an object, sometimes a root array, and - /// sometimes a root scalar. - fn document(&mut self) -> Value { - match self.below(20) { - 0..=11 => self.object(4), - 12..=16 => self.array(3), - _ => self.scalar(), - } - } - - /// A sub-document `v` contains by construction: an object keeps some of - /// its members, an array some of its elements (in any order, and - /// repeated or not, since `jsonb` array containment ignores both), each - /// pruned in turn; a scalar is itself. - fn prune(&mut self, v: &Value) -> Value { - match v { - Value::Object(map) => { - let mut out = Map::new(); - for (key, member) in map { - if self.chance(60) { - let _ = out.insert(key.clone(), self.prune(member)); - } - } - Value::Object(out) - } - Value::Array(items) => { - let mut out = Vec::new(); - if !items.is_empty() { - for _ in 0..self.below(items.len() + 1) { - let item = &items[self.below(items.len())]; - out.push(self.prune(item)); - } - } - Value::Array(out) - } - scalar => scalar.clone(), - } - } - - /// A node somewhere inside `v`, `v` itself included. - fn node<'v>(&mut self, v: &'v Value) -> &'v Value { - if self.chance(40) { - return v; - } - match v { - Value::Object(map) if !map.is_empty() => { - let member = map.values().nth(self.below(map.len())).unwrap(); - self.node(member) - } - Value::Array(items) if !items.is_empty() => { - let item = &items[self.below(items.len())]; - self.node(item) - } - _ => v, - } - } - - /// A needle drawn from the stored document `doc`. Mostly a pruning of - /// the whole document, which `doc` contains by construction (`root`); - /// for a root array, sometimes one of its scalar elements bare, which - /// `jsonb` also counts as contained; otherwise a pruning of a node - /// inside it, which other rows may hold. - fn extract(&mut self, doc: &Value) -> (Value, bool) { - if let Value::Array(items) = doc { - let scalars: Vec<&Value> = items - .iter() - .filter(|v| !v.is_object() && !v.is_array()) - .collect(); - if !scalars.is_empty() && self.chance(25) { - return (scalars[self.below(scalars.len())].clone(), true); - } - } - if self.chance(70) { - (self.prune(doc), true) - } else { - let node = self.node(doc); - (self.prune(node), false) - } - } - - /// One edit at a node inside `v`: change a leaf, add a key, add an - /// element, or change a node's kind. - fn mutate(&mut self, v: &mut Value) { - if self.chance(60) { - match v { - Value::Object(map) if !map.is_empty() => { - let i = self.below(map.len()); - let member = map.values_mut().nth(i).unwrap(); - return self.mutate(member); - } - Value::Array(items) if !items.is_empty() => { - let i = self.below(items.len()); - return self.mutate(&mut items[i]); - } - _ => {} - } - } - match (self.below(4), &mut *v) { - (0, Value::Object(map)) => { - let key = self.pick(KEYS).to_owned(); - let _ = map.insert(key, self.value(1)); - } - (1, Value::Array(items)) => items.push(self.value(1)), - (2, scalar) if !scalar.is_object() && !scalar.is_array() => { - let mut other = self.scalar(); - while other == *scalar { - other = self.scalar(); - } - *scalar = other; - } - (_, Value::Object(map)) => *v = Value::Array(map.values().cloned().collect()), - (_, Value::Array(items)) => { - let key = self.pick(KEYS).to_owned(); - *v = json!({ key: items.first().cloned().unwrap_or(Value::Null) }); - } - (_, scalar) => { - *v = if self.chance(50) { - json!([scalar.clone()]) - } else { - json!({ self.pick(KEYS): scalar.clone() }) - }; - } - } - } -} - -/// The features the generator is meant to cover, counted over the stored -/// documents, so a change to it that stops producing one fails here -/// instead of quietly testing less. -#[derive(Debug, Default)] -struct Census { - root_objects: usize, - root_arrays: usize, - root_scalars: usize, - empty_objects: usize, - empty_arrays: usize, - arrays_of_objects: usize, - arrays_of_arrays: usize, - duplicate_elements: usize, - keys_on_two_levels: usize, - float_spelled_integers: usize, - numeric_strings: usize, - booleans: usize, - nulls: usize, - max_depth: usize, -} - -impl Census { - fn of(docs: &[Value]) -> Self { - let mut census = Census::default(); - for doc in docs { - match doc { - Value::Object(_) => census.root_objects += 1, - Value::Array(_) => census.root_arrays += 1, - _ => census.root_scalars += 1, - } - let mut keys = - std::collections::BTreeMap::<&str, std::collections::BTreeSet>::new(); - census.walk(doc, 0, &mut keys); - if keys.values().any(|depths| depths.len() > 1) { - census.keys_on_two_levels += 1; - } - } - census - } - - fn walk<'v>( - &mut self, - v: &'v Value, - depth: usize, - keys: &mut std::collections::BTreeMap<&'v str, std::collections::BTreeSet>, - ) { - self.max_depth = self.max_depth.max(depth); - match v { - Value::Object(map) => { - if map.is_empty() { - self.empty_objects += 1; - } - for (key, member) in map { - let _ = keys.entry(key).or_default().insert(depth); - self.walk(member, depth + 1, keys); - } - } - Value::Array(items) => { - if items.is_empty() { - self.empty_arrays += 1; - } - if items.iter().any(Value::is_object) { - self.arrays_of_objects += 1; - } - if items.iter().any(Value::is_array) { - self.arrays_of_arrays += 1; - } - if items - .iter() - .enumerate() - .any(|(i, a)| items[i + 1..].contains(a)) - { - self.duplicate_elements += 1; - } - for item in items { - self.walk(item, depth + 1, keys); - } - } - Value::Number(n) => { - if n.is_f64() && n.as_f64().is_some_and(|f| f.fract() == 0.0) { - self.float_spelled_integers += 1; - } - } - Value::String(s) => { - if s.parse::().is_ok() { - self.numeric_strings += 1; - } - } - Value::Bool(_) => self.booleans += 1, - Value::Null => self.nulls += 1, - } - } - - fn assert_covers_the_generator(&self) { - let counts = [ - ("root objects", self.root_objects), - ("root arrays", self.root_arrays), - ("root scalars", self.root_scalars), - ("empty objects", self.empty_objects), - ("empty arrays", self.empty_arrays), - ("arrays of objects", self.arrays_of_objects), - ("arrays of arrays", self.arrays_of_arrays), - ("duplicate array elements", self.duplicate_elements), - ("a key on two levels", self.keys_on_two_levels), - ("integers spelled as floats", self.float_spelled_integers), - ("strings spelled as numbers", self.numeric_strings), - ("booleans", self.booleans), - ("nulls", self.nulls), - ]; - for (what, count) in counts { - assert!( - count > 0, - "the generated documents hold no {what}: {self:?}" - ); - } - assert!(self.max_depth >= 4, "nested a few levels: {self:?}"); - } -} - -/// Where a needle came from, for the failure message. -#[derive(Clone, Copy)] -enum Origin { - Random, - /// A pruning of the stored document's root, so that row holds it. - Contained(usize), - /// A pruning of a node inside the stored document. - Inside(usize), - /// A contained needle with one edit. - NearMiss(usize), -} - -impl std::fmt::Display for Origin { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Origin::Random => write!(f, "random"), - Origin::Contained(row) => write!(f, "pruned from row {row}"), - Origin::Inside(row) => write!(f, "pruned from a node of row {row}"), - Origin::NearMiss(row) => write!(f, "near miss of row {row}"), - } - } -} - -struct Needle { - origin: Origin, - value: Value, - contains: String, - contained_by: String, -} - -/// Needs an EQL v4 installation. The property behind the fixed corpus -/// above, over generated documents: a few hundred stored encrypted beside -/// their plaintexts, and several hundred needles (random values, -/// sub-documents of stored documents, and those sub-documents with one -/// edit). For every needle, the rows the encrypted `@>` and `<@` select are -/// the rows PostgreSQL's own `jsonb` operators select over the plaintexts, -/// once with the planner held to the GIN index and once with every index -/// path turned off, each plan confirmed by `EXPLAIN`. `<@` differs from -/// `jsonb` only where the SQL documents it does, a stored root scalar -/// inside a needle root array, and the test asserts that is the whole -/// difference. Then `->` at generated key paths: the entry is there exactly -/// where `jsonb #>` finds a node, and a leaf opens to the value `jsonb` -/// holds there. -/// -/// The seed is fixed and logged; `EQL_JSON_TREE_SEED` replaces it. A failure -/// names the seed, the needle and each stored document the two answers -/// disagree on. -#[test] -#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] -fn postgres_containment_over_generated_documents_is_jsonb_containment() { - let seed = match std::env::var(SEED_ENV) { - Ok(text) => text - .trim() - .parse::() - .unwrap_or_else(|e| panic!("{SEED_ENV}={text:?}: {e}")), - Err(_) => DEFAULT_SEED, - }; - let replay = format!("seed {seed}; replay with {SEED_ENV}={seed}"); - eprintln!("json_tree generated containment: {replay}"); - let started = std::time::Instant::now(); - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let (cipher, _) = wait(&runtime, common::cipher()); - let keyset = cipher.default_keyset(); - let plan = plan(); - let mut db = connect(); - let mut rng = Gen(seed); - - // The stored documents, sealed, then written in one statement. - let docs: Vec = (0..STORED_DOCUMENTS).map(|_| rng.document()).collect(); - let census = Census::of(&docs); - census.assert_covers_the_generator(); - let ids: Vec = (0..docs.len() as i32).collect(); - let plains: Vec = docs.iter().map(Value::to_string).collect(); - let sealed: Vec = docs - .iter() - .map(|doc| serde_json::to_string(&wait(&runtime, seal(&cipher, doc))).unwrap()) - .collect(); - db.batch_execute( - "CREATE TEMP TABLE json_tree_generated (id int PRIMARY KEY, plain jsonb NOT NULL, doc public.eql_v4_json_tree NOT NULL)", - ) - .unwrap(); - db.execute( - "INSERT INTO json_tree_generated - SELECT id, plain::jsonb, doc::jsonb::public.eql_v4_json_tree - FROM unnest($1::int[], $2::text[], $3::text[]) AS u(id, plain, doc)", - &[&ids, &plains, &sealed], - ) - .unwrap(); - db.batch_execute( - "CREATE INDEX json_tree_generated_doc ON json_tree_generated USING gin ((eql_v4.json_tree_query(doc)) jsonb_path_ops); - ANALYZE json_tree_generated;", - ) - .unwrap(); - let stored_at = started.elapsed(); - - // The needles, each with both operands. - let mut drawn: Vec<(Origin, Value)> = Vec::new(); - for _ in 0..RANDOM_NEEDLES { - drawn.push((Origin::Random, rng.value(2))); - } - for _ in 0..EXTRACTED_NEEDLES { - let from = rng.below(docs.len()); - let (value, root) = rng.extract(&docs[from]); - let origin = if root { - Origin::Contained(from) - } else { - Origin::Inside(from) - }; - drawn.push((origin, value)); - } - for _ in 0..NEAR_MISS_NEEDLES { - let from = rng.below(docs.len()); - let mut value = rng.prune(&docs[from]); - rng.mutate(&mut value); - drawn.push((Origin::NearMiss(from), value)); - } - let operand = |query| serde_json::to_string(&JsonTreeQuery::try_from(&query).unwrap()).unwrap(); - let needles: Vec = drawn - .into_iter() - .map(|(origin, value)| Needle { - contains: operand( - wait(&runtime, cipher.query(op::Contains, &value).on(&plan)).unwrap(), - ), - contained_by: operand( - wait(&runtime, cipher.query(op::ContainedBy, &value).on(&plan)).unwrap(), - ), - origin, - value, - }) - .collect(); - - // The oracle: PostgreSQL's `jsonb @>` and `<@` over the plaintexts, for - // every needle, in one statement. - let plain_needles: Vec = needles.iter().map(|n| n.value.to_string()).collect(); - let oracle: Vec<(Vec, Vec)> = db - .query( - "SELECT coalesce(array_agg(g.id ORDER BY g.id) FILTER (WHERE g.plain @> n.v), '{}'), - coalesce(array_agg(g.id ORDER BY g.id) FILTER (WHERE g.plain <@ n.v), '{}') - FROM (SELECT i, t::jsonb AS v FROM unnest($1::text[]) WITH ORDINALITY AS u(t, i)) AS n - CROSS JOIN json_tree_generated AS g - GROUP BY n.i ORDER BY n.i", - &[&plain_needles], - ) - .unwrap() - .iter() - .map(|r| (r.get(0), r.get(1))) - .collect(); - assert_eq!(oracle.len(), needles.len()); - - // The generator's own claims, checked by the oracle: a pruned root is - // contained by the row it came from, and the edits miss often enough - // to be near misses rather than more hits. - let mut near_misses_missing = 0; - for (needle, (contains, _)) in needles.iter().zip(&oracle) { - match needle.origin { - Origin::Contained(from) => assert!( - contains.contains(&(from as i32)), - "{replay}: row {from} should contain its own sub-document {}\nstored: {}", - needle.value, - docs[from] - ), - Origin::NearMiss(from) if !contains.contains(&(from as i32)) => { - near_misses_missing += 1; - } - _ => {} - } - } - assert!( - near_misses_missing * 4 >= NEAR_MISS_NEEDLES, - "{replay}: only {near_misses_missing} of {NEAR_MISS_NEEDLES} near misses miss their source row" - ); - let matched: usize = oracle.iter().map(|(c, _)| c.len()).sum(); - let hit = oracle.iter().filter(|(c, _)| !c.is_empty()).count(); - - let show = |ids: &[i32]| -> String { - ids.iter() - .map(|id| format!("\n row {id}: {}", docs[*id as usize])) - .collect() - }; - let mut failures: Vec = Vec::new(); - let compare = |failures: &mut Vec, - what: &str, - needle: &Needle, - index: usize, - expected: &[i32], - encrypted: &[i32]| { - if expected != encrypted { - let only_jsonb: Vec = expected - .iter() - .copied() - .filter(|id| !encrypted.contains(id)) - .collect(); - let only_encrypted: Vec = encrypted - .iter() - .copied() - .filter(|id| !expected.contains(id)) - .collect(); - failures.push(format!( - "{what}, needle {index} ({}): {}\n jsonb: {expected:?}\n encrypted: {encrypted:?}\n only jsonb:{}\n only encrypted:{}", - needle.origin, - needle.value, - show(&only_jsonb), - show(&only_encrypted), - )); - } - }; - - let ids_of = |db: &mut postgres::Client, sql: &str, param: &str| -> Vec { - db.query(sql, &[¶m]) - .unwrap_or_else(|e| panic!("{replay}: {sql}: {e}")) - .iter() - .map(|r| r.get(0)) - .collect() - }; - let explain = |db: &mut postgres::Client, sql: &str, param: &str| -> String { - db.query(&format!("EXPLAIN {sql}"), &[¶m]) - .unwrap_or_else(|e| panic!("{replay}: EXPLAIN {sql}: {e}")) - .iter() - .map(|r| r.get::<_, String>(0)) - .collect::>() - .join("\n") - }; - const CONTAINS: &str = - "SELECT id FROM json_tree_generated WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree ORDER BY id"; - const CONTAINED_BY: &str = - "SELECT id FROM json_tree_generated WHERE doc <@ $1::text::jsonb::eql_v4.query_json_tree ORDER BY id"; - - // Both execution paths. The GIN index on the projection serves `@>` - // (`jsonb_path_ops` has no `<@` strategy, so `<@` filters a scan on both - // paths); every `@>` plan is checked to be the one the path names. - let mut excluded = 0; - for (path, settings) in [ - ("index", "SET enable_seqscan = off"), - ( - "seqscan", - "SET enable_seqscan = on; SET enable_bitmapscan = off; SET enable_indexscan = off; SET enable_indexonlyscan = off", - ), - ] { - db.batch_execute(settings).unwrap(); - for (index, (needle, (contains, contained_by))) in needles.iter().zip(&oracle).enumerate() { - let text = explain(&mut db, CONTAINS, &needle.contains); - let planned = match path { - "index" => text.contains("Bitmap Index Scan on json_tree_generated_doc"), - _ => text.contains("Seq Scan on json_tree_generated") && !text.contains("Index"), - }; - assert!( - planned, - "{replay}: `@>` for needle {index} on the {path} path planned otherwise:\n{text}" - ); - let encrypted = ids_of(&mut db, CONTAINS, &needle.contains); - compare(&mut failures, &format!("@> ({path})"), needle, index, contains, &encrypted); - - // `<@`: everything `jsonb` selects, less a stored root scalar - // when the needle is a root array, the one case the SQL - // documents it cannot answer. - let expected: Vec = contained_by - .iter() - .copied() - .filter(|id| { - let stored = &docs[*id as usize]; - !(needle.value.is_array() && !stored.is_object() && !stored.is_array()) - }) - .collect(); - if path == "index" { - excluded += contained_by.len() - expected.len(); - } - let encrypted = ids_of(&mut db, CONTAINED_BY, &needle.contained_by); - compare(&mut failures, &format!("<@ ({path})"), needle, index, &expected, &encrypted); - } - db.batch_execute("RESET ALL").unwrap(); - } - let compared_at = started.elapsed(); - - // `->` at generated key paths: the entry is there exactly where `jsonb` - // finds a node at the path (`#>` reads a key of an array as no node, as - // a strict path does), and a leaf opens to the value `jsonb` holds there, - // compared by PostgreSQL's `jsonb =`, which compares numbers by value. - let mut paths: Vec> = PATH_KEYS.iter().map(|k| vec![*k]).collect(); - for a in PATH_KEYS { - for b in PATH_KEYS { - paths.push(vec![*a, *b]); - } - } - let mut opened: Vec = Vec::new(); - let mut held: Vec = Vec::new(); - let mut at: Vec<(String, i32)> = Vec::new(); - let mut containers = 0; - for steps in &paths { - let path = format!("$.{}", steps.join(".")); - let selector = wait( - &runtime, - keyset.json_selector(&JsonPath::parse(&path).unwrap(), column()), - ) - .unwrap() - .to_string(); - let keys: Vec = steps.iter().map(|s| (*s).to_owned()).collect(); - for row in db - .query( - "SELECT id, (plain #> $2::text[])::text, jsonb_typeof(plain #> $2::text[]), (doc -> $1::text)::jsonb::text - FROM json_tree_generated ORDER BY id", - &[&selector, &keys], - ) - .unwrap() - { - let id: i32 = row.get(0); - let want: Option = row.get(1); - let kind: Option = row.get(2); - let got: Option = row.get(3); - match (want, got) { - (None, None) => {} - (Some(_), Some(_)) if matches!(kind.as_deref(), Some("object" | "array")) => { - containers += 1; - } - (Some(want), Some(got)) => { - let entry: JsonTreeEntry = serde_json::from_str(&got).unwrap(); - match wait(&runtime, cipher.decrypt_as::(entry, column())) { - Ok(value) => { - opened.push(value.to_string()); - held.push(want); - at.push((path.clone(), id)); - } - Err(error) => failures.push(format!( - "-> {path}, row {id}: the entry did not open: {error}\n stored: {}", - docs[id as usize] - )), - } - } - (want, got) => failures.push(format!( - "-> {path}, row {id}: jsonb has {want:?}, the encrypted column {}\n stored: {}", - if got.is_some() { "an entry" } else { "none" }, - docs[id as usize] - )), - } - } - } - let differing: Vec = db - .query( - "SELECT i FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS u(opened, held, i) - WHERE opened::jsonb IS DISTINCT FROM held::jsonb ORDER BY i", - &[&opened, &held], - ) - .unwrap() - .iter() - .map(|r| r.get(0)) - .collect(); - for i in differing { - let i = i as usize - 1; - let (path, id) = &at[i]; - failures.push(format!( - "-> {path}, row {id}: opened {}, jsonb holds {}\n stored: {}", - opened[i], held[i], docs[*id as usize] - )); - } - eprintln!( - "json_tree generated containment: {replay}; {} documents, {} needles ({RANDOM_NEEDLES} random, {EXTRACTED_NEEDLES} extracted, {NEAR_MISS_NEEDLES} near misses, {near_misses_missing} of them missing their source row); {hit} needles match some row, {matched} matches in all; {excluded} root scalars excluded from `<@`; `->` opened {} leaves and found {containers} containers at {} paths; stored in {stored_at:?}, containment compared in {:?}, total {:?}", - docs.len(), - needles.len(), - opened.len(), - paths.len(), - compared_at - stored_at, - started.elapsed(), - ); - assert!( - failures.is_empty(), - "{replay}: {} disagreements with jsonb; the first ten:\n{}", - failures.len(), - failures - .iter() - .take(10) - .cloned() - .collect::>() - .join("\n") - ); - assert!( - excluded > 0, - "{replay}: no stored root scalar sat inside a needle root array, so the `<@` exception went unexercised" - ); - assert!( - opened.len() > 50 && containers > 10, - "{replay}: the paths should reach many leaves and containers: {} leaves, {containers} containers", - opened.len() - ); -} +/// The containment oracle again, over documents and needles proptest +/// generates. A crate root's modules resolve beside it, and a file there would +/// be a test target of its own, hence the path. +#[allow(clippy::unwrap_used)] +#[path = "json_tree/generated.rs"] +mod generated; /// Needs an EQL v4 installation. Extraction by selector opens to the leaf; /// the subtree walk by a client-derived token returns exactly the nodes diff --git a/packages/eql/tests/encryption/tests/json_tree/generated.rs b/packages/eql/tests/encryption/tests/json_tree/generated.rs new file mode 100644 index 000000000..192e0d7e5 --- /dev/null +++ b/packages/eql/tests/encryption/tests/json_tree/generated.rs @@ -0,0 +1,1126 @@ +//! The containment oracle over generated documents. proptest draws a case: a +//! batch of stored documents and a batch of needles (random values, values +//! derived from a stored document, and those with one edit). The documents +//! are stored encrypted beside their plaintexts, and every needle is asked +//! of both columns in a few set-based statements, so PostgreSQL's own +//! `jsonb` operators are the oracle and a case costs a handful of round +//! trips, not one per needle. +//! +//! **Replaying.** The RNG is ChaCha with a fixed seed ([`DEFAULT_SEED`]), +//! logged on every run. `PROPTEST_RNG_SEED` replaces the seed and +//! `PROPTEST_CASES` the case count ([`DEFAULT_CASES`]), as for any proptest +//! run. A failure shrinks to a small case and prints it, and proptest +//! appends its seed to `proptest-regressions/json_tree.txt` in this crate, +//! which every later run replays first. Commit that line with the fix. + +use std::cell::{Cell, RefCell}; +use std::fmt; + +use proptest::collection::vec as vec_of; +use proptest::prelude::*; +use proptest::sample::{select, Index}; +use proptest::strategy::ValueTree; +use proptest::test_runner::{ + Config, FileFailurePersistence, RngAlgorithm, RngSeed, TestCaseError, TestRunner, +}; + +use super::*; + +/// The seed the run uses when `PROPTEST_RNG_SEED` is unset. Fixed, so every +/// CI run checks the same cases. +const DEFAULT_SEED: u64 = 1336; + +/// The cases a run checks when `PROPTEST_CASES` is unset. +const DEFAULT_CASES: u32 = 64; + +/// The largest batches a case holds. +const MAX_DOCUMENTS: usize = 64; +const MAX_NEEDLES: usize = 96; + +/// Where proptest records a failing case's seed. `Direct` with the crate's +/// own path: the default (`SourceParallel`) resolves the test's source file +/// from the current directory, which for this nested workspace member is not +/// where `file!()` is relative to. +const REGRESSIONS: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/proptest-regressions/json_tree.txt" +); + +/// Numbers, several of them one value to `jsonb` in different +/// representations: `1`, `1.0` and `1e0`; `100` and `1e2` (an integer and a +/// float); `0`, `0.0` and `-0.0`; `0.5` and `5e-1`. A value is stored as +/// `serde_json` parsed it and PostgreSQL is sent `serde_json`'s spelling of +/// the same value, so each side sees one plaintext. +const NUMBERS: &[&str] = &[ + "0", + "0.0", + "-0.0", + "1", + "1.0", + "1e0", + "2", + "2.00", + "-1", + "-1.0", + "10", + "1e1", + "100", + "1e2", + "0.5", + "5e-1", + "1.5", + "123456789012", + "1e20", +]; + +/// Strings, some spelled like a number, a boolean or null, so that a kind +/// confused with another shows as a wrong answer. +const STRINGS: &[&str] = &["x", "y", "", "1", "1.0", "2", "true", "null", "a b", "ü"]; + +/// Object keys: few, so they repeat across levels and siblings. The `->` +/// paths are built from the first four. +const KEYS: &[&str] = &["a", "b", "c", "x", "1", ""]; +const PATH_KEYS: &[&str] = &["a", "b", "c", "x"]; + +/// The run's configuration: proptest's defaults, which read +/// `PROPTEST_CASES` and `PROPTEST_RNG_SEED`, with this test's own default +/// for each one left unset. +fn config() -> Config { + let mut config = Config::default(); + if std::env::var_os("PROPTEST_CASES").is_none() { + config.cases = DEFAULT_CASES; + } + if config.rng_seed == RngSeed::Random { + config.rng_seed = RngSeed::Fixed(DEFAULT_SEED); + } + config.rng_algorithm = RngAlgorithm::ChaCha; + config.failure_persistence = Some(Box::new(FileFailurePersistence::Direct(REGRESSIONS))); + // A shrink step is a round trip to the database: bound the search. + config.max_shrink_time = 120_000; + config +} + +/// How to replay the run, for the log and every failure. +fn replay(config: &Config) -> String { + format!( + "seed {seed}, {cases} cases; replay with PROPTEST_RNG_SEED={seed} PROPTEST_CASES={cases}", + seed = config.rng_seed, + cases = config.cases + ) +} + +// ============================================================================= +// The generated values +// ============================================================================= + +/// One edit that turns a contained needle into a near miss. `at` picks the +/// node, in document order; an edit that does not fit the node's kind +/// changes its kind instead. +#[derive(Clone, Debug)] +enum Edit { + Leaf { + at: Index, + to: Value, + }, + Key { + at: Index, + key: String, + value: Value, + }, + Element { + at: Index, + value: Value, + }, + Kind { + at: Index, + }, +} + +/// A needle as proptest draws it: derived ones name their stored document +/// by an [`Index`] into the case's documents, so they follow the documents +/// as those shrink. `keep` says, one member or element at a time in +/// document order, what a pruning keeps; once it runs out, nothing more is +/// kept, so shrinking it makes the needle smaller. +#[derive(Clone, Debug)] +enum NeedleSpec { + Random(Value), + Pruned { + from: Index, + keep: Vec, + at: Option, + }, + NearMiss { + from: Index, + keep: Vec, + edit: Edit, + }, +} + +/// Where a needle came from, for a failure message. +#[derive(Clone, Copy)] +enum Origin { + Random, + /// A pruning of the row's whole document: the row contains it. + PrunedRoot(usize), + /// A pruning of a node inside the row's document. + PrunedNode(usize), + NearMiss(usize), +} + +impl fmt::Display for Origin { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Origin::Random => write!(f, "random"), + Origin::PrunedRoot(row) => write!(f, "pruned from row {row}"), + Origin::PrunedNode(row) => write!(f, "pruned from a node of row {row}"), + Origin::NearMiss(row) => write!(f, "near miss of row {row}"), + } + } +} + +struct Needle { + origin: Origin, + value: Value, +} + +/// One generated case: the documents stored, and the needles asked. +#[derive(Clone)] +struct Case { + docs: Vec, + needles: Vec, +} + +impl Case { + /// The needles as values, each derived from the case's documents. + fn needles(&self) -> Vec { + self.needles + .iter() + .map(|spec| match spec { + NeedleSpec::Random(value) => Needle { + origin: Origin::Random, + value: value.clone(), + }, + NeedleSpec::Pruned { from, keep, at } => { + let row = from.index(self.docs.len()); + let doc = &self.docs[row]; + let pointer = at.as_ref().map_or_else(String::new, |at| { + let pointers = pointers(doc); + at.get(&pointers).clone() + }); + let node = doc.pointer(&pointer).unwrap(); + Needle { + origin: if pointer.is_empty() { + Origin::PrunedRoot(row) + } else { + Origin::PrunedNode(row) + }, + value: prune(node, &mut keep.iter()), + } + } + NeedleSpec::NearMiss { from, keep, edit } => { + let row = from.index(self.docs.len()); + let mut value = prune(&self.docs[row], &mut keep.iter()); + apply(edit, &mut value); + Needle { + origin: Origin::NearMiss(row), + value, + } + } + }) + .collect() + } +} + +/// Printed as JSON, one document and one needle per line: what proptest +/// shows for the minimal failing case. +impl fmt::Debug for Case { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "{} stored documents:", self.docs.len())?; + for (row, doc) in self.docs.iter().enumerate() { + writeln!(f, " row {row}: {doc}")?; + } + writeln!(f, "{} needles:", self.needles.len())?; + for (i, needle) in self.needles().iter().enumerate() { + writeln!(f, " needle {i} ({}): {}", needle.origin, needle.value)?; + } + Ok(()) + } +} + +/// A sub-document `v` contains: an object keeps the members, and an array +/// the elements, that `keep` says to keep, each pruned in turn; a scalar is +/// itself. +fn prune(v: &Value, keep: &mut std::slice::Iter<'_, bool>) -> Value { + match v { + Value::Object(map) => { + let mut out = Map::new(); + for (key, member) in map { + if keep.next().copied().unwrap_or(false) { + let _ = out.insert(key.clone(), prune(member, keep)); + } + } + Value::Object(out) + } + Value::Array(items) => { + let mut out = Vec::new(); + for item in items { + if keep.next().copied().unwrap_or(false) { + out.push(prune(item, keep)); + } + } + Value::Array(out) + } + scalar => scalar.clone(), + } +} + +/// Every node's JSON pointer, in document order, the root (`""`) first. +fn pointers(v: &Value) -> Vec { + fn walk(v: &Value, at: String, out: &mut Vec) { + out.push(at.clone()); + match v { + Value::Object(map) => { + for (key, member) in map { + let key = key.replace('~', "~0").replace('/', "~1"); + walk(member, format!("{at}/{key}"), out); + } + } + Value::Array(items) => { + for (i, item) in items.iter().enumerate() { + walk(item, format!("{at}/{i}"), out); + } + } + _ => {} + } + } + let mut out = Vec::new(); + walk(v, String::new(), &mut out); + out +} + +fn apply(edit: &Edit, needle: &mut Value) { + let at = match edit { + Edit::Leaf { at, .. } + | Edit::Key { at, .. } + | Edit::Element { at, .. } + | Edit::Kind { at } => at, + }; + let pointer = at.get(&pointers(needle)).clone(); + let node = needle.pointer_mut(&pointer).unwrap(); + match edit { + Edit::Leaf { to, .. } if !node.is_object() && !node.is_array() && node != to => { + *node = to.clone(); + } + Edit::Key { key, value, .. } if node.is_object() => { + let _ = node + .as_object_mut() + .unwrap() + .insert(key.clone(), value.clone()); + } + Edit::Element { value, .. } if node.is_array() => { + node.as_array_mut().unwrap().push(value.clone()); + } + _ => { + *node = match node.take() { + Value::Object(map) => Value::Array(map.into_iter().map(|(_, v)| v).collect()), + Value::Array(items) => { + json!({ "a": items.into_iter().next().unwrap_or(Value::Null) }) + } + scalar => Value::Array(vec![scalar]), + }; + } + } +} + +mod strategies { + use super::*; + + pub(super) fn scalar() -> BoxedStrategy { + prop_oneof![ + 4 => select(NUMBERS).prop_map(|text| serde_json::from_str::(text).unwrap()), + 3 => select(STRINGS).prop_map(|text| Value::String(text.to_owned())), + 2 => any::().prop_map(Value::Bool), + 1 => Just(Value::Null), + ] + .boxed() + } + + fn key() -> BoxedStrategy { + select(KEYS).prop_map(str::to_owned).boxed() + } + + /// An object of up to four members. A key drawn twice keeps its later + /// value, as `jsonb` does. + fn object_of(inner: BoxedStrategy) -> BoxedStrategy { + vec_of((key(), inner), 0..5) + .prop_map(|members| Value::Object(members.into_iter().collect())) + .boxed() + } + + /// An array: of anything, of objects, of arrays, or with some elements + /// repeated. + fn array_of(inner: BoxedStrategy) -> BoxedStrategy { + prop_oneof![ + 3 => vec_of(inner.clone(), 0..5).prop_map(Value::Array), + 1 => vec_of(object_of(inner.clone()), 1..4).prop_map(Value::Array), + 1 => vec_of(vec_of(inner.clone(), 0..3).prop_map(Value::Array), 1..4) + .prop_map(Value::Array), + 1 => (vec_of(inner, 1..4), vec_of(any::(), 1..3)).prop_map( + |(mut items, repeats)| { + for repeat in repeats { + let copy = repeat.get(&items).clone(); + items.push(copy); + } + Value::Array(items) + } + ), + ] + .boxed() + } + + /// A value with up to `depth` levels of recursion below it. + pub(super) fn value(depth: u32) -> BoxedStrategy { + scalar() + .prop_recursive(depth, 48, 4, |inner| { + prop_oneof![object_of(inner.clone()), array_of(inner)] + }) + .boxed() + } + + /// A stored document: mostly an object, sometimes a root array, and + /// sometimes a root scalar. + pub(super) fn document() -> BoxedStrategy { + prop_oneof![ + 6 => object_of(value(3)), + 3 => array_of(value(2)), + 1 => scalar(), + ] + .boxed() + } + + fn keep() -> BoxedStrategy> { + vec_of(proptest::bool::weighted(0.7), 0..32).boxed() + } + + fn edit() -> BoxedStrategy { + prop_oneof![ + (any::(), scalar()).prop_map(|(at, to)| Edit::Leaf { at, to }), + (any::(), key(), value(1)).prop_map(|(at, key, value)| Edit::Key { + at, + key, + value + }), + (any::(), value(1)).prop_map(|(at, value)| Edit::Element { at, value }), + any::().prop_map(|at| Edit::Kind { at }), + ] + .boxed() + } + + fn needle() -> BoxedStrategy { + prop_oneof![ + 1 => value(2).prop_map(NeedleSpec::Random), + 2 => (any::(), keep(), proptest::option::weighted(0.3, any::())) + .prop_map(|(from, keep, at)| NeedleSpec::Pruned { from, keep, at }), + 2 => (any::(), keep(), edit()) + .prop_map(|(from, keep, edit)| NeedleSpec::NearMiss { from, keep, edit }), + ] + .boxed() + } + + pub(super) fn case() -> BoxedStrategy { + ( + vec_of(document(), 1..=MAX_DOCUMENTS), + vec_of(needle(), 1..=MAX_NEEDLES), + ) + .prop_map(|(docs, needles)| Case { docs, needles }) + .boxed() + } +} + +// ============================================================================= +// The database +// ============================================================================= + +/// The two execution paths every containment answer is asked on. +#[derive(Clone, Copy, Debug)] +enum Scan { + /// Sequential scans off: the planner takes the GIN index where it can. + Index, + /// Every index path off. + Sequential, +} + +impl Scan { + const ALL: [Scan; 2] = [Scan::Index, Scan::Sequential]; + + fn settings(self) -> &'static str { + match self { + Scan::Index => "SET enable_seqscan = off", + Scan::Sequential => { + "SET enable_seqscan = on; SET enable_bitmapscan = off; SET enable_indexscan = off; SET enable_indexonlyscan = off" + } + } + } +} + +/// The containment operators, with the SQL operator each spells. +#[derive(Clone, Copy, Debug)] +enum Containment { + Contains, + ContainedBy, +} + +impl Containment { + fn sql(self) -> &'static str { + match self { + Containment::Contains => "@>", + Containment::ContainedBy => "<@", + } + } +} + +/// One database session, cipher and runtime, built once for a whole run: +/// proptest's runner is synchronous, so the async work is driven on this +/// runtime. +struct Scenario { + runtime: tokio::runtime::Runtime, + cipher: Cipher, + plan: ValuePlan>, + db: RefCell, +} + +fn scenario() -> Scenario { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let mut db = connect(); + // No primary key: the GIN index is the table's only index, so with + // sequential scans off the planner has no other index to walk whole + // and filter. + db.batch_execute( + "CREATE TEMP TABLE json_tree_generated (id int NOT NULL, plain jsonb NOT NULL, doc public.eql_v4_json_tree NOT NULL); + CREATE INDEX json_tree_generated_doc ON json_tree_generated USING gin ((eql_v4.json_tree_query(doc)) jsonb_path_ops);", + ) + .unwrap(); + Scenario { + runtime, + cipher, + plan: plan(), + db: RefCell::new(db), + } +} + +impl Scenario { + /// Replace the table's rows with the documents, each sealed beside its + /// plaintext, in one statement. Row `i` is document `i`. + fn store(&self, docs: &[Value]) { + let ids: Vec = (0..docs.len() as i32).collect(); + let plains: Vec = docs.iter().map(Value::to_string).collect(); + let sealed: Vec = docs + .iter() + .map(|doc| { + serde_json::to_string(&wait(&self.runtime, seal(&self.cipher, doc))).unwrap() + }) + .collect(); + let mut db = self.db.borrow_mut(); + db.batch_execute("TRUNCATE json_tree_generated").unwrap(); + let _ = db + .execute( + "INSERT INTO json_tree_generated + SELECT id, plain::jsonb, doc::jsonb::public.eql_v4_json_tree + FROM unnest($1::int[], $2::text[], $3::text[]) AS u(id, plain, doc)", + &[&ids, &plains, &sealed], + ) + .unwrap(); + db.batch_execute("ANALYZE json_tree_generated").unwrap(); + } + + /// Each needle's operand for `containment`, as the SQL takes it. + fn operands(&self, containment: Containment, needles: &[Needle]) -> Vec { + needles + .iter() + .map(|needle| { + let query = match containment { + Containment::Contains => wait( + &self.runtime, + self.cipher + .query(op::Contains, &needle.value) + .on(&self.plan), + ), + Containment::ContainedBy => wait( + &self.runtime, + self.cipher + .query(op::ContainedBy, &needle.value) + .on(&self.plan), + ), + } + .unwrap(); + serde_json::to_string(&JsonTreeQuery::try_from(&query).unwrap()).unwrap() + }) + .collect() + } + + /// For each needle, the ids of the rows that `jsonb` selects over the + /// plaintexts: the oracle. + fn jsonb(&self, containment: Containment, needles: &[Needle]) -> Vec> { + let texts: Vec = needles.iter().map(|n| n.value.to_string()).collect(); + self.rows_per_needle(&format!("g.plain {} n.t::jsonb", containment.sql()), &texts) + } + + /// For each operand, the ids of the rows the encrypted column selects, + /// and the plan the statement ran under. + fn encrypted( + &self, + containment: Containment, + scan: Scan, + operands: &[String], + ) -> (String, Vec>) { + let predicate = format!( + "g.doc {} n.t::jsonb::eql_v4.query_json_tree", + containment.sql() + ); + self.db.borrow_mut().batch_execute(scan.settings()).unwrap(); + let explain = self + .db + .borrow_mut() + .query( + &format!("EXPLAIN {}", Self::per_needle(&predicate)), + &[&operands], + ) + .unwrap() + .iter() + .map(|r| r.get::<_, String>(0)) + .collect::>() + .join("\n"); + let rows = self.rows_per_needle(&predicate, operands); + self.db.borrow_mut().batch_execute("RESET ALL").unwrap(); + (explain, rows) + } + + /// The statement that answers every needle at once: one correlated + /// subquery per needle, which the planner can serve from the index. + fn per_needle(predicate: &str) -> String { + format!( + "SELECT coalesce((SELECT array_agg(g.id ORDER BY g.id) FROM json_tree_generated g WHERE {predicate}), '{{}}') + FROM unnest($1::text[]) WITH ORDINALITY AS n(t, i) ORDER BY n.i" + ) + } + + fn rows_per_needle(&self, predicate: &str, params: &[String]) -> Vec> { + let sql = Self::per_needle(predicate); + self.db + .borrow_mut() + .query(&sql, &[¶ms]) + .unwrap_or_else(|e| panic!("{sql}: {e}")) + .iter() + .map(|r| r.get(0)) + .collect() + } +} + +/// Run `body` over a whole proptest run against one [`Scenario`], log what +/// ran, and fail with the minimal case and the way to replay it. +fn run(name: &str, body: impl Fn(&Scenario, &Case) -> Result<(), TestCaseError>) { + let config = config(); + let replay = replay(&config); + eprintln!("json_tree generated {name}: {replay}"); + let started = std::time::Instant::now(); + let scenario = scenario(); + let mut runner = TestRunner::new(config); + let result = runner.run(&strategies::case(), |case| body(&scenario, &case)); + if let Err(error) = result { + panic!("json_tree generated {name}: {replay}\n{error}"); + } + eprintln!( + "json_tree generated {name}: passed in {:?}", + started.elapsed() + ); +} + +// ============================================================================= +// Tests +// ============================================================================= + +mod given_the_document_strategy { + use super::*; + + /// Every feature the oracle relies on, counted over a run's worth of + /// generated documents, so a change to the generator that stops + /// producing one fails here, without a database. + #[test] + fn produces_every_feature_the_oracle_relies_on() { + let mut runner = TestRunner::new(config()); + let strategy = strategies::document(); + let docs: Vec = (0..400) + .map(|_| strategy.new_tree(&mut runner).unwrap().current()) + .collect(); + assertions::covers_every_feature(&assertions::Census::of(&docs)); + } +} + +mod given_generated_documents_in_postgres { + use super::*; + + /// Needs an EQL v4 installation. For every needle of every case, the + /// rows encrypted `@>` selects are the rows `jsonb @>` selects over the + /// plaintexts, on the index plan and on the sequential plan, each + /// confirmed by `EXPLAIN`. A needle pruned from a row's whole document + /// is checked to be contained by that row, and the near misses are + /// checked to miss their row often. + #[test] + #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] + fn contains_selects_the_rows_jsonb_contains_on_both_plans() { + let (needles_asked, near_misses, near_misses_missing) = + (Cell::new(0), Cell::new(0), Cell::new(0)); + run("@>", |scenario, case| { + scenario.store(&case.docs); + let needles = case.needles(); + let jsonb = scenario.jsonb(Containment::Contains, &needles); + assertions::pruned_roots_are_contained(&needles, &jsonb)?; + for (needle, rows) in needles.iter().zip(&jsonb) { + if let Origin::NearMiss(row) = needle.origin { + near_misses.set(near_misses.get() + 1); + if !rows.contains(&(row as i32)) { + near_misses_missing.set(near_misses_missing.get() + 1); + } + } + } + let operands = scenario.operands(Containment::Contains, &needles); + for scan in Scan::ALL { + let (plan, encrypted) = scenario.encrypted(Containment::Contains, scan, &operands); + assertions::planned_as(&plan, scan)?; + assertions::rows_match( + &format!("@> ({scan:?} plan)"), + case, + &needles, + &jsonb, + &encrypted, + )?; + } + needles_asked.set(needles_asked.get() + needles.len()); + Ok(()) + }); + eprintln!( + "json_tree generated @>: {} needles asked, {} near misses, {} of them missing their row", + needles_asked.get(), + near_misses.get(), + near_misses_missing.get() + ); + assert!( + near_misses_missing.get() * 4 >= near_misses.get(), + "the near misses should mostly miss the row they were made from: {} of {}", + near_misses_missing.get(), + near_misses.get() + ); + } + + /// Needs an EQL v4 installation. For every needle of every case, the + /// rows encrypted `<@` selects are the rows `jsonb <@` selects, less a + /// stored root scalar when the needle is a root array: the one case the + /// SQL documents it cannot answer, since a stored document carries no + /// root-array alternative. That case is the whole difference, and the + /// run must meet it. `jsonb_path_ops` has no `<@` strategy, so both + /// plans scan; both are asked all the same. + #[test] + #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] + fn contained_by_selects_the_rows_jsonb_does_except_a_root_scalar_in_a_root_array() { + let excluded = Cell::new(0); + run("<@", |scenario, case| { + scenario.store(&case.docs); + let needles = case.needles(); + let jsonb = scenario.jsonb(Containment::ContainedBy, &needles); + let expected: Vec> = needles + .iter() + .zip(&jsonb) + .map(|(needle, rows)| { + rows.iter() + .copied() + .filter(|id| { + let stored = &case.docs[*id as usize]; + !(needle.value.is_array() && !stored.is_object() && !stored.is_array()) + }) + .collect() + }) + .collect(); + excluded.set( + excluded.get() + jsonb.iter().map(Vec::len).sum::() + - expected.iter().map(Vec::len).sum::(), + ); + let operands = scenario.operands(Containment::ContainedBy, &needles); + for scan in Scan::ALL { + let (_, encrypted) = scenario.encrypted(Containment::ContainedBy, scan, &operands); + assertions::rows_match( + &format!("<@ ({scan:?} plan)"), + case, + &needles, + &expected, + &encrypted, + )?; + } + Ok(()) + }); + eprintln!( + "json_tree generated <@: {} rows excluded as a root scalar in a root array", + excluded.get() + ); + assert!( + excluded.get() > 0, + "no stored root scalar sat inside a needle root array, so the `<@` exception went unexercised" + ); + } + + /// Needs an EQL v4 installation. At every key path of one or two steps + /// over the generated keys, the encrypted column has an entry exactly + /// where `jsonb #>` finds a node (`#>` reads a key of an array as no + /// node, as a strict path does), and a leaf's entry opens to the value + /// `jsonb` holds there, compared by PostgreSQL's `jsonb =`, which + /// compares numbers by value. + #[test] + #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] + fn arrow_opens_a_leaf_to_the_value_jsonb_holds() { + let mut paths: Vec> = PATH_KEYS.iter().map(|k| vec![*k]).collect(); + for a in PATH_KEYS { + for b in PATH_KEYS { + paths.push(vec![*a, *b]); + } + } + let (leaves, containers) = (Cell::new(0), Cell::new(0)); + run("->", |scenario, case| { + scenario.store(&case.docs); + let keyset = scenario.cipher.default_keyset(); + let selectors: Vec = paths + .iter() + .map(|steps| { + let path = JsonPath::parse(&format!("$.{}", steps.join("."))).unwrap(); + wait(&scenario.runtime, keyset.json_selector(&path, column())) + .unwrap() + .to_string() + }) + .collect(); + let keys: Vec = paths + .iter() + .map(|steps| format!("{{{}}}", steps.join(","))) + .collect(); + let rows = scenario + .db + .borrow_mut() + .query( + "SELECT p.i, g.id, (g.plain #> p.k::text[])::text, jsonb_typeof(g.plain #> p.k::text[]), (g.doc -> p.s)::jsonb::text + FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS p(s, k, i) + CROSS JOIN json_tree_generated g ORDER BY p.i, g.id", + &[&selectors, &keys], + ) + .unwrap(); + let mut opened: Vec = Vec::new(); + let mut held: Vec = Vec::new(); + let mut at: Vec<(String, i32)> = Vec::new(); + for row in rows { + let path = format!("$.{}", paths[row.get::<_, i64>(0) as usize - 1].join(".")); + let id: i32 = row.get(1); + let want: Option = row.get(2); + let kind: Option = row.get(3); + let got: Option = row.get(4); + assertions::entry_where_jsonb_has_a_node( + &path, + id, + case, + want.is_some(), + got.is_some(), + )?; + if let (Some(want), Some(got)) = (want, got) { + if matches!(kind.as_deref(), Some("object" | "array")) { + containers.set(containers.get() + 1); + continue; + } + let entry: JsonTreeEntry = serde_json::from_str(&got).unwrap(); + let value = wait( + &scenario.runtime, + scenario.cipher.decrypt_as::(entry, column()), + ) + .map_err(|e| { + TestCaseError::fail(format!( + "-> {path}, row {id}: the entry did not open: {e}" + )) + })?; + opened.push(value.to_string()); + held.push(want); + at.push((path, id)); + } + } + let differing: Vec = scenario + .db + .borrow_mut() + .query( + "SELECT i FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS u(opened, held, i) + WHERE opened::jsonb IS DISTINCT FROM held::jsonb ORDER BY i", + &[&opened, &held], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + assertions::leaves_open_to_the_jsonb_value(&differing, &at, &opened, &held, case)?; + leaves.set(leaves.get() + opened.len()); + Ok(()) + }); + eprintln!( + "json_tree generated ->: {} leaves opened, {} containers found at {} paths", + leaves.get(), + containers.get(), + paths.len() + ); + assert!( + leaves.get() > 50 && containers.get() > 10, + "the paths should reach many leaves and containers: {} leaves, {} containers", + leaves.get(), + containers.get() + ); + } +} + +pub(super) mod assertions { + use super::*; + + /// The rows `encrypted` selected for each needle are the rows `expected` + /// says; every disagreement is named with its needle and the stored + /// documents on each side. + pub(super) fn rows_match( + what: &str, + case: &Case, + needles: &[Needle], + expected: &[Vec], + encrypted: &[Vec], + ) -> Result<(), TestCaseError> { + let show = |ids: Vec| -> String { + ids.iter() + .map(|id| format!("\n row {id}: {}", case.docs[*id as usize])) + .collect() + }; + let disagreements: Vec = needles + .iter() + .zip(expected.iter().zip(encrypted)) + .enumerate() + .filter(|(_, (_, (expected, encrypted)))| expected != encrypted) + .map(|(i, (needle, (expected, encrypted)))| { + let only_jsonb = expected.iter().copied().filter(|id| !encrypted.contains(id)).collect(); + let only_encrypted = encrypted.iter().copied().filter(|id| !expected.contains(id)).collect(); + format!( + "{what}, needle {i} ({}): {}\n jsonb: {expected:?}\n encrypted: {encrypted:?}\n only jsonb:{}\n only encrypted:{}", + needle.origin, + needle.value, + show(only_jsonb), + show(only_encrypted), + ) + }) + .collect(); + prop_assert!( + disagreements.is_empty(), + "{} needles answered differently from jsonb:\n{}", + disagreements.len(), + disagreements.join("\n") + ); + Ok(()) + } + + /// The plan the statement ran under is the one `scan` asks for: a + /// bitmap scan over the GIN index, or a sequential scan and no index. + pub(super) fn planned_as(plan: &str, scan: Scan) -> Result<(), TestCaseError> { + let as_asked = match scan { + Scan::Index => plan.contains("Bitmap Index Scan on json_tree_generated_doc"), + Scan::Sequential => { + plan.contains("Seq Scan on json_tree_generated") + && !plan.contains("Index Scan") + && !plan.contains("Bitmap") + } + }; + prop_assert!( + as_asked, + "the {scan:?} plan should be the one asked for:\n{plan}" + ); + Ok(()) + } + + /// A needle pruned from a row's whole document is contained by that + /// row: the generator's own claim, checked by the oracle. + pub(super) fn pruned_roots_are_contained( + needles: &[Needle], + jsonb: &[Vec], + ) -> Result<(), TestCaseError> { + for (needle, rows) in needles.iter().zip(jsonb) { + if let Origin::PrunedRoot(row) = needle.origin { + prop_assert!( + rows.contains(&(row as i32)), + "row {row} should contain its own pruning {}, but jsonb selects {rows:?}", + needle.value + ); + } + } + Ok(()) + } + + pub(super) fn entry_where_jsonb_has_a_node( + path: &str, + id: i32, + case: &Case, + jsonb_has_a_node: bool, + encrypted_has_an_entry: bool, + ) -> Result<(), TestCaseError> { + prop_assert_eq!( + encrypted_has_an_entry, + jsonb_has_a_node, + "-> {}, row {}: the encrypted column should have an entry exactly where jsonb has a node\n stored: {}", + path, + id, + &case.docs[id as usize] + ); + Ok(()) + } + + pub(super) fn leaves_open_to_the_jsonb_value( + differing: &[i64], + at: &[(String, i32)], + opened: &[String], + held: &[String], + case: &Case, + ) -> Result<(), TestCaseError> { + let report: Vec = differing + .iter() + .map(|i| { + let i = *i as usize - 1; + let (path, id) = &at[i]; + format!( + "-> {path}, row {id}: opened {}, jsonb holds {}\n stored: {}", + opened[i], held[i], case.docs[*id as usize] + ) + }) + .collect(); + prop_assert!( + report.is_empty(), + "leaves should open to the value jsonb holds:\n{}", + report.join("\n") + ); + Ok(()) + } + + /// The features the generator is meant to cover, counted. + #[derive(Debug, Default)] + pub(super) struct Census { + root_objects: usize, + root_arrays: usize, + root_scalars: usize, + empty_objects: usize, + empty_arrays: usize, + arrays_of_objects: usize, + arrays_of_arrays: usize, + duplicate_elements: usize, + keys_on_two_levels: usize, + float_spelled_integers: usize, + numeric_strings: usize, + booleans: usize, + nulls: usize, + max_depth: usize, + } + + impl Census { + pub(super) fn of(docs: &[Value]) -> Self { + let mut census = Census::default(); + for doc in docs { + match doc { + Value::Object(_) => census.root_objects += 1, + Value::Array(_) => census.root_arrays += 1, + _ => census.root_scalars += 1, + } + let mut keys = + std::collections::BTreeMap::<&str, std::collections::BTreeSet>::new(); + census.walk(doc, 0, &mut keys); + if keys.values().any(|depths| depths.len() > 1) { + census.keys_on_two_levels += 1; + } + } + census + } + + fn walk<'v>( + &mut self, + v: &'v Value, + depth: usize, + keys: &mut std::collections::BTreeMap<&'v str, std::collections::BTreeSet>, + ) { + self.max_depth = self.max_depth.max(depth); + match v { + Value::Object(map) => { + if map.is_empty() { + self.empty_objects += 1; + } + for (key, member) in map { + let _ = keys.entry(key).or_default().insert(depth); + self.walk(member, depth + 1, keys); + } + } + Value::Array(items) => { + if items.is_empty() { + self.empty_arrays += 1; + } + if items.iter().any(Value::is_object) { + self.arrays_of_objects += 1; + } + if items.iter().any(Value::is_array) { + self.arrays_of_arrays += 1; + } + if items + .iter() + .enumerate() + .any(|(i, a)| items[i + 1..].contains(a)) + { + self.duplicate_elements += 1; + } + for item in items { + self.walk(item, depth + 1, keys); + } + } + Value::Number(n) => { + if n.is_f64() && n.as_f64().is_some_and(|f| f.fract() == 0.0) { + self.float_spelled_integers += 1; + } + } + Value::String(s) => { + if s.parse::().is_ok() { + self.numeric_strings += 1; + } + } + Value::Bool(_) => self.booleans += 1, + Value::Null => self.nulls += 1, + } + } + } + + pub(super) fn covers_every_feature(census: &Census) { + let counts = [ + ("root objects", census.root_objects), + ("root arrays", census.root_arrays), + ("root scalars", census.root_scalars), + ("empty objects", census.empty_objects), + ("empty arrays", census.empty_arrays), + ("arrays of objects", census.arrays_of_objects), + ("arrays of arrays", census.arrays_of_arrays), + ("duplicate array elements", census.duplicate_elements), + ("a key on two levels", census.keys_on_two_levels), + ("integers spelled as floats", census.float_spelled_integers), + ("strings spelled as numbers", census.numeric_strings), + ("booleans", census.booleans), + ("nulls", census.nulls), + ]; + for (what, count) in counts { + assert!( + count > 0, + "the generated documents should hold {what}: {census:?}" + ); + } + assert!( + census.max_depth >= 4, + "the generated documents should nest at least four levels: {census:?}" + ); + } +} From f2e998c8fe7f486080cec380ef24b484258e63ac Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sat, 24 Oct 2026 12:37:36 +1100 Subject: [PATCH 3/3] test(stack-encrypt): restructure the JSON index tests and generate containment with proptest tests/json_index.rs held 28 flat tests named for whole sentences, with bare assertions and a hand-rolled Lcg for its only randomized check. It now follows the repository's Rust testing conventions: one given_ module per situation and a test named for what it expects, a scenario() helper for the cipher and plan, an assertions module for the refusals and round trips the tests repeat, a message on every assertion, and unwrap allowed at the crate root, which is the test module of an integration test. No assertion was dropped. The longest tests were split along the expectations they made (an accessor's matching, ordering and refusals are five tests now; a subtree key's derivation, gates and root handling are three), and four near-identical assertions became loops. The run goes from 28 tests to 47. The random containment check becomes two proptest properties, one for @> and one for <@, over a generated stored document and a needle: a random value, the document itself, a pruning of it, or a pruning with one edit. They hold the projection to the in-file model of jsonb, which given_the_containment_model pins to PostgreSQL's answers. A failure shrinks to a small pair: breaking the <@ exception shrinks to stored 0 and the needle [0]. The RNG is ChaCha, seed 1336, 512 cases per property. PROPTEST_RNG_SEED and PROPTEST_CASES replace them. Failures persist to the crate's proptest-regressions/json_index.txt. A census test fails if the document strategy stops producing a feature the property relies on. The strategy duplicates the EQL live oracle's, which sits in another Cargo workspace. proptest is a dev-dependency with default features off, as stack-guest-abi takes it, resolving to the 1.11.0 already in the lock. The plan document's description of both oracles is updated to match. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- Cargo.lock | 1 + docs/plans/2026-10-17-json-tree-index.md | 2 +- packages/stack-encrypt/Cargo.toml | 4 + .../proptest-regressions/json_index.txt | 6 + packages/stack-encrypt/tests/json_index.rs | 4542 ++++++++++------- 5 files changed, 2783 insertions(+), 1772 deletions(-) create mode 100644 packages/stack-encrypt/proptest-regressions/json_index.txt diff --git a/Cargo.lock b/Cargo.lock index 5bf1172ff..30979e6a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3368,6 +3368,7 @@ dependencies = [ "cllw-ore", "miette", "ore-rs", + "proptest", "serde", "serde_json", "sha2 0.11.0", diff --git a/docs/plans/2026-10-17-json-tree-index.md b/docs/plans/2026-10-17-json-tree-index.md index dc2d58018..de5909093 100644 --- a/docs/plans/2026-10-17-json-tree-index.md +++ b/docs/plans/2026-10-17-json-tree-index.md @@ -359,7 +359,7 @@ Each phase is a PR against `next`, where #1209 landed. The index, the SQL and the live oracle landed in the pull request stacked on this document; what follows records where the code is and where it departs from the design above, so the two can be read against each other. Phase 0 was folded into Phase 2, and Phases 3 and 4 remain open. -**Where it is.** The engine half is `packages/stack-encrypt/src/json/`: `Json` is the index (`Index`, term `JsonTerms`), `target::json_document` the sealing operation, `JsonDocument` the stored term and an `EncryptFrom` / `DecryptInto` target, `JsonQuery` the needle for `Contains`, `ContainedBy` and `AtPath`, `JsonLeafQuery` the operand for the orderings at a path, and `JsonSelector` / `JsonSubtreeKey` the extraction keys, derived through `KeysetCipher::json_selector` and `json_subtree_key`. `IndexSpec::Json(JsonOptions)` is the data form, with the key `"json"` and `{"json": {"sealed_root": …}}` as its wire form; a data plan reads it and refuses to run it (`Reason::JsonIndexPending`). The SQL is `packages/eql/src/v3/json_tree/`, confined to EQL v4; the EQL types are `packages/eql/crates/eql-bindings/src/v4/json_tree.rs`; the oracle is `packages/eql/tests/encryption/tests/json_tree.rs`, which stores the design's corpus and random documents encrypted beside their plaintexts and checks that `@>` and `<@` over the encrypted column select the rows PostgreSQL's own `jsonb` operators select over the plaintext column, with the GIN index shown in `EXPLAIN`. The pure-Rust half of that oracle is `packages/stack-encrypt/tests/json_index.rs`, against a model of `jsonb @>` pinned to the answers in `jsonb-semantics.sql`. +**Where it is.** The engine half is `packages/stack-encrypt/src/json/`: `Json` is the index (`Index`, term `JsonTerms`), `target::json_document` the sealing operation, `JsonDocument` the stored term and an `EncryptFrom` / `DecryptInto` target, `JsonQuery` the needle for `Contains`, `ContainedBy` and `AtPath`, `JsonLeafQuery` the operand for the orderings at a path, and `JsonSelector` / `JsonSubtreeKey` the extraction keys, derived through `KeysetCipher::json_selector` and `json_subtree_key`. `IndexSpec::Json(JsonOptions)` is the data form, with the key `"json"` and `{"json": {"sealed_root": …}}` as its wire form; a data plan reads it and refuses to run it (`Reason::JsonIndexPending`). The SQL is `packages/eql/src/v3/json_tree/`, confined to EQL v4; the EQL types are `packages/eql/crates/eql-bindings/src/v4/json_tree.rs`; the oracle is `packages/eql/tests/encryption/tests/json_tree.rs`, which stores the design's corpus, and batches of documents proptest generates (`json_tree/generated.rs`), encrypted beside their plaintexts and checks that `@>` and `<@` over the encrypted column select the rows PostgreSQL's own `jsonb` operators select over the plaintext column, on the GIN index and on a sequential scan, each plan shown in `EXPLAIN`. The pure-Rust half of that oracle is `packages/stack-encrypt/tests/json_index.rs` (`given_the_design_corpus` and the proptest properties in `given_generated_documents`), against a model of `jsonb @>` pinned to the answers in `jsonb-semantics.sql`. **Decisions taken by the code.** Of the eleven listed below: 1, hex for every sixteen-byte field and base64 for `c` and `h`, pending Phase 3 (taken on 2026-10-23, Phase 3d: base64 for every field the SQL reads, base85 for `c`); 2, leaf ordering terms bound to the column only, numbers as the `f64` they round to and strings as text, under the scalar `Ope` index's derivation; 3, wildcard arrays only; 4, group tags on, and only on container array elements; 5, the child count is in every container node's ciphertext, and a truncated subtree is `OpenedType`, not a shorter array; 7, plain SHA-256; 8, the option is the boolean `sealed_root`, on by default, recorded in the header's flags; 9, the projection is built from the tags server-side (`eql_v4.json_tree_query`) and `groups_ok` was not needed; 10, position-keyed parent tags on, the ordinal stored as `o`, object members ranked under the parent's token; 11, keyed: in this first form a child's token was `PRF(json/token-step ‖ column, PAE(t(parent), 0x01, key))` (`0x02, LE64(index)` for an element) and an object member's rank `PRF(json/rank ‖ column, PAE(t(parent), key))`, each under its own PAE domain, so a token's holder could neither test key names against the child tokens the walk unwraps nor rank guessed names against the ordinals. Decision 6 is out of scope, as proposed. (Both steps were first hashed unkeyed, as the token chain section above writes them, and were corrected before any release.) diff --git a/packages/stack-encrypt/Cargo.toml b/packages/stack-encrypt/Cargo.toml index 065609938..8cd117435 100644 --- a/packages/stack-encrypt/Cargo.toml +++ b/packages/stack-encrypt/Cargo.toml @@ -105,6 +105,10 @@ zerokms-protocol = { workspace = true } tokio = { workspace = true, features = ["rt", "macros"] } # Compile-fail tests for the derive diagnostics (`tests/ui`). trybuild = "1" +# The generated containment property in tests/json_index.rs. Default features +# off, as stack-guest-abi takes it: `std` carries the failure persistence and +# the ChaCha RNG the run is seeded with, and nothing else is needed. +proptest = { version = "1.7", default-features = false, features = ["std"] } # The derive diagnostics are recorded with every feature on, as `test:unit` # and CI run them. They cannot be relied on to hold under both feature sets diff --git a/packages/stack-encrypt/proptest-regressions/json_index.txt b/packages/stack-encrypt/proptest-regressions/json_index.txt new file mode 100644 index 000000000..341ba3ff3 --- /dev/null +++ b/packages/stack-encrypt/proptest-regressions/json_index.txt @@ -0,0 +1,6 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. diff --git a/packages/stack-encrypt/tests/json_index.rs b/packages/stack-encrypt/tests/json_index.rs index 890075533..4fbba8f06 100644 --- a/packages/stack-encrypt/tests/json_index.rs +++ b/packages/stack-encrypt/tests/json_index.rs @@ -1,18 +1,28 @@ //! The JSON index end to end, with no database: a document seals into one //! entry per node under one data key and opens again; the token chain //! reaches every child from its parent's token and ordinal, and nothing -//! from any other token; -//! a moved, reordered or regrafted entry fails to open; and containment -//! over the element-nested projection agrees with PostgreSQL's `jsonb @>` -//! on the design's corpus and on random documents, through a Rust model -//! of `jsonb @>` written here from the rules PostgreSQL documents. +//! from any other token; a moved, reordered or regrafted entry fails to +//! open; and containment over the element-nested projection agrees with +//! PostgreSQL's `jsonb @>` on the design's corpus and on documents proptest +//! generates, through a Rust model of `jsonb @>` written here from the +//! rules PostgreSQL documents (the live oracle against PostgreSQL itself is +//! `packages/eql/tests/encryption/tests/json_tree.rs`). +//! +//! Each `given_` module is one situation, and each test in it is +//! named for what it expects. Shared setup is [`scenario`] and the helpers +//! below it; repeated checks are in [`assertions`]. The generated cases are +//! in [`given_generated_documents`]: a fixed ChaCha seed that +//! `PROPTEST_RNG_SEED` replaces, and a case count `PROPTEST_CASES` replaces. + +// `.unwrap()` is how a test says "this cannot fail here". +#![allow(clippy::unwrap_used)] mod common; use common::stack_cipher; use serde_json::{json, Map, Value}; use stack_encrypt::json::{ - JsonDocument, JsonNode, JsonOptions, JsonPath, JsonQuery, JsonSubtree, JsonSubtreeKey, + JsonDocument, JsonNode, JsonOptions, JsonPath, JsonQuery, JsonSubtree, JsonSubtreeKey, Tag, }; use stack_encrypt::op; use stack_encrypt::sem::TermError; @@ -22,6 +32,31 @@ use stack_encrypt::target::{ use stack_encrypt::{nonempty, Error, Plan, PlanError, StackCipher}; use stack_kms::FakeDataKeySource; +// ============================================================================= +// Setup +// ============================================================================= + +type Cipher = StackCipher; + +type JsonPlan = stack_encrypt::plan::ValuePlan>; + +type OrderedPlan = + stack_encrypt::plan::ValuePlan>; + +/// What most tests start from: a cipher over the fake key source and the +/// default plan for a JSON column under `users/profile`. +struct Scenario { + cipher: Cipher, + plan: JsonPlan, +} + +async fn scenario() -> Scenario { + Scenario { + cipher: stack_cipher().await, + plan: plan(), + } +} + fn context() -> CallerContext { CallerContext::from(nonempty!("users").with("profile")) } @@ -30,30 +65,22 @@ fn aead() -> AeadContext { AeadContext::from(nonempty!("users").with("profile")) } -async fn seal(cipher: &StackCipher, value: &Value) -> JsonDocument { - cipher - .default_keyset() - .encrypt_as::(value, context()) - .await - .expect("seals") +fn plan() -> JsonPlan { + Plan::value::() + .context("users/profile") + .encrypt_into::() + .build() + .unwrap() } -async fn seal_with( - cipher: &StackCipher, - value: &Value, - options: &JsonOptions, -) -> JsonDocument { - cipher - .default_keyset() - .run( - stack_encrypt::target::json_document::<_, stack_encrypt::target::Borrowed>( - options.clone(), - ), - value, - context(), - ) - .await - .expect("seals") +/// The plan laid out by the JSON index with `$.hello`, `$.number` and +/// `$.flag` ordered. +fn ordered_plan() -> OrderedPlan { + Plan::value::() + .context("users/profile") + .encrypt_as(Json::new(ordering(&["$.hello", "$.number", "$.flag"]))) + .build() + .unwrap() } /// The options with the root open. @@ -69,11 +96,10 @@ fn ordering(paths: &[&str]) -> JsonOptions { JsonOptions::default().ordering(paths).unwrap() } -async fn open(cipher: &StackCipher, doc: JsonDocument) -> Result { - cipher.decrypt_as::(doc, aead()).await -} - -/// The design's fixture document. +/// The design's fixture document. Its entries, by ordinal: `$`, `$.hello`, +/// `$.nested`, `$.nested.deep`, `$.nested.list`, `[0]`, `[0].x`, `[1]`, +/// `[1].x`, `$.number`. The tests pin those ordinals, so the document is +/// fixed rather than generated. fn fixture() -> Value { json!({"hello": "world-1", "nested": {"deep": "constant", "list": [{"x": 1}, {"x": 2}]}, "number": 1}) } @@ -87,215 +113,43 @@ fn nodes(value: &Value) -> usize { } } -// ============================================================================= -// Round trips -// ============================================================================= - -#[tokio::test] -async fn a_document_seals_one_entry_per_node_and_opens_again() { - let cipher = stack_cipher().await; - for value in [ - fixture(), - json!({}), - json!([]), - json!("a"), - json!(2.5), - json!(null), - json!(true), - json!([[1, 2], [3, [4, {"a": null}]]]), - json!({"a": {"b": {"c": {"d": [1, "x", false]}}}}), - json!({"a": "b", "a b": 1, "é": [], "0": {}}), - json!({"n": 18446744073709551615u64, "m": -9223372036854775808i64, "f": 1e300}), - ] { - for options in [JsonOptions::default(), open_root()] { - let doc = seal_with(&cipher, &value, &options).await; - assert_eq!( - doc.entries().len(), - nodes(&value), - "{value}: one entry per node" - ); - assert_eq!(doc.header().sealed_root(), options.sealed_root); - let back = open(&cipher, doc).await.unwrap(); - assert_eq!(back, value, "{value} under {options:?}"); - } - } +/// Seal through the typed target, as `encrypt_as` does. +async fn seal(cipher: &Cipher, value: &Value) -> JsonDocument { + cipher + .default_keyset() + .encrypt_as::(value, context()) + .await + .expect("seals") } -/// The deepest document the index takes opens again under either root -/// option, and one level deeper is refused when it is sealed, not when it is -/// read. A sealed root's plaintext is the document as JSON, which -/// `serde_json` reads back only to 128 levels of nesting. -#[tokio::test] -async fn the_deepest_document_the_index_takes_opens_again() { - let cipher = stack_cipher().await; - let arrays = |n: usize| (0..n).fold(json!(1), |inner, _| json!([inner])); - let objects = |n: usize| (0..n).fold(json!("leaf"), |inner, _| json!({"k": inner})); - for nested in [arrays, objects] { - for options in [JsonOptions::default(), open_root()] { - let deepest = nested(127); - let doc = seal_with(&cipher, &deepest, &options).await; - assert_eq!( - open(&cipher, doc).await.unwrap(), - deepest, - "127 levels under {options:?}" - ); - let refused = cipher - .default_keyset() - .run( - stack_encrypt::target::json_document::<_, stack_encrypt::target::Borrowed>( - options.clone(), - ), - &nested(128), - context(), - ) - .await; - assert!( - matches!(refused, Err(Error::Term(TermError::JsonTooDeep))), - "128 levels under {options:?} is refused when sealed" - ); - } - } +/// Seal through the JSON document target under `options`. +async fn seal_with(cipher: &Cipher, value: &Value, options: &JsonOptions) -> JsonDocument { + try_seal(cipher, value, options).await.expect("seals") } -#[tokio::test] -async fn the_wire_form_is_what_the_sql_domain_stores() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let wire = serde_json::to_value(&doc).unwrap(); - let keys: Vec<&str> = wire - .as_object() - .unwrap() - .keys() - .map(String::as_str) - .collect(); - assert_eq!(keys, ["h", "q", "sv"]); - // `q` is the document's own projection, written once: the needle a - // `Contains` of the whole document derives has the same pairs in the - // same groups, and nothing else. - let needle = cipher - .query(op::Contains, &fixture()) - .on(&plan()) +async fn try_seal( + cipher: &Cipher, + value: &Value, + options: &JsonOptions, +) -> Result { + cipher + .default_keyset() + .run( + stack_encrypt::target::json_document::<_, stack_encrypt::target::Borrowed>( + options.clone(), + ), + value, + context(), + ) .await - .unwrap(); - assert_eq!(wire["q"], serde_json::to_value(&needle).unwrap()); - assert!( - serde_json::from_value::(json!({"h": wire["h"], "sv": wire["sv"]})).is_err(), - "a document without `q` is not a document: the projection is the only place the value selectors live" - ); - let back: JsonDocument = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!( - serde_json::to_value(&back).unwrap(), - wire, - "`q` round-trips as stored" - ); - assert_eq!( - serde_json::to_value(back.projection()).unwrap(), - wire["q"], - "the projection read back is the stored one" - ); - for (j, entry) in wire["sv"].as_array().unwrap().iter().enumerate() { - let entry = entry.as_object().unwrap(); - for key in entry.keys() { - assert!( - ["s", "c", "p", "op"].contains(&key.as_str()), - "entry {j} carries an unknown key {key}" - ); - } - for (tag, chars, bytes) in [("s", 24, 16), ("p", 12, 8)] { - if let Some(text) = entry.get(tag) { - assert_eq!( - text.as_str().unwrap().len(), - chars, - "entry {j}'s {tag} is {bytes} bytes of padded base64" - ); - } - } - let c = entry["c"].as_str().unwrap(); - assert!( - c.bytes() - .all(|b| b.is_ascii_graphic() && b != b'"' && b != b'\\'), - "entry {j}'s c is base85 in an alphabet a JSON string carries unescaped" - ); - } - let back: JsonDocument = serde_json::from_value(wire).unwrap(); - assert_eq!(back, doc); } -#[tokio::test] -async fn the_chain_fields_sit_where_the_design_says() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let entries = doc.entries(); - // $, $.hello, $.nested, $.nested.deep, $.nested.list, [0], [0].x, [1], [1].x, $.number - assert_eq!(entries.len(), 10); - let root = &entries[0]; - assert!(root.parent_tag().is_none()); - for head in [&entries[1], &entries[2], &entries[9]] { - assert!( - head.parent_tag().is_none(), - "a head has no parent tag under the sealed root" - ); - } - for below in &entries[3..9] { - assert!(below.parent_tag().is_some()); - } - // Every parent tag is distinct: siblings are not linkable at rest. - let tags: std::collections::HashSet<_> = entries[3..9] - .iter() - .map(|e| *e.parent_tag().unwrap()) - .collect(); - assert_eq!(tags.len(), 6); - // The projection groups the two list elements: each is a nested group - // holding its own pair and its `x`. - let q = serde_json::to_value(doc.projection()).unwrap(); - assert_eq!(q["el"].as_array().unwrap().len(), 2); - assert_eq!(q["el"][0]["sv"].as_array().unwrap().len(), 2); - assert_eq!( - q["sv"].as_array().unwrap().len(), - 6, - "$, hello, nested, deep, list, number" - ); - // No path is ordered by default, so no entry carries a term. - assert!(entries.iter().all(|e| e.ordering_term().is_none())); - // Equal values at one path share selectors but not ciphertexts; the - // value selectors, in the projection, tell equal values from different. - let pair = |doc: &JsonDocument, i: usize| -> (String, String) { - let q = serde_json::to_value(doc.projection()).unwrap(); - ( - q["sv"][i]["s"].as_str().unwrap().to_owned(), - q["sv"][i]["vs"].as_str().unwrap().to_owned(), - ) - }; - let doc2 = seal(&cipher, &json!([1, 1])).await; - let (a, b) = (&doc2.entries()[1], &doc2.entries()[2]); - assert_eq!(a.selector(), b.selector()); - assert_eq!(pair(&doc2, 1), pair(&doc2, 2)); - assert_ne!(a.ciphertext(), b.ciphertext()); - let doc3 = seal(&cipher, &json!([1, 2])).await; - let (one, two) = (&doc3.entries()[1], &doc3.entries()[2]); - assert_eq!(one.selector(), two.selector(), "one path"); - assert_ne!( - pair(&doc3, 1), - pair(&doc3, 2), - "different values at one path have different value selectors" - ); - assert_eq!( - pair(&doc3, 1), - pair(&doc2, 1), - "the value selector names the value at its path, in any document" - ); - - // Without the sealed root, the top-level fields chain from the root. - let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; - assert!(open_doc.entries()[1].parent_tag().is_some()); +async fn open(cipher: &Cipher, doc: JsonDocument) -> Result { + cipher.decrypt_as::(doc, aead()).await } -// ============================================================================= -// The token chain -// ============================================================================= - async fn subtree_key( - cipher: &StackCipher, + cipher: &Cipher, path: &str, options: JsonOptions, ) -> Result { @@ -311,1652 +165,2798 @@ async fn subtree_key( /// `rows` as the key derived for `path` opens them: the only way to build /// a [`JsonSubtree`]. -async fn keyed( - cipher: &StackCipher, - path: &str, - rows: Vec, -) -> JsonSubtree { +async fn keyed(cipher: &Cipher, path: &str, rows: Vec) -> JsonSubtree { subtree_key(cipher, path, JsonOptions::default()) .await .unwrap() .open(rows) } -/// An ordering term sits on a number or string leaf at an ordered path and -/// nowhere else: a wildcard covers every element. A position is refused, -/// since `->` reads it as the wildcard and no ordering could reach the leaf -/// it names. -#[tokio::test] -async fn an_ordering_term_sits_only_at_an_ordered_path() { - let cipher = stack_cipher().await; - // $, $.hello, $.nested, $.nested.deep, $.nested.list, [0], [0].x, [1], [1].x, $.number - let with_term = |doc: &JsonDocument| -> Vec { - doc.entries() - .iter() - .enumerate() - .filter(|(_, e)| e.ordering_term().is_some()) - .map(|(j, _)| j) - .collect() - }; - let doc = seal_with(&cipher, &fixture(), &ordering(&["$.hello", "$.number"])).await; - assert_eq!(with_term(&doc), [1, 9]); - let wild = seal_with(&cipher, &fixture(), &ordering(&["$.nested.list[*].x"])).await; - assert_eq!(with_term(&wild), [6, 8], "a wildcard covers every element"); - assert!(matches!( - JsonOptions::default().ordering(["$.nested.list[1].x"]), - Err(TermError::JsonOrderingAtPosition) - )); - assert!(matches!( - JsonOptions::default().ordering(["nested"]), - Err(TermError::JsonPath(_)) - )); - // Set on the field, which `ordering` guards, a position refuses the - // seal rather than put a term no ordering reaches on one element. - let mut positional = JsonOptions::default(); - let _ = positional - .ordered - .insert(JsonPath::parse("$.nested.list[1].x").unwrap()); - let refused = cipher - .default_keyset() - .run( - stack_encrypt::target::json_document::<_, stack_encrypt::target::Borrowed>(positional), - &fixture(), - context(), - ) - .await; - assert!( - matches!(refused, Err(Error::Term(TermError::JsonOrderingAtPosition))), - "{refused:?}" - ); - // A container, a boolean or a null at an ordered path carries none. - let kinds = seal_with( - &cipher, - &json!({"a": true, "b": null, "c": {"d": 1}, "e": [1, "x"]}), - &ordering(&["$.a", "$.b", "$.c", "$.e", "$.e[*]"]), - ) - .await; - // $, $.a, $.b, $.c, $.c.d, $.e, $.e[0], $.e[1] - assert_eq!(with_term(&kinds), [6, 7]); - // The root scalar itself, at `$`. - let scalar = seal_with(&cipher, &json!(7), &ordering(&["$"])).await; - assert_eq!(with_term(&scalar), [0]); - // The documents open regardless. - assert_eq!(open(&cipher, doc).await.unwrap(), fixture()); - assert_eq!(open(&cipher, wild).await.unwrap(), fixture()); +/// Padded standard base64, as the SQL reads and writes it. +fn base64(bytes: &[u8]) -> String { + use base64ct::{Base64, Encoding}; + Base64::encode_string(bytes) } -#[tokio::test] -async fn a_token_derived_from_the_path_alone_unlocks_exactly_its_subtree() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) - .await - .unwrap(); - assert!( - key.gates().is_empty(), - "a chain head's token is derived, nothing to gate" - ); - assert_eq!( - key.path(), - &JsonPath::parse("$.nested").unwrap(), - "the key carries the path it was derived from" - ); - let under = doc.subtree(key.token_bytes()); - let found: Vec = under.iter().map(|(j, _, _)| *j).collect(); - assert_eq!( - found, - [3, 4, 5, 6, 7, 8], - "deep, list, both elements, both leaves" - ); - let depths: Vec = under.iter().map(|(_, _, d)| *d).collect(); - assert_eq!(depths, [1, 1, 2, 3, 2, 3]); - let garbage = doc.subtree(&[7; 16]); - assert!(garbage.is_empty(), "a wrong token unlocks nothing"); +/// The ordinals the walk from `token` reaches. +fn subtree_ordinals(doc: &JsonDocument, token: &Tag) -> Vec { + doc.subtree(token).iter().map(|(j, _, _)| *j).collect() +} - // Below a head the client derives nothing but array positions: a key - // step becomes a gate, and the walk from the head takes it. - let deeper = subtree_key(&cipher, "$.nested.list[1]", JsonOptions::default()) - .await - .unwrap(); - assert_eq!( - deeper.token_bytes(), - key.token_bytes(), - "anchored at the head" - ); - assert_eq!(deeper.gates().len(), 2, "list, then [1]"); - assert_eq!( - deeper.gates()[1], - format!("#1:{}", base64(doc.entries()[5].selector())), - "the position, and the selector every element of the list carries" - ); - let list_token = doc - .children(key.token_bytes()) - .into_iter() - .find(|(j, _, _)| *j == 4) - .map(|(_, _, t)| t) - .expect("the list is a child of nested"); - let second = doc - .children(&list_token) +/// The nodes the walk from `token` reaches, as the server returns them. +fn subtree_nodes(doc: &JsonDocument, token: &Tag) -> Vec { + doc.subtree(token) + .iter() + .map(|(j, _, _)| doc.node(*j).unwrap()) + .collect() +} + +/// The token of the child at ordinal `j` under `token`, as the server +/// derives it. +fn child_token(doc: &JsonDocument, token: &Tag, j: usize) -> Tag { + doc.children(token) .into_iter() - .find(|(j, _, _)| *j == 7) + .find(|(ordinal, _, _)| *ordinal == j) .map(|(_, _, t)| t) - .expect("the second element"); - let found: Vec = doc.subtree(&second).iter().map(|(j, _, _)| *j).collect(); - assert_eq!(found, [8], "the second element's x"); + .unwrap_or_else(|| panic!("entry {j} is a child of the token's node")) +} - // The children of a node, by ordinal: the list's two elements. - let children: Vec = doc - .children(&list_token) - .iter() - .map(|(j, _, _)| *j) - .collect(); - assert_eq!(children, [5, 7]); - // An array element's token is derivable from the path once the array - // itself is reached by token: under an open root every node is. - - // The sealed root has no token; the open root's unlocks every node. - assert!(matches!( - subtree_key(&cipher, "$", JsonOptions::default()).await, - Err(Error::Term(TermError::JsonSealedRoot)) - )); - assert!(matches!( - subtree_key(&cipher, "$[*].x", JsonOptions::default()).await, - Err(Error::Term(TermError::JsonSealedRoot)) - )); - let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; - let root = subtree_key(&cipher, "$", open_root()).await.unwrap(); - assert_eq!( - open_doc.subtree(root.token_bytes()).len(), - 9, - "every entry but the root" - ); +/// Node `j` as a walk returns it, `d` levels below where it started. +fn with_depth(doc: &JsonDocument, j: usize, d: u32) -> JsonNode { + let mut wire = serde_json::to_value(doc.node(j).unwrap()).unwrap(); + wire["d"] = json!(d); + serde_json::from_value(wire).unwrap() +} - // A wildcard path anchors at the head and gates on each step's - // selector: the list's, then every element's (the wildcard's), then x's. - let gated = subtree_key(&cipher, "$.nested.list[*].x", JsonOptions::default()) - .await - .unwrap(); - assert_eq!(gated.token_bytes(), key.token_bytes()); - assert_eq!( - gated.gates(), - [ - base64(doc.entries()[4].selector()), - base64(doc.entries()[5].selector()), - base64(doc.entries()[6].selector()), - ] - ); - // Under an open root the whole chain is derivable down to the first - // key step: `$[0]` would be a position, `$.nested` is a gate. - let open_key = subtree_key(&cipher, "$.nested.list[1]", open_root()) - .await - .unwrap(); - assert_eq!(open_key.token_bytes(), root.token_bytes()); - assert_eq!(open_key.gates().len(), 3); - let array_doc = seal_with(&cipher, &json!([[1, 2], [3]]), &open_root()).await; - let second_row = subtree_key(&cipher, "$[1]", open_root()).await.unwrap(); - assert!(second_row.gates().is_empty(), "positions derive"); - let found: Vec = array_doc - .subtree(second_row.token_bytes()) - .iter() - .map(|(j, _, _)| *j) - .collect(); - assert_eq!(found, [5], "$, [0], [0][0], [0][1], [1], [1][0]"); +/// The rows `json_tree_subtree_at(doc, token, gates)` returns for `key`, +/// walked here as the SQL walks them: from the key's token, through each +/// gate (a selector the entry at that depth must carry, or `#n:`, +/// the child at ordinal `n` carrying it), then everything beneath each node +/// the gates reach, each row with its depth below that node. +fn gated_walk(doc: &JsonDocument, key: &JsonSubtreeKey) -> Vec { + let mut frontier = vec![(None, *key.token_bytes())]; + for gate in key.gates() { + let mut next = Vec::new(); + for (_, token) in &frontier { + let children = doc.children(token); + match gate.strip_prefix('#') { + Some(slot) => { + let (n, selector) = slot.split_once(':').unwrap(); + if let Some((j, entry, t)) = children.get(n.parse::().unwrap()) { + if base64(entry.selector()) == selector { + next.push((Some(*j), *t)); + } + } + } + None => next.extend( + children + .iter() + .filter(|(_, entry, _)| base64(entry.selector()) == gate) + .map(|(j, _, t)| (Some(*j), *t)), + ), + } + } + frontier = next; + } + let mut rows = Vec::new(); + for (reached, token) in frontier { + let below = u32::from(reached.is_some()); + if let Some(j) = reached { + rows.push(with_depth(doc, j, 0)); + } + rows.extend( + doc.subtree(&token) + .iter() + .map(|(j, _, d)| with_depth(doc, *j, d - 1 + below)), + ); + } + rows.sort_by_key(JsonNode::ordinal); + rows } -#[tokio::test] -async fn a_selector_is_the_stored_one_and_folds_positions_into_the_wildcard() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let keyset = cipher.default_keyset(); - let selector = |path: &str| { - let path = JsonPath::parse(path).unwrap(); - keyset.json_selector(&path, nonempty!("users").with("profile")) - }; - assert_eq!( - selector("$.nested.deep").await.unwrap().as_bytes(), - doc.entries()[3].selector() - ); - assert_eq!( - selector("$.nested.list[*].x").await.unwrap().as_bytes(), - doc.entries()[6].selector() - ); - assert_eq!( - selector("$.nested.list[1].x").await.unwrap().as_bytes(), - doc.entries()[8].selector() - ); - assert_eq!( - selector("$").await.unwrap().as_bytes(), - doc.entries()[0].selector() - ); - assert_ne!( - selector("$.hello").await.unwrap().as_bytes(), - keyset - .json_selector( - &JsonPath::parse("$.hello").unwrap(), - nonempty!("users").with("bio") - ) - .await - .unwrap() - .as_bytes(), - "a selector is bound to its column" - ); +/// PostgreSQL's `jsonb @>`, from its documented rules: an object contains +/// another when it has every key with a containing value; an array +/// contains another when every needle element is contained by some stored +/// element; a top-level array contains a scalar that is one of its +/// elements; scalars contain equal scalars, numbers by value. +fn pg_contains(stored: &Value, needle: &Value, top: bool) -> bool { + match (stored, needle) { + (Value::Object(a), Value::Object(b)) => b + .iter() + .all(|(k, vb)| a.get(k).is_some_and(|va| pg_contains(va, vb, false))), + (Value::Array(a), Value::Array(b)) => b + .iter() + .all(|vb| a.iter().any(|va| pg_contains(va, vb, false))), + (Value::Array(a), scalar) if top && !scalar.is_object() && !scalar.is_array() => { + a.iter().any(|va| pg_contains(va, scalar, false)) + } + (Value::Number(a), Value::Number(b)) => a.as_f64() == b.as_f64(), + (Value::Object(_) | Value::Array(_), _) | (_, Value::Object(_) | Value::Array(_)) => false, + (a, b) => a == b, + } +} + +/// `<@` compares the same two projections the other way round. The one +/// case it cannot answer is a stored root scalar inside a needle root +/// array: a stored document carries no alternative pair, so the SQL answers +/// false where `jsonb` answers true, as the design's leakage table records +/// for v3's `@>`. +fn scalar_in_array(stored: &Value, needle: &Value) -> bool { + !stored.is_object() && !stored.is_array() && needle.is_array() } // ============================================================================= -// Extraction +// Round trips // ============================================================================= -#[tokio::test] -async fn an_extracted_node_opens_to_its_scalar_and_a_subtree_to_its_value() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let node = |j: usize| doc.node(j).unwrap(); - let scalar = |j: usize| cipher.decrypt_as::(node(j), aead()); - assert_eq!(scalar(1).await.unwrap(), json!("world-1")); - assert_eq!(scalar(6).await.unwrap(), json!(1)); - assert_eq!( - scalar(0).await.unwrap(), - fixture(), - "the sealed root is the document" - ); - assert!( - matches!(scalar(2).await, Err(Error::OpenedType { .. })), - "a container node holds no value of its own" - ); +mod given_a_document { + use super::*; + + /// A document of every shape the index takes. + fn values() -> Vec { + vec![ + fixture(), + json!({}), + json!([]), + json!("a"), + json!(2.5), + json!(null), + json!(true), + json!([[1, 2], [3, [4, {"a": null}]]]), + json!({"a": {"b": {"c": {"d": [1, "x", false]}}}}), + json!({"a": "b", "a b": 1, "é": [], "0": {}}), + json!({"n": 18446744073709551615u64, "m": -9223372036854775808i64, "f": 1e300}), + ] + } - let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) - .await - .unwrap(); - let rows: Vec = doc - .subtree(key.token_bytes()) - .iter() - .map(|(j, _, _)| node(*j)) - .collect(); - let nested = cipher - .decrypt_as::(key.open(rows.clone()), aead()) - .await - .unwrap(); - assert_eq!(nested, fixture()["nested"]); - let forest = cipher - .decrypt_as::, JsonSubtree>(key.open(rows), aead()) - .await - .unwrap(); - assert_eq!(forest, vec![json!("constant"), json!([{"x": 1}, {"x": 2}])]); + #[tokio::test] + async fn seals_one_entry_per_node_and_opens_again() { + let Scenario { cipher, .. } = scenario().await; + for value in values() { + for options in [JsonOptions::default(), open_root()] { + let what = format!("{value} under {options:?}"); + let doc = seal_with(&cipher, &value, &options).await; + assertions::one_entry_per_node(&doc, &value, &what); + assert_eq!( + doc.header().sealed_root(), + options.sealed_root, + "{what}: the header records whether the root is sealed" + ); + assertions::opens_to(&cipher, doc, &value, &what).await; + } + } + } +} - // The list's token is the server's to derive, one level under nested. - let list = doc - .children(key.token_bytes()) - .into_iter() - .find(|(j, _, _)| *j == 4) - .map(|(_, _, t)| t) - .unwrap(); - let rows: Vec = doc - .subtree(&list) - .iter() - .map(|(j, _, _)| node(*j)) - .collect(); - let elements = cipher - .decrypt_as::(keyed(&cipher, "$.nested.list", rows).await, aead()) - .await - .unwrap(); - assert_eq!(elements, json!([{"x": 1}, {"x": 2}])); - - // An extracted node round-trips through its wire form with the - // envelope fields the SQL grafts beside it. - let mut wire = serde_json::to_value(node(6)).unwrap(); - wire["i"] = json!({"t": "users", "c": "profile"}); - wire["v"] = json!(4); - let back: JsonNode = serde_json::from_value(wire).unwrap(); - assert_eq!(back.ordinal(), 6); - assert_eq!( - cipher - .decrypt_as::(back, aead()) - .await - .unwrap(), - json!(1) - ); -} +mod given_an_open_root { + use super::*; -/// The rows of a walk come from the server, which can repeat an entry or -/// pick which entries to return. Every entry is authentic, but the rebuild -/// nests the containers it is handed: a repeated row is refused, and so is -/// a line of containers deeper than any document the index seals, as a -/// shape mismatch rather than a recursion as deep as the rows. -#[tokio::test] -async fn subtree_rows_the_server_repeats_or_lines_up_are_refused() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let node = |j: usize| doc.node(j).unwrap(); - let repeated = keyed(&cipher, "$.nested", vec![node(1), node(1)]).await; - assert!(matches!( - cipher - .decrypt_as::, JsonSubtree>(repeated, aead()) - .await, - Err(Error::OpenedType { .. }) - )); - - // A wide document of one-member objects: its objects, returned as rows - // with the last leaf after them, each take the next as their member, - // and would nest 200 deep. - let wide = Value::Object( - (0..200) - .map(|i| (format!("k{i:03}"), json!({"x": 1}))) - .collect(), - ); - let doc = seal(&cipher, &wide).await; - let last = doc.entries().len() - 1; - let containers: Vec = (1..doc.entries().len()) - .filter(|j| j % 2 == 1 || *j == last) - .map(|j| doc.node(j).unwrap()) - .collect(); - assert_eq!(containers.len(), 201); - assert!(matches!( - cipher - .decrypt_as::, JsonSubtree>( - keyed(&cipher, "$.k000", containers).await, - aead() - ) - .await, - Err(Error::OpenedType { .. }) - )); - - // The same through arrays: one outer array, then every key's inner - // one-element array, each taken as the element of the one before it. - let wide = Value::Object( - (0..200) - .map(|i| (format!("k{i:03}"), json!([[0]]))) - .collect(), - ); - let doc = seal(&cipher, &wide).await; - let last = doc.entries().len() - 1; - let arrays: Vec = (1..doc.entries().len()) - .filter(|j| *j == 1 || j % 3 == 2 || *j == last) - .map(|j| doc.node(j).unwrap()) - .collect(); - assert_eq!(arrays.len(), 202); - assert!(matches!( - cipher - .decrypt_as::, JsonSubtree>(keyed(&cipher, "$.k000", arrays).await, aead()) - .await, - Err(Error::OpenedType { .. }) - )); + /// A document sealed without the sealed root keeps the root as an + /// ordinary node, and opens from its entries. + #[tokio::test] + async fn keeps_the_root_as_an_entry_and_opens_from_the_entries() { + let Scenario { cipher, .. } = scenario().await; + for value in [fixture(), json!({"": {}}), json!([[], {}, ""]), json!("a")] { + let doc = seal_with(&cipher, &value, &open_root()).await; + assert!( + !doc.header().sealed_root(), + "{value}: the header records the open root" + ); + assertions::one_entry_per_node(&doc, &value, &value.to_string()); + assertions::opens_to(&cipher, doc, &value, &value.to_string()).await; + } + } } -// ============================================================================= -// Tamper evidence -// ============================================================================= - -#[tokio::test] -async fn a_moved_reordered_or_regrafted_entry_fails_to_open() { - let cipher = stack_cipher().await; - let value = json!({"a": "x", "b": "y"}); - let doc = seal_with(&cipher, &value, &open_root()).await; - let other = seal_with(&cipher, &value, &open_root()).await; - let tamper = |mut wire: Value, f: &dyn Fn(&mut Vec)| { - let sv = wire["sv"].as_array_mut().unwrap(); - f(sv); - serde_json::from_value::(wire).unwrap() - }; - let wire = serde_json::to_value(&doc).unwrap(); - let other_wire = serde_json::to_value(&other).unwrap(); - - let reordered = tamper(wire.clone(), &|sv| sv.swap(1, 2)); - assert!( - matches!(open(&cipher, reordered).await, Err(Error::Aead)), - "reordered" - ); - - let regrafted = tamper(wire.clone(), &|sv| sv[1] = other_wire["sv"][1].clone()); - assert!( - matches!(open(&cipher, regrafted).await, Err(Error::Aead)), - "regrafted from another document" - ); - - let truncated = tamper(wire.clone(), &|sv| { - let _ = sv.pop(); - }); - assert!( - matches!( - open(&cipher, truncated).await, - Err(Error::OpenedType { .. }) - ), - "truncated" - ); - - let flipped = tamper(wire.clone(), &|sv| { - let c = sv[1]["c"].as_str().unwrap().to_owned(); - let mut bytes = c.into_bytes(); - bytes[3] = if bytes[3] == b'A' { b'B' } else { b'A' }; - sv[1]["c"] = Value::String(String::from_utf8(bytes).unwrap()); - }); - assert!( - matches!(open(&cipher, flipped).await, Err(Error::Aead)), - "a changed byte" - ); - - let wrong_context = cipher - .decrypt_as::(doc, AeadContext::from(nonempty!("users").with("bio"))) - .await; - assert!(matches!(wrong_context, Err(Error::Aead)), "another column"); - - // A node lifted out of its document fails under another ordinal. - let sealed = seal(&cipher, &value).await; - let mut lifted = serde_json::to_value(sealed.node(1).unwrap()).unwrap(); - lifted["j"] = json!(2); - let lifted: JsonNode = serde_json::from_value(lifted).unwrap(); - assert!(matches!( - cipher.decrypt_as::(lifted, aead()).await, - Err(Error::Aead) - )); +mod given_the_deepest_document { + use super::*; + + /// The deepest document the index takes opens again under either root + /// option, and one level deeper is refused when it is sealed, not when + /// it is read. A sealed root's plaintext is the document as JSON, which + /// `serde_json` reads back only to 128 levels of nesting. + #[tokio::test] + async fn opens_again_and_one_level_deeper_is_refused_when_sealed() { + let Scenario { cipher, .. } = scenario().await; + let arrays = |n: usize| (0..n).fold(json!(1), |inner, _| json!([inner])); + let objects = |n: usize| (0..n).fold(json!("leaf"), |inner, _| json!({"k": inner})); + for nested in [arrays, objects] { + for options in [JsonOptions::default(), open_root()] { + let deepest = nested(127); + let doc = seal_with(&cipher, &deepest, &options).await; + assertions::opens_to( + &cipher, + doc, + &deepest, + &format!("127 levels under {options:?}"), + ) + .await; + let refused = try_seal(&cipher, &nested(128), &options).await; + assertions::refused_by_term( + &refused, + |e| matches!(e, TermError::JsonTooDeep), + &format!("128 levels under {options:?} is refused when sealed"), + ); + } + } + } } // ============================================================================= -// Queries +// The stored form // ============================================================================= -type JsonPlan = stack_encrypt::plan::ValuePlan>; - -fn plan() -> JsonPlan { - Plan::value::() - .context("users/profile") - .encrypt_into::() - .build() - .unwrap() -} - -type OrderedPlan = - stack_encrypt::plan::ValuePlan>; +mod given_the_fixture_document { + use super::*; -/// The plan laid out by the JSON index with `$.hello`, `$.number` and -/// `$.flag` ordered. -fn ordered_plan() -> OrderedPlan { - Plan::value::() - .context("users/profile") - .encrypt_as(Json::new(ordering(&["$.hello", "$.number", "$.flag"]))) - .build() - .unwrap() -} + #[tokio::test] + async fn stores_the_wire_form_the_sql_domain_checks() { + let Scenario { cipher, plan } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let wire = serde_json::to_value(&doc).unwrap(); + let keys: Vec<&str> = wire + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + assert_eq!(keys, ["h", "q", "sv"], "the document's wire keys"); + // `q` is the document's own projection, written once: the needle a + // `Contains` of the whole document derives has the same pairs in the + // same groups, and nothing else. + let needle = cipher + .query(op::Contains, &fixture()) + .on(&plan) + .await + .unwrap(); + assert_eq!( + wire["q"], + serde_json::to_value(&needle).unwrap(), + "`q` is the needle a `Contains` of the whole document derives" + ); + assert!( + serde_json::from_value::(json!({"h": wire["h"], "sv": wire["sv"]})).is_err(), + "a document without `q` is not a document: the projection is the only place the value selectors live" + ); + let back: JsonDocument = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!( + serde_json::to_value(&back).unwrap(), + wire, + "`q` round-trips as stored" + ); + assert_eq!( + serde_json::to_value(back.projection()).unwrap(), + wire["q"], + "the projection read back is the stored one" + ); + for (j, entry) in wire["sv"].as_array().unwrap().iter().enumerate() { + let entry = entry.as_object().unwrap(); + for key in entry.keys() { + assert!( + ["s", "c", "p", "op"].contains(&key.as_str()), + "entry {j} carries an unknown key {key}" + ); + } + for (tag, chars, bytes) in [("s", 24, 16), ("p", 12, 8)] { + if let Some(text) = entry.get(tag) { + assert_eq!( + text.as_str().unwrap().len(), + chars, + "entry {j}'s {tag} is {bytes} bytes of padded base64" + ); + } + } + let c = entry["c"].as_str().unwrap(); + assert!( + c.bytes() + .all(|b| b.is_ascii_graphic() && b != b'"' && b != b'\\'), + "entry {j}'s c is base85 in an alphabet a JSON string carries unescaped: {c:?}" + ); + } + let back: JsonDocument = serde_json::from_value(wire).unwrap(); + assert_eq!(back, doc, "the wire form reads back to the document"); + } -/// PostgreSQL's `jsonb @>`, from its documented rules: an object contains -/// another when it has every key with a containing value; an array -/// contains another when every needle element is contained by some stored -/// element; a top-level array contains a scalar that is one of its -/// elements; scalars contain equal scalars, numbers by value. -fn pg_contains(stored: &Value, needle: &Value, top: bool) -> bool { - match (stored, needle) { - (Value::Object(a), Value::Object(b)) => b - .iter() - .all(|(k, vb)| a.get(k).is_some_and(|va| pg_contains(va, vb, false))), - (Value::Array(a), Value::Array(b)) => b - .iter() - .all(|vb| a.iter().any(|va| pg_contains(va, vb, false))), - (Value::Array(a), scalar) if top && !scalar.is_object() && !scalar.is_array() => { - a.iter().any(|va| pg_contains(va, scalar, false)) + #[tokio::test] + async fn places_the_chain_fields_where_the_design_says() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let entries = doc.entries(); + assert_eq!(entries.len(), 10, "one entry per node of the fixture"); + assert!( + entries[0].parent_tag().is_none(), + "the root has no parent tag" + ); + for head in [&entries[1], &entries[2], &entries[9]] { + assert!( + head.parent_tag().is_none(), + "a head has no parent tag under the sealed root" + ); } - (Value::Number(a), Value::Number(b)) => a.as_f64() == b.as_f64(), - (Value::Object(_) | Value::Array(_), _) | (_, Value::Object(_) | Value::Array(_)) => false, - (a, b) => a == b, + for (j, below) in entries[3..9].iter().enumerate() { + assert!( + below.parent_tag().is_some(), + "entry {} below a head carries a parent tag", + j + 3 + ); + } + // Every parent tag is distinct: siblings are not linkable at rest. + let tags: std::collections::HashSet<_> = entries[3..9] + .iter() + .map(|e| *e.parent_tag().unwrap()) + .collect(); + assert_eq!(tags.len(), 6, "every parent tag is distinct"); + // The projection groups the two list elements: each is a nested group + // holding its own pair and its `x`. + let q = serde_json::to_value(doc.projection()).unwrap(); + assert_eq!( + q["el"].as_array().unwrap().len(), + 2, + "one group per list element" + ); + assert_eq!( + q["el"][0]["sv"].as_array().unwrap().len(), + 2, + "an element's group holds its own pair and its x" + ); + assert_eq!( + q["sv"].as_array().unwrap().len(), + 6, + "$, hello, nested, deep, list, number" + ); + assert!( + entries.iter().all(|e| e.ordering_term().is_none()), + "no path is ordered by default, so no entry carries a term" + ); } -} -/// The design's two corpora: `jsonb-semantics.sql` and -/// `containment-oracle.sql`, as `(stored, needle)` pairs. -fn corpus() -> Vec<(Value, Value)> { - [ - (r#"["a"]"#, r#""a""#), - (r#""a""#, r#"["a"]"#), - (r#"{"a":[1]}"#, r#"{"a":1}"#), - (r#"{"a":1}"#, r#"{"a":[1]}"#), - (r#"[1,[2]]"#, r#"[2]"#), - (r#"[1,[2]]"#, r#"[[2]]"#), - (r#"{"a":{"b":1}}"#, r#"{"a":{}}"#), - (r#"{"a":{}}"#, r#"{"a":{"b":1}}"#), - (r#"{"a":[1,2]}"#, r#"{"a":[]}"#), - (r#"{"a":1}"#, r#"{}"#), - (r#"[1]"#, r#"{}"#), - (r#"[1]"#, r#"[]"#), - (r#"[1,2]"#, r#"[1,1,1]"#), - ( - r#"{"a":[{"x":1,"y":5},{"x":7,"y":2}]}"#, - r#"{"a":[{"x":1,"y":2}]}"#, - ), - (r#"{"a":[{"x":1,"y":2,"z":3}]}"#, r#"{"a":[{"x":1,"y":2}]}"#), - (r#"{"a":[{"x":1}]}"#, r#"{"a":[{"x":1},{"x":1}]}"#), - ( - r#"{"a":[{"b":[{"x":1},{"y":2}]}]}"#, - r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, - ), - ( - r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, - r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, - ), - (r#"{"a":[["x"]]}"#, r#"{"a":["x"]}"#), - (r#""a""#, r#""a""#), - (r#"2"#, r#"2.0"#), - (r#"{"a":null}"#, r#"{"a":null}"#), - (r#"[["a"]]"#, r#""a""#), - ( - r#"{"a":[{"b":[{"x":1,"y":2}],"c":3},{"b":[{"x":1,"y":9}],"c":4}]}"#, - r#"{"a":[{"b":[{"x":1,"y":2}],"c":4}]}"#, - ), - ( - r#"{"a":[{"b":[{"x":1,"y":2}],"c":3},{"b":[{"x":1,"y":9}],"c":4}]}"#, - r#"{"a":[{"b":[{"x":1,"y":2}],"c":3}]}"#, - ), - ( - r#"{"a":[{"b":{"c":1,"d":2}},{"b":{"c":3,"d":2}}]}"#, - r#"{"a":[{"b":{"c":1,"d":2}}]}"#, - ), - ( - r#"{"a":[{"b":{"c":1,"d":5}},{"b":{"c":3,"d":2}}]}"#, - r#"{"a":[{"b":{"c":1,"d":2}}]}"#, - ), - (r#"{"a":[[1,2],[3,4]]}"#, r#"{"a":[[1,4]]}"#), - (r#"{"a":[[1,2],[3,4]]}"#, r#"{"a":[[1,2]]}"#), - (r#"{"a":[[1,2],[3,4]]}"#, r#"{"a":[[1],[4]]}"#), - (r#"{"a":"2"}"#, r#"{"a":2}"#), - (r#"{"a":2}"#, r#"{"a":2.0}"#), - (r#"{"a":[{"x":1}]}"#, r#"{"a":[{}]}"#), - (r#"{"a":[1]}"#, r#"{"a":[{}]}"#), - ( - r#"{"a":[{"t":[{"u":1,"v":2}]},{"t":[{"u":1},{"v":2}]}]}"#, - r#"{"a":[{"t":[{"u":1,"v":2}]}]}"#, - ), - ( - r#"{"a":[{"t":[{"u":1},{"v":2}]}]}"#, - r#"{"a":[{"t":[{"u":1,"v":2}]}]}"#, - ), - (r#"{"a":[{}]}"#, r#"{"a":[{}]}"#), - (r#"{"a":[[]]}"#, r#"{"a":[[]]}"#), - (r#"{"a":[[]]}"#, r#"{"a":[{}]}"#), - (r#"[{"a":1},{"b":2}]"#, r#"[{"a":1,"b":2}]"#), - (r#"[{"a":1,"b":2}]"#, r#"[{"a":1},{"b":2}]"#), - (r#"{"a":[true,false,null]}"#, r#"{"a":[null,true]}"#), - (r#"{"a":[true]}"#, r#"{"a":[false]}"#), - (r#"{"a":[{"x":[1,2]},{"x":[3]}]}"#, r#"{"a":[{"x":[2,3]}]}"#), - (r#"{"a":[{"x":[1,2]},{"x":[3]}]}"#, r#"{"a":[{"x":[1,2]}]}"#), - ] - .iter() - .map(|(a, b)| { - ( - serde_json::from_str(a).unwrap(), - serde_json::from_str(b).unwrap(), - ) - }) - .collect() -} + /// Equal values at one path share selectors but not ciphertexts; the + /// value selectors, in the projection, tell equal values from different. + #[tokio::test] + async fn gives_equal_values_at_one_path_one_value_selector_and_two_ciphertexts() { + let Scenario { cipher, .. } = scenario().await; + let pair = |doc: &JsonDocument, i: usize| -> (String, String) { + let q = serde_json::to_value(doc.projection()).unwrap(); + ( + q["sv"][i]["s"].as_str().unwrap().to_owned(), + q["sv"][i]["vs"].as_str().unwrap().to_owned(), + ) + }; + let same = seal(&cipher, &json!([1, 1])).await; + let (a, b) = (&same.entries()[1], &same.entries()[2]); + assert_eq!(a.selector(), b.selector(), "two elements share a path"); + assert_eq!( + pair(&same, 1), + pair(&same, 2), + "equal values at one path share a value selector" + ); + assert_ne!( + a.ciphertext(), + b.ciphertext(), + "equal values do not share a ciphertext" + ); + let different = seal(&cipher, &json!([1, 2])).await; + let (one, two) = (&different.entries()[1], &different.entries()[2]); + assert_eq!(one.selector(), two.selector(), "one path"); + assert_ne!( + pair(&different, 1), + pair(&different, 2), + "different values at one path have different value selectors" + ); + assert_eq!( + pair(&different, 1), + pair(&same, 1), + "the value selector names the value at its path, in any document" + ); + } -/// The pinned answers of `jsonb-semantics.sql`, so the model above is held -/// to PostgreSQL and not only to itself. -#[test] -fn the_containment_model_gives_postgresqls_answers() { - let expect = [ - (r#"["a"]"#, r#""a""#, true), - (r#""a""#, r#"["a"]"#, false), - (r#"{"a":[1]}"#, r#"{"a":1}"#, false), - (r#"{"a":1}"#, r#"{"a":[1]}"#, false), - (r#"[1,[2]]"#, r#"[2]"#, false), - (r#"[1,[2]]"#, r#"[[2]]"#, true), - (r#"{"a":{"b":1}}"#, r#"{"a":{}}"#, true), - (r#"{"a":{}}"#, r#"{"a":{"b":1}}"#, false), - (r#"{"a":[1,2]}"#, r#"{"a":[]}"#, true), - (r#"{"a":1}"#, r#"{}"#, true), - (r#"[1]"#, r#"{}"#, false), - (r#"[1]"#, r#"[]"#, true), - (r#"[1,2]"#, r#"[1,1,1]"#, true), - ( - r#"{"a":[{"x":1,"y":5},{"x":7,"y":2}]}"#, - r#"{"a":[{"x":1,"y":2}]}"#, - false, - ), - ( - r#"{"a":[{"x":1,"y":2,"z":3}]}"#, - r#"{"a":[{"x":1,"y":2}]}"#, - true, - ), - (r#"{"a":[{"x":1}]}"#, r#"{"a":[{"x":1},{"x":1}]}"#, true), - ( - r#"{"a":[{"b":[{"x":1},{"y":2}]}]}"#, - r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, - false, - ), - ( - r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, - r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, - true, - ), - (r#"{"a":[["x"]]}"#, r#"{"a":["x"]}"#, false), - (r#""a""#, r#""a""#, true), - (r#"2"#, r#"2.0"#, true), - (r#"{"a":null}"#, r#"{"a":null}"#, true), - (r#"[["a"]]"#, r#""a""#, false), - ]; - for (stored, needle, answer) in expect { - let (a, b): (Value, Value) = ( - serde_json::from_str(stored).unwrap(), - serde_json::from_str(needle).unwrap(), + #[tokio::test] + async fn chains_the_top_level_fields_from_an_open_root() { + let Scenario { cipher, .. } = scenario().await; + let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; + assert!( + open_doc.entries()[1].parent_tag().is_some(), + "without the sealed root, the top-level fields chain from the root" ); - assert_eq!(pg_contains(&a, &b, true), answer, "{stored} @> {needle}"); } -} -/// A small deterministic generator of documents over a few keys and -/// values, so needles drawn from it overlap stored documents often. -struct Lcg(u64); - -impl Lcg { - fn next(&mut self) -> u64 { - self.0 = self - .0 - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); - self.0 >> 33 - } - fn below(&mut self, n: u64) -> u64 { - self.next() % n - } - fn value(&mut self, depth: u32) -> Value { - match self.below(if depth == 0 { 4 } else { 6 }) { - 0 => json!(self.below(3)), - 1 => json!(["x", "y", "z"][self.below(3) as usize]), - 2 => json!(self.below(2) == 0), - 3 => Value::Null, - 4 => Value::Array((0..self.below(4)).map(|_| self.value(depth - 1)).collect()), - _ => { - let mut map = Map::new(); - for _ in 0..self.below(4) { - let key = ["a", "b", "c"][self.below(3) as usize].to_owned(); - let _ = map.insert(key, self.value(depth - 1)); - } - Value::Object(map) - } + #[tokio::test] + async fn derives_the_stored_selector_from_a_path_folding_positions_into_the_wildcard() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let keyset = cipher.default_keyset(); + let selector = |path: &str| { + let path = JsonPath::parse(path).unwrap(); + keyset.json_selector(&path, nonempty!("users").with("profile")) + }; + for (path, j) in [ + ("$.nested.deep", 3), + ("$.nested.list[*].x", 6), + ("$.nested.list[1].x", 8), + ("$", 0), + ] { + assert_eq!( + selector(path).await.unwrap().as_bytes(), + doc.entries()[j].selector(), + "{path} derives entry {j}'s stored selector" + ); } + assert_ne!( + selector("$.hello").await.unwrap().as_bytes(), + keyset + .json_selector( + &JsonPath::parse("$.hello").unwrap(), + nonempty!("users").with("bio") + ) + .await + .unwrap() + .as_bytes(), + "a selector is bound to its column" + ); } -} -#[tokio::test] -async fn containment_over_the_projection_is_postgresqls_containment() { - let cipher = stack_cipher().await; - let plan = plan(); - let mut cases = corpus(); - let mut rng = Lcg(42); - for _ in 0..150 { - let stored = rng.value(3); - let needle = rng.value(2); - cases.push((stored.clone(), needle)); - // A needle taken from the stored document itself is always contained. - cases.push((stored.clone(), stored)); - } - for (stored, needle) in cases { - let doc = cipher.encrypt(&stored).using(&plan).await.unwrap(); - let projection = doc.projection(); - let query: JsonQuery = cipher.query(op::Contains, &needle).on(&plan).await.unwrap(); - assert_eq!( - query.matches(projection), - pg_contains(&stored, &needle, true), - "{stored} @> {needle}" - ); - // `<@` compares the same two projections the other way round. The - // one case it cannot answer is a stored root scalar inside a needle - // root array: a stored document carries no alternative pair, so the - // SQL answers false where `jsonb` answers true, as the design's - // leakage table records for v3's `@>`. - let reverse: JsonQuery = cipher - .query(op::ContainedBy, &needle) - .on(&plan) + /// No token is stored: an entry below a head carries only the parent + /// tag, and no tag in the document is a token, or a parent's hash of one + /// without the document's salt. + #[tokio::test] + async fn stores_no_token() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let nested = subtree_key(&cipher, "$.nested", JsonOptions::default()) .await .unwrap(); - let scalar_in_array = !stored.is_object() && !stored.is_array() && needle.is_array(); + let mut tokens = vec![*nested.token_bytes()]; + let mut frontier = vec![*nested.token_bytes()]; + while let Some(token) = frontier.pop() { + assert!( + tokens.len() <= doc.entries().len(), + "the chain is a tree: no token is reached twice" + ); + for (_, _, child) in doc.children(&token) { + tokens.push(child); + frontier.push(child); + } + } + assert_eq!(tokens.len(), 7, "nested and its six descendants"); + for (j, entry) in doc.entries().iter().enumerate() { + if let Some(p) = entry.parent_tag() { + assert!( + !tokens.iter().any(|t| t.starts_with(p)), + "entry {j}'s parent tag is not a prefix of a token" + ); + } + assert!( + !tokens.contains(entry.selector()), + "entry {j}'s selector is not a token" + ); + } assert_eq!( - reverse.projection().contains(projection), - pg_contains(&needle, &stored, true) && !scalar_in_array, - "{stored} <@ {needle}" + nested.token(), + base64(nested.token_bytes()), + "a token's text form is its padded base64" ); } } -#[tokio::test] -async fn a_query_term_serializes_as_the_needle_operand_and_derives_no_key() { - let (cipher, generates, _) = common::counting_cipher().await; - let plan = plan(); - let query = cipher - .query(op::Contains, &json!({"a": [{"x": 1}]})) - .on(&plan) - .await - .unwrap(); - let wire = serde_json::to_value(&query).unwrap(); - assert_eq!( - wire["sv"].as_array().unwrap().len(), - 2, - "$ and $.a sit outside the element" - ); - assert_eq!(wire["el"].as_array().unwrap().len(), 1); - assert_eq!( - wire["el"][0]["sv"].as_array().unwrap().len(), - 2, - "the element and its x" - ); - assert!(wire.get("alt").is_none()); - assert_eq!( - generates.load(std::sync::atomic::Ordering::Relaxed), - 0, - "a query mints no key" - ); - - let scalar = cipher - .query(op::Contains, &json!("a")) - .on(&plan) - .await - .unwrap(); - let wire = serde_json::to_value(&scalar).unwrap(); - assert!( - wire.get("alt").is_some(), - "a root scalar carries its root-array alternative" - ); -} - -#[tokio::test] -async fn an_accessor_asks_at_a_path() { - let cipher = stack_cipher().await; - let plan = plan(); - let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); - let projection = doc.projection(); - - let hit: JsonQuery = cipher - .query(op::AtPath(op::Eq), &("$.nested.list[*].x", 2)) - .on(&plan) - .await - .unwrap(); - assert!(hit.matches(projection)); - let miss: JsonQuery = cipher - .query(op::AtPath(op::Eq), &("$.nested.list[*].x", 3)) - .on(&plan) - .await - .unwrap(); - assert!(!miss.matches(projection)); - let deep: JsonQuery = cipher - .query(op::AtPath(op::Eq), &("$.nested.deep", "constant")) - .on(&plan) - .await - .unwrap(); - assert!(deep.matches(projection)); - let wrong_kind: JsonQuery = cipher - .query(op::AtPath(op::Eq), &("$.number", "1")) - .on(&plan) - .await - .unwrap(); - assert!( - !wrong_kind.matches(projection), - "a string is not the number" - ); +mod given_ordered_paths { + use super::*; - // An ordering at a path: the path's selector and a leaf term that - // orders as the stored leaf's does, under native byte comparison. The - // plan declares the paths it orders, and the query runs under them. - let plan = ordered_plan(); - let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); - let bound = cipher - .query(op::AtPath(op::Gte), &("$.number", 0)) - .on(&plan) - .await - .unwrap(); - assert_eq!(bound.selector().as_bytes(), doc.entries()[9].selector()); - assert!( - bound.ordering_term() < doc.entries()[9].ordering_term().unwrap(), - "0 < 1" - ); - let above = cipher - .query(op::AtPath(op::Lt), &("$.number", 5)) - .on(&plan) - .await - .unwrap(); - assert!( - above.ordering_term() > doc.entries()[9].ordering_term().unwrap(), - "5 > 1" - ); - let text = cipher - .query(op::AtPath(op::Gt), &("$.hello", "world-0")) - .on(&plan) - .await - .unwrap(); - assert!(text.ordering_term() < doc.entries()[1].ordering_term().unwrap()); - let range = cipher - .query(op::AtPath(op::Between), &("$.number", (0, 5))) - .on(&plan) - .await - .unwrap(); - assert!(range.lo().ordering_term() < range.hi().ordering_term()); - assert_eq!( - range.lo().selector().as_bytes(), - doc.entries()[9].selector() - ); - let wire = serde_json::to_value(range.lo()).unwrap(); - assert_eq!( - wire.as_object().unwrap().keys().collect::>(), - ["op", "s"] - ); + /// Which entries carry an ordering term. + fn with_term(doc: &JsonDocument) -> Vec { + doc.entries() + .iter() + .enumerate() + .filter(|(_, e)| e.ordering_term().is_some()) + .map(|(j, _)| j) + .collect() + } - // Refusals, before any term is derived. - let bad_path = cipher - .query(op::AtPath(op::Eq), &("address.city", "Sydney")) - .on(&plan) - .await; - assert!(matches!(bad_path, Err(Error::Term(TermError::JsonPath(_))))); - let unordered = cipher - .query(op::AtPath(op::Gte), &("$.flag", true)) - .on(&plan) - .await; - assert!(matches!( - unordered, - Err(Error::Term(TermError::JsonLeafNotOrdered)) - )); - // A path the plan does not order is refused: no stored leaf there - // carries a term, so the comparison would match no row. Under the - // default plan that is every path. - let not_ordered = cipher - .query(op::AtPath(op::Lt), &("$.nested.deep", "z")) - .on(&plan) - .await; - assert!( - matches!(not_ordered, Err(Error::Term(TermError::JsonPathNotOrdered))), - "{not_ordered:?}" - ); - let default_plan = self::plan(); - let not_ordered = cipher - .query(op::AtPath(op::Between), &("$.number", (0, 5))) - .on(&default_plan) + /// An ordering term sits on a number or string leaf at an ordered path + /// and nowhere else: a wildcard covers every element. + #[tokio::test] + async fn puts_a_term_only_on_a_number_or_string_leaf_at_an_ordered_path() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal_with(&cipher, &fixture(), &ordering(&["$.hello", "$.number"])).await; + assert_eq!(with_term(&doc), [1, 9], "$.hello and $.number"); + let wild = seal_with(&cipher, &fixture(), &ordering(&["$.nested.list[*].x"])).await; + assert_eq!(with_term(&wild), [6, 8], "a wildcard covers every element"); + // A container, a boolean or a null at an ordered path carries none. + let kinds = seal_with( + &cipher, + &json!({"a": true, "b": null, "c": {"d": 1}, "e": [1, "x"]}), + &ordering(&["$.a", "$.b", "$.c", "$.e", "$.e[*]"]), + ) .await; - assert!( - matches!(not_ordered, Err(Error::Term(TermError::JsonPathNotOrdered))), - "{not_ordered:?}" - ); - // An ordering at a wildcard path is covered by the wildcard. One at a - // position is refused whatever the plan orders: `->` reads a position - // as the wildcard and would compare the first element, not the one - // named. - let by_element = Plan::value::() - .context("users/profile") - .encrypt_as(Json::new(ordering(&["$.nested.list[*].x"]))) - .build() - .unwrap(); - assert!(cipher - .query(op::AtPath(op::Gt), &("$.nested.list[*].x", 0)) - .on(&by_element) - .await - .is_ok()); - for path in ["$.nested.list[1].x", "$.nested.list[0].x"] { - let refused = cipher - .query(op::AtPath(op::Gt), &(path, 0)) - .on(&by_element) - .await; + // $, $.a, $.b, $.c, $.c.d, $.e, $.e[0], $.e[1] + assert_eq!(with_term(&kinds), [6, 7], "only the number and the string"); + let scalar = seal_with(&cipher, &json!(7), &ordering(&["$"])).await; + assert_eq!(with_term(&scalar), [0], "the root scalar itself, at `$`"); + assertions::opens_to(&cipher, doc, &fixture(), "ordered at $.hello and $.number").await; + assertions::opens_to(&cipher, wild, &fixture(), "ordered at a wildcard").await; + } + + /// A position is refused, since `->` reads it as the wildcard and no + /// ordering could reach the leaf it names; so is a text that is not a + /// path. + #[tokio::test] + async fn refuses_a_position_or_a_text_that_is_not_a_path() { + let Scenario { cipher, .. } = scenario().await; + let position = JsonOptions::default().ordering(["$.nested.list[1].x"]); assert!( - matches!(refused, Err(Error::Term(TermError::JsonOrderingAtPosition))), - "{path}: {refused:?}" + matches!(position, Err(TermError::JsonOrderingAtPosition)), + "a position is refused as an ordered path: {position:?}" ); - let refused = cipher - .query(op::AtPath(op::Between), &(path, (0, 5))) - .on(&by_element) - .await; + let not_a_path = JsonOptions::default().ordering(["nested"]); assert!( - matches!(refused, Err(Error::Term(TermError::JsonOrderingAtPosition))), - "{path}: {refused:?}" + matches!(not_a_path, Err(TermError::JsonPath(_))), + "a text that is not a path is refused: {not_a_path:?}" ); - } - // Equality at a path compares one value: an object or an array would be - // containment, which `op::Contains` asks for. - for container in [json!({"deep": "constant"}), json!([]), json!([{"x": 1}])] { - let refused = cipher - .query(op::AtPath(op::Eq), &("$.nested", container.clone())) - .on(&plan) - .await; - assert!( - matches!(refused, Err(Error::Term(TermError::JsonValueNotScalar))), - "{container}" + // Set on the field, which `ordering` guards, a position refuses the + // seal rather than put a term no ordering reaches on one element. + let mut positional = JsonOptions::default(); + let _ = positional + .ordered + .insert(JsonPath::parse("$.nested.list[1].x").unwrap()); + let refused = try_seal(&cipher, &fixture(), &positional).await; + assertions::refused_by_term( + &refused, + |e| matches!(e, TermError::JsonOrderingAtPosition), + "a position set on the field refuses the seal", ); } } -/// A plan laid out by the index opens what it wrote, one document or a -/// batch, and reports its layout. -#[tokio::test] -async fn a_laid_out_plan_opens_what_it_wrote() { - let cipher = stack_cipher().await; - let plan = ordered_plan(); - let options = ordering(&["$.hello", "$.number", "$.flag"]); - assert_eq!(plan.layout().options(), &options); - assert_eq!(plan.specs(), [IndexSpec::Json(options.clone())]); - assert_eq!( - stack_encrypt::target::Layout::::specs(&Json::new(options.clone())), - [IndexSpec::Json(options)] - ); - assert!( - format!("{plan:?}").contains("Laid(Json"), - "{plan:?}" - ); - let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); - let back: Value = cipher.open(doc.clone()).using(&plan).await.unwrap(); - assert_eq!(back, fixture()); - let docs: Vec = cipher - .open(vec![doc.clone(), doc.clone()]) - .using(&plan) - .await - .unwrap(); - assert_eq!(docs, [fixture(), fixture()]); - // The plan was built with its context, so an open that names one is - // refused, one document or a batch, before any key is requested. - let clash = cipher - .open(doc.clone()) - .context("users/other") - .using(&plan) - .await; - assert!( - matches!(clash, Err(Error::Plan(PlanError::TwoContextSources { .. }))), - "{clash:?}" - ); - let clash = cipher - .open(vec![doc.clone()]) - .context("users/other") - .using(&plan) - .await; - assert!( - matches!(clash, Err(Error::Plan(PlanError::TwoContextSources { .. }))), - "{clash:?}" - ); - // The index named on the query says only which: the plan's own, with - // its options, answers. - let through_default = cipher - .query(op::AtPath(op::Gte), &("$.number", 0)) - .via(Json) - .on(&plan) - .await - .unwrap(); - let chosen = cipher - .query(op::AtPath(op::Gte), &("$.number", 0)) - .on(&plan) - .await - .unwrap(); - assert_eq!(through_default.selector(), chosen.selector()); -} - -/// At the root, equality is of the root value: `("$", 1)` matches a stored -/// `1` and not a stored `[1]`, which containment of the scalar `1` would. -#[tokio::test] -async fn equality_at_the_root_is_not_containment() { - let cipher = stack_cipher().await; - let plan = plan(); - let at_root: JsonQuery = cipher - .query(op::AtPath(op::Eq), &("$", 1)) - .on(&plan) - .await - .unwrap(); - let contains: JsonQuery = cipher - .query(op::Contains, &json!(1)) - .on(&plan) - .await - .unwrap(); - let scalar = cipher.encrypt(&json!(1)).using(&plan).await.unwrap(); - let array = cipher.encrypt(&json!([1])).using(&plan).await.unwrap(); - assert!(at_root.matches(scalar.projection())); - assert!(!at_root.matches(array.projection())); - assert!(contains.matches(array.projection()), "jsonb: [1] @> 1"); - assert!(serde_json::to_value(&at_root).unwrap().get("alt").is_none()); -} - -#[tokio::test] -async fn a_query_under_other_options_is_refused_by_the_plan() { - let cipher = stack_cipher().await; - let plan = plan(); - let refused = cipher - .query(op::Contains, &json!({})) - .via(Json::new(open_root())) - .on(&plan) - .await; - assert!( - matches!(refused, Err(Error::Plan(PlanError::IndexOptions { .. }))), - "{refused:?}" - ); - assert_eq!( - >::indexes(), - [IndexSpec::Json(JsonOptions::default())] - ); - assert_eq!(Json, Json::default(), "the value is the default index"); -} - -#[tokio::test] -async fn opening_through_the_plan_is_the_typed_open() { - let cipher = stack_cipher().await; - let plan = plan(); - let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); - let back: Value = cipher.open(doc).using(&plan).await.unwrap(); - assert_eq!(back, fixture()); -} - -#[cfg(feature = "dynamic")] -#[test] -fn the_json_index_has_a_wire_form() { - use stack_encrypt::dynamic::{not_admitted, Reason, ValueKind}; - assert_eq!( - IndexSpec::parse("json"), - Some(IndexSpec::Json(JsonOptions::default())) - ); - assert_eq!(IndexSpec::Json(JsonOptions::default()).key(), "json"); - assert_eq!( - serde_json::to_value(IndexSpec::Json(JsonOptions::default())).unwrap(), - json!("json") - ); - assert_eq!( - serde_json::to_value(IndexSpec::Json(open_root())).unwrap(), - json!({"json": {"sealed_root": false, "ordered": []}}), - "every option is written when any is not the default" - ); - let frame = - vitaminc_aead_frame::serde::to_vec(&json!({"json": {"sealed_root": false}})).unwrap(); - assert_eq!( - IndexSpec::from_frame(&frame).unwrap(), - IndexSpec::Json(open_root()) - ); - let frame = vitaminc_aead_frame::serde::to_vec(&json!({"json": {"nope": true}})).unwrap(); - assert!(IndexSpec::from_frame(&frame).is_err()); - // The ordered paths ride in `ordered`, written in their canonical - // order and read back parsed; a text that is not a path refuses the - // index. - let ordered = IndexSpec::Json(ordering(&["$.orders[*].total", "$.age", "$.age"])); - assert_eq!( - serde_json::to_value(&ordered).unwrap(), - json!({"json": {"sealed_root": true, "ordered": ["$.age", "$.orders[*].total"]}}) - ); - let frame = vitaminc_aead_frame::serde::to_vec( - &json!({"json": {"ordered": ["$.age", "$.orders[*].total"]}}), - ) - .unwrap(); - assert_eq!(IndexSpec::from_frame(&frame).unwrap(), ordered); - let frame = vitaminc_aead_frame::serde::to_vec(&json!({"json": {"ordered": ["age"]}})).unwrap(); - assert!(IndexSpec::from_frame(&frame).is_err()); - // A position in an ordered path refuses the index too: no ordering can - // reach the leaf it names. - let frame = - vitaminc_aead_frame::serde::to_vec(&json!({"json": {"ordered": ["$.scores[2]"]}})).unwrap(); - assert!(IndexSpec::from_frame(&frame).is_err()); - assert_eq!( - not_admitted(ValueKind::Object, &IndexSpec::Json(JsonOptions::default())), - Some(Reason::JsonIndexPending) - ); -} - // ============================================================================= -// What the suite pins beyond the round trips +// The token chain // ============================================================================= -/// No token is stored: an entry below a head carries only the parent tag, -/// and no tag in the document is a token, or a parent's hash of one -/// without the document's salt. -#[tokio::test] -async fn no_stored_tag_is_a_token() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let nested = subtree_key(&cipher, "$.nested", JsonOptions::default()) - .await - .unwrap(); - let mut tokens = vec![*nested.token_bytes()]; - let mut frontier = vec![*nested.token_bytes()]; - while let Some(token) = frontier.pop() { +mod given_a_subtree_key { + use super::*; + + #[tokio::test] + async fn unlocks_exactly_its_subtree_from_the_path_alone() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) + .await + .unwrap(); assert!( - tokens.len() <= doc.entries().len(), - "the chain is a tree: no token is reached twice" + key.gates().is_empty(), + "a chain head's token is derived, nothing to gate" + ); + assert_eq!( + key.path(), + &JsonPath::parse("$.nested").unwrap(), + "the key carries the path it was derived from" + ); + let under = doc.subtree(key.token_bytes()); + let found: Vec = under.iter().map(|(j, _, _)| *j).collect(); + assert_eq!( + found, + [3, 4, 5, 6, 7, 8], + "deep, list, both elements, both leaves" + ); + let depths: Vec = under.iter().map(|(_, _, d)| *d).collect(); + assert_eq!( + depths, + [1, 1, 2, 3, 2, 3], + "each row's depth below $.nested" + ); + assert!( + doc.subtree(&[7; 16]).is_empty(), + "a wrong token unlocks nothing" ); - for (_, _, child) in doc.children(&token) { - tokens.push(child); - frontier.push(child); - } } - assert_eq!(tokens.len(), 7, "nested and its six descendants"); - for entry in doc.entries() { - if let Some(p) = entry.parent_tag() { - assert!( - !tokens.iter().any(|t| t.starts_with(p)), - "a parent tag is not a prefix of a token" + + /// Below a head the client derives nothing but array positions: a key + /// step becomes a gate, and the walk from the head takes it. + #[tokio::test] + async fn gates_each_key_step_below_its_head() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) + .await + .unwrap(); + let deeper = subtree_key(&cipher, "$.nested.list[1]", JsonOptions::default()) + .await + .unwrap(); + assert_eq!( + deeper.token_bytes(), + key.token_bytes(), + "anchored at the head" + ); + assert_eq!(deeper.gates().len(), 2, "list, then [1]"); + assert_eq!( + deeper.gates()[1], + format!("#1:{}", base64(doc.entries()[5].selector())), + "the position, and the selector every element of the list carries" + ); + let list_token = child_token(&doc, key.token_bytes(), 4); + let second = child_token(&doc, &list_token, 7); + assert_eq!( + subtree_ordinals(&doc, &second), + [8], + "the second element's x" + ); + let children: Vec = doc + .children(&list_token) + .iter() + .map(|(j, _, _)| *j) + .collect(); + assert_eq!( + children, + [5, 7], + "the children of a node, by ordinal: the list's two elements" + ); + // A wildcard path anchors at the head and gates on each step's + // selector: the list's, then every element's (the wildcard's), then x's. + let gated = subtree_key(&cipher, "$.nested.list[*].x", JsonOptions::default()) + .await + .unwrap(); + assert_eq!( + gated.token_bytes(), + key.token_bytes(), + "a wildcard path anchors at the head" + ); + assert_eq!( + gated.gates(), + [ + base64(doc.entries()[4].selector()), + base64(doc.entries()[5].selector()), + base64(doc.entries()[6].selector()), + ], + "the list's selector, every element's, then x's" + ); + } + + /// The sealed root has no token; the open root's unlocks every node, and + /// under an open root the chain is derivable down to the first key step. + #[tokio::test] + async fn has_no_token_at_a_sealed_root_and_derives_from_an_open_one() { + let Scenario { cipher, .. } = scenario().await; + for path in ["$", "$[*].x"] { + let refused = subtree_key(&cipher, path, JsonOptions::default()).await; + assertions::refused_by_term( + &refused, + |e| matches!(e, TermError::JsonSealedRoot), + &format!("{path} under the sealed root"), ); } - assert!(!tokens.contains(entry.selector())); + let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; + let root = subtree_key(&cipher, "$", open_root()).await.unwrap(); + assert_eq!( + open_doc.subtree(root.token_bytes()).len(), + 9, + "every entry but the root" + ); + // `$[0]` would be a position, `$.nested` is a gate. + let open_key = subtree_key(&cipher, "$.nested.list[1]", open_root()) + .await + .unwrap(); + assert_eq!( + open_key.token_bytes(), + root.token_bytes(), + "anchored at the open root" + ); + assert_eq!(open_key.gates().len(), 3, "nested, list, then [1]"); + let array_doc = seal_with(&cipher, &json!([[1, 2], [3]]), &open_root()).await; + let second_row = subtree_key(&cipher, "$[1]", open_root()).await.unwrap(); + assert!(second_row.gates().is_empty(), "positions derive"); + assert_eq!( + subtree_ordinals(&array_doc, second_row.token_bytes()), + [5], + "$, [0], [0][0], [0][1], [1], [1][0]" + ); } - assert_eq!(nested.token(), base64(nested.token_bytes())); } -/// A position names an array element and nothing else. A column's rows -/// need not agree on a node's kind, and an object member's ordinal is a -/// rank among its siblings, so a token derived for `$.a[0]` must not open -/// the rank-0 member of a row whose `$.a` is an object: `jsonb` has no -/// node at `{a,0}` there, and neither does the index. -#[tokio::test] -async fn a_position_reaches_no_object_member() { - let cipher = stack_cipher().await; - let first = subtree_key(&cipher, "$.a[0]", JsonOptions::default()) - .await - .unwrap(); - assert!( - first.gates().is_empty(), - "a position after the head derives" - ); - let array = seal(&cipher, &json!({"a": [{"x": 2}]})).await; - let found: Vec = array - .subtree(first.token_bytes()) - .iter() - .map(|(j, _, _)| *j) - .collect(); - assert_eq!(found, [3], "$, $.a, $.a[0], $.a[0].x"); - for object in [ - json!({"a": {"k": {"x": 1}}}), - json!({"a": {"k": [1, 2]}}), - json!({"a": {"0": {"x": 1}}}), - ] { - let doc = seal(&cipher, &object).await; +mod given_a_position { + use super::*; + + /// A position names an array element and nothing else. A column's rows + /// need not agree on a node's kind, and an object member's ordinal is a + /// rank among its siblings, so a token derived for `$.a[0]` must not + /// open the rank-0 member of a row whose `$.a` is an object: `jsonb` has + /// no node at `{a,0}` there, and neither does the index. + #[tokio::test] + async fn reaches_no_object_member() { + let Scenario { cipher, .. } = scenario().await; + let first = subtree_key(&cipher, "$.a[0]", JsonOptions::default()) + .await + .unwrap(); assert!( - doc.subtree(first.token_bytes()).is_empty(), - "{object}: an object member is not an element" + first.gates().is_empty(), + "a position after the head derives" + ); + let array = seal(&cipher, &json!({"a": [{"x": 2}]})).await; + assert_eq!( + subtree_ordinals(&array, first.token_bytes()), + [3], + "$, $.a, $.a[0], $.a[0].x" ); + for object in [ + json!({"a": {"k": {"x": 1}}}), + json!({"a": {"k": [1, 2]}}), + json!({"a": {"0": {"x": 1}}}), + ] { + let doc = seal(&cipher, &object).await; + assert!( + doc.subtree(first.token_bytes()).is_empty(), + "{object}: an object member is not an element" + ); + } } - // The same below a key step, where the position is a gate: the walk - // from the head reaches `$.h.a` by its selector, and the gate at the - // next step admits an element only. A member at ordinal 0 under an - // object `$.h.a` carries its own selector, not the element's. - let gated = subtree_key(&cipher, "$.h.a[0]", JsonOptions::default()) - .await - .unwrap(); - assert_eq!(gated.gates().len(), 2); - let element = cipher - .default_keyset() - .json_selector( - &JsonPath::parse("$.h.a[*]").unwrap(), - nonempty!("users").with("profile"), - ) - .await - .unwrap(); - assert_eq!(gated.gates()[1], format!("#0:{element}")); -} - -/// Padded standard base64, as the SQL reads and writes it. -fn base64(bytes: &[u8]) -> String { - use base64ct::{Base64, Encoding}; - Base64::encode_string(bytes) -} -/// The rows of a gated walk carry the node the gates reached at depth 0 -/// and its descendants beneath. Opened as a value, they are that node's -/// value; two reached nodes open as a list, or refuse to be one value. -#[tokio::test] -async fn gated_walk_rows_open_to_the_reached_nodes_value() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let nested = subtree_key(&cipher, "$.nested", JsonOptions::default()) - .await - .unwrap(); - let list = doc - .children(nested.token_bytes()) - .into_iter() - .find(|(j, _, _)| *j == 4) - .map(|(_, _, t)| t) - .unwrap(); - // What `json_tree_subtree_at(doc, t(nested), [s(list), #1])` returns: - // the second element at d = 0, its x at d = 1. - let rows_at = |reached: &[usize], below: &[(usize, u32)]| -> Vec { - reached - .iter() - .map(|j| (*j, 0u32)) - .chain(below.iter().copied()) - .map(|(j, d)| { - let mut wire = serde_json::to_value(doc.node(j).unwrap()).unwrap(); - wire["d"] = json!(d); - serde_json::from_value(wire).unwrap() - }) - .collect() - }; - let second = keyed(&cipher, "$.nested.list[1]", rows_at(&[7], &[(8, 1)])).await; - assert!( - !second.is_sealed_root(), - "a reached node is not the fallback" - ); - let value = cipher - .decrypt_as::(second.clone(), aead()) - .await - .unwrap(); - assert_eq!(value, json!({"x": 2}), "the reached node's own value"); - let values = cipher - .decrypt_as::, JsonSubtree>(second, aead()) - .await - .unwrap(); - assert_eq!(values, vec![json!({"x": 2})]); - // A scalar reached by a gate opens to the scalar, where the ungated - // walk of a scalar yields no rows and no value. - let x = cipher - .decrypt_as::( - keyed(&cipher, "$.nested.list[1].x", rows_at(&[8], &[])).await, - aead(), - ) - .await - .unwrap(); - assert_eq!(x, json!(2)); - // Two reached nodes (a wildcard): a list, never one value. - let both = keyed(&cipher, "$.nested.list[*].x", rows_at(&[6, 8], &[])).await; - let values = cipher - .decrypt_as::, JsonSubtree>(both.clone(), aead()) - .await - .unwrap(); - assert_eq!(values, vec![json!(1), json!(2)]); - match cipher.decrypt_as::(both, aead()).await { - Err(Error::OpenedType { expected }) => { - assert_eq!(expected, "one node at the walked path") - } - other => panic!("{other:?}"), + /// The same below a key step, where the position is a gate: the walk + /// from the head reaches `$.h.a` by its selector, and the gate at the + /// next step admits an element only. A member at ordinal 0 under an + /// object `$.h.a` carries its own selector, not the element's. + #[tokio::test] + async fn gates_on_the_elements_selector_below_a_key_step() { + let Scenario { cipher, .. } = scenario().await; + let gated = subtree_key(&cipher, "$.h.a[0]", JsonOptions::default()) + .await + .unwrap(); + assert_eq!(gated.gates().len(), 2, "a, then [0]"); + let element = cipher + .default_keyset() + .json_selector( + &JsonPath::parse("$.h.a[*]").unwrap(), + nonempty!("users").with("profile"), + ) + .await + .unwrap(); + assert_eq!( + gated.gates()[1], + format!("#0:{element}"), + "the position gate names the elements' selector" + ); } - // Without a d = 0 row the same entries are the children of one node and - // assemble into it. - let rows: Vec = doc - .subtree(&list) - .iter() - .map(|(j, _, _)| doc.node(*j).unwrap()) - .collect(); - let elements = cipher - .decrypt_as::(keyed(&cipher, "$.nested.list", rows).await, aead()) - .await - .unwrap(); - assert_eq!(elements, json!([{"x": 1}, {"x": 2}])); } -/// A document, an extracted node and a walked subtree are each the one -/// recoverable field of a derived record: each opens through -/// `DecryptField`, as a record holding one would. -#[tokio::test] -async fn each_stored_form_opens_as_a_record_field() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let node = doc.node(1).unwrap(); - let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) - .await - .unwrap(); - let list = doc - .children(key.token_bytes()) - .into_iter() - .find(|(j, _, _)| *j == 4) - .map(|(_, _, t)| t) - .unwrap(); - let rows: Vec = doc - .subtree(&list) - .iter() - .map(|(j, _, _)| doc.node(*j).unwrap()) - .collect(); - let subtree = keyed(&cipher, "$.nested.list", rows.clone()).await; - assert_eq!(subtree.nodes(), rows.as_slice()); - assert_eq!(subtree.nodes().len(), 4, "both elements and both leaves"); - - let opening: Option> = doc.decryption_field(aead()); - let opened = cipher - .run_decryption(opening.expect("a document opens")) - .await - .unwrap(); - assert_eq!(opened, fixture()); - let opening: Option> = node.decryption_field(aead()); - let opened = cipher - .run_decryption(opening.expect("a node opens")) - .await - .unwrap(); - assert_eq!(opened, json!("world-1")); - let opening: Option> = subtree.decryption_field(aead()); - let opened = cipher - .run_decryption(opening.expect("a subtree opens")) - .await - .unwrap(); - assert_eq!(opened, json!([{"x": 1}, {"x": 2}])); -} +// ============================================================================= +// Extraction +// ============================================================================= -/// Past its budget the server returns the sealed root alone, at depth 0, -/// in place of the subtree: the rows open to the whole document and the -/// key's path selects over it. A path that reaches nothing, or a wildcard -/// that reaches two nodes where one value is asked for, is refused; a root -/// at depth 0 under an unsealed document is not the fallback. A fallback -/// with no path cannot be built: the key is the only constructor. -#[tokio::test] -async fn the_sealed_root_a_walk_fell_back_to_opens_at_the_path() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let fallback = |doc: &JsonDocument| -> JsonNode { - let mut wire = serde_json::to_value(doc.node(0).unwrap()).unwrap(); - wire["d"] = json!(0); - serde_json::from_value(wire).unwrap() - }; - let root = fallback(&doc); - let rows = keyed(&cipher, "$.nested", vec![root.clone()]).await; - assert!(rows.is_sealed_root()); - assert_eq!(rows.path().to_string(), "$.nested"); +mod given_an_extracted_node { + use super::*; - let nested = cipher - .decrypt_as::(rows, aead()) - .await - .unwrap(); - assert_eq!(nested, fixture()["nested"]); - let xs = cipher - .decrypt_as::, JsonSubtree>( - keyed(&cipher, "$.nested.list[*].x", vec![root.clone()]).await, - aead(), - ) - .await - .unwrap(); - assert_eq!(xs, vec![json!(1), json!(2)]); - let none = cipher - .decrypt_as::, JsonSubtree>( - keyed(&cipher, "$.missing[*]", vec![root.clone()]).await, - aead(), - ) - .await - .unwrap(); - assert!(none.is_empty(), "a path that reaches nothing is no subtree"); - - // Each refusal names what it expected: nothing at the path, or one node - // where a wildcard reached two. - for (path, expected) in [ - ("$.missing", "a node at the walked path"), - ("$.nested.list[*].x", "one node at the walked path"), - ] { - let refused = cipher + #[tokio::test] + async fn opens_to_its_scalar_and_a_walked_subtree_to_its_value() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let node = |j: usize| doc.node(j).unwrap(); + let scalar = |j: usize| cipher.decrypt_as::(node(j), aead()); + assert_eq!( + scalar(1).await.unwrap(), + json!("world-1"), + "$.hello opens to its scalar" + ); + assert_eq!( + scalar(6).await.unwrap(), + json!(1), + "[0].x opens to its scalar" + ); + assert_eq!( + scalar(0).await.unwrap(), + fixture(), + "the sealed root is the document" + ); + assertions::refused_as_opened_type( + &scalar(2).await, + None, + "a container node holds no value of its own", + ); + + let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) + .await + .unwrap(); + let rows = subtree_nodes(&doc, key.token_bytes()); + let nested = cipher + .decrypt_as::(key.open(rows.clone()), aead()) + .await + .unwrap(); + assert_eq!( + nested, + fixture()["nested"], + "the walk under $.nested opens to its value" + ); + let forest = cipher + .decrypt_as::, JsonSubtree>(key.open(rows), aead()) + .await + .unwrap(); + assert_eq!( + forest, + vec![json!("constant"), json!([{"x": 1}, {"x": 2}])], + "as a list, the walk opens to $.nested's members" + ); + + // The list's token is the server's to derive, one level under nested. + let list = child_token(&doc, key.token_bytes(), 4); + let elements = cipher .decrypt_as::( - keyed(&cipher, path, vec![root.clone()]).await, + keyed(&cipher, "$.nested.list", subtree_nodes(&doc, &list)).await, aead(), ) - .await; - match refused { - Err(Error::OpenedType { expected: got }) => assert_eq!(got, expected, "{path}"), - other => panic!("{path}: {other:?}"), - } + .await + .unwrap(); + assert_eq!( + elements, + json!([{"x": 1}, {"x": 2}]), + "the walk under the list opens to the list" + ); } - // The same root at depth 1, or at depth 0 under an unsealed document, - // is a walk's row and not the fallback; neither assembles. - let mut as_walked = serde_json::to_value(doc.node(0).unwrap()).unwrap(); - as_walked["d"] = json!(1); - let as_walked: JsonNode = serde_json::from_value(as_walked).unwrap(); - let rows = keyed(&cipher, "$.nested", vec![as_walked]).await; - assert!(!rows.is_sealed_root()); - let refused = cipher.decrypt_as::(rows, aead()).await; - assert!(matches!(refused, Err(Error::OpenedType { .. }))); - let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; - assert!(!keyed(&cipher, "$.nested", vec![fallback(&open_doc)]) - .await - .is_sealed_root()); - assert!(!keyed(&cipher, "$.nested", vec![]).await.is_sealed_root()); - assert!(!keyed(&cipher, "$.nested", vec![root.clone(), root]) - .await - .is_sealed_root()); -} - -/// The rows `json_tree_subtree_at(doc, token, gates)` returns for `key`, -/// walked here as the SQL walks them: from the key's token, through each -/// gate (a selector the entry at that depth must carry, or `#n:`, -/// the child at ordinal `n` carrying it), then everything beneath each node -/// the gates reach, each row with its depth below that node. -fn gated_walk(doc: &JsonDocument, key: &JsonSubtreeKey) -> Vec { - let with_depth = |j: usize, d: u32| -> JsonNode { - let mut wire = serde_json::to_value(doc.node(j).unwrap()).unwrap(); - wire["d"] = json!(d); - serde_json::from_value(wire).unwrap() - }; - let mut frontier = vec![(None, *key.token_bytes())]; - for gate in key.gates() { - let mut next = Vec::new(); - for (_, token) in &frontier { - let children = doc.children(token); - match gate.strip_prefix('#') { - Some(slot) => { - let (n, selector) = slot.split_once(':').unwrap(); - if let Some((j, entry, t)) = children.get(n.parse::().unwrap()) { - if base64(entry.selector()) == selector { - next.push((Some(*j), *t)); - } - } - } - None => next.extend( - children - .iter() - .filter(|(_, entry, _)| base64(entry.selector()) == gate) - .map(|(j, _, t)| (Some(*j), *t)), - ), - } - } - frontier = next; + /// An extracted node round-trips through its wire form with the + /// envelope fields the SQL grafts beside it. + #[tokio::test] + async fn round_trips_with_the_envelope_the_sql_grafts() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let mut wire = serde_json::to_value(doc.node(6).unwrap()).unwrap(); + wire["i"] = json!({"t": "users", "c": "profile"}); + wire["v"] = json!(4); + let back: JsonNode = serde_json::from_value(wire).unwrap(); + assert_eq!(back.ordinal(), 6, "the node keeps its ordinal"); + assert_eq!( + cipher + .decrypt_as::(back, aead()) + .await + .unwrap(), + json!(1), + "the grafted node opens to its scalar" + ); } - let mut rows = Vec::new(); - for (reached, token) in frontier { - let below = u32::from(reached.is_some()); - if let Some(j) = reached { - rows.push(with_depth(j, 0)); - } - rows.extend( - doc.subtree(&token) - .iter() - .map(|(j, _, d)| with_depth(*j, d - 1 + below)), + + /// The walk reports each row's depth below where it started, in `d`; an + /// extracted node carries none. + #[tokio::test] + async fn carries_no_depth_until_a_walk_gives_it_one() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + assert_eq!( + doc.node(6).unwrap().depth(), + None, + "an extracted node has no depth" + ); + let mut wire = serde_json::to_value(doc.node(6).unwrap()).unwrap(); + assert!( + wire.get("d").is_none(), + "an extracted node writes no `d`: {wire}" + ); + wire["d"] = json!(3); + let walked: JsonNode = serde_json::from_value(wire).unwrap(); + assert_eq!(walked.depth(), Some(3), "a walked row reads its depth"); + assert_eq!( + serde_json::to_value(&walked).unwrap()["d"], + json!(3), + "a walked row writes its depth back" ); } - rows.sort_by_key(JsonNode::ordinal); - rows } -/// The sealed root a walk fell back to opens to exactly what the walked -/// rows open to, so a caller's result does not change shape when a -/// document grows past the budget, and no caller says which walk the rows -/// came from: the key does, by its gates. A key with none keyed -/// `json_tree_subtree`, whose rows are the descendants of the path's node, -/// so a `Vec` opens to that node's children; a key with gates keyed -/// `json_tree_subtree_at`, whose rows are the nodes the path reaches. -#[tokio::test] -async fn the_key_built_fallback_opens_to_what_the_walk_opens_to() { - let document = json!({ - "list": [{"x": 1}, {"x": 2}], - "obj": {"a": "constant", "b": [1, 2], "c": {"d": [true, null]}}, - "hello": "world" - }); - let cipher = stack_cipher().await; - let doc = seal(&cipher, &document).await; - let mut root = serde_json::to_value(doc.node(0).unwrap()).unwrap(); - root["d"] = json!(0); - let root: JsonNode = serde_json::from_value(root).unwrap(); - let sorted = |mut values: Vec| { - values.sort_by_key(ToString::to_string); - values - }; - for (text, gated) in [ - ("$.list", false), - ("$.obj", false), - ("$.hello", false), - ("$.list[1]", false), - ("$.obj.b", true), - ("$.obj.c.d", true), - ("$.obj.b[0]", true), - ("$.list[*].x", true), - ("$.list[1].x", true), - ("$.obj.c", true), - ] { - let key = subtree_key(&cipher, text, JsonOptions::default()) +mod given_gated_walk_rows { + use super::*; + + /// The rows of a gated walk carry the node the gates reached at depth 0 + /// and its descendants beneath. Opened as a value, they are that node's + /// value; two reached nodes open as a list, or refuse to be one value. + #[tokio::test] + async fn open_to_the_reached_nodes_value() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let nested = subtree_key(&cipher, "$.nested", JsonOptions::default()) .await .unwrap(); - assert_eq!(!key.gates().is_empty(), gated, "{text}"); - let walked = if gated { - gated_walk(&doc, &key) - } else { - doc.subtree(key.token_bytes()) + let list = child_token(&doc, nested.token_bytes(), 4); + // What `json_tree_subtree_at(doc, t(nested), [s(list), #1])` returns: + // the second element at d = 0, its x at d = 1. + let rows_at = |reached: &[usize], below: &[(usize, u32)]| -> Vec { + reached .iter() - .map(|(j, _, _)| doc.node(*j).unwrap()) + .map(|j| (*j, 0u32)) + .chain(below.iter().copied()) + .map(|(j, d)| with_depth(&doc, j, d)) .collect() }; - let fallen_back = key.open(vec![root.clone()]); - assert!(fallen_back.is_sealed_root(), "{text}"); - let as_walked = cipher - .decrypt_as::, JsonSubtree>(key.open(walked.clone()), aead()) + let second = keyed(&cipher, "$.nested.list[1]", rows_at(&[7], &[(8, 1)])).await; + assert!( + !second.is_sealed_root(), + "a reached node is not the fallback" + ); + let value = cipher + .decrypt_as::(second.clone(), aead()) .await .unwrap(); - let as_fallen = cipher - .decrypt_as::, JsonSubtree>(fallen_back.clone(), aead()) + assert_eq!(value, json!({"x": 2}), "the reached node's own value"); + let values = cipher + .decrypt_as::, JsonSubtree>(second, aead()) .await .unwrap(); - // An object's members come back in an order neither form fixes. - assert_eq!(sorted(as_fallen), sorted(as_walked), "{text}, as a list"); - // A walk with no rows (a scalar's, or an empty container's) is - // never past a budget, so it never falls back: nothing to compare. - if walked.is_empty() { - continue; - } - let one_walked = cipher - .decrypt_as::(key.open(walked), aead()) - .await; - let one_fallen = cipher - .decrypt_as::(fallen_back, aead()) - .await; - match (one_walked, one_fallen) { - (Ok(a), Ok(b)) => assert_eq!(a, b, "{text}, as one value"), - (Err(_), Err(_)) => {} - (a, b) => panic!("{text}: the walk opened to {a:?}, the fallback to {b:?}"), - } - } - // As one value an ungated walk opens to its node, either way. - let key = subtree_key(&cipher, "$.obj", JsonOptions::default()) - .await - .unwrap(); - assert_eq!( - cipher - .decrypt_as::(key.open(vec![root.clone()]), aead()) + assert_eq!( + values, + vec![json!({"x": 2})], + "as a list, the one reached node" + ); + // A scalar reached by a gate opens to the scalar, where the ungated + // walk of a scalar yields no rows and no value. + let x = cipher + .decrypt_as::( + keyed(&cipher, "$.nested.list[1].x", rows_at(&[8], &[])).await, + aead(), + ) .await - .unwrap(), - document["obj"] - ); - // An ungated key names one node: a fallback with nothing at the path, - // or a position under a wildcard head, is refused. - let missing = subtree_key(&cipher, "$.missing", JsonOptions::default()) - .await - .unwrap(); - match cipher - .decrypt_as::, JsonSubtree>(missing.open(vec![root]), aead()) - .await - { - Err(Error::OpenedType { expected }) => { - assert_eq!(expected, "one node at the walked path") - } - other => panic!("{other:?}"), + .unwrap(); + assert_eq!(x, json!(2), "a scalar reached by a gate opens to it"); + // Two reached nodes (a wildcard): a list, never one value. + let both = keyed(&cipher, "$.nested.list[*].x", rows_at(&[6, 8], &[])).await; + let values = cipher + .decrypt_as::, JsonSubtree>(both.clone(), aead()) + .await + .unwrap(); + assert_eq!( + values, + vec![json!(1), json!(2)], + "two reached nodes open as a list" + ); + assertions::refused_as_opened_type( + &cipher.decrypt_as::(both, aead()).await, + Some("one node at the walked path"), + "two reached nodes are not one value", + ); + // Without a d = 0 row the same entries are the children of one node and + // assemble into it. + let elements = cipher + .decrypt_as::( + keyed(&cipher, "$.nested.list", subtree_nodes(&doc, &list)).await, + aead(), + ) + .await + .unwrap(); + assert_eq!( + elements, + json!([{"x": 1}, {"x": 2}]), + "without a d = 0 row the rows assemble into their parent" + ); } } -/// The walk reports each row's depth below where it started, in `d`; an -/// extracted node carries none. -#[tokio::test] -async fn a_walked_row_carries_its_depth() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - assert_eq!(doc.node(6).unwrap().depth(), None); - let mut wire = serde_json::to_value(doc.node(6).unwrap()).unwrap(); - assert!(wire.get("d").is_none()); - wire["d"] = json!(3); - let walked: JsonNode = serde_json::from_value(wire).unwrap(); - assert_eq!(walked.depth(), Some(3)); - assert_eq!(serde_json::to_value(&walked).unwrap()["d"], json!(3)); -} +mod given_a_derived_record { + use super::*; + + /// A document, an extracted node and a walked subtree are each the one + /// recoverable field of a derived record: each opens through + /// `DecryptField`, as a record holding one would. + #[tokio::test] + async fn opens_each_stored_form_as_its_field() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let node = doc.node(1).unwrap(); + let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) + .await + .unwrap(); + let list = child_token(&doc, key.token_bytes(), 4); + let rows = subtree_nodes(&doc, &list); + let subtree = keyed(&cipher, "$.nested.list", rows.clone()).await; + assert_eq!( + subtree.nodes(), + rows.as_slice(), + "a subtree holds the rows it was opened with" + ); + assert_eq!(subtree.nodes().len(), 4, "both elements and both leaves"); -/// A document sealed without the sealed root keeps the root as an ordinary -/// node, and opens from its entries. -#[tokio::test] -async fn a_document_without_the_sealed_root_opens_from_its_entries() { - let cipher = stack_cipher().await; - for value in [fixture(), json!({"": {}}), json!([[], {}, ""]), json!("a")] { - let doc = seal_with(&cipher, &value, &open_root()).await; - assert!(!doc.header().sealed_root()); - assert_eq!(doc.entries().len(), nodes(&value)); - assert_eq!(open(&cipher, doc).await.unwrap(), value); + let opening: Option> = doc.decryption_field(aead()); + let opened = cipher + .run_decryption(opening.expect("a document opens")) + .await + .unwrap(); + assert_eq!(opened, fixture(), "the document field"); + let opening: Option> = node.decryption_field(aead()); + let opened = cipher + .run_decryption(opening.expect("a node opens")) + .await + .unwrap(); + assert_eq!(opened, json!("world-1"), "the node field"); + let opening: Option> = + subtree.decryption_field(aead()); + let opened = cipher + .run_decryption(opening.expect("a subtree opens")) + .await + .unwrap(); + assert_eq!(opened, json!([{"x": 1}, {"x": 2}]), "the subtree field"); } } -/// An accessor whose path or bounds the index refuses is refused when the -/// query is awaited, before the keyset it names is loaded. -#[tokio::test] -async fn an_accessor_is_refused_before_its_keyset_loads() { - let (cipher, loads) = common::loads_counting_cipher().await; - let plan = ordered_plan(); - let eq = cipher - .query(op::AtPath(op::Eq), &("address.city", "Sydney")) - .on(&plan) - .keyset("tenant") - .await; - assert!( - matches!(eq, Err(Error::Term(TermError::JsonPath(_)))), - "{eq:?}" - ); - let between = cipher - .query(op::AtPath(op::Between), &("address.age", (1, 2))) - .on(&plan) - .keyset("tenant") - .await; - assert!( - matches!(between, Err(Error::Term(TermError::JsonPath(_)))), - "{between:?}" - ); - let unordered = cipher - .query(op::AtPath(op::Between), &("$.flag", (false, true))) - .on(&plan) - .keyset("tenant") - .await; - assert!( - matches!(unordered, Err(Error::Term(TermError::JsonLeafNotOrdered))), - "{unordered:?}" - ); - let not_ordered = cipher - .query(op::AtPath(op::Lt), &("$.nested.deep", "z")) - .on(&plan) - .keyset("tenant") - .await; - assert!( - matches!(not_ordered, Err(Error::Term(TermError::JsonPathNotOrdered))), - "{not_ordered:?}" - ); - let at_position = cipher - .query(op::AtPath(op::Gte), &("$.nested.list[0].x", 1)) - .on(&plan) - .keyset("tenant") - .await; - assert!( - matches!( - at_position, - Err(Error::Term(TermError::JsonOrderingAtPosition)) - ), - "{at_position:?}" - ); - // An open that names a context the plan already has is refused the - // same way, one document or a batch, before the keyset loads. - let doc = seal(&stack_cipher().await, &fixture()).await; - let clash = cipher - .open(doc.clone()) - .context("users/other") - .using(&plan) - .keyset("tenant") - .await; - assert!( - matches!(clash, Err(Error::Plan(PlanError::TwoContextSources { .. }))), - "{clash:?}" - ); - let clash = cipher - .open(vec![doc]) - .context("users/other") - .using(&plan) - .keyset("tenant") - .await; - assert!( - matches!(clash, Err(Error::Plan(PlanError::TwoContextSources { .. }))), - "{clash:?}" - ); - assert_eq!( - loads.load(std::sync::atomic::Ordering::SeqCst), - 0, - "no keyset was loaded for a refused query or open" +mod given_a_fallback_root { + use super::*; + + /// The document's root as a walk past its budget returns it: alone, at + /// depth 0. + fn fallback(doc: &JsonDocument) -> JsonNode { + with_depth(doc, 0, 0) + } + + /// Past its budget the server returns the sealed root alone, at depth + /// 0, in place of the subtree: the rows open to the whole document and + /// the key's path selects over it. + #[tokio::test] + async fn opens_at_the_keys_path() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let root = fallback(&doc); + let rows = keyed(&cipher, "$.nested", vec![root.clone()]).await; + assert!( + rows.is_sealed_root(), + "the root alone at d = 0 is the fallback" + ); + assert_eq!( + rows.path().to_string(), + "$.nested", + "the fallback carries the key's path" + ); + let nested = cipher + .decrypt_as::(rows, aead()) + .await + .unwrap(); + assert_eq!(nested, fixture()["nested"], "the value at the path"); + let xs = cipher + .decrypt_as::, JsonSubtree>( + keyed(&cipher, "$.nested.list[*].x", vec![root.clone()]).await, + aead(), + ) + .await + .unwrap(); + assert_eq!( + xs, + vec![json!(1), json!(2)], + "a wildcard path selects every node over the document" + ); + let none = cipher + .decrypt_as::, JsonSubtree>( + keyed(&cipher, "$.missing[*]", vec![root]).await, + aead(), + ) + .await + .unwrap(); + assert!( + none.is_empty(), + "a path that reaches nothing is no subtree: {none:?}" + ); + } + + /// A path that reaches nothing, or a wildcard that reaches two nodes + /// where one value is asked for, is refused, and each refusal names + /// what it expected. + #[tokio::test] + async fn refuses_one_value_where_the_path_reaches_none_or_two() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let root = fallback(&doc); + for (path, expected) in [ + ("$.missing", "a node at the walked path"), + ("$.nested.list[*].x", "one node at the walked path"), + ] { + let refused = cipher + .decrypt_as::( + keyed(&cipher, path, vec![root.clone()]).await, + aead(), + ) + .await; + assertions::refused_as_opened_type(&refused, Some(expected), path); + } + } + + /// The same root at depth 1, or at depth 0 under an unsealed document, + /// is a walk's row and not the fallback; neither assembles. A fallback + /// with no path cannot be built: the key is the only constructor. + #[tokio::test] + async fn is_only_the_sealed_root_alone_at_depth_zero() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let root = fallback(&doc); + let rows = keyed(&cipher, "$.nested", vec![with_depth(&doc, 0, 1)]).await; + assert!( + !rows.is_sealed_root(), + "the root at depth 1 is a walk's row" + ); + assertions::refused_as_opened_type( + &cipher.decrypt_as::(rows, aead()).await, + None, + "the root at depth 1 does not assemble", + ); + let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; + assert!( + !keyed(&cipher, "$.nested", vec![fallback(&open_doc)]) + .await + .is_sealed_root(), + "an unsealed root at depth 0 is not the fallback" + ); + assert!( + !keyed(&cipher, "$.nested", vec![]).await.is_sealed_root(), + "no rows are not the fallback" + ); + assert!( + !keyed(&cipher, "$.nested", vec![root.clone(), root]) + .await + .is_sealed_root(), + "two roots are not the fallback" + ); + } + + /// The sealed root a walk fell back to opens to exactly what the walked + /// rows open to, so a caller's result does not change shape when a + /// document grows past the budget, and no caller says which walk the + /// rows came from: the key does, by its gates. A key with none keyed + /// `json_tree_subtree`, whose rows are the descendants of the path's + /// node, so a `Vec` opens to that node's children; a key with gates + /// keyed `json_tree_subtree_at`, whose rows are the nodes the path + /// reaches. + #[tokio::test] + async fn opens_to_what_the_walk_opens_to() { + let document = json!({ + "list": [{"x": 1}, {"x": 2}], + "obj": {"a": "constant", "b": [1, 2], "c": {"d": [true, null]}}, + "hello": "world" + }); + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &document).await; + let root = fallback(&doc); + let sorted = |mut values: Vec| { + values.sort_by_key(ToString::to_string); + values + }; + for (text, gated) in [ + ("$.list", false), + ("$.obj", false), + ("$.hello", false), + ("$.list[1]", false), + ("$.obj.b", true), + ("$.obj.c.d", true), + ("$.obj.b[0]", true), + ("$.list[*].x", true), + ("$.list[1].x", true), + ("$.obj.c", true), + ] { + let key = subtree_key(&cipher, text, JsonOptions::default()) + .await + .unwrap(); + assert_eq!( + !key.gates().is_empty(), + gated, + "{text}: whether the key gates" + ); + let walked = if gated { + gated_walk(&doc, &key) + } else { + subtree_nodes(&doc, key.token_bytes()) + }; + let fallen_back = key.open(vec![root.clone()]); + assert!(fallen_back.is_sealed_root(), "{text}: the fallback"); + let as_walked = cipher + .decrypt_as::, JsonSubtree>(key.open(walked.clone()), aead()) + .await + .unwrap(); + let as_fallen = cipher + .decrypt_as::, JsonSubtree>(fallen_back.clone(), aead()) + .await + .unwrap(); + // An object's members come back in an order neither form fixes. + assert_eq!(sorted(as_fallen), sorted(as_walked), "{text}, as a list"); + // A walk with no rows (a scalar's, or an empty container's) is + // never past a budget, so it never falls back: nothing to compare. + if walked.is_empty() { + continue; + } + let one_walked = cipher + .decrypt_as::(key.open(walked), aead()) + .await; + let one_fallen = cipher + .decrypt_as::(fallen_back, aead()) + .await; + match (one_walked, one_fallen) { + (Ok(a), Ok(b)) => assert_eq!(a, b, "{text}, as one value"), + (Err(_), Err(_)) => {} + (a, b) => panic!("{text}: the walk opened to {a:?}, the fallback to {b:?}"), + } + } + // As one value an ungated walk opens to its node, either way. + let key = subtree_key(&cipher, "$.obj", JsonOptions::default()) + .await + .unwrap(); + assert_eq!( + cipher + .decrypt_as::(key.open(vec![root.clone()]), aead()) + .await + .unwrap(), + document["obj"], + "an ungated fallback opens to its node as one value" + ); + // An ungated key names one node: a fallback with nothing at the path + // is refused. + let missing = subtree_key(&cipher, "$.missing", JsonOptions::default()) + .await + .unwrap(); + assertions::refused_as_opened_type( + &cipher + .decrypt_as::, JsonSubtree>(missing.open(vec![root]), aead()) + .await, + Some("one node at the walked path"), + "an ungated fallback with nothing at the path", + ); + } +} + +// ============================================================================= +// Tamper evidence +// ============================================================================= + +mod given_rows_the_server_chose { + use super::*; + + /// The rows of a walk come from the server, which can repeat an entry or + /// pick which entries to return. Every entry is authentic, but the + /// rebuild nests the containers it is handed: a repeated row is refused, + /// and so is a line of containers deeper than any document the index + /// seals, as a shape mismatch rather than a recursion as deep as the + /// rows. + #[tokio::test] + async fn refuses_a_repeated_row_or_a_line_of_containers() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let node = |j: usize| doc.node(j).unwrap(); + let repeated = keyed(&cipher, "$.nested", vec![node(1), node(1)]).await; + assertions::refused_as_opened_type( + &cipher + .decrypt_as::, JsonSubtree>(repeated, aead()) + .await, + None, + "a repeated row", + ); + + // A wide document of one-member objects: its objects, returned as rows + // with the last leaf after them, each take the next as their member, + // and would nest 200 deep. + let wide = Value::Object( + (0..200) + .map(|i| (format!("k{i:03}"), json!({"x": 1}))) + .collect(), + ); + let doc = seal(&cipher, &wide).await; + let last = doc.entries().len() - 1; + let containers: Vec = (1..doc.entries().len()) + .filter(|j| j % 2 == 1 || *j == last) + .map(|j| doc.node(j).unwrap()) + .collect(); + assert_eq!(containers.len(), 201, "200 objects and the last leaf"); + assertions::refused_as_opened_type( + &cipher + .decrypt_as::, JsonSubtree>( + keyed(&cipher, "$.k000", containers).await, + aead(), + ) + .await, + None, + "a line of 200 objects", + ); + + // The same through arrays: one outer array, then every key's inner + // one-element array, each taken as the element of the one before it. + let wide = Value::Object( + (0..200) + .map(|i| (format!("k{i:03}"), json!([[0]]))) + .collect(), + ); + let doc = seal(&cipher, &wide).await; + let last = doc.entries().len() - 1; + let arrays: Vec = (1..doc.entries().len()) + .filter(|j| *j == 1 || j % 3 == 2 || *j == last) + .map(|j| doc.node(j).unwrap()) + .collect(); + assert_eq!( + arrays.len(), + 202, + "the outer array, 200 inner and the last leaf" + ); + assertions::refused_as_opened_type( + &cipher + .decrypt_as::, JsonSubtree>( + keyed(&cipher, "$.k000", arrays).await, + aead(), + ) + .await, + None, + "a line of 202 arrays", + ); + } +} + +mod given_a_tampered_document { + use super::*; + + #[tokio::test] + async fn fails_to_open_a_moved_reordered_or_regrafted_entry() { + let Scenario { cipher, .. } = scenario().await; + let value = json!({"a": "x", "b": "y"}); + let doc = seal_with(&cipher, &value, &open_root()).await; + let other = seal_with(&cipher, &value, &open_root()).await; + let tamper = |mut wire: Value, f: &dyn Fn(&mut Vec)| { + let sv = wire["sv"].as_array_mut().unwrap(); + f(sv); + serde_json::from_value::(wire).unwrap() + }; + let wire = serde_json::to_value(&doc).unwrap(); + let other_wire = serde_json::to_value(&other).unwrap(); + + let reordered = tamper(wire.clone(), &|sv| sv.swap(1, 2)); + assertions::refused_as_tampered(&open(&cipher, reordered).await, "reordered"); + + let regrafted = tamper(wire.clone(), &|sv| sv[1] = other_wire["sv"][1].clone()); + assertions::refused_as_tampered( + &open(&cipher, regrafted).await, + "regrafted from another document", + ); + + let truncated = tamper(wire.clone(), &|sv| { + let _ = sv.pop(); + }); + assertions::refused_as_opened_type(&open(&cipher, truncated).await, None, "truncated"); + + let flipped = tamper(wire.clone(), &|sv| { + let c = sv[1]["c"].as_str().unwrap().to_owned(); + let mut bytes = c.into_bytes(); + bytes[3] = if bytes[3] == b'A' { b'B' } else { b'A' }; + sv[1]["c"] = Value::String(String::from_utf8(bytes).unwrap()); + }); + assertions::refused_as_tampered(&open(&cipher, flipped).await, "a changed byte"); + + let wrong_context = cipher + .decrypt_as::( + doc, + AeadContext::from(nonempty!("users").with("bio")), + ) + .await; + assertions::refused_as_tampered(&wrong_context, "another column"); + + // A node lifted out of its document fails under another ordinal. + let sealed = seal(&cipher, &value).await; + let mut lifted = serde_json::to_value(sealed.node(1).unwrap()).unwrap(); + lifted["j"] = json!(2); + let lifted: JsonNode = serde_json::from_value(lifted).unwrap(); + assertions::refused_as_tampered( + &cipher.decrypt_as::(lifted, aead()).await, + "a node under another ordinal", + ); + } +} + +// ============================================================================= +// Containment +// ============================================================================= + +mod given_the_containment_model { + use super::*; + + /// The pinned answers of `jsonb-semantics.sql`, so the model is held to + /// PostgreSQL and not only to itself. + #[test] + fn gives_postgresqls_pinned_answers() { + let expect = [ + (r#"["a"]"#, r#""a""#, true), + (r#""a""#, r#"["a"]"#, false), + (r#"{"a":[1]}"#, r#"{"a":1}"#, false), + (r#"{"a":1}"#, r#"{"a":[1]}"#, false), + (r#"[1,[2]]"#, r#"[2]"#, false), + (r#"[1,[2]]"#, r#"[[2]]"#, true), + (r#"{"a":{"b":1}}"#, r#"{"a":{}}"#, true), + (r#"{"a":{}}"#, r#"{"a":{"b":1}}"#, false), + (r#"{"a":[1,2]}"#, r#"{"a":[]}"#, true), + (r#"{"a":1}"#, r#"{}"#, true), + (r#"[1]"#, r#"{}"#, false), + (r#"[1]"#, r#"[]"#, true), + (r#"[1,2]"#, r#"[1,1,1]"#, true), + ( + r#"{"a":[{"x":1,"y":5},{"x":7,"y":2}]}"#, + r#"{"a":[{"x":1,"y":2}]}"#, + false, + ), + ( + r#"{"a":[{"x":1,"y":2,"z":3}]}"#, + r#"{"a":[{"x":1,"y":2}]}"#, + true, + ), + (r#"{"a":[{"x":1}]}"#, r#"{"a":[{"x":1},{"x":1}]}"#, true), + ( + r#"{"a":[{"b":[{"x":1},{"y":2}]}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, + false, + ), + ( + r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, + true, + ), + (r#"{"a":[["x"]]}"#, r#"{"a":["x"]}"#, false), + (r#""a""#, r#""a""#, true), + (r#"2"#, r#"2.0"#, true), + (r#"{"a":null}"#, r#"{"a":null}"#, true), + (r#"[["a"]]"#, r#""a""#, false), + ]; + for (stored, needle, answer) in expect { + let (a, b): (Value, Value) = ( + serde_json::from_str(stored).unwrap(), + serde_json::from_str(needle).unwrap(), + ); + assert_eq!(pg_contains(&a, &b, true), answer, "{stored} @> {needle}"); + } + } +} + +mod given_the_design_corpus { + use super::*; + + /// The design's two corpora: `jsonb-semantics.sql` and + /// `containment-oracle.sql`, as `(stored, needle)` pairs. + fn corpus() -> Vec<(Value, Value)> { + [ + (r#"["a"]"#, r#""a""#), + (r#""a""#, r#"["a"]"#), + (r#"{"a":[1]}"#, r#"{"a":1}"#), + (r#"{"a":1}"#, r#"{"a":[1]}"#), + (r#"[1,[2]]"#, r#"[2]"#), + (r#"[1,[2]]"#, r#"[[2]]"#), + (r#"{"a":{"b":1}}"#, r#"{"a":{}}"#), + (r#"{"a":{}}"#, r#"{"a":{"b":1}}"#), + (r#"{"a":[1,2]}"#, r#"{"a":[]}"#), + (r#"{"a":1}"#, r#"{}"#), + (r#"[1]"#, r#"{}"#), + (r#"[1]"#, r#"[]"#), + (r#"[1,2]"#, r#"[1,1,1]"#), + ( + r#"{"a":[{"x":1,"y":5},{"x":7,"y":2}]}"#, + r#"{"a":[{"x":1,"y":2}]}"#, + ), + (r#"{"a":[{"x":1,"y":2,"z":3}]}"#, r#"{"a":[{"x":1,"y":2}]}"#), + (r#"{"a":[{"x":1}]}"#, r#"{"a":[{"x":1},{"x":1}]}"#), + ( + r#"{"a":[{"b":[{"x":1},{"y":2}]}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, + ), + ( + r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, + ), + (r#"{"a":[["x"]]}"#, r#"{"a":["x"]}"#), + (r#""a""#, r#""a""#), + (r#"2"#, r#"2.0"#), + (r#"{"a":null}"#, r#"{"a":null}"#), + (r#"[["a"]]"#, r#""a""#), + ( + r#"{"a":[{"b":[{"x":1,"y":2}],"c":3},{"b":[{"x":1,"y":9}],"c":4}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}],"c":4}]}"#, + ), + ( + r#"{"a":[{"b":[{"x":1,"y":2}],"c":3},{"b":[{"x":1,"y":9}],"c":4}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}],"c":3}]}"#, + ), + ( + r#"{"a":[{"b":{"c":1,"d":2}},{"b":{"c":3,"d":2}}]}"#, + r#"{"a":[{"b":{"c":1,"d":2}}]}"#, + ), + ( + r#"{"a":[{"b":{"c":1,"d":5}},{"b":{"c":3,"d":2}}]}"#, + r#"{"a":[{"b":{"c":1,"d":2}}]}"#, + ), + (r#"{"a":[[1,2],[3,4]]}"#, r#"{"a":[[1,4]]}"#), + (r#"{"a":[[1,2],[3,4]]}"#, r#"{"a":[[1,2]]}"#), + (r#"{"a":[[1,2],[3,4]]}"#, r#"{"a":[[1],[4]]}"#), + (r#"{"a":"2"}"#, r#"{"a":2}"#), + (r#"{"a":2}"#, r#"{"a":2.0}"#), + (r#"{"a":[{"x":1}]}"#, r#"{"a":[{}]}"#), + (r#"{"a":[1]}"#, r#"{"a":[{}]}"#), + ( + r#"{"a":[{"t":[{"u":1,"v":2}]},{"t":[{"u":1},{"v":2}]}]}"#, + r#"{"a":[{"t":[{"u":1,"v":2}]}]}"#, + ), + ( + r#"{"a":[{"t":[{"u":1},{"v":2}]}]}"#, + r#"{"a":[{"t":[{"u":1,"v":2}]}]}"#, + ), + (r#"{"a":[{}]}"#, r#"{"a":[{}]}"#), + (r#"{"a":[[]]}"#, r#"{"a":[[]]}"#), + (r#"{"a":[[]]}"#, r#"{"a":[{}]}"#), + (r#"[{"a":1},{"b":2}]"#, r#"[{"a":1,"b":2}]"#), + (r#"[{"a":1,"b":2}]"#, r#"[{"a":1},{"b":2}]"#), + (r#"{"a":[true,false,null]}"#, r#"{"a":[null,true]}"#), + (r#"{"a":[true]}"#, r#"{"a":[false]}"#), + (r#"{"a":[{"x":[1,2]},{"x":[3]}]}"#, r#"{"a":[{"x":[2,3]}]}"#), + (r#"{"a":[{"x":[1,2]},{"x":[3]}]}"#, r#"{"a":[{"x":[1,2]}]}"#), + ] + .iter() + .map(|(a, b)| { + ( + serde_json::from_str(a).unwrap(), + serde_json::from_str(b).unwrap(), + ) + }) + .collect() + } + + /// Every pair of the corpus, both ways round: `@>` over the projections + /// is `jsonb @>`, and `<@` is `jsonb <@` but for a stored root scalar in + /// a needle root array. + #[tokio::test] + async fn answers_containment_both_ways_as_jsonb_does() { + let Scenario { cipher, plan } = scenario().await; + for (stored, needle) in corpus() { + let doc = cipher.encrypt(&stored).using(&plan).await.unwrap(); + let contains: JsonQuery = cipher.query(op::Contains, &needle).on(&plan).await.unwrap(); + let contained_by: JsonQuery = cipher + .query(op::ContainedBy, &needle) + .on(&plan) + .await + .unwrap(); + assertions::contains_as_jsonb(&contains, &doc, &stored, &needle); + assertions::is_contained_by_as_jsonb(&contained_by, &doc, &stored, &needle); + } + } +} + +mod given_generated_documents { + //! The containment property over documents and needles proptest + //! generates. A case is one stored document and one needle: a random + //! value, the document itself or a pruning of it (which it contains by + //! construction), or such a pruning with one edit. A failure shrinks to a small + //! document and needle, and proptest appends its seed to + //! `proptest-regressions/json_index.txt` in this crate, which every + //! later run replays first. The strategies mirror the EQL live oracle's + //! (`packages/eql/tests/encryption/tests/json_tree/generated.rs`), which + //! sits in another Cargo workspace. + + use std::cell::Cell; + use std::fmt; + use std::future::IntoFuture; + + use proptest::collection::vec as vec_of; + use proptest::prelude::*; + use proptest::sample::{select, Index}; + use proptest::strategy::ValueTree; + use proptest::test_runner::{ + Config, FileFailurePersistence, RngAlgorithm, RngSeed, TestCaseError, TestRunner, + }; + + use super::*; + + /// The seed the run uses when `PROPTEST_RNG_SEED` is unset. Fixed, so + /// every run checks the same cases. + const DEFAULT_SEED: u64 = 1336; + + /// The cases a run checks when `PROPTEST_CASES` is unset. + const DEFAULT_CASES: u32 = 512; + + /// Where proptest records a failing case's seed: `Direct`, with the + /// crate's own path, rather than resolving `file!()` from the working + /// directory. + const REGRESSIONS: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/proptest-regressions/json_index.txt" ); - // The same query with a path the index takes does load it. - let _ = cipher - .query(op::AtPath(op::Eq), &("$.address.city", "Sydney")) - .on(&plan) - .keyset("tenant") - .await + + /// Numbers, several of them one value to `jsonb` in different + /// representations: `1`, `1.0` and `1e0`; `100` and `1e2`; `0`, `0.0` + /// and `-0.0`; `0.5` and `5e-1`. + const NUMBERS: &[&str] = &[ + "0", + "0.0", + "-0.0", + "1", + "1.0", + "1e0", + "2", + "2.00", + "-1", + "-1.0", + "10", + "1e1", + "100", + "1e2", + "0.5", + "5e-1", + "1.5", + "123456789012", + "1e20", + ]; + + /// Strings, some spelled like a number, a boolean or null. + const STRINGS: &[&str] = &["x", "y", "", "1", "1.0", "2", "true", "null", "a b", "ü"]; + + /// Object keys: few, so they repeat across levels and siblings. + const KEYS: &[&str] = &["a", "b", "c", "x", "1", ""]; + + /// proptest's defaults, which read `PROPTEST_CASES` and + /// `PROPTEST_RNG_SEED`, with this suite's own default for each one left + /// unset. + fn config() -> Config { + let mut config = Config::default(); + if std::env::var_os("PROPTEST_CASES").is_none() { + config.cases = DEFAULT_CASES; + } + if config.rng_seed == RngSeed::Random { + config.rng_seed = RngSeed::Fixed(DEFAULT_SEED); + } + config.rng_algorithm = RngAlgorithm::ChaCha; + config.failure_persistence = Some(Box::new(FileFailurePersistence::Direct(REGRESSIONS))); + config + } + + /// One edit that turns a contained needle into a near miss. `at` picks + /// the node, in document order; an edit that does not fit the node's + /// kind changes its kind instead. + #[derive(Clone, Debug)] + enum Edit { + Leaf { + at: Index, + to: Value, + }, + Key { + at: Index, + key: String, + value: Value, + }, + Element { + at: Index, + value: Value, + }, + Kind { + at: Index, + }, + } + + /// A needle as proptest draws it. `keep` says, one member or element at + /// a time in document order, what a pruning keeps; once it runs out, + /// nothing more is kept, so shrinking it makes the needle smaller. + #[derive(Clone, Debug)] + enum NeedleSpec { + Random(Value), + /// The stored document itself, which contains itself. + Itself, + Pruned { + keep: Vec, + at: Option, + }, + NearMiss { + keep: Vec, + edit: Edit, + }, + } + + /// One generated case: a stored document and a needle derived from it. + #[derive(Clone)] + struct Case { + stored: Value, + needle: NeedleSpec, + } + + impl Case { + /// The needle as a value, and whether the stored document contains + /// it by construction. + fn needle(&self) -> (Value, bool) { + match &self.needle { + NeedleSpec::Random(value) => (value.clone(), false), + NeedleSpec::Itself => (self.stored.clone(), true), + NeedleSpec::Pruned { keep, at } => { + let pointer = at + .as_ref() + .map_or_else(String::new, |at| at.get(&pointers(&self.stored)).clone()); + let node = self.stored.pointer(&pointer).unwrap(); + (prune(node, &mut keep.iter()), pointer.is_empty()) + } + NeedleSpec::NearMiss { keep, edit } => { + let mut value = prune(&self.stored, &mut keep.iter()); + apply(edit, &mut value); + (value, false) + } + } + } + } + + /// Printed as JSON: what proptest shows for the minimal failing case. + impl fmt::Debug for Case { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (needle, _) = self.needle(); + write!( + f, + "stored {}, needle {needle} ({:?})", + self.stored, self.needle + ) + } + } + + fn prune(v: &Value, keep: &mut std::slice::Iter<'_, bool>) -> Value { + match v { + Value::Object(map) => { + let mut out = Map::new(); + for (key, member) in map { + if keep.next().copied().unwrap_or(false) { + let _ = out.insert(key.clone(), prune(member, keep)); + } + } + Value::Object(out) + } + Value::Array(items) => { + let mut out = Vec::new(); + for item in items { + if keep.next().copied().unwrap_or(false) { + out.push(prune(item, keep)); + } + } + Value::Array(out) + } + scalar => scalar.clone(), + } + } + + /// Every node's JSON pointer, in document order, the root (`""`) first. + fn pointers(v: &Value) -> Vec { + fn walk(v: &Value, at: String, out: &mut Vec) { + out.push(at.clone()); + match v { + Value::Object(map) => { + for (key, member) in map { + let key = key.replace('~', "~0").replace('/', "~1"); + walk(member, format!("{at}/{key}"), out); + } + } + Value::Array(items) => { + for (i, item) in items.iter().enumerate() { + walk(item, format!("{at}/{i}"), out); + } + } + _ => {} + } + } + let mut out = Vec::new(); + walk(v, String::new(), &mut out); + out + } + + fn apply(edit: &Edit, needle: &mut Value) { + let at = match edit { + Edit::Leaf { at, .. } + | Edit::Key { at, .. } + | Edit::Element { at, .. } + | Edit::Kind { at } => at, + }; + let pointer = at.get(&pointers(needle)).clone(); + let node = needle.pointer_mut(&pointer).unwrap(); + match edit { + Edit::Leaf { to, .. } if !node.is_object() && !node.is_array() && node != to => { + *node = to.clone(); + } + Edit::Key { key, value, .. } if node.is_object() => { + let _ = node + .as_object_mut() + .unwrap() + .insert(key.clone(), value.clone()); + } + Edit::Element { value, .. } if node.is_array() => { + node.as_array_mut().unwrap().push(value.clone()); + } + _ => { + *node = match node.take() { + Value::Object(map) => Value::Array(map.into_iter().map(|(_, v)| v).collect()), + Value::Array(items) => { + json!({ "a": items.into_iter().next().unwrap_or(Value::Null) }) + } + scalar => Value::Array(vec![scalar]), + }; + } + } + } + + mod strategies { + use super::*; + + pub(super) fn scalar() -> BoxedStrategy { + prop_oneof![ + 4 => select(NUMBERS).prop_map(|text| serde_json::from_str::(text).unwrap()), + 3 => select(STRINGS).prop_map(|text| Value::String(text.to_owned())), + 2 => any::().prop_map(Value::Bool), + 1 => Just(Value::Null), + ] + .boxed() + } + + fn key() -> BoxedStrategy { + select(KEYS).prop_map(str::to_owned).boxed() + } + + /// An object of up to four members. A key drawn twice keeps its + /// later value, as `jsonb` does. + fn object_of(inner: BoxedStrategy) -> BoxedStrategy { + vec_of((key(), inner), 0..5) + .prop_map(|members| Value::Object(members.into_iter().collect())) + .boxed() + } + + /// An array: of anything, of objects, of arrays, or with some + /// elements repeated. + fn array_of(inner: BoxedStrategy) -> BoxedStrategy { + prop_oneof![ + 3 => vec_of(inner.clone(), 0..5).prop_map(Value::Array), + 1 => vec_of(object_of(inner.clone()), 1..4).prop_map(Value::Array), + 1 => vec_of(vec_of(inner.clone(), 0..3).prop_map(Value::Array), 1..4) + .prop_map(Value::Array), + 1 => (vec_of(inner, 1..4), vec_of(any::(), 1..3)).prop_map( + |(mut items, repeats)| { + for repeat in repeats { + let copy = repeat.get(&items).clone(); + items.push(copy); + } + Value::Array(items) + } + ), + ] + .boxed() + } + + /// A value with up to `depth` levels of recursion below it. + pub(super) fn value(depth: u32) -> BoxedStrategy { + scalar() + .prop_recursive(depth, 48, 4, |inner| { + prop_oneof![object_of(inner.clone()), array_of(inner)] + }) + .boxed() + } + + /// A stored document: mostly an object, sometimes a root array, and + /// sometimes a root scalar. + pub(super) fn document() -> BoxedStrategy { + prop_oneof![ + 6 => object_of(value(3)), + 3 => array_of(value(2)), + 1 => scalar(), + ] + .boxed() + } + + fn keep() -> BoxedStrategy> { + vec_of(proptest::bool::weighted(0.7), 0..32).boxed() + } + + fn edit() -> BoxedStrategy { + prop_oneof![ + (any::(), scalar()).prop_map(|(at, to)| Edit::Leaf { at, to }), + (any::(), key(), value(1)).prop_map(|(at, key, value)| Edit::Key { + at, + key, + value + }), + (any::(), value(1)).prop_map(|(at, value)| Edit::Element { at, value }), + any::().prop_map(|at| Edit::Kind { at }), + ] + .boxed() + } + + fn needle() -> BoxedStrategy { + prop_oneof![ + 1 => value(2).prop_map(NeedleSpec::Random), + 1 => Just(NeedleSpec::Itself), + 2 => (keep(), proptest::option::weighted(0.3, any::())) + .prop_map(|(keep, at)| NeedleSpec::Pruned { keep, at }), + 2 => (keep(), edit()).prop_map(|(keep, edit)| NeedleSpec::NearMiss { keep, edit }), + ] + .boxed() + } + + pub(super) fn case() -> BoxedStrategy { + (document(), needle()) + .prop_map(|(stored, needle)| Case { stored, needle }) + .boxed() + } + } + + /// Run `body` over a whole proptest run on one runtime and cipher, and + /// fail with the minimal case and the way to replay it. + fn run( + name: &str, + body: impl Fn(&Scenario, &tokio::runtime::Runtime, &Case) -> Result<(), TestCaseError>, + ) { + let config = config(); + let replay = format!( + "seed {seed}, {cases} cases; replay with PROPTEST_RNG_SEED={seed} PROPTEST_CASES={cases}", + seed = config.rng_seed, + cases = config.cases + ); + eprintln!("json_index generated {name}: {replay}"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let scenario = runtime.block_on(scenario()); + let mut runner = TestRunner::new(config); + if let Err(error) = runner.run(&strategies::case(), |case| body(&scenario, &runtime, &case)) + { + panic!("json_index generated {name}: {replay}\n{error}"); + } + } + + /// The stored document, sealed through the plan. + fn sealed( + scenario: &Scenario, + runtime: &tokio::runtime::Runtime, + stored: &Value, + ) -> JsonDocument { + runtime + .block_on( + scenario + .cipher + .encrypt(stored) + .using(&scenario.plan) + .into_future(), + ) + .unwrap() + } + + #[test] + fn contains_as_jsonb_contains() { + run("@>", |scenario, runtime, case| { + let (needle, contained) = case.needle(); + let doc = sealed(scenario, runtime, &case.stored); + let query: JsonQuery = runtime + .block_on( + scenario + .cipher + .query(op::Contains, &needle) + .on(&scenario.plan) + .into_future(), + ) + .unwrap(); + prop_assert!( + !contained || pg_contains(&case.stored, &needle, true), + "a pruning of the whole document should be contained by it: {} @> {needle}", + case.stored + ); + prop_assert_eq!( + query.matches(doc.projection()), + pg_contains(&case.stored, &needle, true), + "{} @> {}", + &case.stored, + &needle + ); + Ok(()) + }); + } + + #[test] + fn is_contained_by_as_jsonb_except_a_root_scalar_in_a_root_array() { + let excluded = Cell::new(0); + run("<@", |scenario, runtime, case| { + let (needle, _) = case.needle(); + let doc = sealed(scenario, runtime, &case.stored); + let query: JsonQuery = runtime + .block_on( + scenario + .cipher + .query(op::ContainedBy, &needle) + .on(&scenario.plan) + .into_future(), + ) + .unwrap(); + let jsonb = pg_contains(&needle, &case.stored, true); + if jsonb && scalar_in_array(&case.stored, &needle) { + excluded.set(excluded.get() + 1); + } + prop_assert_eq!( + query.projection().contains(doc.projection()), + jsonb && !scalar_in_array(&case.stored, &needle), + "{} <@ {}", + &case.stored, + &needle + ); + Ok(()) + }); + assert!( + excluded.get() > 0, + "no stored root scalar sat inside a needle root array, so the `<@` exception went unexercised" + ); + } + + mod given_the_document_strategy { + use super::*; + + /// Every feature the property relies on, counted over a run's worth + /// of generated documents, so a change to the generator that stops + /// producing one fails here. + #[test] + fn produces_every_feature_the_property_relies_on() { + let mut runner = TestRunner::new(config()); + let strategy = strategies::document(); + let docs: Vec = (0..400) + .map(|_| strategy.new_tree(&mut runner).unwrap().current()) + .collect(); + assertions::covers_every_feature(&docs); + } + } +} + +// ============================================================================= +// Queries +// ============================================================================= + +mod given_a_containment_query { + use super::*; + + #[tokio::test] + async fn serializes_as_the_needle_operand_and_derives_no_key() { + let (cipher, generates, _) = common::counting_cipher().await; + let plan = plan(); + let query = cipher + .query(op::Contains, &json!({"a": [{"x": 1}]})) + .on(&plan) + .await + .unwrap(); + let wire = serde_json::to_value(&query).unwrap(); + assert_eq!( + wire["sv"].as_array().unwrap().len(), + 2, + "$ and $.a sit outside the element" + ); + assert_eq!( + wire["el"].as_array().unwrap().len(), + 1, + "one group for the one element" + ); + assert_eq!( + wire["el"][0]["sv"].as_array().unwrap().len(), + 2, + "the element and its x" + ); + assert!( + wire.get("alt").is_none(), + "an object needle carries no alternative: {wire}" + ); + assert_eq!( + generates.load(std::sync::atomic::Ordering::Relaxed), + 0, + "a query mints no key" + ); + + let scalar = cipher + .query(op::Contains, &json!("a")) + .on(&plan) + .await + .unwrap(); + let wire = serde_json::to_value(&scalar).unwrap(); + assert!( + wire.get("alt").is_some(), + "a root scalar carries its root-array alternative" + ); + } +} + +mod given_equality_at_the_root { + use super::*; + + /// At the root, equality is of the root value: `("$", 1)` matches a + /// stored `1` and not a stored `[1]`, which containment of the scalar + /// `1` would. + #[tokio::test] + async fn matches_the_root_value_and_not_an_array_holding_it() { + let Scenario { cipher, plan } = scenario().await; + let at_root: JsonQuery = cipher + .query(op::AtPath(op::Eq), &("$", 1)) + .on(&plan) + .await + .unwrap(); + let contains: JsonQuery = cipher + .query(op::Contains, &json!(1)) + .on(&plan) + .await + .unwrap(); + let scalar = cipher.encrypt(&json!(1)).using(&plan).await.unwrap(); + let array = cipher.encrypt(&json!([1])).using(&plan).await.unwrap(); + assert!( + at_root.matches(scalar.projection()), + "(\"$\", 1) matches a stored 1" + ); + assert!( + !at_root.matches(array.projection()), + "(\"$\", 1) does not match a stored [1]" + ); + assert!(contains.matches(array.projection()), "jsonb: [1] @> 1"); + assert!( + serde_json::to_value(&at_root).unwrap().get("alt").is_none(), + "equality at the root carries no alternative" + ); + } +} + +mod given_an_accessor { + use super::*; + + #[tokio::test] + async fn matches_the_value_at_its_path() { + let Scenario { cipher, plan } = scenario().await; + let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); + let projection = doc.projection(); + for (path, value, matches, what) in [ + ("$.nested.list[*].x", json!(2), true, "an element's x is 2"), + ("$.nested.list[*].x", json!(3), false, "no element's x is 3"), + ("$.nested.deep", json!("constant"), true, "the deep string"), + ("$.number", json!("1"), false, "a string is not the number"), + ] { + let query: JsonQuery = cipher + .query(op::AtPath(op::Eq), &(path, value)) + .on(&plan) + .await + .unwrap(); + assert_eq!(query.matches(projection), matches, "{what}"); + } + } + + /// An ordering at a path: the path's selector and a leaf term that + /// orders as the stored leaf's does, under native byte comparison. The + /// plan declares the paths it orders, and the query runs under them. + #[tokio::test] + async fn orders_as_the_stored_leaf_at_an_ordered_path() { + let Scenario { cipher, .. } = scenario().await; + let plan = ordered_plan(); + let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); + let number = &doc.entries()[9]; + let bound = cipher + .query(op::AtPath(op::Gte), &("$.number", 0)) + .on(&plan) + .await + .unwrap(); + assert_eq!( + bound.selector().as_bytes(), + number.selector(), + "the bound asks at $.number" + ); + assert!( + bound.ordering_term() < number.ordering_term().unwrap(), + "0 < 1" + ); + let above = cipher + .query(op::AtPath(op::Lt), &("$.number", 5)) + .on(&plan) + .await + .unwrap(); + assert!( + above.ordering_term() > number.ordering_term().unwrap(), + "5 > 1" + ); + let text = cipher + .query(op::AtPath(op::Gt), &("$.hello", "world-0")) + .on(&plan) + .await + .unwrap(); + assert!( + text.ordering_term() < doc.entries()[1].ordering_term().unwrap(), + "world-0 < world-1" + ); + let range = cipher + .query(op::AtPath(op::Between), &("$.number", (0, 5))) + .on(&plan) + .await + .unwrap(); + assert!( + range.lo().ordering_term() < range.hi().ordering_term(), + "the range's bounds are in order" + ); + assert_eq!( + range.lo().selector().as_bytes(), + number.selector(), + "the range asks at $.number" + ); + let wire = serde_json::to_value(range.lo()).unwrap(); + assert_eq!( + wire.as_object().unwrap().keys().collect::>(), + ["op", "s"], + "a bound's wire keys" + ); + } + + /// Refusals, before any term is derived: a text that is not a path, a + /// leaf kind that has no ordering, and a path the plan does not order + /// (no stored leaf there carries a term, so the comparison would match + /// no row; under the default plan that is every path). + #[tokio::test] + async fn refuses_a_bad_path_an_unordered_leaf_or_a_path_the_plan_does_not_order() { + let Scenario { + cipher, + plan: default_plan, + } = scenario().await; + let plan = ordered_plan(); + let bad_path = cipher + .query(op::AtPath(op::Eq), &("address.city", "Sydney")) + .on(&plan) + .await; + assertions::refused_by_term( + &bad_path, + |e| matches!(e, TermError::JsonPath(_)), + "a text that is not a path", + ); + let unordered = cipher + .query(op::AtPath(op::Gte), &("$.flag", true)) + .on(&plan) + .await; + assertions::refused_by_term( + &unordered, + |e| matches!(e, TermError::JsonLeafNotOrdered), + "a boolean has no ordering", + ); + let not_ordered = cipher + .query(op::AtPath(op::Lt), &("$.nested.deep", "z")) + .on(&plan) + .await; + assertions::refused_by_term( + ¬_ordered, + |e| matches!(e, TermError::JsonPathNotOrdered), + "a path the plan does not order", + ); + let not_ordered = cipher + .query(op::AtPath(op::Between), &("$.number", (0, 5))) + .on(&default_plan) + .await; + assertions::refused_by_term( + ¬_ordered, + |e| matches!(e, TermError::JsonPathNotOrdered), + "the default plan orders no path", + ); + } + + /// An ordering at a wildcard path is covered by the wildcard. One at a + /// position is refused whatever the plan orders: `->` reads a position + /// as the wildcard and would compare the first element, not the one + /// named. + #[tokio::test] + async fn orders_at_a_wildcard_and_refuses_a_position() { + let Scenario { cipher, .. } = scenario().await; + let by_element = Plan::value::() + .context("users/profile") + .encrypt_as(Json::new(ordering(&["$.nested.list[*].x"]))) + .build() + .unwrap(); + let wildcard = cipher + .query(op::AtPath(op::Gt), &("$.nested.list[*].x", 0)) + .on(&by_element) + .await; + assert!( + wildcard.is_ok(), + "the wildcard the plan orders: {wildcard:?}" + ); + for path in ["$.nested.list[1].x", "$.nested.list[0].x"] { + let refused = cipher + .query(op::AtPath(op::Gt), &(path, 0)) + .on(&by_element) + .await; + assertions::refused_by_term( + &refused, + |e| matches!(e, TermError::JsonOrderingAtPosition), + &format!("{path} with Gt"), + ); + let refused = cipher + .query(op::AtPath(op::Between), &(path, (0, 5))) + .on(&by_element) + .await; + assertions::refused_by_term( + &refused, + |e| matches!(e, TermError::JsonOrderingAtPosition), + &format!("{path} with Between"), + ); + } + } + + /// Equality at a path compares one value: an object or an array would + /// be containment, which `op::Contains` asks for. + #[tokio::test] + async fn refuses_a_container_for_equality() { + let Scenario { cipher, .. } = scenario().await; + let plan = ordered_plan(); + for container in [json!({"deep": "constant"}), json!([]), json!([{"x": 1}])] { + let refused = cipher + .query(op::AtPath(op::Eq), &("$.nested", container.clone())) + .on(&plan) + .await; + assertions::refused_by_term( + &refused, + |e| matches!(e, TermError::JsonValueNotScalar), + &container.to_string(), + ); + } + } + + /// An accessor whose path or bounds the index refuses is refused when + /// the query is awaited, before the keyset it names is loaded; so is an + /// open that names a context the plan already has. + #[tokio::test] + async fn is_refused_before_its_keyset_loads() { + let (cipher, loads) = common::loads_counting_cipher().await; + let plan = ordered_plan(); + let eq = cipher + .query(op::AtPath(op::Eq), &("address.city", "Sydney")) + .on(&plan) + .keyset("tenant") + .await; + assertions::refused_by_term( + &eq, + |e| matches!(e, TermError::JsonPath(_)), + "Eq at a bad path", + ); + let between = cipher + .query(op::AtPath(op::Between), &("address.age", (1, 2))) + .on(&plan) + .keyset("tenant") + .await; + assertions::refused_by_term( + &between, + |e| matches!(e, TermError::JsonPath(_)), + "Between at a bad path", + ); + let unordered = cipher + .query(op::AtPath(op::Between), &("$.flag", (false, true))) + .on(&plan) + .keyset("tenant") + .await; + assertions::refused_by_term( + &unordered, + |e| matches!(e, TermError::JsonLeafNotOrdered), + "Between over booleans", + ); + let not_ordered = cipher + .query(op::AtPath(op::Lt), &("$.nested.deep", "z")) + .on(&plan) + .keyset("tenant") + .await; + assertions::refused_by_term( + ¬_ordered, + |e| matches!(e, TermError::JsonPathNotOrdered), + "a path the plan does not order", + ); + let at_position = cipher + .query(op::AtPath(op::Gte), &("$.nested.list[0].x", 1)) + .on(&plan) + .keyset("tenant") + .await; + assertions::refused_by_term( + &at_position, + |e| matches!(e, TermError::JsonOrderingAtPosition), + "an ordering at a position", + ); + let doc = seal(&stack_cipher().await, &fixture()).await; + let clash = cipher + .open(doc.clone()) + .context("users/other") + .using(&plan) + .keyset("tenant") + .await; + assertions::refused_for_two_contexts(&clash, "one document"); + let clash = cipher + .open(vec![doc]) + .context("users/other") + .using(&plan) + .keyset("tenant") + .await; + assertions::refused_for_two_contexts(&clash, "a batch"); + assert_eq!( + loads.load(std::sync::atomic::Ordering::SeqCst), + 0, + "no keyset was loaded for a refused query or open" + ); + // The same query with a path the index takes does load it. + let _ = cipher + .query(op::AtPath(op::Eq), &("$.address.city", "Sydney")) + .on(&plan) + .keyset("tenant") + .await + .unwrap(); + assert_eq!( + loads.load(std::sync::atomic::Ordering::SeqCst), + 1, + "a query the index takes loads the keyset" + ); + } +} + +// ============================================================================= +// Plans +// ============================================================================= + +mod given_a_plan { + use super::*; + + #[tokio::test] + async fn refuses_a_query_under_other_options() { + let Scenario { cipher, plan } = scenario().await; + let refused = cipher + .query(op::Contains, &json!({})) + .via(Json::new(open_root())) + .on(&plan) + .await; + assert!( + matches!(refused, Err(Error::Plan(PlanError::IndexOptions { .. }))), + "a query under other options is refused: {refused:?}" + ); + assert_eq!( + >::indexes(), + [IndexSpec::Json(JsonOptions::default())], + "a JSON document's index is the default JSON index" + ); + assert_eq!(Json, Json::default(), "the value is the default index"); + } + + #[tokio::test] + async fn opens_what_it_wrote_as_the_typed_open_does() { + let Scenario { cipher, plan } = scenario().await; + let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); + let typed = open(&cipher, doc.clone()).await.unwrap(); + let back: Value = cipher.open(doc).using(&plan).await.unwrap(); + assert_eq!(back, fixture(), "the plan opens what it wrote"); + assert_eq!(back, typed, "and the typed open agrees"); + } +} + +mod given_a_laid_out_plan { + use super::*; + + #[test] + fn reports_its_layout() { + let plan = ordered_plan(); + let options = ordering(&["$.hello", "$.number", "$.flag"]); + assert_eq!(plan.layout().options(), &options, "the layout's options"); + assert_eq!( + plan.specs(), + [IndexSpec::Json(options.clone())], + "the plan's index specs" + ); + assert_eq!( + stack_encrypt::target::Layout::::specs(&Json::new(options.clone())), + [IndexSpec::Json(options)], + "the layout's index specs" + ); + assert!( + format!("{plan:?}").contains("Laid(Json"), + "the plan's Debug names its layout: {plan:?}" + ); + } + + /// A plan laid out by the index opens what it wrote, one document or a + /// batch. + #[tokio::test] + async fn opens_what_it_wrote_one_document_or_a_batch() { + let Scenario { cipher, .. } = scenario().await; + let plan = ordered_plan(); + let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); + let back: Value = cipher.open(doc.clone()).using(&plan).await.unwrap(); + assert_eq!(back, fixture(), "one document"); + let docs: Vec = cipher + .open(vec![doc.clone(), doc]) + .using(&plan) + .await + .unwrap(); + assert_eq!(docs, [fixture(), fixture()], "a batch"); + } + + /// The plan was built with its context, so an open that names one is + /// refused, one document or a batch, before any key is requested. + #[tokio::test] + async fn refuses_an_open_that_names_another_context() { + let Scenario { cipher, .. } = scenario().await; + let plan = ordered_plan(); + let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); + let clash = cipher + .open(doc.clone()) + .context("users/other") + .using(&plan) + .await; + assertions::refused_for_two_contexts(&clash, "one document"); + let clash = cipher + .open(vec![doc]) + .context("users/other") + .using(&plan) + .await; + assertions::refused_for_two_contexts(&clash, "a batch"); + } + + /// The index named on the query says only which: the plan's own, with + /// its options, answers. + #[tokio::test] + async fn answers_a_named_index_with_its_own_options() { + let Scenario { cipher, .. } = scenario().await; + let plan = ordered_plan(); + let through_default = cipher + .query(op::AtPath(op::Gte), &("$.number", 0)) + .via(Json) + .on(&plan) + .await + .unwrap(); + let chosen = cipher + .query(op::AtPath(op::Gte), &("$.number", 0)) + .on(&plan) + .await + .unwrap(); + assert_eq!( + through_default.selector(), + chosen.selector(), + "the default index named on the query answers as the plan's own" + ); + } +} + +#[cfg(feature = "dynamic")] +mod given_the_json_index_spec { + use super::*; + + /// The spec's wire form is a fixed contract with stored data and the Go + /// guest, so its strings are pinned literally. + #[test] + fn has_a_wire_form() { + use stack_encrypt::dynamic::{not_admitted, Reason, ValueKind}; + assert_eq!( + IndexSpec::parse("json"), + Some(IndexSpec::Json(JsonOptions::default())), + "`json` parses as the default index" + ); + assert_eq!( + IndexSpec::Json(JsonOptions::default()).key(), + "json", + "the default index's key" + ); + assert_eq!( + serde_json::to_value(IndexSpec::Json(JsonOptions::default())).unwrap(), + json!("json"), + "the default index is written as its key" + ); + assert_eq!( + serde_json::to_value(IndexSpec::Json(open_root())).unwrap(), + json!({"json": {"sealed_root": false, "ordered": []}}), + "every option is written when any is not the default" + ); + let frame = + vitaminc_aead_frame::serde::to_vec(&json!({"json": {"sealed_root": false}})).unwrap(); + assert_eq!( + IndexSpec::from_frame(&frame).unwrap(), + IndexSpec::Json(open_root()), + "an option read from a frame" + ); + let frame = vitaminc_aead_frame::serde::to_vec(&json!({"json": {"nope": true}})).unwrap(); + assert!( + IndexSpec::from_frame(&frame).is_err(), + "an unknown option refuses the index" + ); + // The ordered paths ride in `ordered`, written in their canonical + // order and read back parsed; a text that is not a path refuses the + // index. + let ordered = IndexSpec::Json(ordering(&["$.orders[*].total", "$.age", "$.age"])); + assert_eq!( + serde_json::to_value(&ordered).unwrap(), + json!({"json": {"sealed_root": true, "ordered": ["$.age", "$.orders[*].total"]}}), + "ordered paths are written once, in canonical order" + ); + let frame = vitaminc_aead_frame::serde::to_vec( + &json!({"json": {"ordered": ["$.age", "$.orders[*].total"]}}), + ) .unwrap(); - assert_eq!(loads.load(std::sync::atomic::Ordering::SeqCst), 1); + assert_eq!( + IndexSpec::from_frame(&frame).unwrap(), + ordered, + "ordered paths read back parsed" + ); + let frame = + vitaminc_aead_frame::serde::to_vec(&json!({"json": {"ordered": ["age"]}})).unwrap(); + assert!( + IndexSpec::from_frame(&frame).is_err(), + "a text that is not a path refuses the index" + ); + // A position in an ordered path refuses the index too: no ordering + // can reach the leaf it names. + let frame = + vitaminc_aead_frame::serde::to_vec(&json!({"json": {"ordered": ["$.scores[2]"]}})) + .unwrap(); + assert!( + IndexSpec::from_frame(&frame).is_err(), + "a position in an ordered path refuses the index" + ); + assert_eq!( + not_admitted(ValueKind::Object, &IndexSpec::Json(JsonOptions::default())), + Some(Reason::JsonIndexPending), + "the data plan does not admit the JSON index yet" + ); + } +} + +// ============================================================================= +// Assertions +// ============================================================================= + +mod assertions { + use std::fmt::Debug; + + use super::*; + + pub(super) fn one_entry_per_node(doc: &JsonDocument, value: &Value, what: &str) { + assert_eq!( + doc.entries().len(), + nodes(value), + "{what}: one entry per node" + ); + } + + pub(super) async fn opens_to(cipher: &Cipher, doc: JsonDocument, expected: &Value, what: &str) { + let opened = open(cipher, doc).await; + assert!( + matches!(&opened, Ok(value) if value == expected), + "{what}: should open to {expected}, opened {opened:?}" + ); + } + + /// `result` is the term error `is` picks out. + pub(super) fn refused_by_term( + result: &Result, + is: impl Fn(&TermError) -> bool, + what: &str, + ) { + assert!( + matches!(result, Err(Error::Term(error)) if is(error)), + "{what}: refused with the wrong outcome: {result:?}" + ); + } + + /// `result` is a shape mismatch, naming `expected` when one is given. + pub(super) fn refused_as_opened_type( + result: &Result, + expected: Option<&str>, + what: &str, + ) { + assert!( + matches!(result, Err(Error::OpenedType { expected: got }) if expected.is_none_or(|e| e == *got)), + "{what}: should be refused as a shape mismatch expecting {expected:?}, got {result:?}" + ); + } + + /// `result` is tamper evidence: the authentication failed. + pub(super) fn refused_as_tampered(result: &Result, what: &str) { + assert!( + matches!(result, Err(Error::Aead)), + "{what}: should fail authentication, got {result:?}" + ); + } + + /// `result` is the refusal of an open that names a second context. + pub(super) fn refused_for_two_contexts(result: &Result, what: &str) { + assert!( + matches!( + result, + Err(Error::Plan(PlanError::TwoContextSources { .. })) + ), + "{what}: an open naming another context should be refused, got {result:?}" + ); + } + + /// `@>` over the projections is the model's `jsonb @>`. + pub(super) fn contains_as_jsonb( + query: &JsonQuery, + doc: &JsonDocument, + stored: &Value, + needle: &Value, + ) { + assert_eq!( + query.matches(doc.projection()), + pg_contains(stored, needle, true), + "{stored} @> {needle}" + ); + } + + /// `<@` is the model's `jsonb <@`, but for a stored root scalar in a + /// needle root array. + pub(super) fn is_contained_by_as_jsonb( + query: &JsonQuery, + doc: &JsonDocument, + stored: &Value, + needle: &Value, + ) { + assert_eq!( + query.projection().contains(doc.projection()), + pg_contains(needle, stored, true) && !scalar_in_array(stored, needle), + "{stored} <@ {needle}" + ); + } + + /// Every feature the generator is meant to cover appears in `docs`. + pub(super) fn covers_every_feature(docs: &[Value]) { + #[derive(Debug, Default)] + struct Census { + root_objects: usize, + root_arrays: usize, + root_scalars: usize, + empty_objects: usize, + empty_arrays: usize, + arrays_of_objects: usize, + arrays_of_arrays: usize, + duplicate_elements: usize, + keys_on_two_levels: usize, + float_spelled_integers: usize, + numeric_strings: usize, + booleans: usize, + nulls: usize, + max_depth: usize, + } + fn walk<'v>( + census: &mut Census, + v: &'v Value, + depth: usize, + keys: &mut std::collections::BTreeMap<&'v str, std::collections::BTreeSet>, + ) { + census.max_depth = census.max_depth.max(depth); + match v { + Value::Object(map) => { + census.empty_objects += usize::from(map.is_empty()); + for (key, member) in map { + let _ = keys.entry(key).or_default().insert(depth); + walk(census, member, depth + 1, keys); + } + } + Value::Array(items) => { + census.empty_arrays += usize::from(items.is_empty()); + census.arrays_of_objects += usize::from(items.iter().any(Value::is_object)); + census.arrays_of_arrays += usize::from(items.iter().any(Value::is_array)); + census.duplicate_elements += usize::from( + items + .iter() + .enumerate() + .any(|(i, a)| items[i + 1..].contains(a)), + ); + for item in items { + walk(census, item, depth + 1, keys); + } + } + Value::Number(n) => { + census.float_spelled_integers += + usize::from(n.is_f64() && n.as_f64().is_some_and(|f| f.fract() == 0.0)); + } + Value::String(s) => census.numeric_strings += usize::from(s.parse::().is_ok()), + Value::Bool(_) => census.booleans += 1, + Value::Null => census.nulls += 1, + } + } + let mut census = Census::default(); + for doc in docs { + match doc { + Value::Object(_) => census.root_objects += 1, + Value::Array(_) => census.root_arrays += 1, + _ => census.root_scalars += 1, + } + let mut keys = std::collections::BTreeMap::new(); + walk(&mut census, doc, 0, &mut keys); + census.keys_on_two_levels += usize::from( + keys.values() + .any(|depths: &std::collections::BTreeSet| depths.len() > 1), + ); + } + for (what, count) in [ + ("root objects", census.root_objects), + ("root arrays", census.root_arrays), + ("root scalars", census.root_scalars), + ("empty objects", census.empty_objects), + ("empty arrays", census.empty_arrays), + ("arrays of objects", census.arrays_of_objects), + ("arrays of arrays", census.arrays_of_arrays), + ("duplicate array elements", census.duplicate_elements), + ("a key on two levels", census.keys_on_two_levels), + ("integers spelled as floats", census.float_spelled_integers), + ("strings spelled as numbers", census.numeric_strings), + ("booleans", census.booleans), + ("nulls", census.nulls), + ] { + assert!( + count > 0, + "the generated documents should hold {what}: {census:?}" + ); + } + assert!( + census.max_depth >= 4, + "the generated documents should nest at least four levels: {census:?}" + ); + } }