diff --git a/Cargo.lock b/Cargo.lock index 611e066e1..271420eac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3406,6 +3406,7 @@ dependencies = [ "clww-ore", "miette", "ore-rs", + "proptest", "rust_decimal", "serde", "serde_json", diff --git a/docs/plans/2026-10-17-json-tree-index.md b/docs/plans/2026-10-17-json-tree-index.md index 7c78baf30..41b1e06b7 100644 --- a/docs/plans/2026-10-17-json-tree-index.md +++ b/docs/plans/2026-10-17-json-tree-index.md @@ -363,7 +363,7 @@ Each phase is a PR against `next`, where #1209 landed. The index, the SQL and the live oracle landed in the pull request stacked on this document; what follows records where the code is and where it departs from the design above, so the two can be read against each other. Phase 0 was folded into Phase 2, and Phases 3 and 4 remain open. -**Where it is.** The engine half is `packages/stack-encrypt/src/json/`: `Json` is the index (`Index`, term `JsonTerms`), `target::json_document` the sealing operation, `JsonDocument` the stored term and an `EncryptFrom` / `DecryptInto` target, `JsonQuery` the needle for `Contains`, `ContainedBy` and `AtPath`, `JsonLeafQuery` the operand for the orderings at a path, and `JsonSelector` / `JsonSubtreeKey` the extraction keys, derived through `KeysetCipher::json_selector` and `json_subtree_key`. `IndexSpec::Json(JsonOptions)` is the data form, with the key `"json"` and `{"json": {"sealed_root": …}}` as its wire form; a data plan reads it and refuses to run it (`Reason::JsonIndexPending`). The SQL is `packages/eql/src/v3/json_tree/`, confined to EQL v4; the EQL types are `packages/eql/crates/eql-bindings/src/v4/json_tree.rs`; the oracle is `packages/eql/tests/encryption/tests/json_tree.rs`, which stores the design's corpus and random documents encrypted beside their plaintexts and checks that `@>` and `<@` over the encrypted column select the rows PostgreSQL's own `jsonb` operators select over the plaintext column, with the GIN index shown in `EXPLAIN`. The pure-Rust half of that oracle is `packages/stack-encrypt/tests/json_index.rs`, against a model of `jsonb @>` pinned to the answers in `jsonb-semantics.sql`. +**Where it is.** The engine half is `packages/stack-encrypt/src/json/`: `Json` is the index (`Index`, term `JsonTerms`), `target::json_document` the sealing operation, `JsonDocument` the stored term and an `EncryptFrom` / `DecryptInto` target, `JsonQuery` the needle for `Contains`, `ContainedBy` and `AtPath`, `JsonLeafQuery` the operand for the orderings at a path, and `JsonSelector` / `JsonSubtreeKey` the extraction keys, derived through `KeysetCipher::json_selector` and `json_subtree_key`. `IndexSpec::Json(JsonOptions)` is the data form, with the key `"json"` and `{"json": {"sealed_root": …}}` as its wire form; a data plan reads it and refuses to run it (`Reason::JsonIndexPending`). The SQL is `packages/eql/src/v3/json_tree/`, confined to EQL v4; the EQL types are `packages/eql/crates/eql-bindings/src/v4/json_tree.rs`; the oracle is `packages/eql/tests/encryption/tests/json_tree.rs`, which stores the design's corpus, and batches of documents proptest generates (`json_tree/generated.rs`), encrypted beside their plaintexts and checks that `@>` and `<@` over the encrypted column select the rows PostgreSQL's own `jsonb` operators select over the plaintext column, on the GIN index and on a sequential scan, each plan shown in `EXPLAIN`. The pure-Rust half of that oracle is `packages/stack-encrypt/tests/json_index.rs` (`given_the_design_corpus` and the proptest properties in `given_generated_documents`), against a model of `jsonb @>` pinned to the answers in `jsonb-semantics.sql`. **Decisions taken by the code.** Of the eleven listed below: 1, hex for every sixteen-byte field and base64 for `c` and `h`, pending Phase 3 (taken on 2026-10-23, Phase 3d: base64 for every field the SQL reads, base85 for `c`); 2, leaf ordering terms bound to the column only, numbers as the `f64` they round to and strings as text, under the scalar `Ope` index's derivation; 3, wildcard arrays only; 4, group tags on, and only on container array elements; 5, the child count is in every container node's ciphertext, and a truncated subtree is `OpenedType`, not a shorter array; 7, plain SHA-256; 8, the option is the boolean `sealed_root`, on by default, recorded in the header's flags; 9, the projection is built from the tags server-side (`eql_v4.json_tree_query`) and `groups_ok` was not needed; 10, position-keyed parent tags on, the ordinal stored as `o`, object members ranked under the parent's token; 11, keyed: in this first form a child's token was `PRF(json/token-step ‖ column, PAE(t(parent), 0x01, key))` (`0x02, LE64(index)` for an element) and an object member's rank `PRF(json/rank ‖ column, PAE(t(parent), key))`, each under its own PAE domain, so a token's holder could neither test key names against the child tokens the walk unwraps nor rank guessed names against the ordinals. Decision 6 is out of scope, as proposed. (Both steps were first hashed unkeyed, as the token chain section above writes them, and were corrected before any release.) diff --git a/packages/eql/Cargo.lock b/packages/eql/Cargo.lock index a04fac03f..06e657798 100644 --- a/packages/eql/Cargo.lock +++ b/packages/eql/Cargo.lock @@ -1249,6 +1249,7 @@ dependencies = [ "eql-bindings", "eql-domains", "postgres", + "proptest", "rust_decimal", "serde", "serde_json", diff --git a/packages/eql/mise.toml b/packages/eql/mise.toml index 9f1995443..79bea8ab0 100644 --- a/packages/eql/mise.toml +++ b/packages/eql/mise.toml @@ -343,7 +343,11 @@ psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -f release/cipherstash-encrypt- for suite in text_eq v4_types v4_order v4_date_time_numeric v4_range v4_text_search v4_ore_block json_tree; do listing=$(cargo test --locked -p eql-encryption-tests --test "$suite" -- --ignored --list | sed -n 's/: test$//p') [ -n "$listing" ] || { echo "error: '--ignored' selects no test in $suite; the PostgreSQL test has gone missing." >&2; exit 1; } - cargo test --locked -p eql-encryption-tests --test "$suite" -- --ignored + # `--show-output`: a passing test's output is otherwise captured, and the + # generated containment properties in json_tree log the proptest seed and + # case count they ran under (PROPTEST_RNG_SEED and PROPTEST_CASES replay + # them) and what they checked. + cargo test --locked -p eql-encryption-tests --test "$suite" -- --ignored --show-output done """ diff --git a/packages/eql/tests/encryption/Cargo.toml b/packages/eql/tests/encryption/Cargo.toml index 8ee6acd08..787f19abc 100644 --- a/packages/eql/tests/encryption/Cargo.toml +++ b/packages/eql/tests/encryption/Cargo.toml @@ -54,6 +54,10 @@ postgres = "0.19" # compared. # cipherstash-client = "=0.42.3" +# The generated containment oracle (tests/json_tree/generated.rs). The same +# requirement eql-domains and tests/sqlx carry, so the lockfile gains no +# second copy. +proptest = "1" # The tampering test decodes a stored body, changes one byte and re-encodes # it. The same line eql-bindings uses, so the lockfile gains no second copy. base64 = "0.22" diff --git a/packages/eql/tests/encryption/proptest-regressions/json_tree.txt b/packages/eql/tests/encryption/proptest-regressions/json_tree.txt new file mode 100644 index 000000000..341ba3ff3 --- /dev/null +++ b/packages/eql/tests/encryption/proptest-regressions/json_tree.txt @@ -0,0 +1,6 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index e1c4a97aa..21c9fd05e 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -6,7 +6,9 @@ //! `@>` and `<@` over the encrypted column against PostgreSQL's own //! `jsonb @>` over the plaintexts, for the design's corpus and random //! documents, with a GIN index on the projection that the planner is shown -//! to use. +//! to use, and again as proptest properties over generated batches of +//! documents and needles (`json_tree/generated.rs`; `PROPTEST_RNG_SEED` +//! replays a run). mod common; @@ -435,6 +437,17 @@ fn postgres_containment_over_the_encrypted_column_is_jsonb_containment() { assert!(nulls.get::<_, bool>(0)); } +// ============================================================================= +// PostgreSQL: generated documents +// ============================================================================= + +/// The containment oracle again, over documents and needles proptest +/// generates. A crate root's modules resolve beside it, and a file there would +/// be a test target of its own, hence the path. +#[allow(clippy::unwrap_used)] +#[path = "json_tree/generated.rs"] +mod generated; + /// Needs an EQL v4 installation. Extraction by selector opens to the leaf; /// the subtree walk by a client-derived token returns exactly the nodes /// under it and reassembles; the array functions count and list elements; diff --git a/packages/eql/tests/encryption/tests/json_tree/generated.rs b/packages/eql/tests/encryption/tests/json_tree/generated.rs new file mode 100644 index 000000000..192e0d7e5 --- /dev/null +++ b/packages/eql/tests/encryption/tests/json_tree/generated.rs @@ -0,0 +1,1126 @@ +//! The containment oracle over generated documents. proptest draws a case: a +//! batch of stored documents and a batch of needles (random values, values +//! derived from a stored document, and those with one edit). The documents +//! are stored encrypted beside their plaintexts, and every needle is asked +//! of both columns in a few set-based statements, so PostgreSQL's own +//! `jsonb` operators are the oracle and a case costs a handful of round +//! trips, not one per needle. +//! +//! **Replaying.** The RNG is ChaCha with a fixed seed ([`DEFAULT_SEED`]), +//! logged on every run. `PROPTEST_RNG_SEED` replaces the seed and +//! `PROPTEST_CASES` the case count ([`DEFAULT_CASES`]), as for any proptest +//! run. A failure shrinks to a small case and prints it, and proptest +//! appends its seed to `proptest-regressions/json_tree.txt` in this crate, +//! which every later run replays first. Commit that line with the fix. + +use std::cell::{Cell, RefCell}; +use std::fmt; + +use proptest::collection::vec as vec_of; +use proptest::prelude::*; +use proptest::sample::{select, Index}; +use proptest::strategy::ValueTree; +use proptest::test_runner::{ + Config, FileFailurePersistence, RngAlgorithm, RngSeed, TestCaseError, TestRunner, +}; + +use super::*; + +/// The seed the run uses when `PROPTEST_RNG_SEED` is unset. Fixed, so every +/// CI run checks the same cases. +const DEFAULT_SEED: u64 = 1336; + +/// The cases a run checks when `PROPTEST_CASES` is unset. +const DEFAULT_CASES: u32 = 64; + +/// The largest batches a case holds. +const MAX_DOCUMENTS: usize = 64; +const MAX_NEEDLES: usize = 96; + +/// Where proptest records a failing case's seed. `Direct` with the crate's +/// own path: the default (`SourceParallel`) resolves the test's source file +/// from the current directory, which for this nested workspace member is not +/// where `file!()` is relative to. +const REGRESSIONS: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/proptest-regressions/json_tree.txt" +); + +/// Numbers, several of them one value to `jsonb` in different +/// representations: `1`, `1.0` and `1e0`; `100` and `1e2` (an integer and a +/// float); `0`, `0.0` and `-0.0`; `0.5` and `5e-1`. A value is stored as +/// `serde_json` parsed it and PostgreSQL is sent `serde_json`'s spelling of +/// the same value, so each side sees one plaintext. +const NUMBERS: &[&str] = &[ + "0", + "0.0", + "-0.0", + "1", + "1.0", + "1e0", + "2", + "2.00", + "-1", + "-1.0", + "10", + "1e1", + "100", + "1e2", + "0.5", + "5e-1", + "1.5", + "123456789012", + "1e20", +]; + +/// Strings, some spelled like a number, a boolean or null, so that a kind +/// confused with another shows as a wrong answer. +const STRINGS: &[&str] = &["x", "y", "", "1", "1.0", "2", "true", "null", "a b", "ü"]; + +/// Object keys: few, so they repeat across levels and siblings. The `->` +/// paths are built from the first four. +const KEYS: &[&str] = &["a", "b", "c", "x", "1", ""]; +const PATH_KEYS: &[&str] = &["a", "b", "c", "x"]; + +/// The run's configuration: proptest's defaults, which read +/// `PROPTEST_CASES` and `PROPTEST_RNG_SEED`, with this test's own default +/// for each one left unset. +fn config() -> Config { + let mut config = Config::default(); + if std::env::var_os("PROPTEST_CASES").is_none() { + config.cases = DEFAULT_CASES; + } + if config.rng_seed == RngSeed::Random { + config.rng_seed = RngSeed::Fixed(DEFAULT_SEED); + } + config.rng_algorithm = RngAlgorithm::ChaCha; + config.failure_persistence = Some(Box::new(FileFailurePersistence::Direct(REGRESSIONS))); + // A shrink step is a round trip to the database: bound the search. + config.max_shrink_time = 120_000; + config +} + +/// How to replay the run, for the log and every failure. +fn replay(config: &Config) -> String { + format!( + "seed {seed}, {cases} cases; replay with PROPTEST_RNG_SEED={seed} PROPTEST_CASES={cases}", + seed = config.rng_seed, + cases = config.cases + ) +} + +// ============================================================================= +// The generated values +// ============================================================================= + +/// One edit that turns a contained needle into a near miss. `at` picks the +/// node, in document order; an edit that does not fit the node's kind +/// changes its kind instead. +#[derive(Clone, Debug)] +enum Edit { + Leaf { + at: Index, + to: Value, + }, + Key { + at: Index, + key: String, + value: Value, + }, + Element { + at: Index, + value: Value, + }, + Kind { + at: Index, + }, +} + +/// A needle as proptest draws it: derived ones name their stored document +/// by an [`Index`] into the case's documents, so they follow the documents +/// as those shrink. `keep` says, one member or element at a time in +/// document order, what a pruning keeps; once it runs out, nothing more is +/// kept, so shrinking it makes the needle smaller. +#[derive(Clone, Debug)] +enum NeedleSpec { + Random(Value), + Pruned { + from: Index, + keep: Vec, + at: Option, + }, + NearMiss { + from: Index, + keep: Vec, + edit: Edit, + }, +} + +/// Where a needle came from, for a failure message. +#[derive(Clone, Copy)] +enum Origin { + Random, + /// A pruning of the row's whole document: the row contains it. + PrunedRoot(usize), + /// A pruning of a node inside the row's document. + PrunedNode(usize), + NearMiss(usize), +} + +impl fmt::Display for Origin { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Origin::Random => write!(f, "random"), + Origin::PrunedRoot(row) => write!(f, "pruned from row {row}"), + Origin::PrunedNode(row) => write!(f, "pruned from a node of row {row}"), + Origin::NearMiss(row) => write!(f, "near miss of row {row}"), + } + } +} + +struct Needle { + origin: Origin, + value: Value, +} + +/// One generated case: the documents stored, and the needles asked. +#[derive(Clone)] +struct Case { + docs: Vec, + needles: Vec, +} + +impl Case { + /// The needles as values, each derived from the case's documents. + fn needles(&self) -> Vec { + self.needles + .iter() + .map(|spec| match spec { + NeedleSpec::Random(value) => Needle { + origin: Origin::Random, + value: value.clone(), + }, + NeedleSpec::Pruned { from, keep, at } => { + let row = from.index(self.docs.len()); + let doc = &self.docs[row]; + let pointer = at.as_ref().map_or_else(String::new, |at| { + let pointers = pointers(doc); + at.get(&pointers).clone() + }); + let node = doc.pointer(&pointer).unwrap(); + Needle { + origin: if pointer.is_empty() { + Origin::PrunedRoot(row) + } else { + Origin::PrunedNode(row) + }, + value: prune(node, &mut keep.iter()), + } + } + NeedleSpec::NearMiss { from, keep, edit } => { + let row = from.index(self.docs.len()); + let mut value = prune(&self.docs[row], &mut keep.iter()); + apply(edit, &mut value); + Needle { + origin: Origin::NearMiss(row), + value, + } + } + }) + .collect() + } +} + +/// Printed as JSON, one document and one needle per line: what proptest +/// shows for the minimal failing case. +impl fmt::Debug for Case { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "{} stored documents:", self.docs.len())?; + for (row, doc) in self.docs.iter().enumerate() { + writeln!(f, " row {row}: {doc}")?; + } + writeln!(f, "{} needles:", self.needles.len())?; + for (i, needle) in self.needles().iter().enumerate() { + writeln!(f, " needle {i} ({}): {}", needle.origin, needle.value)?; + } + Ok(()) + } +} + +/// A sub-document `v` contains: an object keeps the members, and an array +/// the elements, that `keep` says to keep, each pruned in turn; a scalar is +/// itself. +fn prune(v: &Value, keep: &mut std::slice::Iter<'_, bool>) -> Value { + match v { + Value::Object(map) => { + let mut out = Map::new(); + for (key, member) in map { + if keep.next().copied().unwrap_or(false) { + let _ = out.insert(key.clone(), prune(member, keep)); + } + } + Value::Object(out) + } + Value::Array(items) => { + let mut out = Vec::new(); + for item in items { + if keep.next().copied().unwrap_or(false) { + out.push(prune(item, keep)); + } + } + Value::Array(out) + } + scalar => scalar.clone(), + } +} + +/// Every node's JSON pointer, in document order, the root (`""`) first. +fn pointers(v: &Value) -> Vec { + fn walk(v: &Value, at: String, out: &mut Vec) { + out.push(at.clone()); + match v { + Value::Object(map) => { + for (key, member) in map { + let key = key.replace('~', "~0").replace('/', "~1"); + walk(member, format!("{at}/{key}"), out); + } + } + Value::Array(items) => { + for (i, item) in items.iter().enumerate() { + walk(item, format!("{at}/{i}"), out); + } + } + _ => {} + } + } + let mut out = Vec::new(); + walk(v, String::new(), &mut out); + out +} + +fn apply(edit: &Edit, needle: &mut Value) { + let at = match edit { + Edit::Leaf { at, .. } + | Edit::Key { at, .. } + | Edit::Element { at, .. } + | Edit::Kind { at } => at, + }; + let pointer = at.get(&pointers(needle)).clone(); + let node = needle.pointer_mut(&pointer).unwrap(); + match edit { + Edit::Leaf { to, .. } if !node.is_object() && !node.is_array() && node != to => { + *node = to.clone(); + } + Edit::Key { key, value, .. } if node.is_object() => { + let _ = node + .as_object_mut() + .unwrap() + .insert(key.clone(), value.clone()); + } + Edit::Element { value, .. } if node.is_array() => { + node.as_array_mut().unwrap().push(value.clone()); + } + _ => { + *node = match node.take() { + Value::Object(map) => Value::Array(map.into_iter().map(|(_, v)| v).collect()), + Value::Array(items) => { + json!({ "a": items.into_iter().next().unwrap_or(Value::Null) }) + } + scalar => Value::Array(vec![scalar]), + }; + } + } +} + +mod strategies { + use super::*; + + pub(super) fn scalar() -> BoxedStrategy { + prop_oneof![ + 4 => select(NUMBERS).prop_map(|text| serde_json::from_str::(text).unwrap()), + 3 => select(STRINGS).prop_map(|text| Value::String(text.to_owned())), + 2 => any::().prop_map(Value::Bool), + 1 => Just(Value::Null), + ] + .boxed() + } + + fn key() -> BoxedStrategy { + select(KEYS).prop_map(str::to_owned).boxed() + } + + /// An object of up to four members. A key drawn twice keeps its later + /// value, as `jsonb` does. + fn object_of(inner: BoxedStrategy) -> BoxedStrategy { + vec_of((key(), inner), 0..5) + .prop_map(|members| Value::Object(members.into_iter().collect())) + .boxed() + } + + /// An array: of anything, of objects, of arrays, or with some elements + /// repeated. + fn array_of(inner: BoxedStrategy) -> BoxedStrategy { + prop_oneof![ + 3 => vec_of(inner.clone(), 0..5).prop_map(Value::Array), + 1 => vec_of(object_of(inner.clone()), 1..4).prop_map(Value::Array), + 1 => vec_of(vec_of(inner.clone(), 0..3).prop_map(Value::Array), 1..4) + .prop_map(Value::Array), + 1 => (vec_of(inner, 1..4), vec_of(any::(), 1..3)).prop_map( + |(mut items, repeats)| { + for repeat in repeats { + let copy = repeat.get(&items).clone(); + items.push(copy); + } + Value::Array(items) + } + ), + ] + .boxed() + } + + /// A value with up to `depth` levels of recursion below it. + pub(super) fn value(depth: u32) -> BoxedStrategy { + scalar() + .prop_recursive(depth, 48, 4, |inner| { + prop_oneof![object_of(inner.clone()), array_of(inner)] + }) + .boxed() + } + + /// A stored document: mostly an object, sometimes a root array, and + /// sometimes a root scalar. + pub(super) fn document() -> BoxedStrategy { + prop_oneof![ + 6 => object_of(value(3)), + 3 => array_of(value(2)), + 1 => scalar(), + ] + .boxed() + } + + fn keep() -> BoxedStrategy> { + vec_of(proptest::bool::weighted(0.7), 0..32).boxed() + } + + fn edit() -> BoxedStrategy { + prop_oneof![ + (any::(), scalar()).prop_map(|(at, to)| Edit::Leaf { at, to }), + (any::(), key(), value(1)).prop_map(|(at, key, value)| Edit::Key { + at, + key, + value + }), + (any::(), value(1)).prop_map(|(at, value)| Edit::Element { at, value }), + any::().prop_map(|at| Edit::Kind { at }), + ] + .boxed() + } + + fn needle() -> BoxedStrategy { + prop_oneof![ + 1 => value(2).prop_map(NeedleSpec::Random), + 2 => (any::(), keep(), proptest::option::weighted(0.3, any::())) + .prop_map(|(from, keep, at)| NeedleSpec::Pruned { from, keep, at }), + 2 => (any::(), keep(), edit()) + .prop_map(|(from, keep, edit)| NeedleSpec::NearMiss { from, keep, edit }), + ] + .boxed() + } + + pub(super) fn case() -> BoxedStrategy { + ( + vec_of(document(), 1..=MAX_DOCUMENTS), + vec_of(needle(), 1..=MAX_NEEDLES), + ) + .prop_map(|(docs, needles)| Case { docs, needles }) + .boxed() + } +} + +// ============================================================================= +// The database +// ============================================================================= + +/// The two execution paths every containment answer is asked on. +#[derive(Clone, Copy, Debug)] +enum Scan { + /// Sequential scans off: the planner takes the GIN index where it can. + Index, + /// Every index path off. + Sequential, +} + +impl Scan { + const ALL: [Scan; 2] = [Scan::Index, Scan::Sequential]; + + fn settings(self) -> &'static str { + match self { + Scan::Index => "SET enable_seqscan = off", + Scan::Sequential => { + "SET enable_seqscan = on; SET enable_bitmapscan = off; SET enable_indexscan = off; SET enable_indexonlyscan = off" + } + } + } +} + +/// The containment operators, with the SQL operator each spells. +#[derive(Clone, Copy, Debug)] +enum Containment { + Contains, + ContainedBy, +} + +impl Containment { + fn sql(self) -> &'static str { + match self { + Containment::Contains => "@>", + Containment::ContainedBy => "<@", + } + } +} + +/// One database session, cipher and runtime, built once for a whole run: +/// proptest's runner is synchronous, so the async work is driven on this +/// runtime. +struct Scenario { + runtime: tokio::runtime::Runtime, + cipher: Cipher, + plan: ValuePlan>, + db: RefCell, +} + +fn scenario() -> Scenario { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let mut db = connect(); + // No primary key: the GIN index is the table's only index, so with + // sequential scans off the planner has no other index to walk whole + // and filter. + db.batch_execute( + "CREATE TEMP TABLE json_tree_generated (id int NOT NULL, plain jsonb NOT NULL, doc public.eql_v4_json_tree NOT NULL); + CREATE INDEX json_tree_generated_doc ON json_tree_generated USING gin ((eql_v4.json_tree_query(doc)) jsonb_path_ops);", + ) + .unwrap(); + Scenario { + runtime, + cipher, + plan: plan(), + db: RefCell::new(db), + } +} + +impl Scenario { + /// Replace the table's rows with the documents, each sealed beside its + /// plaintext, in one statement. Row `i` is document `i`. + fn store(&self, docs: &[Value]) { + let ids: Vec = (0..docs.len() as i32).collect(); + let plains: Vec = docs.iter().map(Value::to_string).collect(); + let sealed: Vec = docs + .iter() + .map(|doc| { + serde_json::to_string(&wait(&self.runtime, seal(&self.cipher, doc))).unwrap() + }) + .collect(); + let mut db = self.db.borrow_mut(); + db.batch_execute("TRUNCATE json_tree_generated").unwrap(); + let _ = db + .execute( + "INSERT INTO json_tree_generated + SELECT id, plain::jsonb, doc::jsonb::public.eql_v4_json_tree + FROM unnest($1::int[], $2::text[], $3::text[]) AS u(id, plain, doc)", + &[&ids, &plains, &sealed], + ) + .unwrap(); + db.batch_execute("ANALYZE json_tree_generated").unwrap(); + } + + /// Each needle's operand for `containment`, as the SQL takes it. + fn operands(&self, containment: Containment, needles: &[Needle]) -> Vec { + needles + .iter() + .map(|needle| { + let query = match containment { + Containment::Contains => wait( + &self.runtime, + self.cipher + .query(op::Contains, &needle.value) + .on(&self.plan), + ), + Containment::ContainedBy => wait( + &self.runtime, + self.cipher + .query(op::ContainedBy, &needle.value) + .on(&self.plan), + ), + } + .unwrap(); + serde_json::to_string(&JsonTreeQuery::try_from(&query).unwrap()).unwrap() + }) + .collect() + } + + /// For each needle, the ids of the rows that `jsonb` selects over the + /// plaintexts: the oracle. + fn jsonb(&self, containment: Containment, needles: &[Needle]) -> Vec> { + let texts: Vec = needles.iter().map(|n| n.value.to_string()).collect(); + self.rows_per_needle(&format!("g.plain {} n.t::jsonb", containment.sql()), &texts) + } + + /// For each operand, the ids of the rows the encrypted column selects, + /// and the plan the statement ran under. + fn encrypted( + &self, + containment: Containment, + scan: Scan, + operands: &[String], + ) -> (String, Vec>) { + let predicate = format!( + "g.doc {} n.t::jsonb::eql_v4.query_json_tree", + containment.sql() + ); + self.db.borrow_mut().batch_execute(scan.settings()).unwrap(); + let explain = self + .db + .borrow_mut() + .query( + &format!("EXPLAIN {}", Self::per_needle(&predicate)), + &[&operands], + ) + .unwrap() + .iter() + .map(|r| r.get::<_, String>(0)) + .collect::>() + .join("\n"); + let rows = self.rows_per_needle(&predicate, operands); + self.db.borrow_mut().batch_execute("RESET ALL").unwrap(); + (explain, rows) + } + + /// The statement that answers every needle at once: one correlated + /// subquery per needle, which the planner can serve from the index. + fn per_needle(predicate: &str) -> String { + format!( + "SELECT coalesce((SELECT array_agg(g.id ORDER BY g.id) FROM json_tree_generated g WHERE {predicate}), '{{}}') + FROM unnest($1::text[]) WITH ORDINALITY AS n(t, i) ORDER BY n.i" + ) + } + + fn rows_per_needle(&self, predicate: &str, params: &[String]) -> Vec> { + let sql = Self::per_needle(predicate); + self.db + .borrow_mut() + .query(&sql, &[¶ms]) + .unwrap_or_else(|e| panic!("{sql}: {e}")) + .iter() + .map(|r| r.get(0)) + .collect() + } +} + +/// Run `body` over a whole proptest run against one [`Scenario`], log what +/// ran, and fail with the minimal case and the way to replay it. +fn run(name: &str, body: impl Fn(&Scenario, &Case) -> Result<(), TestCaseError>) { + let config = config(); + let replay = replay(&config); + eprintln!("json_tree generated {name}: {replay}"); + let started = std::time::Instant::now(); + let scenario = scenario(); + let mut runner = TestRunner::new(config); + let result = runner.run(&strategies::case(), |case| body(&scenario, &case)); + if let Err(error) = result { + panic!("json_tree generated {name}: {replay}\n{error}"); + } + eprintln!( + "json_tree generated {name}: passed in {:?}", + started.elapsed() + ); +} + +// ============================================================================= +// Tests +// ============================================================================= + +mod given_the_document_strategy { + use super::*; + + /// Every feature the oracle relies on, counted over a run's worth of + /// generated documents, so a change to the generator that stops + /// producing one fails here, without a database. + #[test] + fn produces_every_feature_the_oracle_relies_on() { + let mut runner = TestRunner::new(config()); + let strategy = strategies::document(); + let docs: Vec = (0..400) + .map(|_| strategy.new_tree(&mut runner).unwrap().current()) + .collect(); + assertions::covers_every_feature(&assertions::Census::of(&docs)); + } +} + +mod given_generated_documents_in_postgres { + use super::*; + + /// Needs an EQL v4 installation. For every needle of every case, the + /// rows encrypted `@>` selects are the rows `jsonb @>` selects over the + /// plaintexts, on the index plan and on the sequential plan, each + /// confirmed by `EXPLAIN`. A needle pruned from a row's whole document + /// is checked to be contained by that row, and the near misses are + /// checked to miss their row often. + #[test] + #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] + fn contains_selects_the_rows_jsonb_contains_on_both_plans() { + let (needles_asked, near_misses, near_misses_missing) = + (Cell::new(0), Cell::new(0), Cell::new(0)); + run("@>", |scenario, case| { + scenario.store(&case.docs); + let needles = case.needles(); + let jsonb = scenario.jsonb(Containment::Contains, &needles); + assertions::pruned_roots_are_contained(&needles, &jsonb)?; + for (needle, rows) in needles.iter().zip(&jsonb) { + if let Origin::NearMiss(row) = needle.origin { + near_misses.set(near_misses.get() + 1); + if !rows.contains(&(row as i32)) { + near_misses_missing.set(near_misses_missing.get() + 1); + } + } + } + let operands = scenario.operands(Containment::Contains, &needles); + for scan in Scan::ALL { + let (plan, encrypted) = scenario.encrypted(Containment::Contains, scan, &operands); + assertions::planned_as(&plan, scan)?; + assertions::rows_match( + &format!("@> ({scan:?} plan)"), + case, + &needles, + &jsonb, + &encrypted, + )?; + } + needles_asked.set(needles_asked.get() + needles.len()); + Ok(()) + }); + eprintln!( + "json_tree generated @>: {} needles asked, {} near misses, {} of them missing their row", + needles_asked.get(), + near_misses.get(), + near_misses_missing.get() + ); + assert!( + near_misses_missing.get() * 4 >= near_misses.get(), + "the near misses should mostly miss the row they were made from: {} of {}", + near_misses_missing.get(), + near_misses.get() + ); + } + + /// Needs an EQL v4 installation. For every needle of every case, the + /// rows encrypted `<@` selects are the rows `jsonb <@` selects, less a + /// stored root scalar when the needle is a root array: the one case the + /// SQL documents it cannot answer, since a stored document carries no + /// root-array alternative. That case is the whole difference, and the + /// run must meet it. `jsonb_path_ops` has no `<@` strategy, so both + /// plans scan; both are asked all the same. + #[test] + #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] + fn contained_by_selects_the_rows_jsonb_does_except_a_root_scalar_in_a_root_array() { + let excluded = Cell::new(0); + run("<@", |scenario, case| { + scenario.store(&case.docs); + let needles = case.needles(); + let jsonb = scenario.jsonb(Containment::ContainedBy, &needles); + let expected: Vec> = needles + .iter() + .zip(&jsonb) + .map(|(needle, rows)| { + rows.iter() + .copied() + .filter(|id| { + let stored = &case.docs[*id as usize]; + !(needle.value.is_array() && !stored.is_object() && !stored.is_array()) + }) + .collect() + }) + .collect(); + excluded.set( + excluded.get() + jsonb.iter().map(Vec::len).sum::() + - expected.iter().map(Vec::len).sum::(), + ); + let operands = scenario.operands(Containment::ContainedBy, &needles); + for scan in Scan::ALL { + let (_, encrypted) = scenario.encrypted(Containment::ContainedBy, scan, &operands); + assertions::rows_match( + &format!("<@ ({scan:?} plan)"), + case, + &needles, + &expected, + &encrypted, + )?; + } + Ok(()) + }); + eprintln!( + "json_tree generated <@: {} rows excluded as a root scalar in a root array", + excluded.get() + ); + assert!( + excluded.get() > 0, + "no stored root scalar sat inside a needle root array, so the `<@` exception went unexercised" + ); + } + + /// Needs an EQL v4 installation. At every key path of one or two steps + /// over the generated keys, the encrypted column has an entry exactly + /// where `jsonb #>` finds a node (`#>` reads a key of an array as no + /// node, as a strict path does), and a leaf's entry opens to the value + /// `jsonb` holds there, compared by PostgreSQL's `jsonb =`, which + /// compares numbers by value. + #[test] + #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] + fn arrow_opens_a_leaf_to_the_value_jsonb_holds() { + let mut paths: Vec> = PATH_KEYS.iter().map(|k| vec![*k]).collect(); + for a in PATH_KEYS { + for b in PATH_KEYS { + paths.push(vec![*a, *b]); + } + } + let (leaves, containers) = (Cell::new(0), Cell::new(0)); + run("->", |scenario, case| { + scenario.store(&case.docs); + let keyset = scenario.cipher.default_keyset(); + let selectors: Vec = paths + .iter() + .map(|steps| { + let path = JsonPath::parse(&format!("$.{}", steps.join("."))).unwrap(); + wait(&scenario.runtime, keyset.json_selector(&path, column())) + .unwrap() + .to_string() + }) + .collect(); + let keys: Vec = paths + .iter() + .map(|steps| format!("{{{}}}", steps.join(","))) + .collect(); + let rows = scenario + .db + .borrow_mut() + .query( + "SELECT p.i, g.id, (g.plain #> p.k::text[])::text, jsonb_typeof(g.plain #> p.k::text[]), (g.doc -> p.s)::jsonb::text + FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS p(s, k, i) + CROSS JOIN json_tree_generated g ORDER BY p.i, g.id", + &[&selectors, &keys], + ) + .unwrap(); + let mut opened: Vec = Vec::new(); + let mut held: Vec = Vec::new(); + let mut at: Vec<(String, i32)> = Vec::new(); + for row in rows { + let path = format!("$.{}", paths[row.get::<_, i64>(0) as usize - 1].join(".")); + let id: i32 = row.get(1); + let want: Option = row.get(2); + let kind: Option = row.get(3); + let got: Option = row.get(4); + assertions::entry_where_jsonb_has_a_node( + &path, + id, + case, + want.is_some(), + got.is_some(), + )?; + if let (Some(want), Some(got)) = (want, got) { + if matches!(kind.as_deref(), Some("object" | "array")) { + containers.set(containers.get() + 1); + continue; + } + let entry: JsonTreeEntry = serde_json::from_str(&got).unwrap(); + let value = wait( + &scenario.runtime, + scenario.cipher.decrypt_as::(entry, column()), + ) + .map_err(|e| { + TestCaseError::fail(format!( + "-> {path}, row {id}: the entry did not open: {e}" + )) + })?; + opened.push(value.to_string()); + held.push(want); + at.push((path, id)); + } + } + let differing: Vec = scenario + .db + .borrow_mut() + .query( + "SELECT i FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS u(opened, held, i) + WHERE opened::jsonb IS DISTINCT FROM held::jsonb ORDER BY i", + &[&opened, &held], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + assertions::leaves_open_to_the_jsonb_value(&differing, &at, &opened, &held, case)?; + leaves.set(leaves.get() + opened.len()); + Ok(()) + }); + eprintln!( + "json_tree generated ->: {} leaves opened, {} containers found at {} paths", + leaves.get(), + containers.get(), + paths.len() + ); + assert!( + leaves.get() > 50 && containers.get() > 10, + "the paths should reach many leaves and containers: {} leaves, {} containers", + leaves.get(), + containers.get() + ); + } +} + +pub(super) mod assertions { + use super::*; + + /// The rows `encrypted` selected for each needle are the rows `expected` + /// says; every disagreement is named with its needle and the stored + /// documents on each side. + pub(super) fn rows_match( + what: &str, + case: &Case, + needles: &[Needle], + expected: &[Vec], + encrypted: &[Vec], + ) -> Result<(), TestCaseError> { + let show = |ids: Vec| -> String { + ids.iter() + .map(|id| format!("\n row {id}: {}", case.docs[*id as usize])) + .collect() + }; + let disagreements: Vec = needles + .iter() + .zip(expected.iter().zip(encrypted)) + .enumerate() + .filter(|(_, (_, (expected, encrypted)))| expected != encrypted) + .map(|(i, (needle, (expected, encrypted)))| { + let only_jsonb = expected.iter().copied().filter(|id| !encrypted.contains(id)).collect(); + let only_encrypted = encrypted.iter().copied().filter(|id| !expected.contains(id)).collect(); + format!( + "{what}, needle {i} ({}): {}\n jsonb: {expected:?}\n encrypted: {encrypted:?}\n only jsonb:{}\n only encrypted:{}", + needle.origin, + needle.value, + show(only_jsonb), + show(only_encrypted), + ) + }) + .collect(); + prop_assert!( + disagreements.is_empty(), + "{} needles answered differently from jsonb:\n{}", + disagreements.len(), + disagreements.join("\n") + ); + Ok(()) + } + + /// The plan the statement ran under is the one `scan` asks for: a + /// bitmap scan over the GIN index, or a sequential scan and no index. + pub(super) fn planned_as(plan: &str, scan: Scan) -> Result<(), TestCaseError> { + let as_asked = match scan { + Scan::Index => plan.contains("Bitmap Index Scan on json_tree_generated_doc"), + Scan::Sequential => { + plan.contains("Seq Scan on json_tree_generated") + && !plan.contains("Index Scan") + && !plan.contains("Bitmap") + } + }; + prop_assert!( + as_asked, + "the {scan:?} plan should be the one asked for:\n{plan}" + ); + Ok(()) + } + + /// A needle pruned from a row's whole document is contained by that + /// row: the generator's own claim, checked by the oracle. + pub(super) fn pruned_roots_are_contained( + needles: &[Needle], + jsonb: &[Vec], + ) -> Result<(), TestCaseError> { + for (needle, rows) in needles.iter().zip(jsonb) { + if let Origin::PrunedRoot(row) = needle.origin { + prop_assert!( + rows.contains(&(row as i32)), + "row {row} should contain its own pruning {}, but jsonb selects {rows:?}", + needle.value + ); + } + } + Ok(()) + } + + pub(super) fn entry_where_jsonb_has_a_node( + path: &str, + id: i32, + case: &Case, + jsonb_has_a_node: bool, + encrypted_has_an_entry: bool, + ) -> Result<(), TestCaseError> { + prop_assert_eq!( + encrypted_has_an_entry, + jsonb_has_a_node, + "-> {}, row {}: the encrypted column should have an entry exactly where jsonb has a node\n stored: {}", + path, + id, + &case.docs[id as usize] + ); + Ok(()) + } + + pub(super) fn leaves_open_to_the_jsonb_value( + differing: &[i64], + at: &[(String, i32)], + opened: &[String], + held: &[String], + case: &Case, + ) -> Result<(), TestCaseError> { + let report: Vec = differing + .iter() + .map(|i| { + let i = *i as usize - 1; + let (path, id) = &at[i]; + format!( + "-> {path}, row {id}: opened {}, jsonb holds {}\n stored: {}", + opened[i], held[i], case.docs[*id as usize] + ) + }) + .collect(); + prop_assert!( + report.is_empty(), + "leaves should open to the value jsonb holds:\n{}", + report.join("\n") + ); + Ok(()) + } + + /// The features the generator is meant to cover, counted. + #[derive(Debug, Default)] + pub(super) struct Census { + root_objects: usize, + root_arrays: usize, + root_scalars: usize, + empty_objects: usize, + empty_arrays: usize, + arrays_of_objects: usize, + arrays_of_arrays: usize, + duplicate_elements: usize, + keys_on_two_levels: usize, + float_spelled_integers: usize, + numeric_strings: usize, + booleans: usize, + nulls: usize, + max_depth: usize, + } + + impl Census { + pub(super) fn of(docs: &[Value]) -> Self { + let mut census = Census::default(); + for doc in docs { + match doc { + Value::Object(_) => census.root_objects += 1, + Value::Array(_) => census.root_arrays += 1, + _ => census.root_scalars += 1, + } + let mut keys = + std::collections::BTreeMap::<&str, std::collections::BTreeSet>::new(); + census.walk(doc, 0, &mut keys); + if keys.values().any(|depths| depths.len() > 1) { + census.keys_on_two_levels += 1; + } + } + census + } + + fn walk<'v>( + &mut self, + v: &'v Value, + depth: usize, + keys: &mut std::collections::BTreeMap<&'v str, std::collections::BTreeSet>, + ) { + self.max_depth = self.max_depth.max(depth); + match v { + Value::Object(map) => { + if map.is_empty() { + self.empty_objects += 1; + } + for (key, member) in map { + let _ = keys.entry(key).or_default().insert(depth); + self.walk(member, depth + 1, keys); + } + } + Value::Array(items) => { + if items.is_empty() { + self.empty_arrays += 1; + } + if items.iter().any(Value::is_object) { + self.arrays_of_objects += 1; + } + if items.iter().any(Value::is_array) { + self.arrays_of_arrays += 1; + } + if items + .iter() + .enumerate() + .any(|(i, a)| items[i + 1..].contains(a)) + { + self.duplicate_elements += 1; + } + for item in items { + self.walk(item, depth + 1, keys); + } + } + Value::Number(n) => { + if n.is_f64() && n.as_f64().is_some_and(|f| f.fract() == 0.0) { + self.float_spelled_integers += 1; + } + } + Value::String(s) => { + if s.parse::().is_ok() { + self.numeric_strings += 1; + } + } + Value::Bool(_) => self.booleans += 1, + Value::Null => self.nulls += 1, + } + } + } + + pub(super) fn covers_every_feature(census: &Census) { + let counts = [ + ("root objects", census.root_objects), + ("root arrays", census.root_arrays), + ("root scalars", census.root_scalars), + ("empty objects", census.empty_objects), + ("empty arrays", census.empty_arrays), + ("arrays of objects", census.arrays_of_objects), + ("arrays of arrays", census.arrays_of_arrays), + ("duplicate array elements", census.duplicate_elements), + ("a key on two levels", census.keys_on_two_levels), + ("integers spelled as floats", census.float_spelled_integers), + ("strings spelled as numbers", census.numeric_strings), + ("booleans", census.booleans), + ("nulls", census.nulls), + ]; + for (what, count) in counts { + assert!( + count > 0, + "the generated documents should hold {what}: {census:?}" + ); + } + assert!( + census.max_depth >= 4, + "the generated documents should nest at least four levels: {census:?}" + ); + } +} diff --git a/packages/stack-encrypt/Cargo.toml b/packages/stack-encrypt/Cargo.toml index ec6812093..5f02bb657 100644 --- a/packages/stack-encrypt/Cargo.toml +++ b/packages/stack-encrypt/Cargo.toml @@ -117,6 +117,10 @@ zerokms-protocol = { workspace = true } tokio = { workspace = true, features = ["rt", "macros"] } # Compile-fail tests for the derive diagnostics (`tests/ui`). trybuild = "1" +# The generated containment property in tests/json_index.rs. Default features +# off, as stack-guest-abi takes it: `std` carries the failure persistence and +# the ChaCha RNG the run is seeded with, and nothing else is needed. +proptest = { version = "1.7", default-features = false, features = ["std"] } # The derive diagnostics are recorded with every feature on, as `test:unit` # and CI run them. They cannot be relied on to hold under both feature sets diff --git a/packages/stack-encrypt/proptest-regressions/json_index.txt b/packages/stack-encrypt/proptest-regressions/json_index.txt new file mode 100644 index 000000000..341ba3ff3 --- /dev/null +++ b/packages/stack-encrypt/proptest-regressions/json_index.txt @@ -0,0 +1,6 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. diff --git a/packages/stack-encrypt/tests/json_index.rs b/packages/stack-encrypt/tests/json_index.rs index 98ec5a831..f4d42856e 100644 --- a/packages/stack-encrypt/tests/json_index.rs +++ b/packages/stack-encrypt/tests/json_index.rs @@ -1,18 +1,28 @@ //! The JSON index end to end, with no database: a document seals into one //! entry per node under one data key and opens again; the token chain //! reaches every child from its parent's token and ordinal, and nothing -//! from any other token; -//! a moved, reordered or regrafted entry fails to open; and containment -//! over the element-nested projection agrees with PostgreSQL's `jsonb @>` -//! on the design's corpus and on random documents, through a Rust model -//! of `jsonb @>` written here from the rules PostgreSQL documents. +//! from any other token; a moved, reordered or regrafted entry fails to +//! open; and containment over the element-nested projection agrees with +//! PostgreSQL's `jsonb @>` on the design's corpus and on documents proptest +//! generates, through a Rust model of `jsonb @>` written here from the +//! rules PostgreSQL documents (the live oracle against PostgreSQL itself is +//! `packages/eql/tests/encryption/tests/json_tree.rs`). +//! +//! Each `given_` module is one situation, and each test in it is +//! named for what it expects. Shared setup is [`scenario`] and the helpers +//! below it; repeated checks are in [`assertions`]. The generated cases are +//! in [`given_generated_documents`]: a fixed ChaCha seed that +//! `PROPTEST_RNG_SEED` replaces, and a case count `PROPTEST_CASES` replaces. + +// `.unwrap()` is how a test says "this cannot fail here". +#![allow(clippy::unwrap_used)] mod common; use common::stack_cipher; use serde_json::{json, Map, Value}; use stack_encrypt::json::{ - JsonDocument, JsonNode, JsonOptions, JsonPath, JsonQuery, JsonSubtree, JsonSubtreeKey, + JsonDocument, JsonNode, JsonOptions, JsonPath, JsonQuery, JsonSubtree, JsonSubtreeKey, Tag, }; use stack_encrypt::op; use stack_encrypt::sem::TermError; @@ -22,6 +32,31 @@ use stack_encrypt::target::{ use stack_encrypt::{nonempty, Error, Plan, PlanError, StackCipher}; use stack_kms::FakeDataKeySource; +// ============================================================================= +// Setup +// ============================================================================= + +type Cipher = StackCipher; + +type JsonPlan = stack_encrypt::plan::ValuePlan>; + +type OrderedPlan = + stack_encrypt::plan::ValuePlan>; + +/// What most tests start from: a cipher over the fake key source and the +/// default plan for a JSON column under `users/profile`. +struct Scenario { + cipher: Cipher, + plan: JsonPlan, +} + +async fn scenario() -> Scenario { + Scenario { + cipher: stack_cipher().await, + plan: plan(), + } +} + fn context() -> CallerContext { CallerContext::from(nonempty!("users").with("profile")) } @@ -30,30 +65,22 @@ fn aead() -> AeadContext { AeadContext::from(nonempty!("users").with("profile")) } -async fn seal(cipher: &StackCipher, value: &Value) -> JsonDocument { - cipher - .default_keyset() - .encrypt_as::(value, context()) - .await - .expect("seals") +fn plan() -> JsonPlan { + Plan::value::() + .context("users/profile") + .encrypt_into::() + .build() + .unwrap() } -async fn seal_with( - cipher: &StackCipher, - value: &Value, - options: &JsonOptions, -) -> JsonDocument { - cipher - .default_keyset() - .run( - stack_encrypt::target::json_document::<_, stack_encrypt::target::Borrowed>( - options.clone(), - ), - value, - context(), - ) - .await - .expect("seals") +/// The plan laid out by the JSON index with `$.hello`, `$.number` and +/// `$.flag` ordered. +fn ordered_plan() -> OrderedPlan { + Plan::value::() + .context("users/profile") + .encrypt_with_layout(Json::new(ordering(&["$.hello", "$.number", "$.flag"]))) + .build() + .unwrap() } /// The options with the root open. @@ -69,11 +96,10 @@ fn ordering(paths: &[&str]) -> JsonOptions { JsonOptions::default().ordering(paths).unwrap() } -async fn open(cipher: &StackCipher, doc: JsonDocument) -> Result { - cipher.decrypt_as::(doc, aead()).await -} - -/// The design's fixture document. +/// The design's fixture document. Its entries, by ordinal: `$`, `$.hello`, +/// `$.nested`, `$.nested.deep`, `$.nested.list`, `[0]`, `[0].x`, `[1]`, +/// `[1].x`, `$.number`. The tests pin those ordinals, so the document is +/// fixed rather than generated. fn fixture() -> Value { json!({"hello": "world-1", "nested": {"deep": "constant", "list": [{"x": 1}, {"x": 2}]}, "number": 1}) } @@ -87,215 +113,43 @@ fn nodes(value: &Value) -> usize { } } -// ============================================================================= -// Round trips -// ============================================================================= - -#[tokio::test] -async fn a_document_seals_one_entry_per_node_and_opens_again() { - let cipher = stack_cipher().await; - for value in [ - fixture(), - json!({}), - json!([]), - json!("a"), - json!(2.5), - json!(null), - json!(true), - json!([[1, 2], [3, [4, {"a": null}]]]), - json!({"a": {"b": {"c": {"d": [1, "x", false]}}}}), - json!({"a": "b", "a b": 1, "é": [], "0": {}}), - json!({"n": 18446744073709551615u64, "m": -9223372036854775808i64, "f": 1e300}), - ] { - for options in [JsonOptions::default(), open_root()] { - let doc = seal_with(&cipher, &value, &options).await; - assert_eq!( - doc.entries().len(), - nodes(&value), - "{value}: one entry per node" - ); - assert_eq!(doc.header().sealed_root(), options.sealed_root); - let back = open(&cipher, doc).await.unwrap(); - assert_eq!(back, value, "{value} under {options:?}"); - } - } +/// Seal through the typed target, as `encrypt_as` does. +async fn seal(cipher: &Cipher, value: &Value) -> JsonDocument { + cipher + .default_keyset() + .encrypt_as::(value, context()) + .await + .expect("seals") } -/// The deepest document the index takes opens again under either root -/// option, and one level deeper is refused when it is sealed, not when it is -/// read. A sealed root's plaintext is the document as JSON, which -/// `serde_json` reads back only to 128 levels of nesting. -#[tokio::test] -async fn the_deepest_document_the_index_takes_opens_again() { - let cipher = stack_cipher().await; - let arrays = |n: usize| (0..n).fold(json!(1), |inner, _| json!([inner])); - let objects = |n: usize| (0..n).fold(json!("leaf"), |inner, _| json!({"k": inner})); - for nested in [arrays, objects] { - for options in [JsonOptions::default(), open_root()] { - let deepest = nested(127); - let doc = seal_with(&cipher, &deepest, &options).await; - assert_eq!( - open(&cipher, doc).await.unwrap(), - deepest, - "127 levels under {options:?}" - ); - let refused = cipher - .default_keyset() - .run( - stack_encrypt::target::json_document::<_, stack_encrypt::target::Borrowed>( - options.clone(), - ), - &nested(128), - context(), - ) - .await; - assert!( - matches!(refused, Err(Error::Term(TermError::JsonTooDeep))), - "128 levels under {options:?} is refused when sealed" - ); - } - } +/// Seal through the JSON document target under `options`. +async fn seal_with(cipher: &Cipher, value: &Value, options: &JsonOptions) -> JsonDocument { + try_seal(cipher, value, options).await.expect("seals") } -#[tokio::test] -async fn the_wire_form_is_what_the_sql_domain_stores() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let wire = serde_json::to_value(&doc).unwrap(); - let keys: Vec<&str> = wire - .as_object() - .unwrap() - .keys() - .map(String::as_str) - .collect(); - assert_eq!(keys, ["h", "q", "sv"]); - // `q` is the document's own projection, written once: the needle a - // `Contains` of the whole document derives has the same pairs in the - // same groups, and nothing else. - let needle = cipher - .query(op::Contains, &fixture()) - .on(&plan()) +async fn try_seal( + cipher: &Cipher, + value: &Value, + options: &JsonOptions, +) -> Result { + cipher + .default_keyset() + .run( + stack_encrypt::target::json_document::<_, stack_encrypt::target::Borrowed>( + options.clone(), + ), + value, + context(), + ) .await - .unwrap(); - assert_eq!(wire["q"], serde_json::to_value(&needle).unwrap()); - assert!( - serde_json::from_value::(json!({"h": wire["h"], "sv": wire["sv"]})).is_err(), - "a document without `q` is not a document: the projection is the only place the value selectors live" - ); - let back: JsonDocument = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!( - serde_json::to_value(&back).unwrap(), - wire, - "`q` round-trips as stored" - ); - assert_eq!( - serde_json::to_value(back.projection()).unwrap(), - wire["q"], - "the projection read back is the stored one" - ); - for (j, entry) in wire["sv"].as_array().unwrap().iter().enumerate() { - let entry = entry.as_object().unwrap(); - for key in entry.keys() { - assert!( - ["s", "c", "p", "op"].contains(&key.as_str()), - "entry {j} carries an unknown key {key}" - ); - } - for (tag, chars, bytes) in [("s", 24, 16), ("p", 12, 8)] { - if let Some(text) = entry.get(tag) { - assert_eq!( - text.as_str().unwrap().len(), - chars, - "entry {j}'s {tag} is {bytes} bytes of padded base64" - ); - } - } - let c = entry["c"].as_str().unwrap(); - assert!( - c.bytes() - .all(|b| b.is_ascii_graphic() && b != b'"' && b != b'\\'), - "entry {j}'s c is base85 in an alphabet a JSON string carries unescaped" - ); - } - let back: JsonDocument = serde_json::from_value(wire).unwrap(); - assert_eq!(back, doc); } -#[tokio::test] -async fn the_chain_fields_sit_where_the_design_says() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let entries = doc.entries(); - // $, $.hello, $.nested, $.nested.deep, $.nested.list, [0], [0].x, [1], [1].x, $.number - assert_eq!(entries.len(), 10); - let root = &entries[0]; - assert!(root.parent_tag().is_none()); - for head in [&entries[1], &entries[2], &entries[9]] { - assert!( - head.parent_tag().is_none(), - "a head has no parent tag under the sealed root" - ); - } - for below in &entries[3..9] { - assert!(below.parent_tag().is_some()); - } - // Every parent tag is distinct: siblings are not linkable at rest. - let tags: std::collections::HashSet<_> = entries[3..9] - .iter() - .map(|e| *e.parent_tag().unwrap()) - .collect(); - assert_eq!(tags.len(), 6); - // The projection groups the two list elements: each is a nested group - // holding its own pair and its `x`. - let q = serde_json::to_value(doc.projection()).unwrap(); - assert_eq!(q["el"].as_array().unwrap().len(), 2); - assert_eq!(q["el"][0]["sv"].as_array().unwrap().len(), 2); - assert_eq!( - q["sv"].as_array().unwrap().len(), - 6, - "$, hello, nested, deep, list, number" - ); - // No path is ordered by default, so no entry carries a term. - assert!(entries.iter().all(|e| e.ordering_term().is_none())); - // Equal values at one path share selectors but not ciphertexts; the - // value selectors, in the projection, tell equal values from different. - let pair = |doc: &JsonDocument, i: usize| -> (String, String) { - let q = serde_json::to_value(doc.projection()).unwrap(); - ( - q["sv"][i]["s"].as_str().unwrap().to_owned(), - q["sv"][i]["vs"].as_str().unwrap().to_owned(), - ) - }; - let doc2 = seal(&cipher, &json!([1, 1])).await; - let (a, b) = (&doc2.entries()[1], &doc2.entries()[2]); - assert_eq!(a.selector(), b.selector()); - assert_eq!(pair(&doc2, 1), pair(&doc2, 2)); - assert_ne!(a.ciphertext(), b.ciphertext()); - let doc3 = seal(&cipher, &json!([1, 2])).await; - let (one, two) = (&doc3.entries()[1], &doc3.entries()[2]); - assert_eq!(one.selector(), two.selector(), "one path"); - assert_ne!( - pair(&doc3, 1), - pair(&doc3, 2), - "different values at one path have different value selectors" - ); - assert_eq!( - pair(&doc3, 1), - pair(&doc2, 1), - "the value selector names the value at its path, in any document" - ); - - // Without the sealed root, the top-level fields chain from the root. - let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; - assert!(open_doc.entries()[1].parent_tag().is_some()); +async fn open(cipher: &Cipher, doc: JsonDocument) -> Result { + cipher.decrypt_as::(doc, aead()).await } -// ============================================================================= -// The token chain -// ============================================================================= - async fn subtree_key( - cipher: &StackCipher, + cipher: &Cipher, path: &str, options: JsonOptions, ) -> Result { @@ -311,1652 +165,2798 @@ async fn subtree_key( /// `rows` as the key derived for `path` opens them: the only way to build /// a [`JsonSubtree`]. -async fn keyed( - cipher: &StackCipher, - path: &str, - rows: Vec, -) -> JsonSubtree { +async fn keyed(cipher: &Cipher, path: &str, rows: Vec) -> JsonSubtree { subtree_key(cipher, path, JsonOptions::default()) .await .unwrap() .open(rows) } -/// An ordering term sits on a number or string leaf at an ordered path and -/// nowhere else: a wildcard covers every element. A position is refused, -/// since `->` reads it as the wildcard and no ordering could reach the leaf -/// it names. -#[tokio::test] -async fn an_ordering_term_sits_only_at_an_ordered_path() { - let cipher = stack_cipher().await; - // $, $.hello, $.nested, $.nested.deep, $.nested.list, [0], [0].x, [1], [1].x, $.number - let with_term = |doc: &JsonDocument| -> Vec { - doc.entries() - .iter() - .enumerate() - .filter(|(_, e)| e.ordering_term().is_some()) - .map(|(j, _)| j) - .collect() - }; - let doc = seal_with(&cipher, &fixture(), &ordering(&["$.hello", "$.number"])).await; - assert_eq!(with_term(&doc), [1, 9]); - let wild = seal_with(&cipher, &fixture(), &ordering(&["$.nested.list[*].x"])).await; - assert_eq!(with_term(&wild), [6, 8], "a wildcard covers every element"); - assert!(matches!( - JsonOptions::default().ordering(["$.nested.list[1].x"]), - Err(TermError::JsonOrderingAtPosition) - )); - assert!(matches!( - JsonOptions::default().ordering(["nested"]), - Err(TermError::JsonPath(_)) - )); - // Set on the field, which `ordering` guards, a position refuses the - // seal rather than put a term no ordering reaches on one element. - let mut positional = JsonOptions::default(); - let _ = positional - .ordered - .insert(JsonPath::parse("$.nested.list[1].x").unwrap()); - let refused = cipher - .default_keyset() - .run( - stack_encrypt::target::json_document::<_, stack_encrypt::target::Borrowed>(positional), - &fixture(), - context(), - ) - .await; - assert!( - matches!(refused, Err(Error::Term(TermError::JsonOrderingAtPosition))), - "{refused:?}" - ); - // A container, a boolean or a null at an ordered path carries none. - let kinds = seal_with( - &cipher, - &json!({"a": true, "b": null, "c": {"d": 1}, "e": [1, "x"]}), - &ordering(&["$.a", "$.b", "$.c", "$.e", "$.e[*]"]), - ) - .await; - // $, $.a, $.b, $.c, $.c.d, $.e, $.e[0], $.e[1] - assert_eq!(with_term(&kinds), [6, 7]); - // The root scalar itself, at `$`. - let scalar = seal_with(&cipher, &json!(7), &ordering(&["$"])).await; - assert_eq!(with_term(&scalar), [0]); - // The documents open regardless. - assert_eq!(open(&cipher, doc).await.unwrap(), fixture()); - assert_eq!(open(&cipher, wild).await.unwrap(), fixture()); +/// Padded standard base64, as the SQL reads and writes it. +fn base64(bytes: &[u8]) -> String { + use base64ct::{Base64, Encoding}; + Base64::encode_string(bytes) } -#[tokio::test] -async fn a_token_derived_from_the_path_alone_unlocks_exactly_its_subtree() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) - .await - .unwrap(); - assert!( - key.gates().is_empty(), - "a chain head's token is derived, nothing to gate" - ); - assert_eq!( - key.path(), - &JsonPath::parse("$.nested").unwrap(), - "the key carries the path it was derived from" - ); - let under = doc.subtree(key.token_bytes()); - let found: Vec = under.iter().map(|(j, _, _)| *j).collect(); - assert_eq!( - found, - [3, 4, 5, 6, 7, 8], - "deep, list, both elements, both leaves" - ); - let depths: Vec = under.iter().map(|(_, _, d)| *d).collect(); - assert_eq!(depths, [1, 1, 2, 3, 2, 3]); - let garbage = doc.subtree(&[7; 16]); - assert!(garbage.is_empty(), "a wrong token unlocks nothing"); +/// The ordinals the walk from `token` reaches. +fn subtree_ordinals(doc: &JsonDocument, token: &Tag) -> Vec { + doc.subtree(token).iter().map(|(j, _, _)| *j).collect() +} - // Below a head the client derives nothing but array positions: a key - // step becomes a gate, and the walk from the head takes it. - let deeper = subtree_key(&cipher, "$.nested.list[1]", JsonOptions::default()) - .await - .unwrap(); - assert_eq!( - deeper.token_bytes(), - key.token_bytes(), - "anchored at the head" - ); - assert_eq!(deeper.gates().len(), 2, "list, then [1]"); - assert_eq!( - deeper.gates()[1], - format!("#1:{}", base64(doc.entries()[5].selector())), - "the position, and the selector every element of the list carries" - ); - let list_token = doc - .children(key.token_bytes()) - .into_iter() - .find(|(j, _, _)| *j == 4) - .map(|(_, _, t)| t) - .expect("the list is a child of nested"); - let second = doc - .children(&list_token) +/// The nodes the walk from `token` reaches, as the server returns them. +fn subtree_nodes(doc: &JsonDocument, token: &Tag) -> Vec { + doc.subtree(token) + .iter() + .map(|(j, _, _)| doc.node(*j).unwrap()) + .collect() +} + +/// The token of the child at ordinal `j` under `token`, as the server +/// derives it. +fn child_token(doc: &JsonDocument, token: &Tag, j: usize) -> Tag { + doc.children(token) .into_iter() - .find(|(j, _, _)| *j == 7) + .find(|(ordinal, _, _)| *ordinal == j) .map(|(_, _, t)| t) - .expect("the second element"); - let found: Vec = doc.subtree(&second).iter().map(|(j, _, _)| *j).collect(); - assert_eq!(found, [8], "the second element's x"); + .unwrap_or_else(|| panic!("entry {j} is a child of the token's node")) +} - // The children of a node, by ordinal: the list's two elements. - let children: Vec = doc - .children(&list_token) - .iter() - .map(|(j, _, _)| *j) - .collect(); - assert_eq!(children, [5, 7]); - // An array element's token is derivable from the path once the array - // itself is reached by token: under an open root every node is. - - // The sealed root has no token; the open root's unlocks every node. - assert!(matches!( - subtree_key(&cipher, "$", JsonOptions::default()).await, - Err(Error::Term(TermError::JsonSealedRoot)) - )); - assert!(matches!( - subtree_key(&cipher, "$[*].x", JsonOptions::default()).await, - Err(Error::Term(TermError::JsonSealedRoot)) - )); - let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; - let root = subtree_key(&cipher, "$", open_root()).await.unwrap(); - assert_eq!( - open_doc.subtree(root.token_bytes()).len(), - 9, - "every entry but the root" - ); +/// Node `j` as a walk returns it, `d` levels below where it started. +fn with_depth(doc: &JsonDocument, j: usize, d: u32) -> JsonNode { + let mut wire = serde_json::to_value(doc.node(j).unwrap()).unwrap(); + wire["d"] = json!(d); + serde_json::from_value(wire).unwrap() +} - // A wildcard path anchors at the head and gates on each step's - // selector: the list's, then every element's (the wildcard's), then x's. - let gated = subtree_key(&cipher, "$.nested.list[*].x", JsonOptions::default()) - .await - .unwrap(); - assert_eq!(gated.token_bytes(), key.token_bytes()); - assert_eq!( - gated.gates(), - [ - base64(doc.entries()[4].selector()), - base64(doc.entries()[5].selector()), - base64(doc.entries()[6].selector()), - ] - ); - // Under an open root the whole chain is derivable down to the first - // key step: `$[0]` would be a position, `$.nested` is a gate. - let open_key = subtree_key(&cipher, "$.nested.list[1]", open_root()) - .await - .unwrap(); - assert_eq!(open_key.token_bytes(), root.token_bytes()); - assert_eq!(open_key.gates().len(), 3); - let array_doc = seal_with(&cipher, &json!([[1, 2], [3]]), &open_root()).await; - let second_row = subtree_key(&cipher, "$[1]", open_root()).await.unwrap(); - assert!(second_row.gates().is_empty(), "positions derive"); - let found: Vec = array_doc - .subtree(second_row.token_bytes()) - .iter() - .map(|(j, _, _)| *j) - .collect(); - assert_eq!(found, [5], "$, [0], [0][0], [0][1], [1], [1][0]"); +/// The rows `json_tree_subtree_at(doc, token, gates)` returns for `key`, +/// walked here as the SQL walks them: from the key's token, through each +/// gate (a selector the entry at that depth must carry, or `#n:`, +/// the child at ordinal `n` carrying it), then everything beneath each node +/// the gates reach, each row with its depth below that node. +fn gated_walk(doc: &JsonDocument, key: &JsonSubtreeKey) -> Vec { + let mut frontier = vec![(None, *key.token_bytes())]; + for gate in key.gates() { + let mut next = Vec::new(); + for (_, token) in &frontier { + let children = doc.children(token); + match gate.strip_prefix('#') { + Some(slot) => { + let (n, selector) = slot.split_once(':').unwrap(); + if let Some((j, entry, t)) = children.get(n.parse::().unwrap()) { + if base64(entry.selector()) == selector { + next.push((Some(*j), *t)); + } + } + } + None => next.extend( + children + .iter() + .filter(|(_, entry, _)| base64(entry.selector()) == gate) + .map(|(j, _, t)| (Some(*j), *t)), + ), + } + } + frontier = next; + } + let mut rows = Vec::new(); + for (reached, token) in frontier { + let below = u32::from(reached.is_some()); + if let Some(j) = reached { + rows.push(with_depth(doc, j, 0)); + } + rows.extend( + doc.subtree(&token) + .iter() + .map(|(j, _, d)| with_depth(doc, *j, d - 1 + below)), + ); + } + rows.sort_by_key(JsonNode::ordinal); + rows } -#[tokio::test] -async fn a_selector_is_the_stored_one_and_folds_positions_into_the_wildcard() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let keyset = cipher.default_keyset(); - let selector = |path: &str| { - let path = JsonPath::parse(path).unwrap(); - keyset.json_selector(&path, nonempty!("users").with("profile")) - }; - assert_eq!( - selector("$.nested.deep").await.unwrap().as_bytes(), - doc.entries()[3].selector() - ); - assert_eq!( - selector("$.nested.list[*].x").await.unwrap().as_bytes(), - doc.entries()[6].selector() - ); - assert_eq!( - selector("$.nested.list[1].x").await.unwrap().as_bytes(), - doc.entries()[8].selector() - ); - assert_eq!( - selector("$").await.unwrap().as_bytes(), - doc.entries()[0].selector() - ); - assert_ne!( - selector("$.hello").await.unwrap().as_bytes(), - keyset - .json_selector( - &JsonPath::parse("$.hello").unwrap(), - nonempty!("users").with("bio") - ) - .await - .unwrap() - .as_bytes(), - "a selector is bound to its column" - ); +/// PostgreSQL's `jsonb @>`, from its documented rules: an object contains +/// another when it has every key with a containing value; an array +/// contains another when every needle element is contained by some stored +/// element; a top-level array contains a scalar that is one of its +/// elements; scalars contain equal scalars, numbers by value. +fn pg_contains(stored: &Value, needle: &Value, top: bool) -> bool { + match (stored, needle) { + (Value::Object(a), Value::Object(b)) => b + .iter() + .all(|(k, vb)| a.get(k).is_some_and(|va| pg_contains(va, vb, false))), + (Value::Array(a), Value::Array(b)) => b + .iter() + .all(|vb| a.iter().any(|va| pg_contains(va, vb, false))), + (Value::Array(a), scalar) if top && !scalar.is_object() && !scalar.is_array() => { + a.iter().any(|va| pg_contains(va, scalar, false)) + } + (Value::Number(a), Value::Number(b)) => a.as_f64() == b.as_f64(), + (Value::Object(_) | Value::Array(_), _) | (_, Value::Object(_) | Value::Array(_)) => false, + (a, b) => a == b, + } +} + +/// `<@` compares the same two projections the other way round. The one +/// case it cannot answer is a stored root scalar inside a needle root +/// array: a stored document carries no alternative pair, so the SQL answers +/// false where `jsonb` answers true, as the design's leakage table records +/// for v3's `@>`. +fn scalar_in_array(stored: &Value, needle: &Value) -> bool { + !stored.is_object() && !stored.is_array() && needle.is_array() } // ============================================================================= -// Extraction +// Round trips // ============================================================================= -#[tokio::test] -async fn an_extracted_node_opens_to_its_scalar_and_a_subtree_to_its_value() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let node = |j: usize| doc.node(j).unwrap(); - let scalar = |j: usize| cipher.decrypt_as::(node(j), aead()); - assert_eq!(scalar(1).await.unwrap(), json!("world-1")); - assert_eq!(scalar(6).await.unwrap(), json!(1)); - assert_eq!( - scalar(0).await.unwrap(), - fixture(), - "the sealed root is the document" - ); - assert!( - matches!(scalar(2).await, Err(Error::OpenedType { .. })), - "a container node holds no value of its own" - ); +mod given_a_document { + use super::*; + + /// A document of every shape the index takes. + fn values() -> Vec { + vec![ + fixture(), + json!({}), + json!([]), + json!("a"), + json!(2.5), + json!(null), + json!(true), + json!([[1, 2], [3, [4, {"a": null}]]]), + json!({"a": {"b": {"c": {"d": [1, "x", false]}}}}), + json!({"a": "b", "a b": 1, "é": [], "0": {}}), + json!({"n": 18446744073709551615u64, "m": -9223372036854775808i64, "f": 1e300}), + ] + } - let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) - .await - .unwrap(); - let rows: Vec = doc - .subtree(key.token_bytes()) - .iter() - .map(|(j, _, _)| node(*j)) - .collect(); - let nested = cipher - .decrypt_as::(key.open(rows.clone()), aead()) - .await - .unwrap(); - assert_eq!(nested, fixture()["nested"]); - let forest = cipher - .decrypt_as::, JsonSubtree>(key.open(rows), aead()) - .await - .unwrap(); - assert_eq!(forest, vec![json!("constant"), json!([{"x": 1}, {"x": 2}])]); + #[tokio::test] + async fn seals_one_entry_per_node_and_opens_again() { + let Scenario { cipher, .. } = scenario().await; + for value in values() { + for options in [JsonOptions::default(), open_root()] { + let what = format!("{value} under {options:?}"); + let doc = seal_with(&cipher, &value, &options).await; + assertions::one_entry_per_node(&doc, &value, &what); + assert_eq!( + doc.header().sealed_root(), + options.sealed_root, + "{what}: the header records whether the root is sealed" + ); + assertions::opens_to(&cipher, doc, &value, &what).await; + } + } + } +} - // The list's token is the server's to derive, one level under nested. - let list = doc - .children(key.token_bytes()) - .into_iter() - .find(|(j, _, _)| *j == 4) - .map(|(_, _, t)| t) - .unwrap(); - let rows: Vec = doc - .subtree(&list) - .iter() - .map(|(j, _, _)| node(*j)) - .collect(); - let elements = cipher - .decrypt_as::(keyed(&cipher, "$.nested.list", rows).await, aead()) - .await - .unwrap(); - assert_eq!(elements, json!([{"x": 1}, {"x": 2}])); - - // An extracted node round-trips through its wire form with the - // envelope fields the SQL grafts beside it. - let mut wire = serde_json::to_value(node(6)).unwrap(); - wire["i"] = json!({"t": "users", "c": "profile"}); - wire["v"] = json!(4); - let back: JsonNode = serde_json::from_value(wire).unwrap(); - assert_eq!(back.ordinal(), 6); - assert_eq!( - cipher - .decrypt_as::(back, aead()) - .await - .unwrap(), - json!(1) - ); -} +mod given_an_open_root { + use super::*; -/// The rows of a walk come from the server, which can repeat an entry or -/// pick which entries to return. Every entry is authentic, but the rebuild -/// nests the containers it is handed: a repeated row is refused, and so is -/// a line of containers deeper than any document the index seals, as a -/// shape mismatch rather than a recursion as deep as the rows. -#[tokio::test] -async fn subtree_rows_the_server_repeats_or_lines_up_are_refused() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let node = |j: usize| doc.node(j).unwrap(); - let repeated = keyed(&cipher, "$.nested", vec![node(1), node(1)]).await; - assert!(matches!( - cipher - .decrypt_as::, JsonSubtree>(repeated, aead()) - .await, - Err(Error::OpenedType { .. }) - )); - - // A wide document of one-member objects: its objects, returned as rows - // with the last leaf after them, each take the next as their member, - // and would nest 200 deep. - let wide = Value::Object( - (0..200) - .map(|i| (format!("k{i:03}"), json!({"x": 1}))) - .collect(), - ); - let doc = seal(&cipher, &wide).await; - let last = doc.entries().len() - 1; - let containers: Vec = (1..doc.entries().len()) - .filter(|j| j % 2 == 1 || *j == last) - .map(|j| doc.node(j).unwrap()) - .collect(); - assert_eq!(containers.len(), 201); - assert!(matches!( - cipher - .decrypt_as::, JsonSubtree>( - keyed(&cipher, "$.k000", containers).await, - aead() - ) - .await, - Err(Error::OpenedType { .. }) - )); - - // The same through arrays: one outer array, then every key's inner - // one-element array, each taken as the element of the one before it. - let wide = Value::Object( - (0..200) - .map(|i| (format!("k{i:03}"), json!([[0]]))) - .collect(), - ); - let doc = seal(&cipher, &wide).await; - let last = doc.entries().len() - 1; - let arrays: Vec = (1..doc.entries().len()) - .filter(|j| *j == 1 || j % 3 == 2 || *j == last) - .map(|j| doc.node(j).unwrap()) - .collect(); - assert_eq!(arrays.len(), 202); - assert!(matches!( - cipher - .decrypt_as::, JsonSubtree>(keyed(&cipher, "$.k000", arrays).await, aead()) - .await, - Err(Error::OpenedType { .. }) - )); + /// A document sealed without the sealed root keeps the root as an + /// ordinary node, and opens from its entries. + #[tokio::test] + async fn keeps_the_root_as_an_entry_and_opens_from_the_entries() { + let Scenario { cipher, .. } = scenario().await; + for value in [fixture(), json!({"": {}}), json!([[], {}, ""]), json!("a")] { + let doc = seal_with(&cipher, &value, &open_root()).await; + assert!( + !doc.header().sealed_root(), + "{value}: the header records the open root" + ); + assertions::one_entry_per_node(&doc, &value, &value.to_string()); + assertions::opens_to(&cipher, doc, &value, &value.to_string()).await; + } + } } -// ============================================================================= -// Tamper evidence -// ============================================================================= - -#[tokio::test] -async fn a_moved_reordered_or_regrafted_entry_fails_to_open() { - let cipher = stack_cipher().await; - let value = json!({"a": "x", "b": "y"}); - let doc = seal_with(&cipher, &value, &open_root()).await; - let other = seal_with(&cipher, &value, &open_root()).await; - let tamper = |mut wire: Value, f: &dyn Fn(&mut Vec)| { - let sv = wire["sv"].as_array_mut().unwrap(); - f(sv); - serde_json::from_value::(wire).unwrap() - }; - let wire = serde_json::to_value(&doc).unwrap(); - let other_wire = serde_json::to_value(&other).unwrap(); - - let reordered = tamper(wire.clone(), &|sv| sv.swap(1, 2)); - assert!( - matches!(open(&cipher, reordered).await, Err(Error::Aead)), - "reordered" - ); - - let regrafted = tamper(wire.clone(), &|sv| sv[1] = other_wire["sv"][1].clone()); - assert!( - matches!(open(&cipher, regrafted).await, Err(Error::Aead)), - "regrafted from another document" - ); - - let truncated = tamper(wire.clone(), &|sv| { - let _ = sv.pop(); - }); - assert!( - matches!( - open(&cipher, truncated).await, - Err(Error::OpenedType { .. }) - ), - "truncated" - ); - - let flipped = tamper(wire.clone(), &|sv| { - let c = sv[1]["c"].as_str().unwrap().to_owned(); - let mut bytes = c.into_bytes(); - bytes[3] = if bytes[3] == b'A' { b'B' } else { b'A' }; - sv[1]["c"] = Value::String(String::from_utf8(bytes).unwrap()); - }); - assert!( - matches!(open(&cipher, flipped).await, Err(Error::Aead)), - "a changed byte" - ); - - let wrong_context = cipher - .decrypt_as::(doc, AeadContext::from(nonempty!("users").with("bio"))) - .await; - assert!(matches!(wrong_context, Err(Error::Aead)), "another column"); - - // A node lifted out of its document fails under another ordinal. - let sealed = seal(&cipher, &value).await; - let mut lifted = serde_json::to_value(sealed.node(1).unwrap()).unwrap(); - lifted["j"] = json!(2); - let lifted: JsonNode = serde_json::from_value(lifted).unwrap(); - assert!(matches!( - cipher.decrypt_as::(lifted, aead()).await, - Err(Error::Aead) - )); +mod given_the_deepest_document { + use super::*; + + /// The deepest document the index takes opens again under either root + /// option, and one level deeper is refused when it is sealed, not when + /// it is read. A sealed root's plaintext is the document as JSON, which + /// `serde_json` reads back only to 128 levels of nesting. + #[tokio::test] + async fn opens_again_and_one_level_deeper_is_refused_when_sealed() { + let Scenario { cipher, .. } = scenario().await; + let arrays = |n: usize| (0..n).fold(json!(1), |inner, _| json!([inner])); + let objects = |n: usize| (0..n).fold(json!("leaf"), |inner, _| json!({"k": inner})); + for nested in [arrays, objects] { + for options in [JsonOptions::default(), open_root()] { + let deepest = nested(127); + let doc = seal_with(&cipher, &deepest, &options).await; + assertions::opens_to( + &cipher, + doc, + &deepest, + &format!("127 levels under {options:?}"), + ) + .await; + let refused = try_seal(&cipher, &nested(128), &options).await; + assertions::refused_by_term( + &refused, + |e| matches!(e, TermError::JsonTooDeep), + &format!("128 levels under {options:?} is refused when sealed"), + ); + } + } + } } // ============================================================================= -// Queries +// The stored form // ============================================================================= -type JsonPlan = stack_encrypt::plan::ValuePlan>; - -fn plan() -> JsonPlan { - Plan::value::() - .context("users/profile") - .encrypt_into::() - .build() - .unwrap() -} - -type OrderedPlan = - stack_encrypt::plan::ValuePlan>; +mod given_the_fixture_document { + use super::*; -/// The plan laid out by the JSON index with `$.hello`, `$.number` and -/// `$.flag` ordered. -fn ordered_plan() -> OrderedPlan { - Plan::value::() - .context("users/profile") - .encrypt_with_layout(Json::new(ordering(&["$.hello", "$.number", "$.flag"]))) - .build() - .unwrap() -} + #[tokio::test] + async fn stores_the_wire_form_the_sql_domain_checks() { + let Scenario { cipher, plan } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let wire = serde_json::to_value(&doc).unwrap(); + let keys: Vec<&str> = wire + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + assert_eq!(keys, ["h", "q", "sv"], "the document's wire keys"); + // `q` is the document's own projection, written once: the needle a + // `Contains` of the whole document derives has the same pairs in the + // same groups, and nothing else. + let needle = cipher + .query(op::Contains, &fixture()) + .on(&plan) + .await + .unwrap(); + assert_eq!( + wire["q"], + serde_json::to_value(&needle).unwrap(), + "`q` is the needle a `Contains` of the whole document derives" + ); + assert!( + serde_json::from_value::(json!({"h": wire["h"], "sv": wire["sv"]})).is_err(), + "a document without `q` is not a document: the projection is the only place the value selectors live" + ); + let back: JsonDocument = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!( + serde_json::to_value(&back).unwrap(), + wire, + "`q` round-trips as stored" + ); + assert_eq!( + serde_json::to_value(back.projection()).unwrap(), + wire["q"], + "the projection read back is the stored one" + ); + for (j, entry) in wire["sv"].as_array().unwrap().iter().enumerate() { + let entry = entry.as_object().unwrap(); + for key in entry.keys() { + assert!( + ["s", "c", "p", "op"].contains(&key.as_str()), + "entry {j} carries an unknown key {key}" + ); + } + for (tag, chars, bytes) in [("s", 24, 16), ("p", 12, 8)] { + if let Some(text) = entry.get(tag) { + assert_eq!( + text.as_str().unwrap().len(), + chars, + "entry {j}'s {tag} is {bytes} bytes of padded base64" + ); + } + } + let c = entry["c"].as_str().unwrap(); + assert!( + c.bytes() + .all(|b| b.is_ascii_graphic() && b != b'"' && b != b'\\'), + "entry {j}'s c is base85 in an alphabet a JSON string carries unescaped: {c:?}" + ); + } + let back: JsonDocument = serde_json::from_value(wire).unwrap(); + assert_eq!(back, doc, "the wire form reads back to the document"); + } -/// PostgreSQL's `jsonb @>`, from its documented rules: an object contains -/// another when it has every key with a containing value; an array -/// contains another when every needle element is contained by some stored -/// element; a top-level array contains a scalar that is one of its -/// elements; scalars contain equal scalars, numbers by value. -fn pg_contains(stored: &Value, needle: &Value, top: bool) -> bool { - match (stored, needle) { - (Value::Object(a), Value::Object(b)) => b - .iter() - .all(|(k, vb)| a.get(k).is_some_and(|va| pg_contains(va, vb, false))), - (Value::Array(a), Value::Array(b)) => b - .iter() - .all(|vb| a.iter().any(|va| pg_contains(va, vb, false))), - (Value::Array(a), scalar) if top && !scalar.is_object() && !scalar.is_array() => { - a.iter().any(|va| pg_contains(va, scalar, false)) + #[tokio::test] + async fn places_the_chain_fields_where_the_design_says() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let entries = doc.entries(); + assert_eq!(entries.len(), 10, "one entry per node of the fixture"); + assert!( + entries[0].parent_tag().is_none(), + "the root has no parent tag" + ); + for head in [&entries[1], &entries[2], &entries[9]] { + assert!( + head.parent_tag().is_none(), + "a head has no parent tag under the sealed root" + ); } - (Value::Number(a), Value::Number(b)) => a.as_f64() == b.as_f64(), - (Value::Object(_) | Value::Array(_), _) | (_, Value::Object(_) | Value::Array(_)) => false, - (a, b) => a == b, + for (j, below) in entries[3..9].iter().enumerate() { + assert!( + below.parent_tag().is_some(), + "entry {} below a head carries a parent tag", + j + 3 + ); + } + // Every parent tag is distinct: siblings are not linkable at rest. + let tags: std::collections::HashSet<_> = entries[3..9] + .iter() + .map(|e| *e.parent_tag().unwrap()) + .collect(); + assert_eq!(tags.len(), 6, "every parent tag is distinct"); + // The projection groups the two list elements: each is a nested group + // holding its own pair and its `x`. + let q = serde_json::to_value(doc.projection()).unwrap(); + assert_eq!( + q["el"].as_array().unwrap().len(), + 2, + "one group per list element" + ); + assert_eq!( + q["el"][0]["sv"].as_array().unwrap().len(), + 2, + "an element's group holds its own pair and its x" + ); + assert_eq!( + q["sv"].as_array().unwrap().len(), + 6, + "$, hello, nested, deep, list, number" + ); + assert!( + entries.iter().all(|e| e.ordering_term().is_none()), + "no path is ordered by default, so no entry carries a term" + ); } -} -/// The design's two corpora: `jsonb-semantics.sql` and -/// `containment-oracle.sql`, as `(stored, needle)` pairs. -fn corpus() -> Vec<(Value, Value)> { - [ - (r#"["a"]"#, r#""a""#), - (r#""a""#, r#"["a"]"#), - (r#"{"a":[1]}"#, r#"{"a":1}"#), - (r#"{"a":1}"#, r#"{"a":[1]}"#), - (r#"[1,[2]]"#, r#"[2]"#), - (r#"[1,[2]]"#, r#"[[2]]"#), - (r#"{"a":{"b":1}}"#, r#"{"a":{}}"#), - (r#"{"a":{}}"#, r#"{"a":{"b":1}}"#), - (r#"{"a":[1,2]}"#, r#"{"a":[]}"#), - (r#"{"a":1}"#, r#"{}"#), - (r#"[1]"#, r#"{}"#), - (r#"[1]"#, r#"[]"#), - (r#"[1,2]"#, r#"[1,1,1]"#), - ( - r#"{"a":[{"x":1,"y":5},{"x":7,"y":2}]}"#, - r#"{"a":[{"x":1,"y":2}]}"#, - ), - (r#"{"a":[{"x":1,"y":2,"z":3}]}"#, r#"{"a":[{"x":1,"y":2}]}"#), - (r#"{"a":[{"x":1}]}"#, r#"{"a":[{"x":1},{"x":1}]}"#), - ( - r#"{"a":[{"b":[{"x":1},{"y":2}]}]}"#, - r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, - ), - ( - r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, - r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, - ), - (r#"{"a":[["x"]]}"#, r#"{"a":["x"]}"#), - (r#""a""#, r#""a""#), - (r#"2"#, r#"2.0"#), - (r#"{"a":null}"#, r#"{"a":null}"#), - (r#"[["a"]]"#, r#""a""#), - ( - r#"{"a":[{"b":[{"x":1,"y":2}],"c":3},{"b":[{"x":1,"y":9}],"c":4}]}"#, - r#"{"a":[{"b":[{"x":1,"y":2}],"c":4}]}"#, - ), - ( - r#"{"a":[{"b":[{"x":1,"y":2}],"c":3},{"b":[{"x":1,"y":9}],"c":4}]}"#, - r#"{"a":[{"b":[{"x":1,"y":2}],"c":3}]}"#, - ), - ( - r#"{"a":[{"b":{"c":1,"d":2}},{"b":{"c":3,"d":2}}]}"#, - r#"{"a":[{"b":{"c":1,"d":2}}]}"#, - ), - ( - r#"{"a":[{"b":{"c":1,"d":5}},{"b":{"c":3,"d":2}}]}"#, - r#"{"a":[{"b":{"c":1,"d":2}}]}"#, - ), - (r#"{"a":[[1,2],[3,4]]}"#, r#"{"a":[[1,4]]}"#), - (r#"{"a":[[1,2],[3,4]]}"#, r#"{"a":[[1,2]]}"#), - (r#"{"a":[[1,2],[3,4]]}"#, r#"{"a":[[1],[4]]}"#), - (r#"{"a":"2"}"#, r#"{"a":2}"#), - (r#"{"a":2}"#, r#"{"a":2.0}"#), - (r#"{"a":[{"x":1}]}"#, r#"{"a":[{}]}"#), - (r#"{"a":[1]}"#, r#"{"a":[{}]}"#), - ( - r#"{"a":[{"t":[{"u":1,"v":2}]},{"t":[{"u":1},{"v":2}]}]}"#, - r#"{"a":[{"t":[{"u":1,"v":2}]}]}"#, - ), - ( - r#"{"a":[{"t":[{"u":1},{"v":2}]}]}"#, - r#"{"a":[{"t":[{"u":1,"v":2}]}]}"#, - ), - (r#"{"a":[{}]}"#, r#"{"a":[{}]}"#), - (r#"{"a":[[]]}"#, r#"{"a":[[]]}"#), - (r#"{"a":[[]]}"#, r#"{"a":[{}]}"#), - (r#"[{"a":1},{"b":2}]"#, r#"[{"a":1,"b":2}]"#), - (r#"[{"a":1,"b":2}]"#, r#"[{"a":1},{"b":2}]"#), - (r#"{"a":[true,false,null]}"#, r#"{"a":[null,true]}"#), - (r#"{"a":[true]}"#, r#"{"a":[false]}"#), - (r#"{"a":[{"x":[1,2]},{"x":[3]}]}"#, r#"{"a":[{"x":[2,3]}]}"#), - (r#"{"a":[{"x":[1,2]},{"x":[3]}]}"#, r#"{"a":[{"x":[1,2]}]}"#), - ] - .iter() - .map(|(a, b)| { - ( - serde_json::from_str(a).unwrap(), - serde_json::from_str(b).unwrap(), - ) - }) - .collect() -} + /// Equal values at one path share selectors but not ciphertexts; the + /// value selectors, in the projection, tell equal values from different. + #[tokio::test] + async fn gives_equal_values_at_one_path_one_value_selector_and_two_ciphertexts() { + let Scenario { cipher, .. } = scenario().await; + let pair = |doc: &JsonDocument, i: usize| -> (String, String) { + let q = serde_json::to_value(doc.projection()).unwrap(); + ( + q["sv"][i]["s"].as_str().unwrap().to_owned(), + q["sv"][i]["vs"].as_str().unwrap().to_owned(), + ) + }; + let same = seal(&cipher, &json!([1, 1])).await; + let (a, b) = (&same.entries()[1], &same.entries()[2]); + assert_eq!(a.selector(), b.selector(), "two elements share a path"); + assert_eq!( + pair(&same, 1), + pair(&same, 2), + "equal values at one path share a value selector" + ); + assert_ne!( + a.ciphertext(), + b.ciphertext(), + "equal values do not share a ciphertext" + ); + let different = seal(&cipher, &json!([1, 2])).await; + let (one, two) = (&different.entries()[1], &different.entries()[2]); + assert_eq!(one.selector(), two.selector(), "one path"); + assert_ne!( + pair(&different, 1), + pair(&different, 2), + "different values at one path have different value selectors" + ); + assert_eq!( + pair(&different, 1), + pair(&same, 1), + "the value selector names the value at its path, in any document" + ); + } -/// The pinned answers of `jsonb-semantics.sql`, so the model above is held -/// to PostgreSQL and not only to itself. -#[test] -fn the_containment_model_gives_postgresqls_answers() { - let expect = [ - (r#"["a"]"#, r#""a""#, true), - (r#""a""#, r#"["a"]"#, false), - (r#"{"a":[1]}"#, r#"{"a":1}"#, false), - (r#"{"a":1}"#, r#"{"a":[1]}"#, false), - (r#"[1,[2]]"#, r#"[2]"#, false), - (r#"[1,[2]]"#, r#"[[2]]"#, true), - (r#"{"a":{"b":1}}"#, r#"{"a":{}}"#, true), - (r#"{"a":{}}"#, r#"{"a":{"b":1}}"#, false), - (r#"{"a":[1,2]}"#, r#"{"a":[]}"#, true), - (r#"{"a":1}"#, r#"{}"#, true), - (r#"[1]"#, r#"{}"#, false), - (r#"[1]"#, r#"[]"#, true), - (r#"[1,2]"#, r#"[1,1,1]"#, true), - ( - r#"{"a":[{"x":1,"y":5},{"x":7,"y":2}]}"#, - r#"{"a":[{"x":1,"y":2}]}"#, - false, - ), - ( - r#"{"a":[{"x":1,"y":2,"z":3}]}"#, - r#"{"a":[{"x":1,"y":2}]}"#, - true, - ), - (r#"{"a":[{"x":1}]}"#, r#"{"a":[{"x":1},{"x":1}]}"#, true), - ( - r#"{"a":[{"b":[{"x":1},{"y":2}]}]}"#, - r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, - false, - ), - ( - r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, - r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, - true, - ), - (r#"{"a":[["x"]]}"#, r#"{"a":["x"]}"#, false), - (r#""a""#, r#""a""#, true), - (r#"2"#, r#"2.0"#, true), - (r#"{"a":null}"#, r#"{"a":null}"#, true), - (r#"[["a"]]"#, r#""a""#, false), - ]; - for (stored, needle, answer) in expect { - let (a, b): (Value, Value) = ( - serde_json::from_str(stored).unwrap(), - serde_json::from_str(needle).unwrap(), + #[tokio::test] + async fn chains_the_top_level_fields_from_an_open_root() { + let Scenario { cipher, .. } = scenario().await; + let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; + assert!( + open_doc.entries()[1].parent_tag().is_some(), + "without the sealed root, the top-level fields chain from the root" ); - assert_eq!(pg_contains(&a, &b, true), answer, "{stored} @> {needle}"); } -} -/// A small deterministic generator of documents over a few keys and -/// values, so needles drawn from it overlap stored documents often. -struct Lcg(u64); - -impl Lcg { - fn next(&mut self) -> u64 { - self.0 = self - .0 - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); - self.0 >> 33 - } - fn below(&mut self, n: u64) -> u64 { - self.next() % n - } - fn value(&mut self, depth: u32) -> Value { - match self.below(if depth == 0 { 4 } else { 6 }) { - 0 => json!(self.below(3)), - 1 => json!(["x", "y", "z"][self.below(3) as usize]), - 2 => json!(self.below(2) == 0), - 3 => Value::Null, - 4 => Value::Array((0..self.below(4)).map(|_| self.value(depth - 1)).collect()), - _ => { - let mut map = Map::new(); - for _ in 0..self.below(4) { - let key = ["a", "b", "c"][self.below(3) as usize].to_owned(); - let _ = map.insert(key, self.value(depth - 1)); - } - Value::Object(map) - } + #[tokio::test] + async fn derives_the_stored_selector_from_a_path_folding_positions_into_the_wildcard() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let keyset = cipher.default_keyset(); + let selector = |path: &str| { + let path = JsonPath::parse(path).unwrap(); + keyset.json_selector(&path, nonempty!("users").with("profile")) + }; + for (path, j) in [ + ("$.nested.deep", 3), + ("$.nested.list[*].x", 6), + ("$.nested.list[1].x", 8), + ("$", 0), + ] { + assert_eq!( + selector(path).await.unwrap().as_bytes(), + doc.entries()[j].selector(), + "{path} derives entry {j}'s stored selector" + ); } + assert_ne!( + selector("$.hello").await.unwrap().as_bytes(), + keyset + .json_selector( + &JsonPath::parse("$.hello").unwrap(), + nonempty!("users").with("bio") + ) + .await + .unwrap() + .as_bytes(), + "a selector is bound to its column" + ); } -} -#[tokio::test] -async fn containment_over_the_projection_is_postgresqls_containment() { - let cipher = stack_cipher().await; - let plan = plan(); - let mut cases = corpus(); - let mut rng = Lcg(42); - for _ in 0..150 { - let stored = rng.value(3); - let needle = rng.value(2); - cases.push((stored.clone(), needle)); - // A needle taken from the stored document itself is always contained. - cases.push((stored.clone(), stored)); - } - for (stored, needle) in cases { - let doc = cipher.encrypt(&stored).using(&plan).await.unwrap(); - let projection = doc.projection(); - let query: JsonQuery = cipher.query(op::Contains, &needle).on(&plan).await.unwrap(); - assert_eq!( - query.matches(projection), - pg_contains(&stored, &needle, true), - "{stored} @> {needle}" - ); - // `<@` compares the same two projections the other way round. The - // one case it cannot answer is a stored root scalar inside a needle - // root array: a stored document carries no alternative pair, so the - // SQL answers false where `jsonb` answers true, as the design's - // leakage table records for v3's `@>`. - let reverse: JsonQuery = cipher - .query(op::ContainedBy, &needle) - .on(&plan) + /// No token is stored: an entry below a head carries only the parent + /// tag, and no tag in the document is a token, or a parent's hash of one + /// without the document's salt. + #[tokio::test] + async fn stores_no_token() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let nested = subtree_key(&cipher, "$.nested", JsonOptions::default()) .await .unwrap(); - let scalar_in_array = !stored.is_object() && !stored.is_array() && needle.is_array(); + let mut tokens = vec![*nested.token_bytes()]; + let mut frontier = vec![*nested.token_bytes()]; + while let Some(token) = frontier.pop() { + assert!( + tokens.len() <= doc.entries().len(), + "the chain is a tree: no token is reached twice" + ); + for (_, _, child) in doc.children(&token) { + tokens.push(child); + frontier.push(child); + } + } + assert_eq!(tokens.len(), 7, "nested and its six descendants"); + for (j, entry) in doc.entries().iter().enumerate() { + if let Some(p) = entry.parent_tag() { + assert!( + !tokens.iter().any(|t| t.starts_with(p)), + "entry {j}'s parent tag is not a prefix of a token" + ); + } + assert!( + !tokens.contains(entry.selector()), + "entry {j}'s selector is not a token" + ); + } assert_eq!( - reverse.projection().contains(projection), - pg_contains(&needle, &stored, true) && !scalar_in_array, - "{stored} <@ {needle}" + nested.token(), + base64(nested.token_bytes()), + "a token's text form is its padded base64" ); } } -#[tokio::test] -async fn a_query_term_serializes_as_the_needle_operand_and_derives_no_key() { - let (cipher, generates, _) = common::counting_cipher().await; - let plan = plan(); - let query = cipher - .query(op::Contains, &json!({"a": [{"x": 1}]})) - .on(&plan) - .await - .unwrap(); - let wire = serde_json::to_value(&query).unwrap(); - assert_eq!( - wire["sv"].as_array().unwrap().len(), - 2, - "$ and $.a sit outside the element" - ); - assert_eq!(wire["el"].as_array().unwrap().len(), 1); - assert_eq!( - wire["el"][0]["sv"].as_array().unwrap().len(), - 2, - "the element and its x" - ); - assert!(wire.get("alt").is_none()); - assert_eq!( - generates.load(std::sync::atomic::Ordering::Relaxed), - 0, - "a query mints no key" - ); - - let scalar = cipher - .query(op::Contains, &json!("a")) - .on(&plan) - .await - .unwrap(); - let wire = serde_json::to_value(&scalar).unwrap(); - assert!( - wire.get("alt").is_some(), - "a root scalar carries its root-array alternative" - ); -} - -#[tokio::test] -async fn an_accessor_asks_at_a_path() { - let cipher = stack_cipher().await; - let plan = plan(); - let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); - let projection = doc.projection(); - - let hit: JsonQuery = cipher - .query(op::AtPath(op::Eq), &("$.nested.list[*].x", 2)) - .on(&plan) - .await - .unwrap(); - assert!(hit.matches(projection)); - let miss: JsonQuery = cipher - .query(op::AtPath(op::Eq), &("$.nested.list[*].x", 3)) - .on(&plan) - .await - .unwrap(); - assert!(!miss.matches(projection)); - let deep: JsonQuery = cipher - .query(op::AtPath(op::Eq), &("$.nested.deep", "constant")) - .on(&plan) - .await - .unwrap(); - assert!(deep.matches(projection)); - let wrong_kind: JsonQuery = cipher - .query(op::AtPath(op::Eq), &("$.number", "1")) - .on(&plan) - .await - .unwrap(); - assert!( - !wrong_kind.matches(projection), - "a string is not the number" - ); +mod given_ordered_paths { + use super::*; - // An ordering at a path: the path's selector and a leaf term that - // orders as the stored leaf's does, under native byte comparison. The - // plan declares the paths it orders, and the query runs under them. - let plan = ordered_plan(); - let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); - let bound = cipher - .query(op::AtPath(op::Gte), &("$.number", 0)) - .on(&plan) - .await - .unwrap(); - assert_eq!(bound.selector().as_bytes(), doc.entries()[9].selector()); - assert!( - bound.ordering_term() < doc.entries()[9].ordering_term().unwrap(), - "0 < 1" - ); - let above = cipher - .query(op::AtPath(op::Lt), &("$.number", 5)) - .on(&plan) - .await - .unwrap(); - assert!( - above.ordering_term() > doc.entries()[9].ordering_term().unwrap(), - "5 > 1" - ); - let text = cipher - .query(op::AtPath(op::Gt), &("$.hello", "world-0")) - .on(&plan) - .await - .unwrap(); - assert!(text.ordering_term() < doc.entries()[1].ordering_term().unwrap()); - let range = cipher - .query(op::AtPath(op::Between), &("$.number", (0, 5))) - .on(&plan) - .await - .unwrap(); - assert!(range.lo().ordering_term() < range.hi().ordering_term()); - assert_eq!( - range.lo().selector().as_bytes(), - doc.entries()[9].selector() - ); - let wire = serde_json::to_value(range.lo()).unwrap(); - assert_eq!( - wire.as_object().unwrap().keys().collect::>(), - ["op", "s"] - ); + /// Which entries carry an ordering term. + fn with_term(doc: &JsonDocument) -> Vec { + doc.entries() + .iter() + .enumerate() + .filter(|(_, e)| e.ordering_term().is_some()) + .map(|(j, _)| j) + .collect() + } - // Refusals, before any term is derived. - let bad_path = cipher - .query(op::AtPath(op::Eq), &("address.city", "Sydney")) - .on(&plan) - .await; - assert!(matches!(bad_path, Err(Error::Term(TermError::JsonPath(_))))); - let unordered = cipher - .query(op::AtPath(op::Gte), &("$.flag", true)) - .on(&plan) - .await; - assert!(matches!( - unordered, - Err(Error::Term(TermError::JsonLeafNotOrdered)) - )); - // A path the plan does not order is refused: no stored leaf there - // carries a term, so the comparison would match no row. Under the - // default plan that is every path. - let not_ordered = cipher - .query(op::AtPath(op::Lt), &("$.nested.deep", "z")) - .on(&plan) - .await; - assert!( - matches!(not_ordered, Err(Error::Term(TermError::JsonPathNotOrdered))), - "{not_ordered:?}" - ); - let default_plan = self::plan(); - let not_ordered = cipher - .query(op::AtPath(op::Between), &("$.number", (0, 5))) - .on(&default_plan) + /// An ordering term sits on a number or string leaf at an ordered path + /// and nowhere else: a wildcard covers every element. + #[tokio::test] + async fn puts_a_term_only_on_a_number_or_string_leaf_at_an_ordered_path() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal_with(&cipher, &fixture(), &ordering(&["$.hello", "$.number"])).await; + assert_eq!(with_term(&doc), [1, 9], "$.hello and $.number"); + let wild = seal_with(&cipher, &fixture(), &ordering(&["$.nested.list[*].x"])).await; + assert_eq!(with_term(&wild), [6, 8], "a wildcard covers every element"); + // A container, a boolean or a null at an ordered path carries none. + let kinds = seal_with( + &cipher, + &json!({"a": true, "b": null, "c": {"d": 1}, "e": [1, "x"]}), + &ordering(&["$.a", "$.b", "$.c", "$.e", "$.e[*]"]), + ) .await; - assert!( - matches!(not_ordered, Err(Error::Term(TermError::JsonPathNotOrdered))), - "{not_ordered:?}" - ); - // An ordering at a wildcard path is covered by the wildcard. One at a - // position is refused whatever the plan orders: `->` reads a position - // as the wildcard and would compare the first element, not the one - // named. - let by_element = Plan::value::() - .context("users/profile") - .encrypt_with_layout(Json::new(ordering(&["$.nested.list[*].x"]))) - .build() - .unwrap(); - assert!(cipher - .query(op::AtPath(op::Gt), &("$.nested.list[*].x", 0)) - .on(&by_element) - .await - .is_ok()); - for path in ["$.nested.list[1].x", "$.nested.list[0].x"] { - let refused = cipher - .query(op::AtPath(op::Gt), &(path, 0)) - .on(&by_element) - .await; + // $, $.a, $.b, $.c, $.c.d, $.e, $.e[0], $.e[1] + assert_eq!(with_term(&kinds), [6, 7], "only the number and the string"); + let scalar = seal_with(&cipher, &json!(7), &ordering(&["$"])).await; + assert_eq!(with_term(&scalar), [0], "the root scalar itself, at `$`"); + assertions::opens_to(&cipher, doc, &fixture(), "ordered at $.hello and $.number").await; + assertions::opens_to(&cipher, wild, &fixture(), "ordered at a wildcard").await; + } + + /// A position is refused, since `->` reads it as the wildcard and no + /// ordering could reach the leaf it names; so is a text that is not a + /// path. + #[tokio::test] + async fn refuses_a_position_or_a_text_that_is_not_a_path() { + let Scenario { cipher, .. } = scenario().await; + let position = JsonOptions::default().ordering(["$.nested.list[1].x"]); assert!( - matches!(refused, Err(Error::Term(TermError::JsonOrderingAtPosition))), - "{path}: {refused:?}" + matches!(position, Err(TermError::JsonOrderingAtPosition)), + "a position is refused as an ordered path: {position:?}" ); - let refused = cipher - .query(op::AtPath(op::Between), &(path, (0, 5))) - .on(&by_element) - .await; + let not_a_path = JsonOptions::default().ordering(["nested"]); assert!( - matches!(refused, Err(Error::Term(TermError::JsonOrderingAtPosition))), - "{path}: {refused:?}" + matches!(not_a_path, Err(TermError::JsonPath(_))), + "a text that is not a path is refused: {not_a_path:?}" ); - } - // Equality at a path compares one value: an object or an array would be - // containment, which `op::Contains` asks for. - for container in [json!({"deep": "constant"}), json!([]), json!([{"x": 1}])] { - let refused = cipher - .query(op::AtPath(op::Eq), &("$.nested", container.clone())) - .on(&plan) - .await; - assert!( - matches!(refused, Err(Error::Term(TermError::JsonValueNotScalar))), - "{container}" + // Set on the field, which `ordering` guards, a position refuses the + // seal rather than put a term no ordering reaches on one element. + let mut positional = JsonOptions::default(); + let _ = positional + .ordered + .insert(JsonPath::parse("$.nested.list[1].x").unwrap()); + let refused = try_seal(&cipher, &fixture(), &positional).await; + assertions::refused_by_term( + &refused, + |e| matches!(e, TermError::JsonOrderingAtPosition), + "a position set on the field refuses the seal", ); } } -/// A plan laid out by the index opens what it wrote, one document or a -/// batch, and reports its layout. -#[tokio::test] -async fn a_laid_out_plan_opens_what_it_wrote() { - let cipher = stack_cipher().await; - let plan = ordered_plan(); - let options = ordering(&["$.hello", "$.number", "$.flag"]); - assert_eq!(plan.layout().options(), &options); - assert_eq!(plan.specs(), [IndexSpec::Json(options.clone())]); - assert_eq!( - stack_encrypt::target::Layout::::specs(&Json::new(options.clone())), - [IndexSpec::Json(options)] - ); - assert!( - format!("{plan:?}").contains("Laid(Json"), - "{plan:?}" - ); - let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); - let back: Value = cipher.open(doc.clone()).using(&plan).await.unwrap(); - assert_eq!(back, fixture()); - let docs: Vec = cipher - .open(vec![doc.clone(), doc.clone()]) - .using(&plan) - .await - .unwrap(); - assert_eq!(docs, [fixture(), fixture()]); - // The plan was built with its context, so an open that names one is - // refused, one document or a batch, before any key is requested. - let clash = cipher - .open(doc.clone()) - .context("users/other") - .using(&plan) - .await; - assert!( - matches!(clash, Err(Error::Plan(PlanError::TwoContextSources { .. }))), - "{clash:?}" - ); - let clash = cipher - .open(vec![doc.clone()]) - .context("users/other") - .using(&plan) - .await; - assert!( - matches!(clash, Err(Error::Plan(PlanError::TwoContextSources { .. }))), - "{clash:?}" - ); - // The index named on the query says only which: the plan's own, with - // its options, answers. - let through_default = cipher - .query(op::AtPath(op::Gte), &("$.number", 0)) - .via(Json) - .on(&plan) - .await - .unwrap(); - let chosen = cipher - .query(op::AtPath(op::Gte), &("$.number", 0)) - .on(&plan) - .await - .unwrap(); - assert_eq!(through_default.selector(), chosen.selector()); -} - -/// At the root, equality is of the root value: `("$", 1)` matches a stored -/// `1` and not a stored `[1]`, which containment of the scalar `1` would. -#[tokio::test] -async fn equality_at_the_root_is_not_containment() { - let cipher = stack_cipher().await; - let plan = plan(); - let at_root: JsonQuery = cipher - .query(op::AtPath(op::Eq), &("$", 1)) - .on(&plan) - .await - .unwrap(); - let contains: JsonQuery = cipher - .query(op::Contains, &json!(1)) - .on(&plan) - .await - .unwrap(); - let scalar = cipher.encrypt(&json!(1)).using(&plan).await.unwrap(); - let array = cipher.encrypt(&json!([1])).using(&plan).await.unwrap(); - assert!(at_root.matches(scalar.projection())); - assert!(!at_root.matches(array.projection())); - assert!(contains.matches(array.projection()), "jsonb: [1] @> 1"); - assert!(serde_json::to_value(&at_root).unwrap().get("alt").is_none()); -} - -#[tokio::test] -async fn a_query_under_other_options_is_refused_by_the_plan() { - let cipher = stack_cipher().await; - let plan = plan(); - let refused = cipher - .query(op::Contains, &json!({})) - .via(Json::new(open_root())) - .on(&plan) - .await; - assert!( - matches!(refused, Err(Error::Plan(PlanError::IndexOptions { .. }))), - "{refused:?}" - ); - assert_eq!( - >::indexes(), - [IndexSpec::Json(JsonOptions::default())] - ); - assert_eq!(Json, Json::default(), "the value is the default index"); -} - -#[tokio::test] -async fn opening_through_the_plan_is_the_typed_open() { - let cipher = stack_cipher().await; - let plan = plan(); - let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); - let back: Value = cipher.open(doc).using(&plan).await.unwrap(); - assert_eq!(back, fixture()); -} - -#[cfg(feature = "dynamic")] -#[test] -fn the_json_index_has_a_wire_form() { - use stack_encrypt::dynamic::{not_admitted, Reason, ValueKind}; - assert_eq!( - IndexSpec::parse("json"), - Some(IndexSpec::Json(JsonOptions::default())) - ); - assert_eq!(IndexSpec::Json(JsonOptions::default()).key(), "json"); - assert_eq!( - serde_json::to_value(IndexSpec::Json(JsonOptions::default())).unwrap(), - json!("json") - ); - assert_eq!( - serde_json::to_value(IndexSpec::Json(open_root())).unwrap(), - json!({"json": {"sealed_root": false, "ordered": []}}), - "every option is written when any is not the default" - ); - let frame = - vitaminc_aead_frame::serde::to_vec(&json!({"json": {"sealed_root": false}})).unwrap(); - assert_eq!( - IndexSpec::from_frame(&frame).unwrap(), - IndexSpec::Json(open_root()) - ); - let frame = vitaminc_aead_frame::serde::to_vec(&json!({"json": {"nope": true}})).unwrap(); - assert!(IndexSpec::from_frame(&frame).is_err()); - // The ordered paths ride in `ordered`, written in their canonical - // order and read back parsed; a text that is not a path refuses the - // index. - let ordered = IndexSpec::Json(ordering(&["$.orders[*].total", "$.age", "$.age"])); - assert_eq!( - serde_json::to_value(&ordered).unwrap(), - json!({"json": {"sealed_root": true, "ordered": ["$.age", "$.orders[*].total"]}}) - ); - let frame = vitaminc_aead_frame::serde::to_vec( - &json!({"json": {"ordered": ["$.age", "$.orders[*].total"]}}), - ) - .unwrap(); - assert_eq!(IndexSpec::from_frame(&frame).unwrap(), ordered); - let frame = vitaminc_aead_frame::serde::to_vec(&json!({"json": {"ordered": ["age"]}})).unwrap(); - assert!(IndexSpec::from_frame(&frame).is_err()); - // A position in an ordered path refuses the index too: no ordering can - // reach the leaf it names. - let frame = - vitaminc_aead_frame::serde::to_vec(&json!({"json": {"ordered": ["$.scores[2]"]}})).unwrap(); - assert!(IndexSpec::from_frame(&frame).is_err()); - assert_eq!( - not_admitted(ValueKind::Object, &IndexSpec::Json(JsonOptions::default())), - Some(Reason::JsonIndexPending) - ); -} - // ============================================================================= -// What the suite pins beyond the round trips +// The token chain // ============================================================================= -/// No token is stored: an entry below a head carries only the parent tag, -/// and no tag in the document is a token, or a parent's hash of one -/// without the document's salt. -#[tokio::test] -async fn no_stored_tag_is_a_token() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let nested = subtree_key(&cipher, "$.nested", JsonOptions::default()) - .await - .unwrap(); - let mut tokens = vec![*nested.token_bytes()]; - let mut frontier = vec![*nested.token_bytes()]; - while let Some(token) = frontier.pop() { +mod given_a_subtree_key { + use super::*; + + #[tokio::test] + async fn unlocks_exactly_its_subtree_from_the_path_alone() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) + .await + .unwrap(); assert!( - tokens.len() <= doc.entries().len(), - "the chain is a tree: no token is reached twice" + key.gates().is_empty(), + "a chain head's token is derived, nothing to gate" + ); + assert_eq!( + key.path(), + &JsonPath::parse("$.nested").unwrap(), + "the key carries the path it was derived from" + ); + let under = doc.subtree(key.token_bytes()); + let found: Vec = under.iter().map(|(j, _, _)| *j).collect(); + assert_eq!( + found, + [3, 4, 5, 6, 7, 8], + "deep, list, both elements, both leaves" + ); + let depths: Vec = under.iter().map(|(_, _, d)| *d).collect(); + assert_eq!( + depths, + [1, 1, 2, 3, 2, 3], + "each row's depth below $.nested" + ); + assert!( + doc.subtree(&[7; 16]).is_empty(), + "a wrong token unlocks nothing" ); - for (_, _, child) in doc.children(&token) { - tokens.push(child); - frontier.push(child); - } } - assert_eq!(tokens.len(), 7, "nested and its six descendants"); - for entry in doc.entries() { - if let Some(p) = entry.parent_tag() { - assert!( - !tokens.iter().any(|t| t.starts_with(p)), - "a parent tag is not a prefix of a token" + + /// Below a head the client derives nothing but array positions: a key + /// step becomes a gate, and the walk from the head takes it. + #[tokio::test] + async fn gates_each_key_step_below_its_head() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) + .await + .unwrap(); + let deeper = subtree_key(&cipher, "$.nested.list[1]", JsonOptions::default()) + .await + .unwrap(); + assert_eq!( + deeper.token_bytes(), + key.token_bytes(), + "anchored at the head" + ); + assert_eq!(deeper.gates().len(), 2, "list, then [1]"); + assert_eq!( + deeper.gates()[1], + format!("#1:{}", base64(doc.entries()[5].selector())), + "the position, and the selector every element of the list carries" + ); + let list_token = child_token(&doc, key.token_bytes(), 4); + let second = child_token(&doc, &list_token, 7); + assert_eq!( + subtree_ordinals(&doc, &second), + [8], + "the second element's x" + ); + let children: Vec = doc + .children(&list_token) + .iter() + .map(|(j, _, _)| *j) + .collect(); + assert_eq!( + children, + [5, 7], + "the children of a node, by ordinal: the list's two elements" + ); + // A wildcard path anchors at the head and gates on each step's + // selector: the list's, then every element's (the wildcard's), then x's. + let gated = subtree_key(&cipher, "$.nested.list[*].x", JsonOptions::default()) + .await + .unwrap(); + assert_eq!( + gated.token_bytes(), + key.token_bytes(), + "a wildcard path anchors at the head" + ); + assert_eq!( + gated.gates(), + [ + base64(doc.entries()[4].selector()), + base64(doc.entries()[5].selector()), + base64(doc.entries()[6].selector()), + ], + "the list's selector, every element's, then x's" + ); + } + + /// The sealed root has no token; the open root's unlocks every node, and + /// under an open root the chain is derivable down to the first key step. + #[tokio::test] + async fn has_no_token_at_a_sealed_root_and_derives_from_an_open_one() { + let Scenario { cipher, .. } = scenario().await; + for path in ["$", "$[*].x"] { + let refused = subtree_key(&cipher, path, JsonOptions::default()).await; + assertions::refused_by_term( + &refused, + |e| matches!(e, TermError::JsonSealedRoot), + &format!("{path} under the sealed root"), ); } - assert!(!tokens.contains(entry.selector())); + let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; + let root = subtree_key(&cipher, "$", open_root()).await.unwrap(); + assert_eq!( + open_doc.subtree(root.token_bytes()).len(), + 9, + "every entry but the root" + ); + // `$[0]` would be a position, `$.nested` is a gate. + let open_key = subtree_key(&cipher, "$.nested.list[1]", open_root()) + .await + .unwrap(); + assert_eq!( + open_key.token_bytes(), + root.token_bytes(), + "anchored at the open root" + ); + assert_eq!(open_key.gates().len(), 3, "nested, list, then [1]"); + let array_doc = seal_with(&cipher, &json!([[1, 2], [3]]), &open_root()).await; + let second_row = subtree_key(&cipher, "$[1]", open_root()).await.unwrap(); + assert!(second_row.gates().is_empty(), "positions derive"); + assert_eq!( + subtree_ordinals(&array_doc, second_row.token_bytes()), + [5], + "$, [0], [0][0], [0][1], [1], [1][0]" + ); } - assert_eq!(nested.token(), base64(nested.token_bytes())); } -/// A position names an array element and nothing else. A column's rows -/// need not agree on a node's kind, and an object member's ordinal is a -/// rank among its siblings, so a token derived for `$.a[0]` must not open -/// the rank-0 member of a row whose `$.a` is an object: `jsonb` has no -/// node at `{a,0}` there, and neither does the index. -#[tokio::test] -async fn a_position_reaches_no_object_member() { - let cipher = stack_cipher().await; - let first = subtree_key(&cipher, "$.a[0]", JsonOptions::default()) - .await - .unwrap(); - assert!( - first.gates().is_empty(), - "a position after the head derives" - ); - let array = seal(&cipher, &json!({"a": [{"x": 2}]})).await; - let found: Vec = array - .subtree(first.token_bytes()) - .iter() - .map(|(j, _, _)| *j) - .collect(); - assert_eq!(found, [3], "$, $.a, $.a[0], $.a[0].x"); - for object in [ - json!({"a": {"k": {"x": 1}}}), - json!({"a": {"k": [1, 2]}}), - json!({"a": {"0": {"x": 1}}}), - ] { - let doc = seal(&cipher, &object).await; +mod given_a_position { + use super::*; + + /// A position names an array element and nothing else. A column's rows + /// need not agree on a node's kind, and an object member's ordinal is a + /// rank among its siblings, so a token derived for `$.a[0]` must not + /// open the rank-0 member of a row whose `$.a` is an object: `jsonb` has + /// no node at `{a,0}` there, and neither does the index. + #[tokio::test] + async fn reaches_no_object_member() { + let Scenario { cipher, .. } = scenario().await; + let first = subtree_key(&cipher, "$.a[0]", JsonOptions::default()) + .await + .unwrap(); assert!( - doc.subtree(first.token_bytes()).is_empty(), - "{object}: an object member is not an element" + first.gates().is_empty(), + "a position after the head derives" + ); + let array = seal(&cipher, &json!({"a": [{"x": 2}]})).await; + assert_eq!( + subtree_ordinals(&array, first.token_bytes()), + [3], + "$, $.a, $.a[0], $.a[0].x" ); + for object in [ + json!({"a": {"k": {"x": 1}}}), + json!({"a": {"k": [1, 2]}}), + json!({"a": {"0": {"x": 1}}}), + ] { + let doc = seal(&cipher, &object).await; + assert!( + doc.subtree(first.token_bytes()).is_empty(), + "{object}: an object member is not an element" + ); + } } - // The same below a key step, where the position is a gate: the walk - // from the head reaches `$.h.a` by its selector, and the gate at the - // next step admits an element only. A member at ordinal 0 under an - // object `$.h.a` carries its own selector, not the element's. - let gated = subtree_key(&cipher, "$.h.a[0]", JsonOptions::default()) - .await - .unwrap(); - assert_eq!(gated.gates().len(), 2); - let element = cipher - .default_keyset() - .json_selector( - &JsonPath::parse("$.h.a[*]").unwrap(), - nonempty!("users").with("profile"), - ) - .await - .unwrap(); - assert_eq!(gated.gates()[1], format!("#0:{element}")); -} - -/// Padded standard base64, as the SQL reads and writes it. -fn base64(bytes: &[u8]) -> String { - use base64ct::{Base64, Encoding}; - Base64::encode_string(bytes) -} -/// The rows of a gated walk carry the node the gates reached at depth 0 -/// and its descendants beneath. Opened as a value, they are that node's -/// value; two reached nodes open as a list, or refuse to be one value. -#[tokio::test] -async fn gated_walk_rows_open_to_the_reached_nodes_value() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let nested = subtree_key(&cipher, "$.nested", JsonOptions::default()) - .await - .unwrap(); - let list = doc - .children(nested.token_bytes()) - .into_iter() - .find(|(j, _, _)| *j == 4) - .map(|(_, _, t)| t) - .unwrap(); - // What `json_tree_subtree_at(doc, t(nested), [s(list), #1])` returns: - // the second element at d = 0, its x at d = 1. - let rows_at = |reached: &[usize], below: &[(usize, u32)]| -> Vec { - reached - .iter() - .map(|j| (*j, 0u32)) - .chain(below.iter().copied()) - .map(|(j, d)| { - let mut wire = serde_json::to_value(doc.node(j).unwrap()).unwrap(); - wire["d"] = json!(d); - serde_json::from_value(wire).unwrap() - }) - .collect() - }; - let second = keyed(&cipher, "$.nested.list[1]", rows_at(&[7], &[(8, 1)])).await; - assert!( - !second.is_sealed_root(), - "a reached node is not the fallback" - ); - let value = cipher - .decrypt_as::(second.clone(), aead()) - .await - .unwrap(); - assert_eq!(value, json!({"x": 2}), "the reached node's own value"); - let values = cipher - .decrypt_as::, JsonSubtree>(second, aead()) - .await - .unwrap(); - assert_eq!(values, vec![json!({"x": 2})]); - // A scalar reached by a gate opens to the scalar, where the ungated - // walk of a scalar yields no rows and no value. - let x = cipher - .decrypt_as::( - keyed(&cipher, "$.nested.list[1].x", rows_at(&[8], &[])).await, - aead(), - ) - .await - .unwrap(); - assert_eq!(x, json!(2)); - // Two reached nodes (a wildcard): a list, never one value. - let both = keyed(&cipher, "$.nested.list[*].x", rows_at(&[6, 8], &[])).await; - let values = cipher - .decrypt_as::, JsonSubtree>(both.clone(), aead()) - .await - .unwrap(); - assert_eq!(values, vec![json!(1), json!(2)]); - match cipher.decrypt_as::(both, aead()).await { - Err(Error::OpenedType { expected }) => { - assert_eq!(expected, "one node at the walked path") - } - other => panic!("{other:?}"), + /// The same below a key step, where the position is a gate: the walk + /// from the head reaches `$.h.a` by its selector, and the gate at the + /// next step admits an element only. A member at ordinal 0 under an + /// object `$.h.a` carries its own selector, not the element's. + #[tokio::test] + async fn gates_on_the_elements_selector_below_a_key_step() { + let Scenario { cipher, .. } = scenario().await; + let gated = subtree_key(&cipher, "$.h.a[0]", JsonOptions::default()) + .await + .unwrap(); + assert_eq!(gated.gates().len(), 2, "a, then [0]"); + let element = cipher + .default_keyset() + .json_selector( + &JsonPath::parse("$.h.a[*]").unwrap(), + nonempty!("users").with("profile"), + ) + .await + .unwrap(); + assert_eq!( + gated.gates()[1], + format!("#0:{element}"), + "the position gate names the elements' selector" + ); } - // Without a d = 0 row the same entries are the children of one node and - // assemble into it. - let rows: Vec = doc - .subtree(&list) - .iter() - .map(|(j, _, _)| doc.node(*j).unwrap()) - .collect(); - let elements = cipher - .decrypt_as::(keyed(&cipher, "$.nested.list", rows).await, aead()) - .await - .unwrap(); - assert_eq!(elements, json!([{"x": 1}, {"x": 2}])); } -/// A document, an extracted node and a walked subtree are each the one -/// recoverable field of a derived record: each opens through -/// `DecryptField`, as a record holding one would. -#[tokio::test] -async fn each_stored_form_opens_as_a_record_field() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let node = doc.node(1).unwrap(); - let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) - .await - .unwrap(); - let list = doc - .children(key.token_bytes()) - .into_iter() - .find(|(j, _, _)| *j == 4) - .map(|(_, _, t)| t) - .unwrap(); - let rows: Vec = doc - .subtree(&list) - .iter() - .map(|(j, _, _)| doc.node(*j).unwrap()) - .collect(); - let subtree = keyed(&cipher, "$.nested.list", rows.clone()).await; - assert_eq!(subtree.nodes(), rows.as_slice()); - assert_eq!(subtree.nodes().len(), 4, "both elements and both leaves"); - - let opening: Option> = doc.decryption_field(aead()); - let opened = cipher - .run_decryption(opening.expect("a document opens")) - .await - .unwrap(); - assert_eq!(opened, fixture()); - let opening: Option> = node.decryption_field(aead()); - let opened = cipher - .run_decryption(opening.expect("a node opens")) - .await - .unwrap(); - assert_eq!(opened, json!("world-1")); - let opening: Option> = subtree.decryption_field(aead()); - let opened = cipher - .run_decryption(opening.expect("a subtree opens")) - .await - .unwrap(); - assert_eq!(opened, json!([{"x": 1}, {"x": 2}])); -} +// ============================================================================= +// Extraction +// ============================================================================= -/// Past its budget the server returns the sealed root alone, at depth 0, -/// in place of the subtree: the rows open to the whole document and the -/// key's path selects over it. A path that reaches nothing, or a wildcard -/// that reaches two nodes where one value is asked for, is refused; a root -/// at depth 0 under an unsealed document is not the fallback. A fallback -/// with no path cannot be built: the key is the only constructor. -#[tokio::test] -async fn the_sealed_root_a_walk_fell_back_to_opens_at_the_path() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - let fallback = |doc: &JsonDocument| -> JsonNode { - let mut wire = serde_json::to_value(doc.node(0).unwrap()).unwrap(); - wire["d"] = json!(0); - serde_json::from_value(wire).unwrap() - }; - let root = fallback(&doc); - let rows = keyed(&cipher, "$.nested", vec![root.clone()]).await; - assert!(rows.is_sealed_root()); - assert_eq!(rows.path().to_string(), "$.nested"); +mod given_an_extracted_node { + use super::*; - let nested = cipher - .decrypt_as::(rows, aead()) - .await - .unwrap(); - assert_eq!(nested, fixture()["nested"]); - let xs = cipher - .decrypt_as::, JsonSubtree>( - keyed(&cipher, "$.nested.list[*].x", vec![root.clone()]).await, - aead(), - ) - .await - .unwrap(); - assert_eq!(xs, vec![json!(1), json!(2)]); - let none = cipher - .decrypt_as::, JsonSubtree>( - keyed(&cipher, "$.missing[*]", vec![root.clone()]).await, - aead(), - ) - .await - .unwrap(); - assert!(none.is_empty(), "a path that reaches nothing is no subtree"); - - // Each refusal names what it expected: nothing at the path, or one node - // where a wildcard reached two. - for (path, expected) in [ - ("$.missing", "a node at the walked path"), - ("$.nested.list[*].x", "one node at the walked path"), - ] { - let refused = cipher + #[tokio::test] + async fn opens_to_its_scalar_and_a_walked_subtree_to_its_value() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let node = |j: usize| doc.node(j).unwrap(); + let scalar = |j: usize| cipher.decrypt_as::(node(j), aead()); + assert_eq!( + scalar(1).await.unwrap(), + json!("world-1"), + "$.hello opens to its scalar" + ); + assert_eq!( + scalar(6).await.unwrap(), + json!(1), + "[0].x opens to its scalar" + ); + assert_eq!( + scalar(0).await.unwrap(), + fixture(), + "the sealed root is the document" + ); + assertions::refused_as_opened_type( + &scalar(2).await, + None, + "a container node holds no value of its own", + ); + + let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) + .await + .unwrap(); + let rows = subtree_nodes(&doc, key.token_bytes()); + let nested = cipher + .decrypt_as::(key.open(rows.clone()), aead()) + .await + .unwrap(); + assert_eq!( + nested, + fixture()["nested"], + "the walk under $.nested opens to its value" + ); + let forest = cipher + .decrypt_as::, JsonSubtree>(key.open(rows), aead()) + .await + .unwrap(); + assert_eq!( + forest, + vec![json!("constant"), json!([{"x": 1}, {"x": 2}])], + "as a list, the walk opens to $.nested's members" + ); + + // The list's token is the server's to derive, one level under nested. + let list = child_token(&doc, key.token_bytes(), 4); + let elements = cipher .decrypt_as::( - keyed(&cipher, path, vec![root.clone()]).await, + keyed(&cipher, "$.nested.list", subtree_nodes(&doc, &list)).await, aead(), ) - .await; - match refused { - Err(Error::OpenedType { expected: got }) => assert_eq!(got, expected, "{path}"), - other => panic!("{path}: {other:?}"), - } + .await + .unwrap(); + assert_eq!( + elements, + json!([{"x": 1}, {"x": 2}]), + "the walk under the list opens to the list" + ); } - // The same root at depth 1, or at depth 0 under an unsealed document, - // is a walk's row and not the fallback; neither assembles. - let mut as_walked = serde_json::to_value(doc.node(0).unwrap()).unwrap(); - as_walked["d"] = json!(1); - let as_walked: JsonNode = serde_json::from_value(as_walked).unwrap(); - let rows = keyed(&cipher, "$.nested", vec![as_walked]).await; - assert!(!rows.is_sealed_root()); - let refused = cipher.decrypt_as::(rows, aead()).await; - assert!(matches!(refused, Err(Error::OpenedType { .. }))); - let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; - assert!(!keyed(&cipher, "$.nested", vec![fallback(&open_doc)]) - .await - .is_sealed_root()); - assert!(!keyed(&cipher, "$.nested", vec![]).await.is_sealed_root()); - assert!(!keyed(&cipher, "$.nested", vec![root.clone(), root]) - .await - .is_sealed_root()); -} - -/// The rows `json_tree_subtree_at(doc, token, gates)` returns for `key`, -/// walked here as the SQL walks them: from the key's token, through each -/// gate (a selector the entry at that depth must carry, or `#n:`, -/// the child at ordinal `n` carrying it), then everything beneath each node -/// the gates reach, each row with its depth below that node. -fn gated_walk(doc: &JsonDocument, key: &JsonSubtreeKey) -> Vec { - let with_depth = |j: usize, d: u32| -> JsonNode { - let mut wire = serde_json::to_value(doc.node(j).unwrap()).unwrap(); - wire["d"] = json!(d); - serde_json::from_value(wire).unwrap() - }; - let mut frontier = vec![(None, *key.token_bytes())]; - for gate in key.gates() { - let mut next = Vec::new(); - for (_, token) in &frontier { - let children = doc.children(token); - match gate.strip_prefix('#') { - Some(slot) => { - let (n, selector) = slot.split_once(':').unwrap(); - if let Some((j, entry, t)) = children.get(n.parse::().unwrap()) { - if base64(entry.selector()) == selector { - next.push((Some(*j), *t)); - } - } - } - None => next.extend( - children - .iter() - .filter(|(_, entry, _)| base64(entry.selector()) == gate) - .map(|(j, _, t)| (Some(*j), *t)), - ), - } - } - frontier = next; + /// An extracted node round-trips through its wire form with the + /// envelope fields the SQL grafts beside it. + #[tokio::test] + async fn round_trips_with_the_envelope_the_sql_grafts() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let mut wire = serde_json::to_value(doc.node(6).unwrap()).unwrap(); + wire["i"] = json!({"t": "users", "c": "profile"}); + wire["v"] = json!(4); + let back: JsonNode = serde_json::from_value(wire).unwrap(); + assert_eq!(back.ordinal(), 6, "the node keeps its ordinal"); + assert_eq!( + cipher + .decrypt_as::(back, aead()) + .await + .unwrap(), + json!(1), + "the grafted node opens to its scalar" + ); } - let mut rows = Vec::new(); - for (reached, token) in frontier { - let below = u32::from(reached.is_some()); - if let Some(j) = reached { - rows.push(with_depth(j, 0)); - } - rows.extend( - doc.subtree(&token) - .iter() - .map(|(j, _, d)| with_depth(*j, d - 1 + below)), + + /// The walk reports each row's depth below where it started, in `d`; an + /// extracted node carries none. + #[tokio::test] + async fn carries_no_depth_until_a_walk_gives_it_one() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + assert_eq!( + doc.node(6).unwrap().depth(), + None, + "an extracted node has no depth" + ); + let mut wire = serde_json::to_value(doc.node(6).unwrap()).unwrap(); + assert!( + wire.get("d").is_none(), + "an extracted node writes no `d`: {wire}" + ); + wire["d"] = json!(3); + let walked: JsonNode = serde_json::from_value(wire).unwrap(); + assert_eq!(walked.depth(), Some(3), "a walked row reads its depth"); + assert_eq!( + serde_json::to_value(&walked).unwrap()["d"], + json!(3), + "a walked row writes its depth back" ); } - rows.sort_by_key(JsonNode::ordinal); - rows } -/// The sealed root a walk fell back to opens to exactly what the walked -/// rows open to, so a caller's result does not change shape when a -/// document grows past the budget, and no caller says which walk the rows -/// came from: the key does, by its gates. A key with none keyed -/// `json_tree_subtree`, whose rows are the descendants of the path's node, -/// so a `Vec` opens to that node's children; a key with gates keyed -/// `json_tree_subtree_at`, whose rows are the nodes the path reaches. -#[tokio::test] -async fn the_key_built_fallback_opens_to_what_the_walk_opens_to() { - let document = json!({ - "list": [{"x": 1}, {"x": 2}], - "obj": {"a": "constant", "b": [1, 2], "c": {"d": [true, null]}}, - "hello": "world" - }); - let cipher = stack_cipher().await; - let doc = seal(&cipher, &document).await; - let mut root = serde_json::to_value(doc.node(0).unwrap()).unwrap(); - root["d"] = json!(0); - let root: JsonNode = serde_json::from_value(root).unwrap(); - let sorted = |mut values: Vec| { - values.sort_by_key(ToString::to_string); - values - }; - for (text, gated) in [ - ("$.list", false), - ("$.obj", false), - ("$.hello", false), - ("$.list[1]", false), - ("$.obj.b", true), - ("$.obj.c.d", true), - ("$.obj.b[0]", true), - ("$.list[*].x", true), - ("$.list[1].x", true), - ("$.obj.c", true), - ] { - let key = subtree_key(&cipher, text, JsonOptions::default()) +mod given_gated_walk_rows { + use super::*; + + /// The rows of a gated walk carry the node the gates reached at depth 0 + /// and its descendants beneath. Opened as a value, they are that node's + /// value; two reached nodes open as a list, or refuse to be one value. + #[tokio::test] + async fn open_to_the_reached_nodes_value() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let nested = subtree_key(&cipher, "$.nested", JsonOptions::default()) .await .unwrap(); - assert_eq!(!key.gates().is_empty(), gated, "{text}"); - let walked = if gated { - gated_walk(&doc, &key) - } else { - doc.subtree(key.token_bytes()) + let list = child_token(&doc, nested.token_bytes(), 4); + // What `json_tree_subtree_at(doc, t(nested), [s(list), #1])` returns: + // the second element at d = 0, its x at d = 1. + let rows_at = |reached: &[usize], below: &[(usize, u32)]| -> Vec { + reached .iter() - .map(|(j, _, _)| doc.node(*j).unwrap()) + .map(|j| (*j, 0u32)) + .chain(below.iter().copied()) + .map(|(j, d)| with_depth(&doc, j, d)) .collect() }; - let fallen_back = key.open(vec![root.clone()]); - assert!(fallen_back.is_sealed_root(), "{text}"); - let as_walked = cipher - .decrypt_as::, JsonSubtree>(key.open(walked.clone()), aead()) + let second = keyed(&cipher, "$.nested.list[1]", rows_at(&[7], &[(8, 1)])).await; + assert!( + !second.is_sealed_root(), + "a reached node is not the fallback" + ); + let value = cipher + .decrypt_as::(second.clone(), aead()) .await .unwrap(); - let as_fallen = cipher - .decrypt_as::, JsonSubtree>(fallen_back.clone(), aead()) + assert_eq!(value, json!({"x": 2}), "the reached node's own value"); + let values = cipher + .decrypt_as::, JsonSubtree>(second, aead()) .await .unwrap(); - // An object's members come back in an order neither form fixes. - assert_eq!(sorted(as_fallen), sorted(as_walked), "{text}, as a list"); - // A walk with no rows (a scalar's, or an empty container's) is - // never past a budget, so it never falls back: nothing to compare. - if walked.is_empty() { - continue; - } - let one_walked = cipher - .decrypt_as::(key.open(walked), aead()) - .await; - let one_fallen = cipher - .decrypt_as::(fallen_back, aead()) - .await; - match (one_walked, one_fallen) { - (Ok(a), Ok(b)) => assert_eq!(a, b, "{text}, as one value"), - (Err(_), Err(_)) => {} - (a, b) => panic!("{text}: the walk opened to {a:?}, the fallback to {b:?}"), - } - } - // As one value an ungated walk opens to its node, either way. - let key = subtree_key(&cipher, "$.obj", JsonOptions::default()) - .await - .unwrap(); - assert_eq!( - cipher - .decrypt_as::(key.open(vec![root.clone()]), aead()) + assert_eq!( + values, + vec![json!({"x": 2})], + "as a list, the one reached node" + ); + // A scalar reached by a gate opens to the scalar, where the ungated + // walk of a scalar yields no rows and no value. + let x = cipher + .decrypt_as::( + keyed(&cipher, "$.nested.list[1].x", rows_at(&[8], &[])).await, + aead(), + ) .await - .unwrap(), - document["obj"] - ); - // An ungated key names one node: a fallback with nothing at the path, - // or a position under a wildcard head, is refused. - let missing = subtree_key(&cipher, "$.missing", JsonOptions::default()) - .await - .unwrap(); - match cipher - .decrypt_as::, JsonSubtree>(missing.open(vec![root]), aead()) - .await - { - Err(Error::OpenedType { expected }) => { - assert_eq!(expected, "one node at the walked path") - } - other => panic!("{other:?}"), + .unwrap(); + assert_eq!(x, json!(2), "a scalar reached by a gate opens to it"); + // Two reached nodes (a wildcard): a list, never one value. + let both = keyed(&cipher, "$.nested.list[*].x", rows_at(&[6, 8], &[])).await; + let values = cipher + .decrypt_as::, JsonSubtree>(both.clone(), aead()) + .await + .unwrap(); + assert_eq!( + values, + vec![json!(1), json!(2)], + "two reached nodes open as a list" + ); + assertions::refused_as_opened_type( + &cipher.decrypt_as::(both, aead()).await, + Some("one node at the walked path"), + "two reached nodes are not one value", + ); + // Without a d = 0 row the same entries are the children of one node and + // assemble into it. + let elements = cipher + .decrypt_as::( + keyed(&cipher, "$.nested.list", subtree_nodes(&doc, &list)).await, + aead(), + ) + .await + .unwrap(); + assert_eq!( + elements, + json!([{"x": 1}, {"x": 2}]), + "without a d = 0 row the rows assemble into their parent" + ); } } -/// The walk reports each row's depth below where it started, in `d`; an -/// extracted node carries none. -#[tokio::test] -async fn a_walked_row_carries_its_depth() { - let cipher = stack_cipher().await; - let doc = seal(&cipher, &fixture()).await; - assert_eq!(doc.node(6).unwrap().depth(), None); - let mut wire = serde_json::to_value(doc.node(6).unwrap()).unwrap(); - assert!(wire.get("d").is_none()); - wire["d"] = json!(3); - let walked: JsonNode = serde_json::from_value(wire).unwrap(); - assert_eq!(walked.depth(), Some(3)); - assert_eq!(serde_json::to_value(&walked).unwrap()["d"], json!(3)); -} +mod given_a_derived_record { + use super::*; + + /// A document, an extracted node and a walked subtree are each the one + /// recoverable field of a derived record: each opens through + /// `DecryptField`, as a record holding one would. + #[tokio::test] + async fn opens_each_stored_form_as_its_field() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let node = doc.node(1).unwrap(); + let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) + .await + .unwrap(); + let list = child_token(&doc, key.token_bytes(), 4); + let rows = subtree_nodes(&doc, &list); + let subtree = keyed(&cipher, "$.nested.list", rows.clone()).await; + assert_eq!( + subtree.nodes(), + rows.as_slice(), + "a subtree holds the rows it was opened with" + ); + assert_eq!(subtree.nodes().len(), 4, "both elements and both leaves"); -/// A document sealed without the sealed root keeps the root as an ordinary -/// node, and opens from its entries. -#[tokio::test] -async fn a_document_without_the_sealed_root_opens_from_its_entries() { - let cipher = stack_cipher().await; - for value in [fixture(), json!({"": {}}), json!([[], {}, ""]), json!("a")] { - let doc = seal_with(&cipher, &value, &open_root()).await; - assert!(!doc.header().sealed_root()); - assert_eq!(doc.entries().len(), nodes(&value)); - assert_eq!(open(&cipher, doc).await.unwrap(), value); + let opening: Option> = doc.decryption_field(aead()); + let opened = cipher + .run_decryption(opening.expect("a document opens")) + .await + .unwrap(); + assert_eq!(opened, fixture(), "the document field"); + let opening: Option> = node.decryption_field(aead()); + let opened = cipher + .run_decryption(opening.expect("a node opens")) + .await + .unwrap(); + assert_eq!(opened, json!("world-1"), "the node field"); + let opening: Option> = + subtree.decryption_field(aead()); + let opened = cipher + .run_decryption(opening.expect("a subtree opens")) + .await + .unwrap(); + assert_eq!(opened, json!([{"x": 1}, {"x": 2}]), "the subtree field"); } } -/// An accessor whose path or bounds the index refuses is refused when the -/// query is awaited, before the keyset it names is loaded. -#[tokio::test] -async fn an_accessor_is_refused_before_its_keyset_loads() { - let (cipher, loads) = common::loads_counting_cipher().await; - let plan = ordered_plan(); - let eq = cipher - .query(op::AtPath(op::Eq), &("address.city", "Sydney")) - .on(&plan) - .keyset("tenant") - .await; - assert!( - matches!(eq, Err(Error::Term(TermError::JsonPath(_)))), - "{eq:?}" - ); - let between = cipher - .query(op::AtPath(op::Between), &("address.age", (1, 2))) - .on(&plan) - .keyset("tenant") - .await; - assert!( - matches!(between, Err(Error::Term(TermError::JsonPath(_)))), - "{between:?}" - ); - let unordered = cipher - .query(op::AtPath(op::Between), &("$.flag", (false, true))) - .on(&plan) - .keyset("tenant") - .await; - assert!( - matches!(unordered, Err(Error::Term(TermError::JsonLeafNotOrdered))), - "{unordered:?}" - ); - let not_ordered = cipher - .query(op::AtPath(op::Lt), &("$.nested.deep", "z")) - .on(&plan) - .keyset("tenant") - .await; - assert!( - matches!(not_ordered, Err(Error::Term(TermError::JsonPathNotOrdered))), - "{not_ordered:?}" - ); - let at_position = cipher - .query(op::AtPath(op::Gte), &("$.nested.list[0].x", 1)) - .on(&plan) - .keyset("tenant") - .await; - assert!( - matches!( - at_position, - Err(Error::Term(TermError::JsonOrderingAtPosition)) - ), - "{at_position:?}" - ); - // An open that names a context the plan already has is refused the - // same way, one document or a batch, before the keyset loads. - let doc = seal(&stack_cipher().await, &fixture()).await; - let clash = cipher - .open(doc.clone()) - .context("users/other") - .using(&plan) - .keyset("tenant") - .await; - assert!( - matches!(clash, Err(Error::Plan(PlanError::TwoContextSources { .. }))), - "{clash:?}" - ); - let clash = cipher - .open(vec![doc]) - .context("users/other") - .using(&plan) - .keyset("tenant") - .await; - assert!( - matches!(clash, Err(Error::Plan(PlanError::TwoContextSources { .. }))), - "{clash:?}" - ); - assert_eq!( - loads.load(std::sync::atomic::Ordering::SeqCst), - 0, - "no keyset was loaded for a refused query or open" +mod given_a_fallback_root { + use super::*; + + /// The document's root as a walk past its budget returns it: alone, at + /// depth 0. + fn fallback(doc: &JsonDocument) -> JsonNode { + with_depth(doc, 0, 0) + } + + /// Past its budget the server returns the sealed root alone, at depth + /// 0, in place of the subtree: the rows open to the whole document and + /// the key's path selects over it. + #[tokio::test] + async fn opens_at_the_keys_path() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let root = fallback(&doc); + let rows = keyed(&cipher, "$.nested", vec![root.clone()]).await; + assert!( + rows.is_sealed_root(), + "the root alone at d = 0 is the fallback" + ); + assert_eq!( + rows.path().to_string(), + "$.nested", + "the fallback carries the key's path" + ); + let nested = cipher + .decrypt_as::(rows, aead()) + .await + .unwrap(); + assert_eq!(nested, fixture()["nested"], "the value at the path"); + let xs = cipher + .decrypt_as::, JsonSubtree>( + keyed(&cipher, "$.nested.list[*].x", vec![root.clone()]).await, + aead(), + ) + .await + .unwrap(); + assert_eq!( + xs, + vec![json!(1), json!(2)], + "a wildcard path selects every node over the document" + ); + let none = cipher + .decrypt_as::, JsonSubtree>( + keyed(&cipher, "$.missing[*]", vec![root]).await, + aead(), + ) + .await + .unwrap(); + assert!( + none.is_empty(), + "a path that reaches nothing is no subtree: {none:?}" + ); + } + + /// A path that reaches nothing, or a wildcard that reaches two nodes + /// where one value is asked for, is refused, and each refusal names + /// what it expected. + #[tokio::test] + async fn refuses_one_value_where_the_path_reaches_none_or_two() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let root = fallback(&doc); + for (path, expected) in [ + ("$.missing", "a node at the walked path"), + ("$.nested.list[*].x", "one node at the walked path"), + ] { + let refused = cipher + .decrypt_as::( + keyed(&cipher, path, vec![root.clone()]).await, + aead(), + ) + .await; + assertions::refused_as_opened_type(&refused, Some(expected), path); + } + } + + /// The same root at depth 1, or at depth 0 under an unsealed document, + /// is a walk's row and not the fallback; neither assembles. A fallback + /// with no path cannot be built: the key is the only constructor. + #[tokio::test] + async fn is_only_the_sealed_root_alone_at_depth_zero() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let root = fallback(&doc); + let rows = keyed(&cipher, "$.nested", vec![with_depth(&doc, 0, 1)]).await; + assert!( + !rows.is_sealed_root(), + "the root at depth 1 is a walk's row" + ); + assertions::refused_as_opened_type( + &cipher.decrypt_as::(rows, aead()).await, + None, + "the root at depth 1 does not assemble", + ); + let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; + assert!( + !keyed(&cipher, "$.nested", vec![fallback(&open_doc)]) + .await + .is_sealed_root(), + "an unsealed root at depth 0 is not the fallback" + ); + assert!( + !keyed(&cipher, "$.nested", vec![]).await.is_sealed_root(), + "no rows are not the fallback" + ); + assert!( + !keyed(&cipher, "$.nested", vec![root.clone(), root]) + .await + .is_sealed_root(), + "two roots are not the fallback" + ); + } + + /// The sealed root a walk fell back to opens to exactly what the walked + /// rows open to, so a caller's result does not change shape when a + /// document grows past the budget, and no caller says which walk the + /// rows came from: the key does, by its gates. A key with none keyed + /// `json_tree_subtree`, whose rows are the descendants of the path's + /// node, so a `Vec` opens to that node's children; a key with gates + /// keyed `json_tree_subtree_at`, whose rows are the nodes the path + /// reaches. + #[tokio::test] + async fn opens_to_what_the_walk_opens_to() { + let document = json!({ + "list": [{"x": 1}, {"x": 2}], + "obj": {"a": "constant", "b": [1, 2], "c": {"d": [true, null]}}, + "hello": "world" + }); + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &document).await; + let root = fallback(&doc); + let sorted = |mut values: Vec| { + values.sort_by_key(ToString::to_string); + values + }; + for (text, gated) in [ + ("$.list", false), + ("$.obj", false), + ("$.hello", false), + ("$.list[1]", false), + ("$.obj.b", true), + ("$.obj.c.d", true), + ("$.obj.b[0]", true), + ("$.list[*].x", true), + ("$.list[1].x", true), + ("$.obj.c", true), + ] { + let key = subtree_key(&cipher, text, JsonOptions::default()) + .await + .unwrap(); + assert_eq!( + !key.gates().is_empty(), + gated, + "{text}: whether the key gates" + ); + let walked = if gated { + gated_walk(&doc, &key) + } else { + subtree_nodes(&doc, key.token_bytes()) + }; + let fallen_back = key.open(vec![root.clone()]); + assert!(fallen_back.is_sealed_root(), "{text}: the fallback"); + let as_walked = cipher + .decrypt_as::, JsonSubtree>(key.open(walked.clone()), aead()) + .await + .unwrap(); + let as_fallen = cipher + .decrypt_as::, JsonSubtree>(fallen_back.clone(), aead()) + .await + .unwrap(); + // An object's members come back in an order neither form fixes. + assert_eq!(sorted(as_fallen), sorted(as_walked), "{text}, as a list"); + // A walk with no rows (a scalar's, or an empty container's) is + // never past a budget, so it never falls back: nothing to compare. + if walked.is_empty() { + continue; + } + let one_walked = cipher + .decrypt_as::(key.open(walked), aead()) + .await; + let one_fallen = cipher + .decrypt_as::(fallen_back, aead()) + .await; + match (one_walked, one_fallen) { + (Ok(a), Ok(b)) => assert_eq!(a, b, "{text}, as one value"), + (Err(_), Err(_)) => {} + (a, b) => panic!("{text}: the walk opened to {a:?}, the fallback to {b:?}"), + } + } + // As one value an ungated walk opens to its node, either way. + let key = subtree_key(&cipher, "$.obj", JsonOptions::default()) + .await + .unwrap(); + assert_eq!( + cipher + .decrypt_as::(key.open(vec![root.clone()]), aead()) + .await + .unwrap(), + document["obj"], + "an ungated fallback opens to its node as one value" + ); + // An ungated key names one node: a fallback with nothing at the path + // is refused. + let missing = subtree_key(&cipher, "$.missing", JsonOptions::default()) + .await + .unwrap(); + assertions::refused_as_opened_type( + &cipher + .decrypt_as::, JsonSubtree>(missing.open(vec![root]), aead()) + .await, + Some("one node at the walked path"), + "an ungated fallback with nothing at the path", + ); + } +} + +// ============================================================================= +// Tamper evidence +// ============================================================================= + +mod given_rows_the_server_chose { + use super::*; + + /// The rows of a walk come from the server, which can repeat an entry or + /// pick which entries to return. Every entry is authentic, but the + /// rebuild nests the containers it is handed: a repeated row is refused, + /// and so is a line of containers deeper than any document the index + /// seals, as a shape mismatch rather than a recursion as deep as the + /// rows. + #[tokio::test] + async fn refuses_a_repeated_row_or_a_line_of_containers() { + let Scenario { cipher, .. } = scenario().await; + let doc = seal(&cipher, &fixture()).await; + let node = |j: usize| doc.node(j).unwrap(); + let repeated = keyed(&cipher, "$.nested", vec![node(1), node(1)]).await; + assertions::refused_as_opened_type( + &cipher + .decrypt_as::, JsonSubtree>(repeated, aead()) + .await, + None, + "a repeated row", + ); + + // A wide document of one-member objects: its objects, returned as rows + // with the last leaf after them, each take the next as their member, + // and would nest 200 deep. + let wide = Value::Object( + (0..200) + .map(|i| (format!("k{i:03}"), json!({"x": 1}))) + .collect(), + ); + let doc = seal(&cipher, &wide).await; + let last = doc.entries().len() - 1; + let containers: Vec = (1..doc.entries().len()) + .filter(|j| j % 2 == 1 || *j == last) + .map(|j| doc.node(j).unwrap()) + .collect(); + assert_eq!(containers.len(), 201, "200 objects and the last leaf"); + assertions::refused_as_opened_type( + &cipher + .decrypt_as::, JsonSubtree>( + keyed(&cipher, "$.k000", containers).await, + aead(), + ) + .await, + None, + "a line of 200 objects", + ); + + // The same through arrays: one outer array, then every key's inner + // one-element array, each taken as the element of the one before it. + let wide = Value::Object( + (0..200) + .map(|i| (format!("k{i:03}"), json!([[0]]))) + .collect(), + ); + let doc = seal(&cipher, &wide).await; + let last = doc.entries().len() - 1; + let arrays: Vec = (1..doc.entries().len()) + .filter(|j| *j == 1 || j % 3 == 2 || *j == last) + .map(|j| doc.node(j).unwrap()) + .collect(); + assert_eq!( + arrays.len(), + 202, + "the outer array, 200 inner and the last leaf" + ); + assertions::refused_as_opened_type( + &cipher + .decrypt_as::, JsonSubtree>( + keyed(&cipher, "$.k000", arrays).await, + aead(), + ) + .await, + None, + "a line of 202 arrays", + ); + } +} + +mod given_a_tampered_document { + use super::*; + + #[tokio::test] + async fn fails_to_open_a_moved_reordered_or_regrafted_entry() { + let Scenario { cipher, .. } = scenario().await; + let value = json!({"a": "x", "b": "y"}); + let doc = seal_with(&cipher, &value, &open_root()).await; + let other = seal_with(&cipher, &value, &open_root()).await; + let tamper = |mut wire: Value, f: &dyn Fn(&mut Vec)| { + let sv = wire["sv"].as_array_mut().unwrap(); + f(sv); + serde_json::from_value::(wire).unwrap() + }; + let wire = serde_json::to_value(&doc).unwrap(); + let other_wire = serde_json::to_value(&other).unwrap(); + + let reordered = tamper(wire.clone(), &|sv| sv.swap(1, 2)); + assertions::refused_as_tampered(&open(&cipher, reordered).await, "reordered"); + + let regrafted = tamper(wire.clone(), &|sv| sv[1] = other_wire["sv"][1].clone()); + assertions::refused_as_tampered( + &open(&cipher, regrafted).await, + "regrafted from another document", + ); + + let truncated = tamper(wire.clone(), &|sv| { + let _ = sv.pop(); + }); + assertions::refused_as_opened_type(&open(&cipher, truncated).await, None, "truncated"); + + let flipped = tamper(wire.clone(), &|sv| { + let c = sv[1]["c"].as_str().unwrap().to_owned(); + let mut bytes = c.into_bytes(); + bytes[3] = if bytes[3] == b'A' { b'B' } else { b'A' }; + sv[1]["c"] = Value::String(String::from_utf8(bytes).unwrap()); + }); + assertions::refused_as_tampered(&open(&cipher, flipped).await, "a changed byte"); + + let wrong_context = cipher + .decrypt_as::( + doc, + AeadContext::from(nonempty!("users").with("bio")), + ) + .await; + assertions::refused_as_tampered(&wrong_context, "another column"); + + // A node lifted out of its document fails under another ordinal. + let sealed = seal(&cipher, &value).await; + let mut lifted = serde_json::to_value(sealed.node(1).unwrap()).unwrap(); + lifted["j"] = json!(2); + let lifted: JsonNode = serde_json::from_value(lifted).unwrap(); + assertions::refused_as_tampered( + &cipher.decrypt_as::(lifted, aead()).await, + "a node under another ordinal", + ); + } +} + +// ============================================================================= +// Containment +// ============================================================================= + +mod given_the_containment_model { + use super::*; + + /// The pinned answers of `jsonb-semantics.sql`, so the model is held to + /// PostgreSQL and not only to itself. + #[test] + fn gives_postgresqls_pinned_answers() { + let expect = [ + (r#"["a"]"#, r#""a""#, true), + (r#""a""#, r#"["a"]"#, false), + (r#"{"a":[1]}"#, r#"{"a":1}"#, false), + (r#"{"a":1}"#, r#"{"a":[1]}"#, false), + (r#"[1,[2]]"#, r#"[2]"#, false), + (r#"[1,[2]]"#, r#"[[2]]"#, true), + (r#"{"a":{"b":1}}"#, r#"{"a":{}}"#, true), + (r#"{"a":{}}"#, r#"{"a":{"b":1}}"#, false), + (r#"{"a":[1,2]}"#, r#"{"a":[]}"#, true), + (r#"{"a":1}"#, r#"{}"#, true), + (r#"[1]"#, r#"{}"#, false), + (r#"[1]"#, r#"[]"#, true), + (r#"[1,2]"#, r#"[1,1,1]"#, true), + ( + r#"{"a":[{"x":1,"y":5},{"x":7,"y":2}]}"#, + r#"{"a":[{"x":1,"y":2}]}"#, + false, + ), + ( + r#"{"a":[{"x":1,"y":2,"z":3}]}"#, + r#"{"a":[{"x":1,"y":2}]}"#, + true, + ), + (r#"{"a":[{"x":1}]}"#, r#"{"a":[{"x":1},{"x":1}]}"#, true), + ( + r#"{"a":[{"b":[{"x":1},{"y":2}]}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, + false, + ), + ( + r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, + true, + ), + (r#"{"a":[["x"]]}"#, r#"{"a":["x"]}"#, false), + (r#""a""#, r#""a""#, true), + (r#"2"#, r#"2.0"#, true), + (r#"{"a":null}"#, r#"{"a":null}"#, true), + (r#"[["a"]]"#, r#""a""#, false), + ]; + for (stored, needle, answer) in expect { + let (a, b): (Value, Value) = ( + serde_json::from_str(stored).unwrap(), + serde_json::from_str(needle).unwrap(), + ); + assert_eq!(pg_contains(&a, &b, true), answer, "{stored} @> {needle}"); + } + } +} + +mod given_the_design_corpus { + use super::*; + + /// The design's two corpora: `jsonb-semantics.sql` and + /// `containment-oracle.sql`, as `(stored, needle)` pairs. + fn corpus() -> Vec<(Value, Value)> { + [ + (r#"["a"]"#, r#""a""#), + (r#""a""#, r#"["a"]"#), + (r#"{"a":[1]}"#, r#"{"a":1}"#), + (r#"{"a":1}"#, r#"{"a":[1]}"#), + (r#"[1,[2]]"#, r#"[2]"#), + (r#"[1,[2]]"#, r#"[[2]]"#), + (r#"{"a":{"b":1}}"#, r#"{"a":{}}"#), + (r#"{"a":{}}"#, r#"{"a":{"b":1}}"#), + (r#"{"a":[1,2]}"#, r#"{"a":[]}"#), + (r#"{"a":1}"#, r#"{}"#), + (r#"[1]"#, r#"{}"#), + (r#"[1]"#, r#"[]"#), + (r#"[1,2]"#, r#"[1,1,1]"#), + ( + r#"{"a":[{"x":1,"y":5},{"x":7,"y":2}]}"#, + r#"{"a":[{"x":1,"y":2}]}"#, + ), + (r#"{"a":[{"x":1,"y":2,"z":3}]}"#, r#"{"a":[{"x":1,"y":2}]}"#), + (r#"{"a":[{"x":1}]}"#, r#"{"a":[{"x":1},{"x":1}]}"#), + ( + r#"{"a":[{"b":[{"x":1},{"y":2}]}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, + ), + ( + r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}]}]}"#, + ), + (r#"{"a":[["x"]]}"#, r#"{"a":["x"]}"#), + (r#""a""#, r#""a""#), + (r#"2"#, r#"2.0"#), + (r#"{"a":null}"#, r#"{"a":null}"#), + (r#"[["a"]]"#, r#""a""#), + ( + r#"{"a":[{"b":[{"x":1,"y":2}],"c":3},{"b":[{"x":1,"y":9}],"c":4}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}],"c":4}]}"#, + ), + ( + r#"{"a":[{"b":[{"x":1,"y":2}],"c":3},{"b":[{"x":1,"y":9}],"c":4}]}"#, + r#"{"a":[{"b":[{"x":1,"y":2}],"c":3}]}"#, + ), + ( + r#"{"a":[{"b":{"c":1,"d":2}},{"b":{"c":3,"d":2}}]}"#, + r#"{"a":[{"b":{"c":1,"d":2}}]}"#, + ), + ( + r#"{"a":[{"b":{"c":1,"d":5}},{"b":{"c":3,"d":2}}]}"#, + r#"{"a":[{"b":{"c":1,"d":2}}]}"#, + ), + (r#"{"a":[[1,2],[3,4]]}"#, r#"{"a":[[1,4]]}"#), + (r#"{"a":[[1,2],[3,4]]}"#, r#"{"a":[[1,2]]}"#), + (r#"{"a":[[1,2],[3,4]]}"#, r#"{"a":[[1],[4]]}"#), + (r#"{"a":"2"}"#, r#"{"a":2}"#), + (r#"{"a":2}"#, r#"{"a":2.0}"#), + (r#"{"a":[{"x":1}]}"#, r#"{"a":[{}]}"#), + (r#"{"a":[1]}"#, r#"{"a":[{}]}"#), + ( + r#"{"a":[{"t":[{"u":1,"v":2}]},{"t":[{"u":1},{"v":2}]}]}"#, + r#"{"a":[{"t":[{"u":1,"v":2}]}]}"#, + ), + ( + r#"{"a":[{"t":[{"u":1},{"v":2}]}]}"#, + r#"{"a":[{"t":[{"u":1,"v":2}]}]}"#, + ), + (r#"{"a":[{}]}"#, r#"{"a":[{}]}"#), + (r#"{"a":[[]]}"#, r#"{"a":[[]]}"#), + (r#"{"a":[[]]}"#, r#"{"a":[{}]}"#), + (r#"[{"a":1},{"b":2}]"#, r#"[{"a":1,"b":2}]"#), + (r#"[{"a":1,"b":2}]"#, r#"[{"a":1},{"b":2}]"#), + (r#"{"a":[true,false,null]}"#, r#"{"a":[null,true]}"#), + (r#"{"a":[true]}"#, r#"{"a":[false]}"#), + (r#"{"a":[{"x":[1,2]},{"x":[3]}]}"#, r#"{"a":[{"x":[2,3]}]}"#), + (r#"{"a":[{"x":[1,2]},{"x":[3]}]}"#, r#"{"a":[{"x":[1,2]}]}"#), + ] + .iter() + .map(|(a, b)| { + ( + serde_json::from_str(a).unwrap(), + serde_json::from_str(b).unwrap(), + ) + }) + .collect() + } + + /// Every pair of the corpus, both ways round: `@>` over the projections + /// is `jsonb @>`, and `<@` is `jsonb <@` but for a stored root scalar in + /// a needle root array. + #[tokio::test] + async fn answers_containment_both_ways_as_jsonb_does() { + let Scenario { cipher, plan } = scenario().await; + for (stored, needle) in corpus() { + let doc = cipher.encrypt(&stored).using(&plan).await.unwrap(); + let contains: JsonQuery = cipher.query(op::Contains, &needle).on(&plan).await.unwrap(); + let contained_by: JsonQuery = cipher + .query(op::ContainedBy, &needle) + .on(&plan) + .await + .unwrap(); + assertions::contains_as_jsonb(&contains, &doc, &stored, &needle); + assertions::is_contained_by_as_jsonb(&contained_by, &doc, &stored, &needle); + } + } +} + +mod given_generated_documents { + //! The containment property over documents and needles proptest + //! generates. A case is one stored document and one needle: a random + //! value, the document itself or a pruning of it (which it contains by + //! construction), or such a pruning with one edit. A failure shrinks to a small + //! document and needle, and proptest appends its seed to + //! `proptest-regressions/json_index.txt` in this crate, which every + //! later run replays first. The strategies mirror the EQL live oracle's + //! (`packages/eql/tests/encryption/tests/json_tree/generated.rs`), which + //! sits in another Cargo workspace. + + use std::cell::Cell; + use std::fmt; + use std::future::IntoFuture; + + use proptest::collection::vec as vec_of; + use proptest::prelude::*; + use proptest::sample::{select, Index}; + use proptest::strategy::ValueTree; + use proptest::test_runner::{ + Config, FileFailurePersistence, RngAlgorithm, RngSeed, TestCaseError, TestRunner, + }; + + use super::*; + + /// The seed the run uses when `PROPTEST_RNG_SEED` is unset. Fixed, so + /// every run checks the same cases. + const DEFAULT_SEED: u64 = 1336; + + /// The cases a run checks when `PROPTEST_CASES` is unset. + const DEFAULT_CASES: u32 = 512; + + /// Where proptest records a failing case's seed: `Direct`, with the + /// crate's own path, rather than resolving `file!()` from the working + /// directory. + const REGRESSIONS: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/proptest-regressions/json_index.txt" ); - // The same query with a path the index takes does load it. - let _ = cipher - .query(op::AtPath(op::Eq), &("$.address.city", "Sydney")) - .on(&plan) - .keyset("tenant") - .await + + /// Numbers, several of them one value to `jsonb` in different + /// representations: `1`, `1.0` and `1e0`; `100` and `1e2`; `0`, `0.0` + /// and `-0.0`; `0.5` and `5e-1`. + const NUMBERS: &[&str] = &[ + "0", + "0.0", + "-0.0", + "1", + "1.0", + "1e0", + "2", + "2.00", + "-1", + "-1.0", + "10", + "1e1", + "100", + "1e2", + "0.5", + "5e-1", + "1.5", + "123456789012", + "1e20", + ]; + + /// Strings, some spelled like a number, a boolean or null. + const STRINGS: &[&str] = &["x", "y", "", "1", "1.0", "2", "true", "null", "a b", "ü"]; + + /// Object keys: few, so they repeat across levels and siblings. + const KEYS: &[&str] = &["a", "b", "c", "x", "1", ""]; + + /// proptest's defaults, which read `PROPTEST_CASES` and + /// `PROPTEST_RNG_SEED`, with this suite's own default for each one left + /// unset. + fn config() -> Config { + let mut config = Config::default(); + if std::env::var_os("PROPTEST_CASES").is_none() { + config.cases = DEFAULT_CASES; + } + if config.rng_seed == RngSeed::Random { + config.rng_seed = RngSeed::Fixed(DEFAULT_SEED); + } + config.rng_algorithm = RngAlgorithm::ChaCha; + config.failure_persistence = Some(Box::new(FileFailurePersistence::Direct(REGRESSIONS))); + config + } + + /// One edit that turns a contained needle into a near miss. `at` picks + /// the node, in document order; an edit that does not fit the node's + /// kind changes its kind instead. + #[derive(Clone, Debug)] + enum Edit { + Leaf { + at: Index, + to: Value, + }, + Key { + at: Index, + key: String, + value: Value, + }, + Element { + at: Index, + value: Value, + }, + Kind { + at: Index, + }, + } + + /// A needle as proptest draws it. `keep` says, one member or element at + /// a time in document order, what a pruning keeps; once it runs out, + /// nothing more is kept, so shrinking it makes the needle smaller. + #[derive(Clone, Debug)] + enum NeedleSpec { + Random(Value), + /// The stored document itself, which contains itself. + Itself, + Pruned { + keep: Vec, + at: Option, + }, + NearMiss { + keep: Vec, + edit: Edit, + }, + } + + /// One generated case: a stored document and a needle derived from it. + #[derive(Clone)] + struct Case { + stored: Value, + needle: NeedleSpec, + } + + impl Case { + /// The needle as a value, and whether the stored document contains + /// it by construction. + fn needle(&self) -> (Value, bool) { + match &self.needle { + NeedleSpec::Random(value) => (value.clone(), false), + NeedleSpec::Itself => (self.stored.clone(), true), + NeedleSpec::Pruned { keep, at } => { + let pointer = at + .as_ref() + .map_or_else(String::new, |at| at.get(&pointers(&self.stored)).clone()); + let node = self.stored.pointer(&pointer).unwrap(); + (prune(node, &mut keep.iter()), pointer.is_empty()) + } + NeedleSpec::NearMiss { keep, edit } => { + let mut value = prune(&self.stored, &mut keep.iter()); + apply(edit, &mut value); + (value, false) + } + } + } + } + + /// Printed as JSON: what proptest shows for the minimal failing case. + impl fmt::Debug for Case { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (needle, _) = self.needle(); + write!( + f, + "stored {}, needle {needle} ({:?})", + self.stored, self.needle + ) + } + } + + fn prune(v: &Value, keep: &mut std::slice::Iter<'_, bool>) -> Value { + match v { + Value::Object(map) => { + let mut out = Map::new(); + for (key, member) in map { + if keep.next().copied().unwrap_or(false) { + let _ = out.insert(key.clone(), prune(member, keep)); + } + } + Value::Object(out) + } + Value::Array(items) => { + let mut out = Vec::new(); + for item in items { + if keep.next().copied().unwrap_or(false) { + out.push(prune(item, keep)); + } + } + Value::Array(out) + } + scalar => scalar.clone(), + } + } + + /// Every node's JSON pointer, in document order, the root (`""`) first. + fn pointers(v: &Value) -> Vec { + fn walk(v: &Value, at: String, out: &mut Vec) { + out.push(at.clone()); + match v { + Value::Object(map) => { + for (key, member) in map { + let key = key.replace('~', "~0").replace('/', "~1"); + walk(member, format!("{at}/{key}"), out); + } + } + Value::Array(items) => { + for (i, item) in items.iter().enumerate() { + walk(item, format!("{at}/{i}"), out); + } + } + _ => {} + } + } + let mut out = Vec::new(); + walk(v, String::new(), &mut out); + out + } + + fn apply(edit: &Edit, needle: &mut Value) { + let at = match edit { + Edit::Leaf { at, .. } + | Edit::Key { at, .. } + | Edit::Element { at, .. } + | Edit::Kind { at } => at, + }; + let pointer = at.get(&pointers(needle)).clone(); + let node = needle.pointer_mut(&pointer).unwrap(); + match edit { + Edit::Leaf { to, .. } if !node.is_object() && !node.is_array() && node != to => { + *node = to.clone(); + } + Edit::Key { key, value, .. } if node.is_object() => { + let _ = node + .as_object_mut() + .unwrap() + .insert(key.clone(), value.clone()); + } + Edit::Element { value, .. } if node.is_array() => { + node.as_array_mut().unwrap().push(value.clone()); + } + _ => { + *node = match node.take() { + Value::Object(map) => Value::Array(map.into_iter().map(|(_, v)| v).collect()), + Value::Array(items) => { + json!({ "a": items.into_iter().next().unwrap_or(Value::Null) }) + } + scalar => Value::Array(vec![scalar]), + }; + } + } + } + + mod strategies { + use super::*; + + pub(super) fn scalar() -> BoxedStrategy { + prop_oneof![ + 4 => select(NUMBERS).prop_map(|text| serde_json::from_str::(text).unwrap()), + 3 => select(STRINGS).prop_map(|text| Value::String(text.to_owned())), + 2 => any::().prop_map(Value::Bool), + 1 => Just(Value::Null), + ] + .boxed() + } + + fn key() -> BoxedStrategy { + select(KEYS).prop_map(str::to_owned).boxed() + } + + /// An object of up to four members. A key drawn twice keeps its + /// later value, as `jsonb` does. + fn object_of(inner: BoxedStrategy) -> BoxedStrategy { + vec_of((key(), inner), 0..5) + .prop_map(|members| Value::Object(members.into_iter().collect())) + .boxed() + } + + /// An array: of anything, of objects, of arrays, or with some + /// elements repeated. + fn array_of(inner: BoxedStrategy) -> BoxedStrategy { + prop_oneof![ + 3 => vec_of(inner.clone(), 0..5).prop_map(Value::Array), + 1 => vec_of(object_of(inner.clone()), 1..4).prop_map(Value::Array), + 1 => vec_of(vec_of(inner.clone(), 0..3).prop_map(Value::Array), 1..4) + .prop_map(Value::Array), + 1 => (vec_of(inner, 1..4), vec_of(any::(), 1..3)).prop_map( + |(mut items, repeats)| { + for repeat in repeats { + let copy = repeat.get(&items).clone(); + items.push(copy); + } + Value::Array(items) + } + ), + ] + .boxed() + } + + /// A value with up to `depth` levels of recursion below it. + pub(super) fn value(depth: u32) -> BoxedStrategy { + scalar() + .prop_recursive(depth, 48, 4, |inner| { + prop_oneof![object_of(inner.clone()), array_of(inner)] + }) + .boxed() + } + + /// A stored document: mostly an object, sometimes a root array, and + /// sometimes a root scalar. + pub(super) fn document() -> BoxedStrategy { + prop_oneof![ + 6 => object_of(value(3)), + 3 => array_of(value(2)), + 1 => scalar(), + ] + .boxed() + } + + fn keep() -> BoxedStrategy> { + vec_of(proptest::bool::weighted(0.7), 0..32).boxed() + } + + fn edit() -> BoxedStrategy { + prop_oneof![ + (any::(), scalar()).prop_map(|(at, to)| Edit::Leaf { at, to }), + (any::(), key(), value(1)).prop_map(|(at, key, value)| Edit::Key { + at, + key, + value + }), + (any::(), value(1)).prop_map(|(at, value)| Edit::Element { at, value }), + any::().prop_map(|at| Edit::Kind { at }), + ] + .boxed() + } + + fn needle() -> BoxedStrategy { + prop_oneof![ + 1 => value(2).prop_map(NeedleSpec::Random), + 1 => Just(NeedleSpec::Itself), + 2 => (keep(), proptest::option::weighted(0.3, any::())) + .prop_map(|(keep, at)| NeedleSpec::Pruned { keep, at }), + 2 => (keep(), edit()).prop_map(|(keep, edit)| NeedleSpec::NearMiss { keep, edit }), + ] + .boxed() + } + + pub(super) fn case() -> BoxedStrategy { + (document(), needle()) + .prop_map(|(stored, needle)| Case { stored, needle }) + .boxed() + } + } + + /// Run `body` over a whole proptest run on one runtime and cipher, and + /// fail with the minimal case and the way to replay it. + fn run( + name: &str, + body: impl Fn(&Scenario, &tokio::runtime::Runtime, &Case) -> Result<(), TestCaseError>, + ) { + let config = config(); + let replay = format!( + "seed {seed}, {cases} cases; replay with PROPTEST_RNG_SEED={seed} PROPTEST_CASES={cases}", + seed = config.rng_seed, + cases = config.cases + ); + eprintln!("json_index generated {name}: {replay}"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let scenario = runtime.block_on(scenario()); + let mut runner = TestRunner::new(config); + if let Err(error) = runner.run(&strategies::case(), |case| body(&scenario, &runtime, &case)) + { + panic!("json_index generated {name}: {replay}\n{error}"); + } + } + + /// The stored document, sealed through the plan. + fn sealed( + scenario: &Scenario, + runtime: &tokio::runtime::Runtime, + stored: &Value, + ) -> JsonDocument { + runtime + .block_on( + scenario + .cipher + .encrypt(stored) + .using(&scenario.plan) + .into_future(), + ) + .unwrap() + } + + #[test] + fn contains_as_jsonb_contains() { + run("@>", |scenario, runtime, case| { + let (needle, contained) = case.needle(); + let doc = sealed(scenario, runtime, &case.stored); + let query: JsonQuery = runtime + .block_on( + scenario + .cipher + .query(op::Contains, &needle) + .on(&scenario.plan) + .into_future(), + ) + .unwrap(); + prop_assert!( + !contained || pg_contains(&case.stored, &needle, true), + "a pruning of the whole document should be contained by it: {} @> {needle}", + case.stored + ); + prop_assert_eq!( + query.matches(doc.projection()), + pg_contains(&case.stored, &needle, true), + "{} @> {}", + &case.stored, + &needle + ); + Ok(()) + }); + } + + #[test] + fn is_contained_by_as_jsonb_except_a_root_scalar_in_a_root_array() { + let excluded = Cell::new(0); + run("<@", |scenario, runtime, case| { + let (needle, _) = case.needle(); + let doc = sealed(scenario, runtime, &case.stored); + let query: JsonQuery = runtime + .block_on( + scenario + .cipher + .query(op::ContainedBy, &needle) + .on(&scenario.plan) + .into_future(), + ) + .unwrap(); + let jsonb = pg_contains(&needle, &case.stored, true); + if jsonb && scalar_in_array(&case.stored, &needle) { + excluded.set(excluded.get() + 1); + } + prop_assert_eq!( + query.projection().contains(doc.projection()), + jsonb && !scalar_in_array(&case.stored, &needle), + "{} <@ {}", + &case.stored, + &needle + ); + Ok(()) + }); + assert!( + excluded.get() > 0, + "no stored root scalar sat inside a needle root array, so the `<@` exception went unexercised" + ); + } + + mod given_the_document_strategy { + use super::*; + + /// Every feature the property relies on, counted over a run's worth + /// of generated documents, so a change to the generator that stops + /// producing one fails here. + #[test] + fn produces_every_feature_the_property_relies_on() { + let mut runner = TestRunner::new(config()); + let strategy = strategies::document(); + let docs: Vec = (0..400) + .map(|_| strategy.new_tree(&mut runner).unwrap().current()) + .collect(); + assertions::covers_every_feature(&docs); + } + } +} + +// ============================================================================= +// Queries +// ============================================================================= + +mod given_a_containment_query { + use super::*; + + #[tokio::test] + async fn serializes_as_the_needle_operand_and_derives_no_key() { + let (cipher, generates, _) = common::counting_cipher().await; + let plan = plan(); + let query = cipher + .query(op::Contains, &json!({"a": [{"x": 1}]})) + .on(&plan) + .await + .unwrap(); + let wire = serde_json::to_value(&query).unwrap(); + assert_eq!( + wire["sv"].as_array().unwrap().len(), + 2, + "$ and $.a sit outside the element" + ); + assert_eq!( + wire["el"].as_array().unwrap().len(), + 1, + "one group for the one element" + ); + assert_eq!( + wire["el"][0]["sv"].as_array().unwrap().len(), + 2, + "the element and its x" + ); + assert!( + wire.get("alt").is_none(), + "an object needle carries no alternative: {wire}" + ); + assert_eq!( + generates.load(std::sync::atomic::Ordering::Relaxed), + 0, + "a query mints no key" + ); + + let scalar = cipher + .query(op::Contains, &json!("a")) + .on(&plan) + .await + .unwrap(); + let wire = serde_json::to_value(&scalar).unwrap(); + assert!( + wire.get("alt").is_some(), + "a root scalar carries its root-array alternative" + ); + } +} + +mod given_equality_at_the_root { + use super::*; + + /// At the root, equality is of the root value: `("$", 1)` matches a + /// stored `1` and not a stored `[1]`, which containment of the scalar + /// `1` would. + #[tokio::test] + async fn matches_the_root_value_and_not_an_array_holding_it() { + let Scenario { cipher, plan } = scenario().await; + let at_root: JsonQuery = cipher + .query(op::AtPath(op::Eq), &("$", 1)) + .on(&plan) + .await + .unwrap(); + let contains: JsonQuery = cipher + .query(op::Contains, &json!(1)) + .on(&plan) + .await + .unwrap(); + let scalar = cipher.encrypt(&json!(1)).using(&plan).await.unwrap(); + let array = cipher.encrypt(&json!([1])).using(&plan).await.unwrap(); + assert!( + at_root.matches(scalar.projection()), + "(\"$\", 1) matches a stored 1" + ); + assert!( + !at_root.matches(array.projection()), + "(\"$\", 1) does not match a stored [1]" + ); + assert!(contains.matches(array.projection()), "jsonb: [1] @> 1"); + assert!( + serde_json::to_value(&at_root).unwrap().get("alt").is_none(), + "equality at the root carries no alternative" + ); + } +} + +mod given_an_accessor { + use super::*; + + #[tokio::test] + async fn matches_the_value_at_its_path() { + let Scenario { cipher, plan } = scenario().await; + let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); + let projection = doc.projection(); + for (path, value, matches, what) in [ + ("$.nested.list[*].x", json!(2), true, "an element's x is 2"), + ("$.nested.list[*].x", json!(3), false, "no element's x is 3"), + ("$.nested.deep", json!("constant"), true, "the deep string"), + ("$.number", json!("1"), false, "a string is not the number"), + ] { + let query: JsonQuery = cipher + .query(op::AtPath(op::Eq), &(path, value)) + .on(&plan) + .await + .unwrap(); + assert_eq!(query.matches(projection), matches, "{what}"); + } + } + + /// An ordering at a path: the path's selector and a leaf term that + /// orders as the stored leaf's does, under native byte comparison. The + /// plan declares the paths it orders, and the query runs under them. + #[tokio::test] + async fn orders_as_the_stored_leaf_at_an_ordered_path() { + let Scenario { cipher, .. } = scenario().await; + let plan = ordered_plan(); + let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); + let number = &doc.entries()[9]; + let bound = cipher + .query(op::AtPath(op::Gte), &("$.number", 0)) + .on(&plan) + .await + .unwrap(); + assert_eq!( + bound.selector().as_bytes(), + number.selector(), + "the bound asks at $.number" + ); + assert!( + bound.ordering_term() < number.ordering_term().unwrap(), + "0 < 1" + ); + let above = cipher + .query(op::AtPath(op::Lt), &("$.number", 5)) + .on(&plan) + .await + .unwrap(); + assert!( + above.ordering_term() > number.ordering_term().unwrap(), + "5 > 1" + ); + let text = cipher + .query(op::AtPath(op::Gt), &("$.hello", "world-0")) + .on(&plan) + .await + .unwrap(); + assert!( + text.ordering_term() < doc.entries()[1].ordering_term().unwrap(), + "world-0 < world-1" + ); + let range = cipher + .query(op::AtPath(op::Between), &("$.number", (0, 5))) + .on(&plan) + .await + .unwrap(); + assert!( + range.lo().ordering_term() < range.hi().ordering_term(), + "the range's bounds are in order" + ); + assert_eq!( + range.lo().selector().as_bytes(), + number.selector(), + "the range asks at $.number" + ); + let wire = serde_json::to_value(range.lo()).unwrap(); + assert_eq!( + wire.as_object().unwrap().keys().collect::>(), + ["op", "s"], + "a bound's wire keys" + ); + } + + /// Refusals, before any term is derived: a text that is not a path, a + /// leaf kind that has no ordering, and a path the plan does not order + /// (no stored leaf there carries a term, so the comparison would match + /// no row; under the default plan that is every path). + #[tokio::test] + async fn refuses_a_bad_path_an_unordered_leaf_or_a_path_the_plan_does_not_order() { + let Scenario { + cipher, + plan: default_plan, + } = scenario().await; + let plan = ordered_plan(); + let bad_path = cipher + .query(op::AtPath(op::Eq), &("address.city", "Sydney")) + .on(&plan) + .await; + assertions::refused_by_term( + &bad_path, + |e| matches!(e, TermError::JsonPath(_)), + "a text that is not a path", + ); + let unordered = cipher + .query(op::AtPath(op::Gte), &("$.flag", true)) + .on(&plan) + .await; + assertions::refused_by_term( + &unordered, + |e| matches!(e, TermError::JsonLeafNotOrdered), + "a boolean has no ordering", + ); + let not_ordered = cipher + .query(op::AtPath(op::Lt), &("$.nested.deep", "z")) + .on(&plan) + .await; + assertions::refused_by_term( + ¬_ordered, + |e| matches!(e, TermError::JsonPathNotOrdered), + "a path the plan does not order", + ); + let not_ordered = cipher + .query(op::AtPath(op::Between), &("$.number", (0, 5))) + .on(&default_plan) + .await; + assertions::refused_by_term( + ¬_ordered, + |e| matches!(e, TermError::JsonPathNotOrdered), + "the default plan orders no path", + ); + } + + /// An ordering at a wildcard path is covered by the wildcard. One at a + /// position is refused whatever the plan orders: `->` reads a position + /// as the wildcard and would compare the first element, not the one + /// named. + #[tokio::test] + async fn orders_at_a_wildcard_and_refuses_a_position() { + let Scenario { cipher, .. } = scenario().await; + let by_element = Plan::value::() + .context("users/profile") + .encrypt_with_layout(Json::new(ordering(&["$.nested.list[*].x"]))) + .build() + .unwrap(); + let wildcard = cipher + .query(op::AtPath(op::Gt), &("$.nested.list[*].x", 0)) + .on(&by_element) + .await; + assert!( + wildcard.is_ok(), + "the wildcard the plan orders: {wildcard:?}" + ); + for path in ["$.nested.list[1].x", "$.nested.list[0].x"] { + let refused = cipher + .query(op::AtPath(op::Gt), &(path, 0)) + .on(&by_element) + .await; + assertions::refused_by_term( + &refused, + |e| matches!(e, TermError::JsonOrderingAtPosition), + &format!("{path} with Gt"), + ); + let refused = cipher + .query(op::AtPath(op::Between), &(path, (0, 5))) + .on(&by_element) + .await; + assertions::refused_by_term( + &refused, + |e| matches!(e, TermError::JsonOrderingAtPosition), + &format!("{path} with Between"), + ); + } + } + + /// Equality at a path compares one value: an object or an array would + /// be containment, which `op::Contains` asks for. + #[tokio::test] + async fn refuses_a_container_for_equality() { + let Scenario { cipher, .. } = scenario().await; + let plan = ordered_plan(); + for container in [json!({"deep": "constant"}), json!([]), json!([{"x": 1}])] { + let refused = cipher + .query(op::AtPath(op::Eq), &("$.nested", container.clone())) + .on(&plan) + .await; + assertions::refused_by_term( + &refused, + |e| matches!(e, TermError::JsonValueNotScalar), + &container.to_string(), + ); + } + } + + /// An accessor whose path or bounds the index refuses is refused when + /// the query is awaited, before the keyset it names is loaded; so is an + /// open that names a context the plan already has. + #[tokio::test] + async fn is_refused_before_its_keyset_loads() { + let (cipher, loads) = common::loads_counting_cipher().await; + let plan = ordered_plan(); + let eq = cipher + .query(op::AtPath(op::Eq), &("address.city", "Sydney")) + .on(&plan) + .keyset("tenant") + .await; + assertions::refused_by_term( + &eq, + |e| matches!(e, TermError::JsonPath(_)), + "Eq at a bad path", + ); + let between = cipher + .query(op::AtPath(op::Between), &("address.age", (1, 2))) + .on(&plan) + .keyset("tenant") + .await; + assertions::refused_by_term( + &between, + |e| matches!(e, TermError::JsonPath(_)), + "Between at a bad path", + ); + let unordered = cipher + .query(op::AtPath(op::Between), &("$.flag", (false, true))) + .on(&plan) + .keyset("tenant") + .await; + assertions::refused_by_term( + &unordered, + |e| matches!(e, TermError::JsonLeafNotOrdered), + "Between over booleans", + ); + let not_ordered = cipher + .query(op::AtPath(op::Lt), &("$.nested.deep", "z")) + .on(&plan) + .keyset("tenant") + .await; + assertions::refused_by_term( + ¬_ordered, + |e| matches!(e, TermError::JsonPathNotOrdered), + "a path the plan does not order", + ); + let at_position = cipher + .query(op::AtPath(op::Gte), &("$.nested.list[0].x", 1)) + .on(&plan) + .keyset("tenant") + .await; + assertions::refused_by_term( + &at_position, + |e| matches!(e, TermError::JsonOrderingAtPosition), + "an ordering at a position", + ); + let doc = seal(&stack_cipher().await, &fixture()).await; + let clash = cipher + .open(doc.clone()) + .context("users/other") + .using(&plan) + .keyset("tenant") + .await; + assertions::refused_for_two_contexts(&clash, "one document"); + let clash = cipher + .open(vec![doc]) + .context("users/other") + .using(&plan) + .keyset("tenant") + .await; + assertions::refused_for_two_contexts(&clash, "a batch"); + assert_eq!( + loads.load(std::sync::atomic::Ordering::SeqCst), + 0, + "no keyset was loaded for a refused query or open" + ); + // The same query with a path the index takes does load it. + let _ = cipher + .query(op::AtPath(op::Eq), &("$.address.city", "Sydney")) + .on(&plan) + .keyset("tenant") + .await + .unwrap(); + assert_eq!( + loads.load(std::sync::atomic::Ordering::SeqCst), + 1, + "a query the index takes loads the keyset" + ); + } +} + +// ============================================================================= +// Plans +// ============================================================================= + +mod given_a_plan { + use super::*; + + #[tokio::test] + async fn refuses_a_query_under_other_options() { + let Scenario { cipher, plan } = scenario().await; + let refused = cipher + .query(op::Contains, &json!({})) + .via(Json::new(open_root())) + .on(&plan) + .await; + assert!( + matches!(refused, Err(Error::Plan(PlanError::IndexOptions { .. }))), + "a query under other options is refused: {refused:?}" + ); + assert_eq!( + >::indexes(), + [IndexSpec::Json(JsonOptions::default())], + "a JSON document's index is the default JSON index" + ); + assert_eq!(Json, Json::default(), "the value is the default index"); + } + + #[tokio::test] + async fn opens_what_it_wrote_as_the_typed_open_does() { + let Scenario { cipher, plan } = scenario().await; + let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); + let typed = open(&cipher, doc.clone()).await.unwrap(); + let back: Value = cipher.open(doc).using(&plan).await.unwrap(); + assert_eq!(back, fixture(), "the plan opens what it wrote"); + assert_eq!(back, typed, "and the typed open agrees"); + } +} + +mod given_a_laid_out_plan { + use super::*; + + #[test] + fn reports_its_layout() { + let plan = ordered_plan(); + let options = ordering(&["$.hello", "$.number", "$.flag"]); + assert_eq!(plan.layout().options(), &options, "the layout's options"); + assert_eq!( + plan.specs(), + [IndexSpec::Json(options.clone())], + "the plan's index specs" + ); + assert_eq!( + stack_encrypt::target::Layout::::specs(&Json::new(options.clone())), + [IndexSpec::Json(options)], + "the layout's index specs" + ); + assert!( + format!("{plan:?}").contains("Laid(Json"), + "the plan's Debug names its layout: {plan:?}" + ); + } + + /// A plan laid out by the index opens what it wrote, one document or a + /// batch. + #[tokio::test] + async fn opens_what_it_wrote_one_document_or_a_batch() { + let Scenario { cipher, .. } = scenario().await; + let plan = ordered_plan(); + let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); + let back: Value = cipher.open(doc.clone()).using(&plan).await.unwrap(); + assert_eq!(back, fixture(), "one document"); + let docs: Vec = cipher + .open(vec![doc.clone(), doc]) + .using(&plan) + .await + .unwrap(); + assert_eq!(docs, [fixture(), fixture()], "a batch"); + } + + /// The plan was built with its context, so an open that names one is + /// refused, one document or a batch, before any key is requested. + #[tokio::test] + async fn refuses_an_open_that_names_another_context() { + let Scenario { cipher, .. } = scenario().await; + let plan = ordered_plan(); + let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); + let clash = cipher + .open(doc.clone()) + .context("users/other") + .using(&plan) + .await; + assertions::refused_for_two_contexts(&clash, "one document"); + let clash = cipher + .open(vec![doc]) + .context("users/other") + .using(&plan) + .await; + assertions::refused_for_two_contexts(&clash, "a batch"); + } + + /// The index named on the query says only which: the plan's own, with + /// its options, answers. + #[tokio::test] + async fn answers_a_named_index_with_its_own_options() { + let Scenario { cipher, .. } = scenario().await; + let plan = ordered_plan(); + let through_default = cipher + .query(op::AtPath(op::Gte), &("$.number", 0)) + .via(Json) + .on(&plan) + .await + .unwrap(); + let chosen = cipher + .query(op::AtPath(op::Gte), &("$.number", 0)) + .on(&plan) + .await + .unwrap(); + assert_eq!( + through_default.selector(), + chosen.selector(), + "the default index named on the query answers as the plan's own" + ); + } +} + +#[cfg(feature = "dynamic")] +mod given_the_json_index_spec { + use super::*; + + /// The spec's wire form is a fixed contract with stored data and the Go + /// guest, so its strings are pinned literally. + #[test] + fn has_a_wire_form() { + use stack_encrypt::dynamic::{not_admitted, Reason, ValueKind}; + assert_eq!( + IndexSpec::parse("json"), + Some(IndexSpec::Json(JsonOptions::default())), + "`json` parses as the default index" + ); + assert_eq!( + IndexSpec::Json(JsonOptions::default()).key(), + "json", + "the default index's key" + ); + assert_eq!( + serde_json::to_value(IndexSpec::Json(JsonOptions::default())).unwrap(), + json!("json"), + "the default index is written as its key" + ); + assert_eq!( + serde_json::to_value(IndexSpec::Json(open_root())).unwrap(), + json!({"json": {"sealed_root": false, "ordered": []}}), + "every option is written when any is not the default" + ); + let frame = + vitaminc_aead_frame::serde::to_vec(&json!({"json": {"sealed_root": false}})).unwrap(); + assert_eq!( + IndexSpec::from_frame(&frame).unwrap(), + IndexSpec::Json(open_root()), + "an option read from a frame" + ); + let frame = vitaminc_aead_frame::serde::to_vec(&json!({"json": {"nope": true}})).unwrap(); + assert!( + IndexSpec::from_frame(&frame).is_err(), + "an unknown option refuses the index" + ); + // The ordered paths ride in `ordered`, written in their canonical + // order and read back parsed; a text that is not a path refuses the + // index. + let ordered = IndexSpec::Json(ordering(&["$.orders[*].total", "$.age", "$.age"])); + assert_eq!( + serde_json::to_value(&ordered).unwrap(), + json!({"json": {"sealed_root": true, "ordered": ["$.age", "$.orders[*].total"]}}), + "ordered paths are written once, in canonical order" + ); + let frame = vitaminc_aead_frame::serde::to_vec( + &json!({"json": {"ordered": ["$.age", "$.orders[*].total"]}}), + ) .unwrap(); - assert_eq!(loads.load(std::sync::atomic::Ordering::SeqCst), 1); + assert_eq!( + IndexSpec::from_frame(&frame).unwrap(), + ordered, + "ordered paths read back parsed" + ); + let frame = + vitaminc_aead_frame::serde::to_vec(&json!({"json": {"ordered": ["age"]}})).unwrap(); + assert!( + IndexSpec::from_frame(&frame).is_err(), + "a text that is not a path refuses the index" + ); + // A position in an ordered path refuses the index too: no ordering + // can reach the leaf it names. + let frame = + vitaminc_aead_frame::serde::to_vec(&json!({"json": {"ordered": ["$.scores[2]"]}})) + .unwrap(); + assert!( + IndexSpec::from_frame(&frame).is_err(), + "a position in an ordered path refuses the index" + ); + assert_eq!( + not_admitted(ValueKind::Object, &IndexSpec::Json(JsonOptions::default())), + Some(Reason::JsonIndexPending), + "the data plan does not admit the JSON index yet" + ); + } +} + +// ============================================================================= +// Assertions +// ============================================================================= + +mod assertions { + use std::fmt::Debug; + + use super::*; + + pub(super) fn one_entry_per_node(doc: &JsonDocument, value: &Value, what: &str) { + assert_eq!( + doc.entries().len(), + nodes(value), + "{what}: one entry per node" + ); + } + + pub(super) async fn opens_to(cipher: &Cipher, doc: JsonDocument, expected: &Value, what: &str) { + let opened = open(cipher, doc).await; + assert!( + matches!(&opened, Ok(value) if value == expected), + "{what}: should open to {expected}, opened {opened:?}" + ); + } + + /// `result` is the term error `is` picks out. + pub(super) fn refused_by_term( + result: &Result, + is: impl Fn(&TermError) -> bool, + what: &str, + ) { + assert!( + matches!(result, Err(Error::Term(error)) if is(error)), + "{what}: refused with the wrong outcome: {result:?}" + ); + } + + /// `result` is a shape mismatch, naming `expected` when one is given. + pub(super) fn refused_as_opened_type( + result: &Result, + expected: Option<&str>, + what: &str, + ) { + assert!( + matches!(result, Err(Error::OpenedType { expected: got }) if expected.is_none_or(|e| e == *got)), + "{what}: should be refused as a shape mismatch expecting {expected:?}, got {result:?}" + ); + } + + /// `result` is tamper evidence: the authentication failed. + pub(super) fn refused_as_tampered(result: &Result, what: &str) { + assert!( + matches!(result, Err(Error::Aead)), + "{what}: should fail authentication, got {result:?}" + ); + } + + /// `result` is the refusal of an open that names a second context. + pub(super) fn refused_for_two_contexts(result: &Result, what: &str) { + assert!( + matches!( + result, + Err(Error::Plan(PlanError::TwoContextSources { .. })) + ), + "{what}: an open naming another context should be refused, got {result:?}" + ); + } + + /// `@>` over the projections is the model's `jsonb @>`. + pub(super) fn contains_as_jsonb( + query: &JsonQuery, + doc: &JsonDocument, + stored: &Value, + needle: &Value, + ) { + assert_eq!( + query.matches(doc.projection()), + pg_contains(stored, needle, true), + "{stored} @> {needle}" + ); + } + + /// `<@` is the model's `jsonb <@`, but for a stored root scalar in a + /// needle root array. + pub(super) fn is_contained_by_as_jsonb( + query: &JsonQuery, + doc: &JsonDocument, + stored: &Value, + needle: &Value, + ) { + assert_eq!( + query.projection().contains(doc.projection()), + pg_contains(needle, stored, true) && !scalar_in_array(stored, needle), + "{stored} <@ {needle}" + ); + } + + /// Every feature the generator is meant to cover appears in `docs`. + pub(super) fn covers_every_feature(docs: &[Value]) { + #[derive(Debug, Default)] + struct Census { + root_objects: usize, + root_arrays: usize, + root_scalars: usize, + empty_objects: usize, + empty_arrays: usize, + arrays_of_objects: usize, + arrays_of_arrays: usize, + duplicate_elements: usize, + keys_on_two_levels: usize, + float_spelled_integers: usize, + numeric_strings: usize, + booleans: usize, + nulls: usize, + max_depth: usize, + } + fn walk<'v>( + census: &mut Census, + v: &'v Value, + depth: usize, + keys: &mut std::collections::BTreeMap<&'v str, std::collections::BTreeSet>, + ) { + census.max_depth = census.max_depth.max(depth); + match v { + Value::Object(map) => { + census.empty_objects += usize::from(map.is_empty()); + for (key, member) in map { + let _ = keys.entry(key).or_default().insert(depth); + walk(census, member, depth + 1, keys); + } + } + Value::Array(items) => { + census.empty_arrays += usize::from(items.is_empty()); + census.arrays_of_objects += usize::from(items.iter().any(Value::is_object)); + census.arrays_of_arrays += usize::from(items.iter().any(Value::is_array)); + census.duplicate_elements += usize::from( + items + .iter() + .enumerate() + .any(|(i, a)| items[i + 1..].contains(a)), + ); + for item in items { + walk(census, item, depth + 1, keys); + } + } + Value::Number(n) => { + census.float_spelled_integers += + usize::from(n.is_f64() && n.as_f64().is_some_and(|f| f.fract() == 0.0)); + } + Value::String(s) => census.numeric_strings += usize::from(s.parse::().is_ok()), + Value::Bool(_) => census.booleans += 1, + Value::Null => census.nulls += 1, + } + } + let mut census = Census::default(); + for doc in docs { + match doc { + Value::Object(_) => census.root_objects += 1, + Value::Array(_) => census.root_arrays += 1, + _ => census.root_scalars += 1, + } + let mut keys = std::collections::BTreeMap::new(); + walk(&mut census, doc, 0, &mut keys); + census.keys_on_two_levels += usize::from( + keys.values() + .any(|depths: &std::collections::BTreeSet| depths.len() > 1), + ); + } + for (what, count) in [ + ("root objects", census.root_objects), + ("root arrays", census.root_arrays), + ("root scalars", census.root_scalars), + ("empty objects", census.empty_objects), + ("empty arrays", census.empty_arrays), + ("arrays of objects", census.arrays_of_objects), + ("arrays of arrays", census.arrays_of_arrays), + ("duplicate array elements", census.duplicate_elements), + ("a key on two levels", census.keys_on_two_levels), + ("integers spelled as floats", census.float_spelled_integers), + ("strings spelled as numbers", census.numeric_strings), + ("booleans", census.booleans), + ("nulls", census.nulls), + ] { + assert!( + count > 0, + "the generated documents should hold {what}: {census:?}" + ); + } + assert!( + census.max_depth >= 4, + "the generated documents should nest at least four levels: {census:?}" + ); + } }