diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index ea4b454da..0ce84f41f 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -113,11 +113,6 @@ exclude_re = [ 'stack-encrypt/src/json/mod\.rs:\d+:\d+: replace \| with \^ in unhex$', # `JsonPath::root` is `Self::default()`. 'stack-encrypt/src/json/path\.rs:\d+:\d+: replace JsonPath::root -> Self with Default::default\(\)$', - # The sealed root's `None` token: with that arm gone, the root falls to - # the arm that wraps its parent's token, and the root has no parent, so - # it is `None` there too. `the_sealed_root_has_no_token_and_its_fields_are_heads` - # pins the value; the arm stays so the match says it. - 'stack-encrypt/src/json/derive\.rs:\d+:\d+: delete match arm \(0, true\) in Walker<\x27_>::visit$', # A document always has a root, so its terms are never empty: `false` is # the value `is_empty` returns. The `true` replacement is still caught. 'stack-encrypt/src/json/derive\.rs:\d+:\d+: replace JsonTerms::is_empty -> bool with false$', diff --git a/.changeset/eql-json-tree-derivable-tokens.md b/.changeset/eql-json-tree-derivable-tokens.md new file mode 100644 index 000000000..f6501623e --- /dev/null +++ b/.changeset/eql-json-tree-derivable-tokens.md @@ -0,0 +1,9 @@ +--- +"@cipherstash/eql": minor +--- + +EQL v4 `json_tree` (unreleased): a subtree read sends the token of exactly the node it reads, and returns that node. Every token is now derivable from its node's path alone: an object member's is `H("t" ‖ t(parent) ‖ s)`, with no ordinal, and an array element's keeps its index. Every stored entry gains a self tag `x`, the first eight bytes of `H("x" ‖ iv ‖ t)`, by which the server finds the node a token names; the entry validator requires it, so a document written by the earlier build of this unreleased family is refused. A read of `$.profile.address.city` used to send `$.profile`'s token, a capability over that whole field in every row, and walk to the leaf through gates; it now sends the leaf's token and nothing that covers any node above it. + +**SQL signature changes.** `eql_v4.json_tree_subtree(doc, tokens text[], budget integer DEFAULT 64)` replaces `json_tree_subtree(doc, token, budget)` and `json_tree_subtree_at(doc, token, gates, budget)`, which is removed: for each token in order it returns the entry whose `x` matches at `d = 0`, then its descendants with `d` their depth below it, so `{}`, `[]`, a scalar and a missing node are four answers, as `jsonb` gives them (#1356). The budget counts rows across all the tokens. `json_tree_children(doc, token)` and `json_tree_child_count(doc, token)` lose their `gates` argument and take the token of any node; `json_tree_child_count` of a token that names no node is NULL, as `jsonb_array_length` of a missing path is (it was 0), and so is the sealed root's, whose top-level fields chain from nothing. `eql_v4_internal.json_tree_child_token` is replaced by `json_tree_member_token`, `json_tree_element_token`, `json_tree_self_tag`, `json_tree_hit_token` and `json_tree_index`, which maps every entry's `x`, and every chained entry's `p`, to its position in `sv`; `json_tree_by_parent` is removed. The chain primitives carry the `eql-inline-critical` marker, so the post-install step leaves their `search_path` unpinned and the planner inlines them: pinned, every probe of a walk was a full SQL-function call, which was most of a read's cost. + +A wildcard (`$.items[*].price`) is expanded by the client into one token per position below a bound, 64 by default, plus a probe at the bound; a row where the probe finds an element fails the read with the bound rather than return part of the list, walked or fallen back to the sealed root. A read of `$` is legal under the sealed root and returns the root entry, which opens to the whole document. Stored documents grow by about a tenth in characters (+9.8% on a nested document, +9.4% on a flat one). `JsonHeader::FORMAT_VERSION` and the document's `v` are unchanged. In `eql-bindings`, `JsonTreeEntry` gains `x` and `JsonTreeSubtree` opens to `Option`; in Stack Encrypt, `JsonSubtreeKey` carries the token list (`tokens()`, `probes()`, `bound()`), `KeysetCipher::json_subtree_key_with_bound` sets the bound, `Error::JsonWildcardTruncated` and `TermError::JsonWildcardTooWide` are new, and `TermError::JsonSealedRoot` is removed. What the database learns is in `docs/reference/json-tree-leakage.md`; the design and the decisions behind it are in `docs/plans/2026-10-17-json-tree-index.md`, "Design change (2026-10-24)". diff --git a/biome.json b/biome.json index 02cdf6475..a9413ab0b 100644 --- a/biome.json +++ b/biome.json @@ -14,6 +14,7 @@ "!packages/stack-encrypt/tests/fixtures/query_operators.json", "!packages/stack-encrypt/tests/fixtures/value_kinds.json", "!packages/stack-encrypt/tests/fixtures/stored_rows_v1.json", + "!packages/stack-encrypt/tests/fixtures/json_tree_vectors.json", "!packages/eql/tests/encryption/fixtures/ste_vec_selectors.json", "!packages/eql/tests/encryption/fixtures/v4_rows.json", "!packages/eql/tests/encryption/fixtures/stored_v4_rows_v1.json", diff --git a/docs/plans/2026-10-17-json-tree-index.md b/docs/plans/2026-10-17-json-tree-index.md index 75d8c6f0b..f8d869b20 100644 --- a/docs/plans/2026-10-17-json-tree-index.md +++ b/docs/plans/2026-10-17-json-tree-index.md @@ -276,7 +276,7 @@ On a flat five-node document the proposal is about 8% larger: the two chain fiel | size of every subtree | yes, each path entry's ciphertext is its subtree | no, only leaf scalar lengths, as the v3 leaf entries already show | | parent and child relation | inferable from subtree sizes and cross-row co-occurrence | hidden at rest; sibling grouping (`p`) and element grouping (`g`) visible at rest as per-document random tags; linkable only with a token the query supplies | | shape equality across rows | selectors are column-deterministic, so yes | tags are salted by the document IV, so no, until a token is supplied | -| which leaves sit in the same array element | no | yes, `g`. The one new leak, required for exact containment, and bounded to array elements | +| which leaves sit in the same array element | yes, whenever elements are indexed: document order plus the per-element `a` flag, or position selectors outright (corrected 2026-10-25; this row first said no) | yes, `g`. Required for exact containment, and bounded to array elements | | equal values at two paths | hidden | hidden | | equal values in sibling elements | ciphertexts equal (selectors already equal) | ciphertexts differ | | key names | hashed into `s` | hashed into `s`; also encrypted in `c`; under a token the query supplied, testable by guessing until the label step is keyed (decision 11) | @@ -522,7 +522,7 @@ Two of the three items that gate a release are closed here; the third, the final ## Design change (2026-10-24): derivable tokens, the self tag, and wildcard expansion on the client -**Status:** agreed with the owner on 2026-10-24 and not yet implemented. The implementation is stacked on this write-up (see "Plan" below), and the leakage page (`packages/eql/docs/reference/json-tree-leakage.md`) is revised here to describe the format after the change, so neither should reach `next` without the other. Tracked in the issue named in the pull request that adds this section. +**Status:** agreed with the owner on 2026-10-24 and implemented in the pull request stacked on this write-up (see "Plan" below), with the open questions this section first carried settled as "Decisions (2026-10-24)" at its end, which override anything above them. The leakage page (`packages/eql/docs/reference/json-tree-leakage.md`) describes the format after the change, so neither should reach `next` without the other. Tracked in #1360. ### Why @@ -588,32 +588,32 @@ eql_v4.json_tree_subtree(doc, tokens text[], budget integer DEFAULT 64) For each token in turn, it looks the entry up by `x` (one map of the document's entries keyed by `x`, built once per call beside the existing map keyed by `p`), returns it at depth 0, then returns its descendants by ordinal probing under the node's own token, deriving each hit's token as section 1 describes. Rows come out in token order, and within a token in document order. A token with no entry yields no row. `json_tree_children(doc, token)` and `json_tree_child_count(doc, token)` lose their gates and take any derivable node's token: they enumerate one level below it, which needs no child token and so no rule choice. -**The budget.** The walk counts rows across all the tokens of one call, the depth-0 rows included. With the root sealed and the count past the budget, it returns the root entry alone at depth 0, as today. The client opens the root to the whole document and evaluates the same concrete paths and the same probes over it. A probe that finds an element there is the same truncation error the walked read raises, so one row gives the same answer under the budget and over it, which is the property the Phase 3b review (#1329) held the fallback to. The client could answer a wildcard in full from the document, but then the answer would depend on the budget; see the open questions. With the root open there is no fallback, as today. +**The budget.** The walk counts rows across all the tokens of one call, the depth-0 rows included. With the root sealed and the count past the budget, it returns the root entry alone at depth 0, as today. The client opens the root to the whole document and evaluates the same concrete paths and the same probes over it. A probe that finds an element there is the same truncation error the walked read raises, so one row gives the same answer under the budget and over it, which is the property the Phase 3b review (#1329) held the fallback to. The client could answer a wildcard in full from the document, but then the answer would depend on the budget; decision 4 below keeps the bound. With the root open there is no fallback, as today. -**The walk's bound needs rewording.** "The bound" in `functions.sql` argues that no two rows share a token because a hit's token is a hash of its probe, which includes the ordinal. A member's token no longer includes it: two entries forged under one parent with the same `s` and the member self tag would derive one token, and the walk would visit the subtree beneath it twice. The walk's other limit, `LIMIT` the entry count plus one, still caps the rows, and the client refuses a repeated position (`j`) as it does today. The comment has to say so. +**The walk's bound needs rewording.** "The bound" in `functions.sql` argues that no two rows share a token because a hit's token is a hash of its probe, which includes the ordinal. A member's token no longer includes it: two entries forged under one parent with the same `s` and the member self tag would derive one token, and the walk would visit the subtree beneath it twice. The walk's other limit, `LIMIT` the entry count plus one, still caps the rows, and the client refuses a repeated position (`j`) as it does today. The comment has to say so. (Decision 9 below makes the read's limit the entry count times the number of tokens; `json_tree_children` and `json_tree_child_count`, which probe one level, keep the entry count plus one.) ### 4. The client's check after the change PR #1358 made the client open a walk's rows only through the key it derived and refuse any row that does not fit (`Error::JsonWrongEntry`). It adapts: - **Each depth-0 row** must carry `x = H8("x" ‖ iv ‖ t)` for one of the tokens sent, and its authenticated selector and decrypted label must agree with that token's path: the path's selector, and the key or index its last step names. -- **Each deeper row** must sit at the next consecutive position `j` within its subtree, and its decrypted label must agree with its selector, as #1358 checks today; and it must carry an `x` (and a `p`) that matches the child token the client recomputes from its parent's token, the row's selector and its label (the key for a member; the index, from the label, for an element). +- **Each deeper row** must sit at the next consecutive position `j` within its subtree, carry the selector of its parent's path extended by its decrypted label, and carry an `x` that matches the child token the client recomputes from its parent's token, the row's selector and its label (the key for a member; the index, from the label, for an element). `p` is not checked (decision 8). - The child counts in each container's ciphertext still catch a dropped row, and a repeated position is still refused. -`x` and `p` are not authenticated. The database holds every token a read sends and every row's `iv`, so it can write any `x` or `p` it likes. Matching them catches a row corrupted by someone who never saw the token, and a server that walks wrongly by mistake, but not a dishonest database. What binds a row to its place against the database is what the AAD covers: the selector, the position `j`, and the label and child count inside `c`. The check that each subtree's rows are consecutive in `sv` is on `j`, so it is one of those. Nothing replaces it: the `x` check is a consistency check, and the bound fields with the consecutive-position rule are the security check, as the leakage page says. Open question 1 shows what the rule refuses that nothing else does. +`x` and `p` are not authenticated. The database holds every token a read sends and every row's `iv`, so it can write any `x` or `p` it likes. Matching them catches a row corrupted by someone who never saw the token, and a server that walks wrongly by mistake, but not a dishonest database. What binds a row to its place against the database is what the AAD covers: the selector, the position `j`, and the label and child count inside `c`. The check that each subtree's rows are consecutive in `sv` is on `j`, so it is one of those, and it stays beside the `x` check (decision 8). Nothing replaces it: the `x` check is a consistency check, and the bound fields with the consecutive-position rule are the security check, as the leakage page says. Decision 8 shows what the rule refuses that nothing else does. Still allowed by design (ADR 0010, `packages/stack-encrypt/docs/adr/0010-what-a-stored-value-authenticates.md`): an entry swapped in from another row under the same column context, returned with that row's header, opens. Row provenance is the database's word. ### 5. Wire -A stored entry becomes `{s, c, x, p?, op?}`: `x` on every entry, `p` below the heads as now. The domain CHECK on `public.eql_v4_json_tree_entry` and the `JsonTreeEntry` newtypes in `packages/eql/crates/eql-bindings/src/v4/json_tree.rs` gain `x` as padded base64 of eight bytes. Nothing else in `h`, `sv` or `q` changes. Counted informally, this is the format's sixth stored form, revision 5 to 6; no field records that number (see the open questions). EQL v4's `json_tree` is unreleased, so nothing stored breaks: every fixture, every pinned vector and every benchmark table is regenerated. +A stored entry becomes `{s, c, x, p?, op?}`: `x` on every entry, `p` below the heads as now. The domain CHECK on `public.eql_v4_json_tree_entry` and the `JsonTreeEntry` newtypes in `packages/eql/crates/eql-bindings/src/v4/json_tree.rs` gain `x` as padded base64 of eight bytes. Nothing else in `h`, `sv` or `q` changes. Counted informally, this is the format's sixth stored form, "revision 5 to 6"; that is a narrative label only, and no field records it: `JsonHeader::FORMAT_VERSION` stays `1` and the document's `v` stays `4` (decision 5). EQL v4's `json_tree` is unreleased, so nothing stored breaks: every fixture, every pinned vector and every benchmark table is regenerated. ### 6. Leakage after the change `packages/eql/docs/reference/json-tree-leakage.md` carries the full statement; in short: - **A subtree read sends exactly the tokens of the nodes asked for.** A read of `$.profile.address.city` sends that leaf's token and nothing that covers `$.profile`. No capability over a top-level field or an array leaves the client unless that field or array is itself what is read. -- **A wildcard read sends `N + 1` tokens for each array it expands**, `N² + N + 1` in all for two levels. The database learns which of them exist in each row, so, up to `N`, which elements hold the rest of the path (each array's length, where the path ends at the element, which the stored entry order already shows, #1357), whether each array is longer than `N`, and the structure beneath each node found. It learns nothing about the elements' sibling fields. One exception: a probe that hits is a token for the element at index `N`, and the server enumerates that element's subtree in each row where it exists, although the read then fails. The exception is in the token, not in what the SQL does with it: whoever holds an element's token can derive that element's subtree in any row, now or in rows written later, so it holds for every variant of the read that sends the element's token (open question 3). +- **A wildcard read sends `N + 1` tokens for each array it expands**, `N² + N + 1` in all for two levels. The database learns which of them exist in each row, so, up to `N`, which elements hold the rest of the path (each array's length, where the path ends at the element, which the stored entry order already shows, #1357), whether each array is longer than `N`, and the structure beneath each node found. It learns nothing about the elements' sibling fields. One exception: a probe that hits is a token for the element at index `N`, and the server enumerates that element's subtree in each row where it exists, although the read then fails. The exception is in the token, not in what the SQL does with it: whoever holds an element's token can derive that element's subtree in any row, now or in rows written later, so it holds for every variant of the read that sends the element's token (decision 3). - **Unchanged:** a token is still column-wide and still does not expire; the `->` extraction path and containment send and reveal what they do today; at rest, `x` adds nothing beyond what `p` already shows; and the stored-order disclosure (#1357) is untouched by this change and stays a separate decision. ### 7. Plan @@ -634,13 +634,68 @@ A stored entry becomes `{s, c, x, p?, op?}`: `x` on every entry, `p` below the h **Merge order.** #1345 (ready) rewrites `packages/stack-encrypt/tests/json_index.rs`, which #1358 also touches, and should merge first; #1358 then rebases, and this branch and the implementation follow it. -### Open questions - -1. **The consecutive-position check stays; `x` does not replace it.** `x` and `p` are unauthenticated, and the database can compute both from the tokens it was sent and the row's `iv`, while `j` is bound by the AAD. So matching them is a consistency check, and the check that each subtree's rows are consecutive in `sv` remains part of the security check. It is what refuses a node spliced in from under another element of the same array. A read of `$.items[0]` returns the element at depth 0 and then its members. Answered with the element's own entry followed by `$.items[1].price` in place of `$.items[0].price`, with `x` rewritten to the self tag the client recomputes for `$.items[0].price`, it passes every other check: every element shares the wildcard's selector, so the selector is right; the label is `price`; and the element's child count is met. Only the position refuses it, because a member must sit at the position after the row before it. The rule does not reach a node that is itself the depth-0 row: a read of `$.items[1].price` is one row with nothing before it to fix its position, so `$.items[0].price` with its `x` rewritten opens in its place, as the leakage page says. Proposal: keep the consecutive check and add the `x` check beside it. Checking `p` adds nothing over `x`, and for a member it needs every sibling's key to recompute the rank, so it could be dropped. -2. **The server needs the self tag to tell a member from an element** (section 1). The two rules leave the walk unable to choose which one derives a hit's token. Section 1 resolves it by testing the member candidate against `x`, at up to four hashes per hit instead of one. The alternative, a stored kind bit, would reveal at rest which entries are array elements. Confirm the choice, and the cost once measured. -3. **A probe that hits unlocks one element the caller did not ask for.** A probe is the token of the element at index `N`, so where that element exists the server enumerates its subtree, and the leakage page has to carve out an exception to "exactly the nodes asked for". The capability is in the token, not in what the SQL does with it. Whoever holds `t($.items[N])` can find that element in any row by its self tag, probe for its children's parent tags, and derive every descendant's token by the public derivations of sections 1 and 2, whatever the function returns, and can keep doing so in other rows and in rows written later. So a separate `probes text[]` argument answered with existence only would not remove the exception: it holds for every variant that sends the element's token. Only a different derivation would. For example, the client could send `H16("e" ‖ t)` in place of the token, with each entry storing a tag the server can match from that value (an existence tag beside `x`, or `x` itself derived from it), so that the server learns whether the element exists in each row but cannot derive its token, its parent-tag probes or anything beneath it. That is a change to the stored format and a future option, not this design. It would also tell the server which values are probes, which it can already pick out: a probe is the last token of its level, and the token count gives the bound away. Proposal: send probes as ordinary tokens and keep the exception. -4. **The fallback's answer to a long array.** Section 3 has the fallback apply the same bound, so one row's answer does not depend on the budget. The client could instead resolve the wildcard in full from the document it holds. The two answer differently for a row with an array longer than `N` under the budget versus over it. Decide which. -5. **No field records the wire revision.** The header's `JsonHeader::FORMAT_VERSION` is still `1` and the document's `v` is `4` through all five earlier stored forms. `packages/stack-encrypt/src/json/mod.rs` says a changed derivation is a new version, never an edit in place, but the chain step is a plain SHA-256 with the domain byte `"t"`, not a PAE domain label that can be bumped. Since nothing is released, the choice is between editing in place (as the earlier forms did) and bumping `FORMAT_VERSION` to `2` so that a reader refuses an old document instead of misreading it. -6. **A read of `$` under the sealed root becomes legal.** Today `json_subtree_key` refuses it (`TermError::JsonSealedRoot`), and the root has no token. With `x` on the root, `$` returns the root entry, which is what the fallback already returns. Confirm that `$` should become a read rather than stay refused. -7. **`json_tree_child_count` of an absent node.** Today it is `0` for a token that names no node. `jsonb_array_length` of a missing path is NULL, and with `x` the function can tell an absent node from an empty one. The reasoning of #1356 suggests NULL. -8. **The leakage page describes the format after the change.** It otherwise describes the format as built, so this branch must not reach `next` ahead of the implementation. Stacking the implementation on this branch is the proposed way to make sure. +### Decisions (2026-10-24) + +The owner settled the questions this section first left open, before implementation. Where a decision differs from the text above, the decision holds, and the implementation follows it. + +1. **Tokens.** `t(member) = H16("t" ‖ t(parent) ‖ s(member))`, with no ordinal; `t(element) = H16("t" ‖ t(array) ‖ s(element) ‖ LE32(i))`, `i` the index; the root's token and, under the sealed root, the heads' are PRFs of their paths, as before. The parent tag `p = H8("p" ‖ iv ‖ t(parent) ‖ LE32(o))` is unchanged, `o` the keyed rank for a member and the index for an element, and is used to enumerate children only. +2. **The self tag, and no kind bit.** `x = H8("x" ‖ iv ‖ t(self))` on every entry, the root and the heads included. Nothing stored says whether a node is a member or an element: when the server enumerates beneath a node and finds a hit at ordinal `o`, it derives the member rule's token and tests it against the hit's `x`, and on a miss tries the element rule's (`eql_v4_internal.json_tree_hit_token`). Up to four hashes per hit; the benchmarks measure what that costs. +3. **Probes are ordinary tokens.** No separate `probes` argument: for each array a wildcard expands the client sends the tokens of the positions `0 … N − 1` and the probe at `N`, the element itself (`t($.items[N])` for `$.items[*].price`). A row at a probe token has an array longer than `N`, and the client fails loudly, `Error::JsonWildcardTruncated { bound }`, never with a partial answer. `N` defaults to 64 (`JsonSubtreeKey::DEFAULT_BOUND`) and is set per read with `KeysetCipher::json_subtree_key_with_bound`; it is a read parameter, not part of the index's options, so the stored form and the data plan do not change with it. Nested wildcards multiply, and a read that would send more than 8,192 tokens is refused before any is derived (`TermError::JsonWildcardTooWide`): every one-level wildcard up to 8,191 elements, two levels up to a bound of 90, never three (the cap was 65,536 until the owner lowered it on 2026-10-25; see the implementation notes below). The exception the leakage page states stands: a probe that hits has the server enumerate that element's subtree, although the read then fails. The exception is in the token, not in what the SQL does with it: whoever holds `t($.items[N])` can find that element in any row by its self tag, probe for its children's parent tags and derive every descendant's token by the public derivations of sections 1 and 2, whatever the function returns, in rows written later too. So a separate argument answered with existence only would not remove it; it holds for every variant that sends the element's token. An existence-only probe needs a different derivation, for example a one-way value such as `H16("e" ‖ t)` sent in place of the token, with each entry storing a tag the server matches from it (beside `x`, or `x` itself derived from it): a change to the stored format and a future option, not this design. +4. **The fallback applies the same bound.** When a read falls back to the sealed root, the client evaluates the same concrete paths and the same probes over the whole document, so one row's answer never depends on the budget, the truncation error included. +5. **No version bump.** `JsonHeader::FORMAT_VERSION` stays `1` and the document's `v` stays `4`; the derivation is edited in place, as the earlier stored forms were. "Revision 5 to 6" above is a narrative label. EQL v4's `json_tree` is unreleased, so nothing stored breaks; every fixture, pinned vector and benchmark table is regenerated. +6. **`$` is a read under the sealed root.** It sends the root's token, which finds the root entry at depth 0, which opens to the whole document. `TermError::JsonSealedRoot` had no other use and is removed; `->` never used it. +7. **Children of an absent node.** `json_tree_child_count` of a token that names no node is NULL (it was 0); `json_tree_children` of one returns no rows. Both take the token of any derivable node. Under the sealed root the root's members chain from nothing, so the root's own token enumerates none: its count is NULL too, not a misleading 0. +8. **The client's check.** The depth-0 row's `s` must be the path's selector and its decrypted label the path's last step (the key, the index, or the root for `$`); each deeper row must sit at the next consecutive `j` within its subtree, with `s` the selector of its parent's path extended by its decrypted label; and every row's stored `x` must be `H8("x" ‖ iv ‖ t)` for the token the client recomputes. The `p` check is dropped. `x` is unauthenticated, so the `x` check is a consistency check that catches server bugs, not substitution; the bound fields `s`, `j` and the label are the security check. The consecutive-position rule is what refuses a node spliced in from under another element of the same array: a read of `$.items[0]` answered with the element's own entry and then `$.items[1].price` in place of `$.items[0].price`, its `x` rewritten, has the right selector, label and child count, and only its position is wrong. Still allowed by design (ADR 0010): an entry from another row, returned with its header; and a node at the same selector under another element when that node is the depth-0 row (`$.items[0].price` served for a read of `$.items[1].price`, with no row before it to fix its position), which no bound field tells apart. Also beyond any check: a database that withholds rows, so that a read opens as no node, or a wildcard read without some elements. The leakage page's "What is authenticated" says all three. +9. **No gates.** `json_tree_subtree_at` is dropped. `json_tree_subtree(e, tokens text[], budget integer DEFAULT 64)` returns, for each token in order, the entry whose `x` matches at `d = 0` and its descendants with `d` their depth below it, in document order within a token. The budget counts rows across all tokens, the depth-0 rows included; past it with the root sealed, the root entry alone at `d = 0`. The recursion is still read through a `LIMIT`, now the entry count times the number of tokens, since forged member tags can make one token's subtree be visited more than once ("The bound" in `functions.sql`). Wanting a whole array is a read of the array's path, which sends the array's own token. +10. **Stacking.** The implementation stacks on this write-up's branch and its pull request targets it. #1345 conflicts with #1358 and merges first. + +**Implemented.** The client's reads, checks and the fallback are in `packages/stack-encrypt/src/json/derive.rs`; the key and its expansion in `derive.rs` and `query.rs`; the SQL in `packages/eql/src/v3/json_tree/functions.sql`. The stored form gains `x` and is otherwise unchanged; a sealed document grows by about a tenth in characters (+9.8% on the nested benchmark shape and +9.4% on the flat one, pinned in `packages/stack-encrypt/tests/json_properties.rs`). The derivations are pinned under the deterministic key source in `packages/stack-encrypt/tests/json_tree_vectors.rs`. The PostgreSQL oracle (`packages/eql/tests/encryption/tests/json_tree.rs`) holds every read with no wildcard to `jsonb_path_query_first` in strict mode, wildcard reads to `jsonb_path_query`, the child functions to `jsonb_object_keys` and `jsonb_array_length`, and every read of a randomly tampered row to either the right answer, a refusal, or one of the allowances of decision 8. + +### Implementation notes (2026-10-25): what the walk was paying for + +The benchmarks of the implementation (cipherstash/benches `report/JSON_TREE_DERIVABLE_TOKENS.md`, summarised on #1363) found the token-specific reads faster than the gated walk and everything that enumerates slower: whole-subtree reads +38 to +46%, a full walk of a 300-element array +78%, `json_tree_child_count` on a small node +59%, with the per-hit cost inside a walk (about 36 µs per member, 80 µs per element) well above what the hashes explain. Reading the plan found four costs, none of them in the format. All four are SQL-only and fixed in #1363; the stored document is unchanged. + +1. **A pinned `search_path` disabled inlining of every chain primitive.** The post-install step (`tasks/pin_search_path_v3.sql`) sets `search_path` on every EQL function that does not carry the `eql-inline-critical` marker or take a jsonb-domain argument, and the planner does not inline a SQL function with a `SET` clause. So `json_tree_h16`, `json_tree_le32`, `json_tree_parent_tag`, `json_tree_self_tag`, `json_tree_member_token`, `json_tree_element_token` and `json_tree_hit_token` each ran as a full SQL-function call, about ten microseconds, where the hashes inside cost under one. A level of the walk makes about fourteen such calls, and `EXPLAIN ANALYZE` put the recursive step at 160 µs per level on the 300-wide corpus. The gated walk paid the same way, which is why the derivation's cost was invisible in the first comparison. The seven now carry the marker; the step is 14 µs per level. + +2. **`json_tree_hit_token` could not inline even unpinned.** Its body held a `FROM` clause, which blocks inlining; rewritten without one it was still refused, twice: as a STRICT function whose body is a `CASE`, which the planner inlines only when every argument is provably non-null, and then because its arguments, in the recursive term, were the expression that locates a hit, which the planner had flattened into the call, and a function that uses an expensive argument more than once is not inlined. It is now not STRICT, and the walk derives a hit's token where it consumes it, in the probe beneath, from the hit's stored `s` and `x` read off the working table, which are plain columns. An `OFFSET 0` fence on the lateral subqueries also inlined it, but introduced a `Memoize` node keyed on the whole document, at 64 µs per row. + +3. **The maps copied every entry, and were built twice.** `json_tree_by_parent` and `json_tree_by_self` each aggregated `elem || {"j": …}` over `sv`, 0.55 ms per map on a 215-entry document, and every read built both; `json_tree_child_count` built them, then called `json_tree_children`, which built them again. One function, `json_tree_index`, now builds both maps as tag → position in one pass (0.14 ms), set-returning with the two maps as columns so the planner inlines it as an aggregate subquery of the caller's `LATERAL`, which is computed once: a scalar function the `doc` CTE named twice (`idx -> 'p'`, `idx -> 'x'`) was evaluated twice when the planner flattened the subquery. An entry is fetched as `sv -> j` only when it is returned. + +4. **Every reference to the argument decompressed the document.** The stored value is TOAST-compressed, and the `doc` CTE referenced `e::jsonb` five times (header, `sv`, metadata, entry count, index), each a decompression of the whole document, about 40 µs for a nested one: 0.2 ms of a 0.6 ms one-token read. Each read now takes `e::jsonb #> '{}'`, the identity path, into a MATERIALIZED CTE and reads that. + +Measured on the benchmark's own pass queries (500 documents per scenario, `id = ANY(…)`, in-server time per document, three interleaved repetitions, medians), the SQL as #1363 first had it against the bundle built from the tree, on the same rows on one cluster; every scenario returned identical rows from both: + +| read | gated walk (bench) | derivable, as benched | derivable, now | change | +|---|--:|--:|--:|--:| +| `read/first_name` | 0.06 ms | 0.10 ms | 0.11 ms | +8.9% | +| `read/company.name` | 1.28 ms | 1.27 ms | 0.39 ms | -69.5% | +| `read/user.contact.emergency_contact.name` | 1.76 ms | 1.27 ms | 0.37 ms | -70.5% | +| `read/orders[2]` | 2.89 ms | 4.26 ms | 0.83 ms | -80.5% | +| `read/orders[2].status` | 1.24 ms | 1.26 ms | 0.37 ms | -70.3% | +| `read/company` | 1.36 ms | 2.06 ms | 0.49 ms | -76.2% | +| `read/user` | 1.91 ms | 2.77 ms | 0.59 ms | -78.6% | +| `read/orders (fallback)` | 5.60 ms | 8.64 ms | 0.84 ms | -90.3% | +| `read/orders (walked)` | 16.76 ms | 24.03 ms | 3.57 ms | -85.1% | +| `read/$ (sealed root)` | – | 1.26 ms | 0.39 ms | -69.1% | +| `read/$ (root open)` | 20.01 ms | 28.21 ms | 4.20 ms | -85.1% | +| `wildcard/orders[*].total N=8` | 4.62 ms | 2.41 ms | 0.60 ms | -75.0% | +| `wildcard/orders[*].total N=64` | 4.47 ms | 1.79 ms | 0.50 ms | -71.9% | +| `wildcard/orders[*].total N=256` | 4.61 ms | 2.81 ms | 0.66 ms | -76.5% | +| `wildcard/orders[*].items[*].price N=8` | 16.06 ms | 3.57 ms | 0.95 ms | -73.3% | +| `wildcard/orders[*].items[*].price N=64` | 16.30 ms | 25.03 ms | 4.08 ms | -83.7% | +| `children/orders (nested)` | – | 1.35 ms | 0.42 ms | -68.6% | +| `child_count/orders (nested)` | – | 2.39 ms | 0.31 ms | -87.1% | +| `children/attrs` | 24.13 ms | 12.33 ms | 5.12 ms | -58.4% | +| `child_count/attrs` | 24.47 ms | 14.27 ms | 1.20 ms | -91.6% | +| `read/attrs (walked)` | 29.82 ms | 40.74 ms | 7.22 ms | -82.3% | +| `children/list` | 24.45 ms | 12.59 ms | 5.08 ms | -59.7% | +| `child_count/list` | 24.28 ms | 14.23 ms | 1.21 ms | -91.5% | +| `read/list (walked)` | 30.16 ms | 55.61 ms | 8.22 ms | -85.2% | + +The gated-walk column is the benchmark's in-server figure for the same scenario, from a different run, and is there for orientation only. Every row the first comparison had as a regression against the gated walk is now faster than it. The `cipherstash/benches` harness measures `json_tree_by_parent` and `json_tree_by_self` as floors; both are gone, so its plan step must move to `json_tree_index` before the next full run. + +**For a future version of the format**, recorded here from the review of #1363 (none is in scope for this one): + +- *The sealed root's children enumerate from nothing.* The top-level entries carry no parent tag, because the format predates the root having a token, so `json_tree_children` at `$` returns no rows and `json_tree_child_count` returns NULL on the default layout, and a developer counts top-level fields by reading `$` and opening it. Chaining the heads from the root's token, `p = H("p" ‖ iv ‖ t($) ‖ LE32(o))`, would make the root enumerate like every other node. **Decided against for now (owner, 2026-10-25):** a walk from the root is a capability over the whole document's shape that the format deliberately withholds, so adding it increases leakage; SteVec offered nothing like it either, and it waits for a customer to ask. +- *Withheld rows are not detected.* Nothing in a row says how many nodes a read should find, so a database that drops rows makes a node look absent, or shortens a wildcard read. This is an integrity property against a lying database, outside the current threat model (confidentiality against an honest-but-curious one); closing it needs an authenticated count, for instance the array's child count read beside a wildcard. +- *The entries × tokens `LIMIT` and the token cap bound a lying database's own work.* Forged member tags can make a walk visit one subtree repeatedly; the client refuses the repeated positions, so only the database's work is at stake, and with the default sealed root and budget the read stops at `budget + 1` rows regardless. The `LIMIT` is the only bound when the budget is 0 or the root is open, and the token cap (`TermError::JsonWildcardTooWide`) scales it: a one-level wildcard with bound N sends N + 1 tokens and two levels send N² + N + 1, so the cap is 8,192 (owner, 2026-10-25; it was 65,536, which no measurement approached, the largest read benchmarked sending 4,161): every one-level read up to 8,191 elements, two levels up to a bound of 90, the default of 64 included, and no third level. diff --git a/packages/eql/crates/eql-bindings/CHANGELOG.md b/packages/eql/crates/eql-bindings/CHANGELOG.md index 69ce58852..aa5882c69 100644 --- a/packages/eql/crates/eql-bindings/CHANGELOG.md +++ b/packages/eql/crates/eql-bindings/CHANGELOG.md @@ -13,17 +13,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 feature). `JsonTreeDocument` is `public.eql_v4_json_tree`, `{v, i, h, sv, q}`: the document's entries, one per JSON node, and `q`, its containment projection, which the server compares with a needle by native `jsonb @>`. - `JsonTreeEntry` is one entry as `->` and the walks return it, + `JsonTreeEntry` is one entry as `->` and the subtree reads return it, + `{s, c, x, p?, op?}` with the extractors' grafted fields, `JsonTreeQuery` the containment needle and `JsonTreeLeafQuery` the - ordering operand at a path. `JsonTreeSubtree` collects a walk's rows; past - the walk's budget the server returns the sealed root instead, which - opens at the path the walk's key was derived from: the rows are built + ordering operand at a path. `JsonTreeSubtree` collects a read's rows + (`eql_v4.json_tree_subtree(doc, key.tokens(), budget)`) and opens to the + value at the path as a `Value`, to `Option` (`None` where the row + has no node there) or to one value per concrete path as a `Vec`; + past the read's budget the server returns the sealed root instead, which + opens to the same answer under the read's key: the rows are built with `JsonTreeSubtree::from_key(&key, rows)`, and there is no constructor without the key. An entry `->` returned opens through the selector that asked for it, `JsonTreeExtracted::from_selector(&selector, entry)`, which refuses an entry carrying another selector (`stack_encrypt::Error::JsonWrongEntry`), - and a walk's rows through the key that keyed it, which checks each row + and a read's rows through the key that keyed it, which checks each row against the path the same way; `JsonTreeEntry` does not open on its own. The document is an `EncryptFrom` and `DecryptInto` target, the extracted entry and the subtree open through `DecryptInto`, and the needle and @@ -33,7 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 seals with `keyset.run(JsonTree::new(options).encryption(), ..)`), such as the paths that carry an ordering term. The wire fields are typed apart from the v3 hex terms, and each names its text encoding: - `JsonTreeSelector` (`s`, and the pairs of `q`), `JsonTreeTag` (`p`), + `JsonTreeSelector` (`s`, and the pairs of `q`), `JsonTreeTag` (`x` and `p`), `JsonTreeOrderingTerm` (`op`) and `JsonTreeKeyHeader` (`h`) hold an `encoding::Base64Text`, padded standard base64, and `JsonTreeCiphertext` (`c`) an `encoding::Base85Text`, RFC 1924 base85. Each encoding newtype diff --git a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs index ba0f77eb7..9364c45bb 100644 --- a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs +++ b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs @@ -48,10 +48,13 @@ pub struct JsonTreeSelector(pub Base64Text); #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, TS, JsonSchema)] pub struct JsonTreeCiphertext(pub Base85Text); -/// An eight-byte parent tag `p`, in [`Base64Text`]: the first eight bytes -/// of the parent-tag hash, per document, salted by the document's `iv`, so -/// unlinkable across rows at rest. `eql_v4.json_tree_subtree_at` derives -/// the same text with `encode(substr(…, 1, 8), 'base64')` to find a child. +/// An eight-byte chain tag in [`Base64Text`]: an entry's self tag `x`, the +/// first eight bytes of `H("x" ‖ iv ‖ t)` for the entry's own token `t`, or +/// its parent tag `p`, the first eight bytes of +/// `H("p" ‖ iv ‖ t(parent) ‖ LE32(o))`. Salted by the document's `iv`, so +/// unlinkable across rows at rest. `eql_v4.json_tree_subtree` derives the +/// same text with `encode(substr(…, 1, 8), 'base64')` to find the node a +/// token names (`x`) and its children (`p`). #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, TS, JsonSchema)] pub struct JsonTreeTag(pub Base64Text); @@ -91,14 +94,15 @@ pub struct JsonTreeDocument { } /// `public.eql_v4_json_tree_entry`: one node. `s` is the path selector, -/// `c` the node's own ciphertext; `p` the parent tag below the chain's -/// heads, which the walk probes by ordinal; `op` the leaf ordering term. +/// `c` the node's own ciphertext, `x` the self tag by which a read finds +/// the node its token names; `p` the parent tag below the chain's heads, +/// which a read probes by ordinal; `op` the leaf ordering term. /// `h`, `j`, `d`, `v` and `i` are what the extractors graft onto an entry /// they return, so that it can be opened. /// /// An entry a query returned is the database's word and does not open on /// its own: open the entry `->` returned through the selector that asked -/// for it ([`JsonTreeExtracted::from_selector`]), and a walk's rows through +/// for it ([`JsonTreeExtracted::from_selector`]), and a read's rows through /// the key that keyed it ([`JsonTreeSubtree::from_key`]). Each checks that /// the rows are the ones it asked for. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] @@ -106,6 +110,7 @@ pub struct JsonTreeDocument { pub struct JsonTreeEntry { pub s: JsonTreeSelector, pub c: JsonTreeCiphertext, + pub x: JsonTreeTag, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] pub p: Option, @@ -207,18 +212,23 @@ impl JsonTreeExtracted { } } -/// The rows a subtree walk returned (`eql_v4.json_tree_subtree` or -/// `json_tree_subtree_at`), collected with the key that keyed the walk: -/// opens to the value under the walked path, or to the list of complete -/// subtrees the rows hold. +/// The rows a subtree read returned, +/// `eql_v4.json_tree_subtree(doc, key.tokens(), budget)`, collected with the +/// key that keyed the read. Every read returns each node a token names at +/// depth `0` with everything beneath it, so the rows open to the value at +/// the path, `{}`, `[]` and scalars included; to `None` as an +/// `Option` when the row has no node there; or to a list, one value +/// per concrete path a wildcard reaches. A wildcard read of an array longer +/// than the key's bound fails with +/// [`Error::JsonWildcardTruncated`](stack_encrypt::Error::JsonWildcardTruncated) +/// rather than open to part of the answer. /// -/// Past its budget a walk returns the sealed root alone, at depth `0`; -/// those rows open to the whole document, and the key's path selects the -/// same value(s) the walked rows open to. The key also says which walk the -/// rows came from (no gates: `json_tree_subtree`, the rows below the path's -/// node; gates: `json_tree_subtree_at`, the nodes the path reaches), so the -/// only constructor, [`from_key`](Self::from_key), takes it, and rows -/// cannot be opened without it. +/// Past its budget a read returns the sealed root alone, at depth `0`; +/// those rows open to the whole document, and the key's concrete paths and +/// probes are evaluated over it to the same answers. The only constructor, +/// [`from_key`](Self::from_key), takes the key, which checks every row +/// against its path when the rows are opened, so rows cannot be opened +/// without it. #[derive(Clone, Debug, PartialEq)] pub struct JsonTreeSubtree { rows: Vec, @@ -226,7 +236,7 @@ pub struct JsonTreeSubtree { } impl JsonTreeSubtree { - /// The rows the walk `key` keyed returned, as the server sent them. + /// The rows the read `key` keyed returned, as the server sent them. pub fn from_key(key: &JsonSubtreeKey, rows: Vec) -> Self { Self { rows, @@ -239,7 +249,7 @@ impl JsonTreeSubtree { &self.rows } - /// The path the walk was keyed from. + /// The path the read was keyed from. pub fn path(&self) -> &JsonPath { self.key.path() } @@ -455,8 +465,8 @@ impl DecryptInto for JsonTreeExtracted { } } -/// The rows of `json_tree_subtree(doc, token)` open to the value under the -/// walked node. +/// The rows of a read open to the value at its path; no node there is an +/// error, which `Option` reads as `None`. impl DecryptInto for JsonTreeSubtree { type Context = NonEmpty; fn decryption(self, context: Self::Context) -> Decryption { @@ -467,8 +477,20 @@ impl DecryptInto for JsonTreeSubtree { } } -/// The rows of a walk open to the complete subtrees they hold, in order: -/// one per node the gates of `json_tree_subtree_at` reached. +/// The rows of a read open to the value at its path, or `None` where the +/// row has no node there: the four answers `jsonb #>` gives. +impl DecryptInto> for JsonTreeSubtree { + type Context = NonEmpty; + fn decryption(self, context: Self::Context) -> Decryption, K> { + match self.subtree() { + Ok(subtree) => subtree.decryption(AeadContext::from(context)), + Err(error) => Decryption::failed(error), + } + } +} + +/// The rows of a read open to a list: one value per concrete path the key +/// named that the row has, in order. impl DecryptInto> for JsonTreeSubtree { type Context = NonEmpty; fn decryption(self, context: Self::Context) -> Decryption, K> { @@ -499,6 +521,11 @@ mod tests { Base64Text::encode([byte; 16]).as_str().to_owned() } + /// Eight bytes of `byte` as a self or parent tag's wire text. + fn b64_8(byte: u8) -> String { + Base64Text::encode([byte; 8]).as_str().to_owned() + } + /// A ciphertext's wire text. fn b85(byte: u8) -> String { Base85Text::encode([byte; 28]).as_str().to_owned() @@ -542,7 +569,7 @@ mod tests { fn a_document_carries_its_projection() { let document = serde_json::json!({ "v": 4, "i": {"t": "users", "c": "profile"}, "h": "AA==", - "sv": [{"s": b64(0xaa), "c": b85(0)}], + "sv": [{"s": b64(0xaa), "c": b85(0), "x": b64_8(0xcc)}], "q": {"sv": [{"s": b64(0xaa), "vs": b64(0xbb)}], "el": []}, }); let stored: JsonTreeDocument = serde_json::from_value(document.clone()).unwrap(); @@ -559,9 +586,11 @@ mod tests { #[test] fn an_entry_with_a_field_in_the_wrong_encoding_is_refused() { for entry in [ - serde_json::json!({"s": "aa", "c": b85(0)}), - serde_json::json!({"s": b64(0), "c": "z"}), - serde_json::json!({"s": b64(0), "c": b85(0), "p": "not base64"}), + serde_json::json!({"s": "aa", "c": b85(0), "x": b64_8(0)}), + serde_json::json!({"s": b64(0), "c": "z", "x": b64_8(0)}), + serde_json::json!({"s": b64(0), "c": b85(0), "x": "not base64"}), + serde_json::json!({"s": b64(0), "c": b85(0), "x": b64_8(0), "p": "not base64"}), + serde_json::json!({"s": b64(0), "c": b85(0)}), ] { assert!( serde_json::from_value::(entry.clone()).is_err(), @@ -573,7 +602,7 @@ mod tests { #[test] fn an_entry_without_the_grafted_fields_does_not_open() { let entry: JsonTreeEntry = serde_json::from_value(serde_json::json!({ - "s": b64(0), "c": b85(0) + "s": b64(0), "c": b85(0), "x": b64_8(0) })) .unwrap(); assert!(entry.node().is_err()); diff --git a/packages/eql/docs/reference/json-tree-leakage.md b/packages/eql/docs/reference/json-tree-leakage.md index 97b38af2c..f7e8ce3c7 100644 --- a/packages/eql/docs/reference/json-tree-leakage.md +++ b/packages/eql/docs/reference/json-tree-leakage.md @@ -18,7 +18,7 @@ Someone who holds the column and no key, such as a database administrator, a bac |---|---| | `s`, the path selector on every node | **Which paths the row has, and how many nodes sit at each.** A selector is a keyed hash of the path, the same in every row of the column, so two rows with a selector in common hold the same path. Key names are inside the hash and not readable. Array elements share their array's selector, so `$.tags[0]` and `$.tags[1]` look alike, but each element is its own entry: the number of entries carrying `$.tags[*]`'s selector is the length of `$.tags` (the total over every array at that path, when the path runs through another array). | | the order of the entries in `sv` | **The row's shape.** Entries are stored in depth-first order, the root first, each node followed by everything beneath it, and an object's members in the byte order of their key names' UTF-8 encodings (the order of serde_json's default map; a build that turns on serde_json's `preserve_order` feature stores them in insertion order instead, which shows the application's field order). So a row's sequence of selectors fixes its shape exactly: two rows with the same sequence have the same tree with the same key names, and two rows that differ show where. Which selector's path sits under which is not written down, but it is the same in every row, so an observer comparing rows learns it, and with it each row's parent and child relations and the node count of every subtree. Siblings stored in key order also show, across rows, **the byte order of sibling key names** (for example that the key behind one selector sorts before the key behind another; byte order, not a locale's alphabetical order, so `Z` sorts before `a`), which narrows guesses at what each selector names. | -| `q`, the containment projection: one `{s, vs}` pair per node, nested by array element, in the same order | **Whether two rows hold the same value at the same path**, through the value selector `vs`, a keyed hash of the path and the value (for an object or an array, of its kind alone). So at each path the database sees which rows agree and how often each value occurs: a frequency histogram of every path, and whether one row's document contains another's. Not the value itself, and not whether two different paths hold the same value (the ordering terms below do show that, at the paths the index orders). **Which nodes sit inside the same array element**, through the nesting, for every element that is an object or an array. That grouping is what makes containment exact for arrays of objects, and it is the one thing this format reveals that v3 did not. | +| `q`, the containment projection: one `{s, vs}` pair per node, nested by array element, in the same order | **Whether two rows hold the same value at the same path**, through the value selector `vs`, a keyed hash of the path and the value (for an object or an array, of its kind alone). So at each path the database sees which rows agree and how often each value occurs: a frequency histogram of every path, and whether one row's document contains another's. Not the value itself, and not whether two different paths hold the same value (the ordering terms below do show that, at the paths the index orders). **Which nodes sit inside the same array element**, through the nesting, for every element that is an object or an array. That grouping is what makes containment exact for arrays of objects. v3 reveals it too, through its entry order and per-element `a` flag (see the comparison below); here it is explicit in the projection. | | `p`, the parent tag below the chain's heads | Nothing beyond the entry order on its own. It is salted by the document's `iv`, so the same position in two rows is unrelated bytes, and it does not say which entry is the parent. Which entries carry no `p` does show: with the sealed root on, those are the root's direct members, and since everything beneath a member follows it, **the node count of each top-level field's subtree** can be read off a single row exactly. With the sealed root off, only the root lacks one. | | `x`, the self tag on every entry | Nothing on its own. It is a hash of the entry's own token salted by the document's `iv`, so it is unrelated bytes in every row and links no entry to another, and since every entry carries one, its presence marks none. Given a token, it is how the database finds that token's node in any row. | | each ciphertext's length, and the number of entries | **The node count**, and for each node the length of its key name (for an object member) plus, for a scalar, the length of its value, exactly, less the AEAD's fixed overhead. A string's length in bytes and a number's length in digits are therefore visible per node. With the sealed root on, the root entry's length is the document's length as JSON. | @@ -37,7 +37,8 @@ A query hands the database a term derived from the plaintext it asks about, and - **A subtree read (`json_tree_subtree`, `json_tree_children`, `json_tree_child_count`).** A **token** for exactly the node the path names, which the client derives from the path alone. Reading `$.profile.address.city` sends that leaf's token and nothing that covers `$.profile`. No token over a top-level field or an array leaves the client unless that field or array is itself what is read. - **A wildcard** (`$.items[*].price`) is expanded by the client into one token per concrete path, up to a bound `N` (64 by default, set by the caller), plus a **probe** token for the element at index `N` of every array a wildcard expands. A probe that finds an element means the array is longer than the bound, and the read fails rather than return part of the answer. So the database learns which of the nodes exist in each row, which shows, up to `N`, which elements hold the rest of the path (each array's length, where the path ends at the element; the entry order already shows it), whether each array is longer than `N`, and the structure beneath each node it finds. It learns nothing about the elements' sibling fields. In a row where a probe finds an element, the database also walks that element's subtree, although the read then fails. That exception is in the token, not in what the SQL function does with it: a probe is the element's own token, so the database can find that element in any row, enumerate beneath it and derive its descendants' tokens, in rows written later too, whatever the function returns. It holds for any read that sends the element's token. Two wildcard levels send `N² + N + 1` tokens, `N + 1` of them probes, one at index `N` of each array the read expands: the `N` inner arrays and the outer one. - **What a token is.** A capability over the structure of its node's subtree in *every row of the column*: whoever holds it can find that node in any row (through its self tag `x`), see how the nodes beneath it nest, link each one to its parent, and derive the tokens of the nodes beneath. It opens no value. Tokens are column-wide and do not expire, so a token sent once is useful to the database for as long as the column lives. Send a token to the database in a query and nowhere else. - - **The root.** With the sealed root on (the default) no token covers a whole document: the root's token finds the root entry, whose ciphertext is the document, and nothing beneath it. With it off, a read of `$` itself sends the root's token, which covers every row's whole document; a read of anything below `$` does not. + - **The root.** A read of `$` sends the root's token, and a read of anything below `$` does not. With the sealed root on (the default) that token covers no document's structure: it finds the root entry, whose ciphertext is the document, and nothing beneath it, since the top-level fields chain from nothing. With it off, the root's token covers every row's whole document. + - **`json_tree_children` and `json_tree_child_count`** take one token, of any node a path names, and send nothing else. Under the sealed root the root's token enumerates nothing (its count is NULL), for the same reason. ## What is authenticated @@ -46,9 +47,15 @@ Only the ciphertext `c`. Opening an entry verifies that it was sealed under this The consequence to design around: tampering with those fields changes **which rows a query returns, where a walk stops, and which entries of a row an extraction or a walk hands back**. An entry the database returns from `->` or a walk carries its own `s` and `j`, and the header it carries names the data key, so a decrypted value is always one that was sealed in this column. The client also checks that it is the entry it asked for, from what it derived for the query, and refuses anything else (`JsonWrongEntry`): - **`->`.** The entry must carry the selector the client derived for the path. Another path's entry, such as the row's `$.bonus` returned for `$.salary`, is refused, and so is the sealed root returned for a field (it opens to the whole document); the root is accepted only when `$` itself was asked for. -- **A subtree read.** The node each token names, returned first, must carry that path's selector, the key or position the path ends in, and the self tag of one of the tokens sent. Every other row must carry the selector of its parent's path extended by its own key (or the wildcard, for an array element), which the client derives from the decrypted key, and a subtree's rows must follow each other in the document with no gap. A row of another path, a dropped or reordered row, and a row whose parent is not among the rows are refused. The self tags are not authenticated, so matching them catches a row corrupted by someone who never held the token, not a dishonest database; the selector, the position and the label are what bind a row to its place. +- **A subtree read.** The node each token names, returned at depth 0, must carry that path's selector and, in its decrypted label, the key or position the path ends in (the root, for `$`). Every row beneath it must sit at the next position `j` after the row before it, with no gap, and carry the selector of its parent's path extended by its own decrypted key (or the wildcard, for an array element). A row of another path, another position of the same array, a dropped, truncated or repeated row, a row whose parent is not among the rows, and rows of two documents are refused. Each row's self tag `x` must also be the one the client recomputes from the token it sent, or, below depth 0, from its parent's token and the row's own selector and label. The self tags are not authenticated, and the database holds every token a read sends and every row's `iv`, so it can write any self tag it likes: the `x` check is a consistency check, which catches a corrupted row or a server that walked wrongly, not a substitution. The selector, the position `j` and the label, which the AAD and the ciphertext cover, are the security check. A read of a wildcard whose array is longer than the bound fails rather than return part of the list, both when the rows are walked and when they fall back to the sealed root. -What the checks do not cover: **which row an entry comes from**. Nothing in an entry names its row, so another row's `$.salary`, returned with that row's header, opens; that is the rule every EQL stored value follows (ADR 0010). And **a position before the last step of a path**: every element of an array shares its selector, so a read of `$.orders[1].items` can be answered with `$.orders[0].items` of the same row. A query's result is not integrity-protected as a whole: each decrypted value is genuine and at the path asked for, but which row it came from is the database's word. +What the checks do not cover, by design: + +- **Which row an entry comes from.** Nothing in an entry names its row, so another row's `$.salary`, returned with that row's header, opens; that is the rule every EQL stored value follows (ADR 0010). +- **Which element a node sits under.** Every element of an array shares its selector, and no field a row binds says which element a node belongs to, so a read of `$.items[1].price` can be answered with `$.items[0].price` of the same row (its self tag rewritten), and opens to that element's price. +- **Which nodes a row holds.** Nothing in a row says how many nodes a read should find, so a database can withhold rows: a read answered with none opens as no node at the path (`None`), and a wildcard read answered with some elements' rows left out opens to the rest. That includes a probe's rows, so an array longer than the bound can open as its first `N` elements instead of failing. A node it does return is checked as above. + +A query's result is not integrity-protected as a whole: each decrypted value is genuine and at a path with the selector asked for, but which row it came from, under which element, and whether anything was left out, is the database's word. ## Compared with EQL v3 (`public.eql_v3_json_search`) @@ -61,7 +68,7 @@ What the checks do not cover: **which row an entry comes from**. Nothing in an e | size of every subtree | yes: each path entry's ciphertext is the subtree under it | no ciphertext spans a subtree, but its node count follows from the entry order, and its size is the sum of its entries' lengths | | parent and child relation | inferable from subtree sizes and co-occurrence across rows | not stored, but inferable from the entry order and co-occurrence across rows; exact for top-level fields under the sealed root | | whether two rows have the same shape | yes, selectors are column-deterministic | yes, selectors are column-deterministic and entries are stored in depth-first order | -| which leaves sit in the same array element | no | yes, through the projection's nesting: the one new disclosure, required for exact containment, bounded to array elements | +| which leaves sit in the same array element | yes, whenever elements are indexed at all. Entries sit in document order and every element's entry carries the `a` flag (`parent_is_array`), so the leaves between two flagged entries belong to the element before them; with position selectors (`array_index_mode` `position` or `all`) each leaf's selector names its element outright. Only `array_index_mode: none`, the stack's default, hides it, by indexing nothing inside an array | yes, through the projection's nesting and the entry order, as v3's order and flag do | | equal values in sibling elements | ciphertexts equal | ciphertexts differ | | ordering terms | on scalar leaves, with no per-path choice | only at the paths the index orders; none by default | | a subtree read | reveals the selector asked | reveals a token for exactly the node asked for (one per concrete path, and one probe for each array a wildcard expands), and with it that node's shape in every row for the life of the column | diff --git a/packages/eql/src/v3/json_tree/functions.sql b/packages/eql/src/v3/json_tree/functions.sql index b325a2e2d..5951aa0e7 100644 --- a/packages/eql/src/v3/json_tree/functions.sql +++ b/packages/eql/src/v3/json_tree/functions.sql @@ -6,7 +6,7 @@ --! @file v3/json_tree/functions.sql --! @brief The JSON tree surface's functions: the chain primitives, the ---! containment projection, extraction by selector, the subtree walk +--! containment projection, extraction by selector, the subtree read --! by token, and the ordering extractors. --! --! Everything here is core PostgreSQL: `sha256(bytea)`, `get_byte` / @@ -33,6 +33,9 @@ AS $$ SELECT substr(sha256(x), 1, 16) $$; +COMMENT ON FUNCTION {{prefix}}_internal.json_tree_h16(bytea) IS + 'eql-inline-critical: chain primitive of the JSON tree family, evaluated per probe and per hit of a walk; must stay inlinable (unpinned search_path)'; + --! @brief A 32-bit integer as four little-endian bytes. --! @internal --! @param n integer The integer, non-negative. @@ -45,6 +48,9 @@ AS $$ 0, n & 255), 1, (n >> 8) & 255), 2, (n >> 16) & 255), 3, (n >> 24) & 255) $$; +COMMENT ON FUNCTION {{prefix}}_internal.json_tree_le32(integer) IS + 'eql-inline-critical: chain primitive of the JSON tree family, evaluated per probe of a walk; must stay inlinable (unpinned search_path)'; + --! @brief The document's ZeroKMS `iv` from its header: the per-document salt --! of every chain tag. --! @internal @@ -75,26 +81,104 @@ AS $$ SELECT encode(substr({{prefix}}_internal.json_tree_h16('p'::bytea || iv || token || {{prefix}}_internal.json_tree_le32(o)), 1, 8), 'base64') $$; ---! @brief The token of the child at ordinal `o` under the node with token ---! `token`, whose selector is `s`: `H("t" ‖ token ‖ s ‖ LE32(o))`. ---! Column-wide like every token, with no document salt, so a client ---! derives an array element's token from the path alone and the walk ---! derives a hit's token from its parent's and the hit's own `s` on ---! the spot. Nothing stores a token. The selector keeps an array ---! element apart from an object member at the same ordinal, which a ---! row whose node at the path is an object would otherwise put there. +COMMENT ON FUNCTION {{prefix}}_internal.json_tree_parent_tag(bytea, bytea, integer) IS + 'eql-inline-critical: the probe of the JSON tree walk, evaluated once per ordinal visited; must stay inlinable (unpinned search_path)'; + +--! @brief The self tag of the node whose token is `token`: the first eight +--! bytes of `H("x" ‖ iv ‖ token)`. Every entry carries its own in +--! `x`, so the server finds the node a client-derived token names by +--! computing this and looking it up. Eight bytes, since the tag is a +--! lookup key within one document and nothing else: an entry's AAD +--! binds its selector and ordinal, not its self tag. +--! @internal +--! @param iv bytea The document's iv. +--! @param token bytea The node's token, sixteen bytes. +--! @return text The tag, padded base64, as the entry's `x` spells it. +CREATE FUNCTION {{prefix}}_internal.json_tree_self_tag(iv bytea, token bytea) + RETURNS text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT encode(substr({{prefix}}_internal.json_tree_h16('x'::bytea || iv || token), 1, 8), 'base64') +$$; + +COMMENT ON FUNCTION {{prefix}}_internal.json_tree_self_tag(bytea, bytea) IS + 'eql-inline-critical: the JSON tree walk evaluates it per token and per hit; must stay inlinable (unpinned search_path)'; + +--! @brief The token of the object member whose selector is `s`, under the +--! object whose token is `token`: `H("t" ‖ token ‖ s)`. Column-wide +--! like every token, with no document salt, and with no ordinal, so a +--! client derives it from the path alone whatever else the object +--! holds. Nothing stores a token. --! @internal --! @param token bytea The parent's token, sixteen bytes. ---! @param s bytea The child's selector, sixteen bytes. ---! @param o integer The child's ordinal. ---! @return bytea The child's token, sixteen bytes. -CREATE FUNCTION {{prefix}}_internal.json_tree_child_token(token bytea, s bytea, o integer) +--! @param s bytea The member's selector, sixteen bytes. +--! @return bytea The member's token, sixteen bytes. +CREATE FUNCTION {{prefix}}_internal.json_tree_member_token(token bytea, s bytea) + RETURNS bytea + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}_internal.json_tree_h16('t'::bytea || token || s) +$$; + +COMMENT ON FUNCTION {{prefix}}_internal.json_tree_member_token(bytea, bytea) IS + 'eql-inline-critical: the JSON tree walk evaluates it per hit; must stay inlinable (unpinned search_path)'; + +--! @brief The token of the array element at index `i`, whose selector is +--! `s`, under the array whose token is `token`: +--! `H("t" ‖ token ‖ s ‖ LE32(i))`. Every element of an array shares +--! one selector, so an element keeps its index; the input is four +--! bytes longer than a member's, so the two rules never meet. +--! @internal +--! @param token bytea The array's token, sixteen bytes. +--! @param s bytea The element's selector, sixteen bytes. +--! @param i integer The element's index. +--! @return bytea The element's token, sixteen bytes. +CREATE FUNCTION {{prefix}}_internal.json_tree_element_token(token bytea, s bytea, i integer) RETURNS bytea LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT {{prefix}}_internal.json_tree_h16('t'::bytea || token || s || {{prefix}}_internal.json_tree_le32(o)) + SELECT {{prefix}}_internal.json_tree_h16('t'::bytea || token || s || {{prefix}}_internal.json_tree_le32(i)) $$; +COMMENT ON FUNCTION {{prefix}}_internal.json_tree_element_token(bytea, bytea, integer) IS + 'eql-inline-critical: the JSON tree walk evaluates it per hit; must stay inlinable (unpinned search_path)'; + +--! @brief The token of a child the walk found at ordinal `o` under the node +--! whose token is `token`: the member rule's if the child's self tag +--! `x` is that token's, else the element rule's if it is that one's, +--! else NULL. Nothing stored says whether a node is a member or an +--! element (its kind is inside `c`, and a selector does not say +--! whether its path ends in a key), so the self tag is what chooses. +--! Three hashes on a member, six on an element: the chosen token is +--! derived twice, which is cheaper than the FROM clause that would +--! hold it, since a FROM clause stops the planner inlining the +--! function. Not STRICT for the same reason: a STRICT function whose +--! body is a CASE is inlined only where every argument is provably +--! non-null. The rules inside are STRICT, so a NULL argument makes +--! both comparisons NULL and the CASE NULL. +--! @internal +--! @param iv bytea The document's iv. +--! @param token bytea The parent's token, sixteen bytes. +--! @param s bytea The child's selector, sixteen bytes. +--! @param o integer The child's ordinal: an element's index, a member's rank. +--! @param x text The child's stored self tag. +--! @return bytea The child's token, or NULL when neither rule's self tag is +--! `x`, and the walk does not descend into it. +CREATE FUNCTION {{prefix}}_internal.json_tree_hit_token(iv bytea, token bytea, s bytea, o integer, x text) + RETURNS bytea + LANGUAGE sql IMMUTABLE PARALLEL SAFE +AS $$ + SELECT CASE + WHEN {{prefix}}_internal.json_tree_self_tag(iv, {{prefix}}_internal.json_tree_member_token(token, s)) = x + THEN {{prefix}}_internal.json_tree_member_token(token, s) + WHEN {{prefix}}_internal.json_tree_self_tag(iv, {{prefix}}_internal.json_tree_element_token(token, s, o)) = x + THEN {{prefix}}_internal.json_tree_element_token(token, s, o) + END +$$; + +COMMENT ON FUNCTION {{prefix}}_internal.json_tree_hit_token(bytea, bytea, bytea, integer, text) IS + 'eql-inline-critical: the JSON tree walk evaluates it per hit; must stay inlinable (unpinned search_path)'; + --! @brief Whether the document's root is sealed: its header's sealed-root --! flag, bit 0 of the flags byte at offset 1 of the frozen layout. --! A sealed root's `c` is the whole document, so a walk may return @@ -127,26 +211,42 @@ $$; COMMENT ON FUNCTION {{prefix}}_internal.json_tree_ope_cllw(jsonb) IS 'eql-inline-critical: raw-jsonb CLLW OPE extractor of the JSON tree family; must stay inlinable (unpinned search_path)'; ---! @brief The chained entries of a document keyed by parent tag: one object ---! whose keys are each entry's `p` and whose values are the entry ---! with its ordinal `j` in `sv` added. A walk looks a child up by ---! the tag it computes, `map -> tag`, instead of scanning `sv` for ---! it, so the document is read once per walk and a probe costs one ---! hash and one lookup, not a pass over the document. +--! @brief The positions of a document's entries by tag: `map`, the parent +--! tag of every chained entry, and `nodes`, the self tag of every +--! entry, each mapped to the entry's ordinal `j` in `sv`. A read +--! finds the node a token names, and a walk each child it probes for, +--! by the tag it computes, `nodes ->> tag` or `map ->> tag`, so a +--! token or a probe costs one hash and one lookup, and the entry +--! itself is fetched as `sv -> j` only when it is used. One pass over +--! `sv` builds both maps. A map of the entries themselves, as the +--! walk first had, copied every entry to build and cost four times as +--! much as this on a document of two hundred entries, which was most +--! of a one-token read. +--! +--! Set-returning, with the two maps as columns, and not STRICT, so +--! that the planner inlines it as a subquery of the caller's +--! `LATERAL`: an aggregate subquery is computed once, where a scalar +--! function the caller's subquery named twice (`idx -> 'p'`, +--! `idx -> 'x'`) was evaluated twice when the planner flattened that +--! subquery, and a STRICT set-returning function is not inlined at +--! all. The aggregate over no rows is one row of two `{}`, for a +--! NULL or empty `sv`. --! @internal --! @param val jsonb The document. ---! @return jsonb The map; `{}` for a document with no chained entry. -CREATE FUNCTION {{prefix}}_internal.json_tree_by_parent(val jsonb) - RETURNS jsonb - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +--! @return TABLE(map jsonb, nodes jsonb) One row: `map` `{p: j, …}` and +--! `nodes` `{x: j, …}`, either `{}` when no entry carries that tag. +CREATE FUNCTION {{prefix}}_internal.json_tree_index(val jsonb) + RETURNS TABLE(map jsonb, nodes jsonb) + LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ - SELECT COALESCE( - jsonb_object_agg(elem ->> 'p', elem || jsonb_build_object('j', ord - 1)), - '{}'::jsonb) + SELECT COALESCE(jsonb_object_agg(elem ->> 'p', ord - 1) FILTER (WHERE elem ? 'p'), '{}'::jsonb), + COALESCE(jsonb_object_agg(elem ->> 'x', ord - 1) FILTER (WHERE elem ? 'x'), '{}'::jsonb) FROM jsonb_array_elements(val -> 'sv') WITH ORDINALITY AS t(elem, ord) - WHERE elem ? 'p' $$; +COMMENT ON FUNCTION {{prefix}}_internal.json_tree_index(jsonb) IS + 'eql-inline-critical: the index every JSON tree read builds, inlined as a subquery so it is built once; must stay inlinable (unpinned search_path)'; + ------------------------------------------------------------------------------ -- Envelope helpers ------------------------------------------------------------------------------ @@ -351,236 +451,297 @@ AS $$ $$; ------------------------------------------------------------------------------ --- The token chain: children and subtrees by token +-- The token chain: nodes, children and subtrees by token ------------------------------------------------------------------------------ ---! The walk, in every function below. A node's children sit at the dense ---! ordinals `0, 1, 2, …` under it, and a child's parent tag is ---! `H("p" ‖ iv ‖ t(parent) ‖ LE32(o))`. So from a token the walk probes ---! ordinal 0, then 1, and so on, looking each tag up in the document's ---! parent map ({{prefix}}_internal.json_tree_by_parent), and stops at the ---! first miss. A hit's own token is `H("t" ‖ t(parent) ‖ s ‖ LE32(o))`, ---! `s` being the hit's own selector, derived on the spot ---! ({{prefix}}_internal.json_tree_child_token), and the walk recurses under ---! it. A subtree of `N` nodes therefore costs about ---! `2N` probes, `N` hits and a miss for every node visited (the end of a ---! parent's children, or a leaf's lack of any), each probe one SHA-256 and ---! one map lookup, plus one SHA-256 per hit for its token: about `3N` ---! hashes whatever the document's width. The map and the header fields ---! are computed once per call, so the document is read once. +--! The read, in every function below. Every node's token derives from its +--! path alone, so the client sends the token of exactly the node it reads, +--! and the server finds that node by its self tag, +--! `x = H("x" ‖ iv ‖ token)`, looked up in the document's index +--! ({{prefix}}_internal.json_tree_index), which maps every entry's `x`, and +--! every chained entry's `p`, to the entry's position in `sv`. The node's +--! children sit at the dense ordinals `0, 1, 2, …` under it, and a child's +--! parent tag is `H("p" ‖ iv ‖ t(parent) ‖ LE32(o))`. So from a node's +--! token the walk probes ordinal 0, then 1, and so on, looking each tag up +--! in the index, and stops at the first miss. A hit's own token is the +--! member rule's, `H("t" ‖ t(parent) ‖ s)`, or the element rule's, +--! `H("t" ‖ t(parent) ‖ s ‖ LE32(o))`, `s` being the hit's own selector: +--! nothing stored says which, so the walk keeps whichever one's self tag is +--! the hit's `x` ({{prefix}}_internal.json_tree_hit_token), and does not +--! descend into a hit that matches neither. A subtree of `N` nodes +--! therefore costs one self-tag lookup, about `2N` probes (`N` hits and a +--! miss for every node visited), each one SHA-256 and one lookup, and three +--! to six SHA-256s per hit to choose its token. The index and the header +--! fields are computed once per call, so the document is read once, and an +--! entry is copied out of `sv` only when it is returned. +--! +--! **Inlining.** Every chain primitive a probe or a hit evaluates (the +--! hashes, the tags and the token rules; not `json_tree_iv` or +--! `json_tree_sealed_root`, read once per call) carries the +--! `eql-inline-critical` marker that keeps its search_path unpinned: the planner does not inline a function with a +--! pinned search_path, and a probe then costs a full SQL-function call, +--! about ten microseconds, against under one for the hashes it performs. +--! For the same reason a hit's token is derived where it is consumed, in +--! the probe beneath it, from the hit's stored `s` and `x` read off the +--! working table: the planner inlines a function that uses an argument more +--! than once only when that argument is a cheap expression, and the +--! expression that locates a hit is not. --! ---! **The bound.** A stored document is untrusted: its parent tags are not ---! authenticated, so whoever holds a token can write entries of any shape ---! under it. The walk still ends after at most one row per entry. A probe ---! `(t(parent), o)` finds at most one entry, since the map keeps one entry ---! per tag and entries sharing a `p` collapse into it; a hit's token is a ---! hash of that probe, so no two rows share a token and no probe is made ---! twice; and a miss adds no row. Each walk also reads its recursion ---! through `LIMIT` the entry count plus one, the seed row, so the bound ---! holds without relying on either fact: PostgreSQL computes a recursive ---! query only as far as its reader pulls rows, so the limit stops the ---! recursion and not only the output, and a walk of a sealed document ---! never reaches it. +--! **One decompression.** The stored document is TOAST-compressed, and +--! every reference to the function's argument decompresses it again: the +--! header, `sv`, the metadata, the entry count and the index each cost a +--! decompression of the whole document, about forty microseconds for a +--! nested document of two hundred entries, a third of a one-token read. +--! So each read first takes `e::jsonb #> '{}'`, the identity path, into a +--! MATERIALIZED CTE: the operator returns the decompressed document, and +--! everything below reads that. --! ---! **Gates.** An object member's ordinal is a function of its siblings' ---! keys, which the client does not know, so the client derives a token ---! only for a chain head and the array positions that directly follow it, ---! and walks the rest through one gate per further step: either the step's selector, ---! which the entry at that depth must carry in `s`, or `#n:`, an ---! array position, which is the one ordinal probed at that depth and whose ---! entry must carry the array elements' selector, so that an object member ---! ranked at ordinal `n` in a row whose node there is an object is not ---! taken for an element. A wildcard step's selector is every element's, so ---! it admits them all. Entries are returned from the depth of the last gate ---! down: the node each gate chain reaches, and everything beneath it. - ---! @brief The children of the node the gates reach from the node whose ---! token is `token`, as rows. +--! **The bound.** A stored document is untrusted: its self tags and parent +--! tags are not authenticated, so whoever holds a token can write entries +--! of any shape under it. A probe `(t(parent), o)` finds at most one entry, +--! since the index keeps one position per tag, and a miss adds no row. But +--! a member's token is a hash of its parent's token and its selector, not +--! of the probe, so two entries forged under one parent with one selector +--! and the member rule's self tag derive one token, and the walk visits the +--! subtree beneath it once for each: forged level after level, the rows +--! would double at every level. So {{prefix}}.json_tree_subtree reads its +--! recursion through `LIMIT` the entry count times the number of tokens: a +--! sealed document's nodes each sit under one token of a read, so an honest +--! read never reaches it, and PostgreSQL computes a recursive query only as +--! far as its reader pulls rows, so the limit stops the recursion and not +--! only the output. A forged walk that hits it returns rows the client +--! refuses: it places them by the authenticated positions and child counts, +--! and refuses a repeated position (`j`). {{prefix}}.json_tree_children and +--! {{prefix}}.json_tree_child_count probe one level and never descend: +--! each ordinal's tag names at most one entry, and an entry carries one +--! parent tag, so the probes hit each entry at most once, and theirs is +--! `LIMIT` the entry count plus one, every entry and the miss. + +--! @brief The children of the node whose token is `token`, as rows. --! ---! What `jsonb_array_elements` is for an encrypted array: one row per ---! element, in the document's order, each opening on its own. With no ---! gates, the children of the token's own node; with gates, the walk ---! descends through them as `json_tree_subtree_at` does and returns one ---! level below the node they reach. +--! What `jsonb_array_elements` is for an encrypted array, and the members of +--! an encrypted object: one row per child, in the document's order, each +--! opening on its own. Enumerating one level needs no child's token, so the +--! rows are the same whichever rule each child's token takes. --! --! @param e public.{{prefix}}_json_tree The stored document. ---! @param token text The start node's token, base64: derived by the client ---! from the path alone, and never stored. ---! @param gates text[] One gate per step below the token, as ---! `json_tree_subtree_at` takes them. Default none. ---! @return SETOF public.{{prefix}}_json_tree_entry The children, each with `d = 1`. -CREATE FUNCTION {{prefix}}.json_tree_children(e public.{{prefix}}_json_tree, token text, gates text[] DEFAULT ARRAY[]::text[]) +--! @param token text The node's token, base64: derived by the client from +--! the node's path alone, and never stored. +--! @return SETOF public.{{prefix}}_json_tree_entry The children, each with +--! `d = 1`; no rows for a token that names no node, for a scalar or +--! an empty container, and for the root of a document whose root is +--! sealed, whose top-level fields chain from nothing. +CREATE FUNCTION {{prefix}}.json_tree_children(e public.{{prefix}}_json_tree, token text) RETURNS SETOF public.{{prefix}}_json_tree_entry LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ WITH RECURSIVE + -- The document, decompressed once (see "One decompression" above). + raw AS MATERIALIZED (SELECT e::jsonb #> '{}'::text[] AS d), doc AS MATERIALIZED ( - SELECT {{prefix}}_internal.json_tree_iv(e::jsonb ->> 'h') AS iv, - {{prefix}}_internal.json_tree_by_parent(e::jsonb) AS map, - {{prefix}}.json_tree_meta(e::jsonb) AS meta, - jsonb_array_length(e::jsonb -> 'sv') AS entries, - COALESCE(array_length(gates, 1), 0) AS gated + SELECT {{prefix}}_internal.json_tree_iv(raw.d ->> 'h') AS iv, + raw.d -> 'sv' AS sv, + idx.map, + idx.nodes, + {{prefix}}.json_tree_meta(raw.d) AS meta, + jsonb_array_length(raw.d -> 'sv') AS entries, + decode(token, 'base64') AS t + FROM raw + CROSS JOIN LATERAL {{prefix}}_internal.json_tree_index(raw.d) AS idx ), - -- The walk of `json_tree_subtree_at`, cut one level below the gates. - walk(depth, pt, o, entry, t, ppass, pass) AS ( - SELECT 0, NULL::bytea, NULL::integer, NULL::jsonb, decode(token, 'base64'), true, true + -- One row per ordinal probed under the node, from 0 to the first miss: + -- `j` the position in `sv` of the entry found there, NULL at the miss. + probe(o, j) AS ( + SELECT 0, (doc.map ->> {{prefix}}_internal.json_tree_parent_tag(doc.iv, doc.t, 0))::integer + FROM doc + WHERE doc.nodes ? {{prefix}}_internal.json_tree_self_tag(doc.iv, doc.t) UNION ALL - SELECT probe.depth, probe.pt, probe.o, hit.entry, - {{prefix}}_internal.json_tree_child_token(probe.pt, decode(hit.entry ->> 's', 'base64'), probe.o), - probe.ppass, - probe.ppass AND ( - probe.depth > doc.gated - OR hit.entry ->> 's' = CASE WHEN gates[probe.depth] LIKE '#%' - THEN split_part(gates[probe.depth], ':', 2) - ELSE gates[probe.depth] END) - FROM walk AS w - CROSS JOIN doc - CROSS JOIN LATERAL ( - SELECT w.depth + 1 AS depth, - w.t AS pt, - CASE WHEN COALESCE(gates[w.depth + 1], '') LIKE '#%' - THEN split_part(substr(gates[w.depth + 1], 2), ':', 1)::integer - ELSE 0 END AS o, - w.pass AS ppass - WHERE w.pass AND w.depth <= doc.gated - UNION ALL - SELECT w.depth, w.pt, w.o + 1, w.ppass - WHERE w.entry IS NOT NULL AND COALESCE(gates[w.depth], '') NOT LIKE '#%' - ) AS probe - CROSS JOIN LATERAL ( - SELECT doc.map -> {{prefix}}_internal.json_tree_parent_tag(doc.iv, probe.pt, probe.o) AS entry - ) AS hit - WHERE hit.entry IS NOT NULL + SELECT p.o + 1, (doc.map ->> {{prefix}}_internal.json_tree_parent_tag(doc.iv, doc.t, p.o + 1))::integer + FROM probe AS p, doc + WHERE p.j IS NOT NULL ), - -- At most one row per entry and the seed (see "The bound" above). + -- At most one row per entry and the miss (see "The bound" above). visited AS MATERIALIZED ( - SELECT w.* FROM walk AS w LIMIT (SELECT doc.entries + 1 FROM doc) + SELECT p.* FROM probe AS p LIMIT (SELECT doc.entries + 1 FROM doc) ) - SELECT (doc.meta || w.entry || '{"d": 1}'::jsonb)::public.{{prefix}}_json_tree_entry - FROM visited AS w, doc - WHERE w.entry IS NOT NULL AND w.pass AND w.depth = doc.gated + 1 - ORDER BY (w.entry ->> 'j')::integer + SELECT (doc.meta || (doc.sv -> p.j) || jsonb_build_object('j', p.j, 'd', 1))::public.{{prefix}}_json_tree_entry + FROM visited AS p, doc + WHERE p.j IS NOT NULL + ORDER BY p.j $$; ---! @brief How many children the node the gates reach from `token` has: ---! the length of an encrypted array, or the member count of an ---! object. +--! @brief How many children the node whose token is `token` has: the length +--! of an encrypted array, or the member count of an object. +--! +--! The same probes as {{prefix}}.json_tree_children, counted: the index is +--! built once, where calling that function after finding the node would +--! build it twice. +--! --! @param e public.{{prefix}}_json_tree The stored document. ---! @param token text The start node's token, base64. ---! @param gates text[] One gate per step below the token. Default none. ---! @return integer The count; `0` for a node with no children, a token that ---! names no node included. -CREATE FUNCTION {{prefix}}.json_tree_child_count(e public.{{prefix}}_json_tree, token text, gates text[] DEFAULT ARRAY[]::text[]) +--! @param token text The node's token, base64. +--! @return integer The count; `0` for a scalar or an empty container; NULL +--! for a token that names no node, as `jsonb_array_length` of a +--! missing path is, and for the root of a document whose root is +--! sealed, whose top-level fields chain from nothing and so cannot +--! be counted from its token. +CREATE FUNCTION {{prefix}}.json_tree_child_count(e public.{{prefix}}_json_tree, token text) RETURNS integer LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT count(*)::integer FROM {{prefix}}.json_tree_children(e, token, gates) + WITH RECURSIVE + -- The document, decompressed once (see "One decompression" above). + raw AS MATERIALIZED (SELECT e::jsonb #> '{}'::text[] AS d), + doc AS MATERIALIZED ( + SELECT src.iv, + idx.map, + -- The position of the node the token names; NULL for none. + (idx.nodes ->> {{prefix}}_internal.json_tree_self_tag(src.iv, src.t))::integer AS node, + jsonb_array_length(src.d -> 'sv') AS entries, + {{prefix}}_internal.json_tree_sealed_root(src.d ->> 'h') AS sealed, + src.t + FROM (SELECT raw.d, + {{prefix}}_internal.json_tree_iv(raw.d ->> 'h') AS iv, + decode(token, 'base64') AS t + FROM raw) AS src + CROSS JOIN LATERAL {{prefix}}_internal.json_tree_index(src.d) AS idx + ), + probe(o, j) AS ( + SELECT 0, (doc.map ->> {{prefix}}_internal.json_tree_parent_tag(doc.iv, doc.t, 0))::integer + FROM doc + WHERE doc.node IS NOT NULL AND NOT (doc.node = 0 AND doc.sealed) + UNION ALL + SELECT p.o + 1, (doc.map ->> {{prefix}}_internal.json_tree_parent_tag(doc.iv, doc.t, p.o + 1))::integer + FROM probe AS p, doc + WHERE p.j IS NOT NULL + ), + visited AS MATERIALIZED ( + SELECT p.* FROM probe AS p LIMIT (SELECT doc.entries + 1 FROM doc) + ) + SELECT CASE + WHEN doc.node IS NULL THEN NULL + WHEN doc.node = 0 AND doc.sealed THEN NULL + ELSE (SELECT count(*)::integer FROM visited AS p WHERE p.j IS NOT NULL) + END + FROM doc $$; ---! @brief The entries under the node the gates reach from the node whose ---! token is `token`, as rows: the recursive walk of the chain, in ---! core SQL. +--! @brief The nodes the tokens name, each with everything beneath it, as +--! rows: the subtree read, in core SQL. --! ---! Each level probes the ordinals under the node in hand until the first ---! miss, derives each hit's token, and recurses. A token that names no node ---! yields no row, and nothing in a row is linkable to anything without the ---! token. At depth `d` (1 below the token) the gate `gates[d]`, if there ---! is one, decides which hits the walk descends through: an entry whose ---! `s` is the gate's selector, or, for a gate `#n:`, the one hit ---! at ordinal `n` if it carries that selector. Rows are returned from the depth of the last gate down: the ---! node a gate chain reaches, with `d = 0`, and everything beneath it, with ---! `d` its depth below that node. The start node is never returned. With ---! no gates, every descendant of the start node, with `d` its depth below ---! it, from 1. +--! For each token in turn, the entry whose self tag is the token's, at +--! `d = 0`, then its descendants, each with `d` its depth below that entry, +--! found by probing the ordinals under each node in hand until the first +--! miss and recursing under each hit's own token. So a read returns the +--! node it names, and `{}`, `[]`, a scalar and an absent node are an empty +--! object's entry, an empty array's entry, the scalar's entry and no rows, +--! the four answers `jsonb #>` gives. Rows come out in token order and, +--! within a token, in document order. A token that names no node yields no +--! row, and nothing in a row is linkable to anything without its token. +--! The client sends one token for a path with no wildcard, and for a +--! wildcard one per position below its bound and a probe at the bound. The +--! server treats a probe like any other token, but can pick it out: it is +--! the last token of its level, the token count gives the bound away, and +--! a hit on it carries the element's selector rather than the leaf's. +--! Wanting a whole array is a read of the array's path, which sends the +--! array's own token. --! ---! **The budget.** A walk returns about two hundred characters per entry, +--! **The budget.** A read returns about two hundred characters per entry, --! where a sealed root holds the whole document in one ciphertext of about ---! the plaintext's size. So when the root is sealed and the walk would ---! return more than `budget` entries, it stops and returns the root entry ---! alone, with `d = 0`: the client opens it to the whole document and takes ---! the path itself, which is cheaper on both sides than the walk past that ---! point. The default of 64 is past the crossover by bytes (about a dozen ---! entries) by a margin that keeps a small subtree arriving as the entries ---! it is. A `budget` of `0` never falls back. A token that names no node ---! yields no row under any budget, and a document whose root is not sealed ---! is always walked in full, since its root holds no document to return. +--! the plaintext's size. So when the root is sealed and the read would +--! return more than `budget` rows, counted across all the tokens and the +--! nodes at `d = 0` included, it stops and returns the root entry alone, +--! with `d = 0`: the client opens it to the whole document and evaluates +--! the same paths over it, which is cheaper on both sides than the read +--! past that point. The default of 64 is past the crossover by bytes +--! (about a dozen entries) by a margin that keeps a small subtree arriving +--! as the entries it is. A `budget` of `0` never falls back. Tokens that +--! name no node yield no row under any budget, and a document whose root +--! is not sealed is always read in full, since its root holds no document +--! to return. --! --! @param e public.{{prefix}}_json_tree The stored document. ---! @param token text The start node's token, base64. ---! @param gates text[] One gate per step below the token: a selector, base64, ---! or `#n:` for the array position `n`, with the array ---! elements' selector. Empty for the whole subtree. ---! @param budget integer The most entries to return before falling back to ---! the sealed root; `0` to always walk out. Default 64. ---! @return SETOF public.{{prefix}}_json_tree_entry The entries, each with its ---! depth `d` below the node the gates reach, in document order; or ---! the root entry alone, with `j = 0` and `d = 0`, past the budget. -CREATE FUNCTION {{prefix}}.json_tree_subtree_at(e public.{{prefix}}_json_tree, token text, gates text[], budget integer DEFAULT 64) +--! @param tokens text[] The tokens of the nodes to read, base64, each +--! derived by the client from a concrete path alone. +--! @param budget integer The most rows to return before falling back to +--! the sealed root; `0` to always read in full. Default 64. +--! @return SETOF public.{{prefix}}_json_tree_entry The rows, each with its +--! depth `d` below the node its token named; or the root entry +--! alone, with `j = 0` and `d = 0`, past the budget. +CREATE FUNCTION {{prefix}}.json_tree_subtree(e public.{{prefix}}_json_tree, tokens text[], budget integer DEFAULT 64) RETURNS SETOF public.{{prefix}}_json_tree_entry LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ WITH RECURSIVE + -- The document, decompressed once (see "One decompression" above). + raw AS MATERIALIZED (SELECT e::jsonb #> '{}'::text[] AS d), doc AS MATERIALIZED ( - SELECT {{prefix}}_internal.json_tree_iv(e::jsonb ->> 'h') AS iv, - {{prefix}}_internal.json_tree_by_parent(e::jsonb) AS map, - {{prefix}}.json_tree_meta(e::jsonb) AS meta, - e::jsonb -> 'sv' -> 0 AS root, - jsonb_array_length(e::jsonb -> 'sv') AS entries, - budget > 0 AND {{prefix}}_internal.json_tree_sealed_root(e::jsonb ->> 'h') AS capped, - COALESCE(array_length(gates, 1), 0) AS gated + SELECT {{prefix}}_internal.json_tree_iv(raw.d ->> 'h') AS iv, + raw.d -> 'sv' AS sv, + idx.map, + idx.nodes, + {{prefix}}.json_tree_meta(raw.d) AS meta, + jsonb_array_length(raw.d -> 'sv') AS entries, + budget > 0 AND {{prefix}}_internal.json_tree_sealed_root(raw.d ->> 'h') AS capped + FROM raw + CROSS JOIN LATERAL {{prefix}}_internal.json_tree_index(raw.d) AS idx ), - -- One row per hit, after the seed row for the start node. `pt` and `o` - -- are where the hit sits, its parent's token and its ordinal; `t` is its - -- own token; `ppass` whether every gate down to its parent admitted the - -- parent, `pass` whether this hit's gate admits it too. - walk(depth, pt, o, entry, t, ppass, pass) AS ( - SELECT 0, NULL::bytea, NULL::integer, NULL::jsonb, decode(token, 'base64'), true, true + -- One row per node found: `k` the place in `tokens` of the token whose + -- node it sits in, `depth` its depth below that node, `pt` and `o` + -- where a hit sits, its parent's token and its ordinal (NULL for the + -- node a token named), `j` its position in `sv` and `entry` the entry + -- there, and `t` the token the node was named by, NULL for a hit: a + -- hit's own token is derived where it is used, in the probe beneath it + -- (see "Inlining" above). + walk(k, depth, pt, o, j, entry, t) AS ( + SELECT given.k, 0, NULL::bytea, NULL::integer, node.j, doc.sv -> node.j, given.t + FROM doc + CROSS JOIN LATERAL ( + SELECT g.ord AS k, decode(g.token, 'base64') AS t + FROM unnest(tokens) WITH ORDINALITY AS g(token, ord) + ) AS given + CROSS JOIN LATERAL ( + SELECT (doc.nodes ->> {{prefix}}_internal.json_tree_self_tag(doc.iv, given.t))::integer AS j + ) AS node + WHERE node.j IS NOT NULL UNION ALL - SELECT probe.depth, probe.pt, probe.o, hit.entry, - {{prefix}}_internal.json_tree_child_token(probe.pt, decode(hit.entry ->> 's', 'base64'), probe.o), - probe.ppass, - probe.ppass AND ( - probe.depth > doc.gated - OR hit.entry ->> 's' = CASE WHEN gates[probe.depth] LIKE '#%' - THEN split_part(gates[probe.depth], ':', 2) - ELSE gates[probe.depth] END) + SELECT w.k, probe.depth, probe.pt, probe.o, pos.j, doc.sv -> pos.j, NULL::bytea FROM walk AS w CROSS JOIN doc CROSS JOIN LATERAL ( - -- Down: the first ordinal under this node, or the one position an - -- array gate names, from a node its gates admitted. - SELECT w.depth + 1 AS depth, - w.t AS pt, - CASE WHEN COALESCE(gates[w.depth + 1], '') LIKE '#%' - THEN split_part(substr(gates[w.depth + 1], 2), ':', 1)::integer - ELSE 0 END AS o, - w.pass AS ppass - WHERE w.pass + -- Down: the first ordinal under this node, by its token: the one + -- it was named by, else the one its own `s` and `x` derive. A hit + -- whose self tag matches neither rule has no token, and is + -- returned but not descended into. + SELECT w.depth + 1 AS depth, own.t AS pt, 0 AS o + FROM ( + SELECT COALESCE(w.t, {{prefix}}_internal.json_tree_hit_token( + doc.iv, w.pt, decode(w.entry ->> 's', 'base64'), w.o, w.entry ->> 'x')) AS t + ) AS own + WHERE own.t IS NOT NULL UNION ALL -- Across: the next ordinal under the same parent, from any hit, - -- since a hit its gate refused still has siblings to find; not - -- when an array gate pins this depth to one position. - SELECT w.depth, w.pt, w.o + 1, w.ppass - WHERE w.entry IS NOT NULL AND COALESCE(gates[w.depth], '') NOT LIKE '#%' + -- since a hit with no token of its own still has siblings. + SELECT w.depth, w.pt, w.o + 1 + WHERE w.depth > 0 ) AS probe CROSS JOIN LATERAL ( - SELECT doc.map -> {{prefix}}_internal.json_tree_parent_tag(doc.iv, probe.pt, probe.o) AS entry - ) AS hit - WHERE hit.entry IS NOT NULL + SELECT (doc.map ->> {{prefix}}_internal.json_tree_parent_tag(doc.iv, probe.pt, probe.o))::integer AS j + ) AS pos + WHERE pos.j IS NOT NULL ), - -- At most one row per entry and the seed (see "The bound" above), + -- At most one row per entry for each token (see "The bound" above), -- whatever the budget: a budget of 0, or an open root, leaves `found` - -- uncapped, and this is then the walk's only limit. + -- uncapped, and this is then the read's only limit. visited AS MATERIALIZED ( - SELECT w.* FROM walk AS w LIMIT (SELECT doc.entries + 1 FROM doc) + SELECT w.* FROM walk AS w + LIMIT (SELECT doc.entries * COALESCE(cardinality(tokens), 0) FROM doc) ), - -- One row past the budget is enough to know the walk went over; a walk - -- under it is the subtree complete. Uncapped, the limit is NULL: all - -- that `visited` holds. + -- One row past the budget is enough to know the read went over; a read + -- under it is complete. Uncapped, the limit is NULL: all that + -- `visited` holds. found AS MATERIALIZED ( - SELECT w.depth, w.entry - FROM visited AS w, doc - WHERE w.entry IS NOT NULL AND w.pass AND w.depth >= doc.gated + SELECT w.k, w.depth, w.j, w.entry + FROM visited AS w LIMIT (SELECT CASE WHEN doc.capped THEN budget + 1 END FROM doc) ), over AS MATERIALIZED ( @@ -588,33 +749,17 @@ AS $$ ) SELECT rows.entry FROM ( - SELECT 0 AS j, - (doc.meta || doc.root || '{"j": 0, "d": 0}'::jsonb)::public.{{prefix}}_json_tree_entry AS entry + SELECT 0::bigint AS k, 0 AS j, + (doc.meta || (doc.sv -> 0) || '{"j": 0, "d": 0}'::jsonb)::public.{{prefix}}_json_tree_entry AS entry FROM doc, over WHERE over.fallback UNION ALL - SELECT (f.entry ->> 'j')::integer, - (doc.meta || f.entry || jsonb_build_object('d', f.depth - doc.gated))::public.{{prefix}}_json_tree_entry + SELECT f.k, f.j, + (doc.meta || f.entry || jsonb_build_object('j', f.j, 'd', f.depth))::public.{{prefix}}_json_tree_entry FROM found AS f, doc, over WHERE NOT over.fallback ) AS rows - ORDER BY rows.j -$$; - ---! @brief Every entry under the node whose token is `token`: ---! `json_tree_subtree_at` with no gates. ---! @param e public.{{prefix}}_json_tree The stored document. ---! @param token text The subtree's token, base64. ---! @param budget integer The most entries to return before falling back to ---! the sealed root; `0` to always walk the subtree out. Default 64. ---! @return SETOF public.{{prefix}}_json_tree_entry The descendants, each with its ---! depth `d` below the node, in document order; or the root entry ---! alone, with `d = 0`, past the budget. -CREATE FUNCTION {{prefix}}.json_tree_subtree(e public.{{prefix}}_json_tree, token text, budget integer DEFAULT 64) - RETURNS SETOF public.{{prefix}}_json_tree_entry - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ - SELECT * FROM {{prefix}}.json_tree_subtree_at(e, token, ARRAY[]::text[], budget) + ORDER BY rows.k, rows.j $$; ------------------------------------------------------------------------------ diff --git a/packages/eql/src/v3/json_tree/types.sql b/packages/eql/src/v3/json_tree/types.sql index a4b36e0d4..52ad558aa 100644 --- a/packages/eql/src/v3/json_tree/types.sql +++ b/packages/eql/src/v3/json_tree/types.sql @@ -12,7 +12,7 @@ --! so the native-jsonb firewall in blockers.sql attaches): --! - public.{{prefix}}_json_tree — storage: an EQL envelope `{v, i, h, sv, q}`. --! - public.{{prefix}}_json_tree_entry — one `sv` element (returned by `->` ---! and the subtree walk). +--! and the subtree read). --! - {{prefix}}.query_json_tree — a containment needle: the element-nested --! projection of the needle's selectors. --! - {{prefix}}.query_json_tree_leaf — an ordering operand at a path: a @@ -25,17 +25,25 @@ --! `c` the node's own ciphertext (RFC 1924 base85, which no SQL function --! decodes): its label, its kind and, for a scalar, its value. The --! root's is the whole document when the root is sealed (the default). +--! `x` the self tag, the first eight bytes of `H("x" ‖ iv ‖ t)` (base64, +--! twelve characters), `t` being the node's own token: on every +--! entry, the root and the chain's heads included. Every token derives +--! from its node's path alone (a member's as `H("t" ‖ t(parent) ‖ s)`, +--! an element's as `H("t" ‖ t(parent) ‖ s ‖ LE32(i))`), so the server +--! finds the node a client-derived token names by its self tag. --! `p` the parent tag, the first eight bytes of --! `H("p" ‖ iv ‖ t(parent) ‖ LE32(o))` (base64), where `o` is the --! node's ordinal under its parent: present below the chain's heads. ---! The ordinals under a node are dense from 0, so a walk holding +--! The ordinals under a node are dense from 0, so a read holding --! `t(parent)` probes them in turn until the first miss, and derives ---! each hit's own token as `H("t" ‖ t(parent) ‖ s ‖ LE32(o))` from ---! the hit's selector. Nothing stores a token or an ordinal. +--! each hit's own token by the member rule or the element rule, +--! whichever the hit's self tag matches. Nothing stores a token or an +--! ordinal. --! `op` the leaf ordering term (CLLW OPE, base64) on number and string leaves. --! The header `h` (base64) carries the document's data key material once; the --! SQL reads only the sealed-root flag at byte offset 1 and the ZeroKMS `iv` ---! at byte offset 22 of it, the per-document salt of the parent tags, and +--! at byte offset 22 of it, the per-document salt of the self and parent +--! tags, and --! otherwise carries it opaquely. `q` is the document's containment --! projection, `{"sv": [{s, vs}, …], "el": [group, …]}`: one `{s, vs}` pair --! per node, `vs` being the value selector of the node's kind and, for a @@ -44,16 +52,16 @@ --! live: the server compares it with a needle by native `jsonb @>`. --! The CHECK validates its shape, not that it agrees with `sv`: containment --! answers from `q` as written, so a row written by anything but Stack ---! Encrypt's JSON index can answer `@>` differently from `->`, the walks +--! Encrypt's JSON index can answer `@>` differently from `->`, the reads --! and decryption, which answer from `sv`. --! @brief Validate one JSON tree entry. --! @internal --! @param val jsonb Candidate entry. ---! @return boolean True when `val` is an object with string `s` and `c`, ---! optional string `p` and `op`, and the metadata the extractors +--! @return boolean True when `val` is an object with string `s`, `c` and +--! `x`, optional string `p` and `op`, and the metadata the extractors --! graft: string `h`, numeric `j` (the entry's ordinal in the ---! document) and `d` (its depth below a walk's start), and the +--! document) and `d` (its depth below the node a read's token named), and the --! envelope's `v` and `i`. No other key. CREATE OR REPLACE FUNCTION public.{{prefix}}_is_valid_json_tree_entry_payload(val jsonb) RETURNS boolean @@ -63,12 +71,13 @@ AS $$ jsonb_typeof(val) = 'object' AND jsonb_typeof(val -> 's') = 'string' AND jsonb_typeof(val -> 'c') = 'string' + AND jsonb_typeof(val -> 'x') = 'string' AND (NOT (val ? 'p') OR jsonb_typeof(val -> 'p') = 'string') AND (NOT (val ? 'op') OR jsonb_typeof(val -> 'op') = 'string') AND (NOT (val ? 'h') OR jsonb_typeof(val -> 'h') = 'string') AND (NOT (val ? 'j') OR jsonb_typeof(val -> 'j') = 'number') AND (NOT (val ? 'd') OR jsonb_typeof(val -> 'd') = 'number') - AND val - ARRAY['s', 'c', 'p', 'op', 'h', 'j', 'd', 'v', 'i']::text[] = '{}'::jsonb, + AND val - ARRAY['s', 'c', 'x', 'p', 'op', 'h', 'j', 'd', 'v', 'i']::text[] = '{}'::jsonb, false ) $$; @@ -194,10 +203,10 @@ $$; --! @brief Domain type for one entry of a JSON tree document. --! ---! What `->`, the path functions and the subtree walk return: the entry with +--! What `->`, the path functions and the subtree read return: the entry with --! the document's `h`, `v` and `i` and its own ordinal `j` grafted on, so it ---! opens on its own. A walk also grafts `d`, the entry's depth below the ---! walk's start. +--! opens on its own. A read also grafts `d`, the entry's depth below the +--! node its token named. --! --! @note The CHECK calls the validator rather than inlining it: an entry is --! constructed by the extractors, not cast on the query hot path. diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index 2903bb005..fe4ea4f19 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -18,7 +18,9 @@ use eql_bindings::v4::json_tree::{ }; use eql_bindings::Identifier; use serde_json::{json, Map, Value}; -use stack_encrypt::json::{JsonDocument, JsonOptions, JsonPath, JsonSelector, JsonSubtreeKey}; +use stack_encrypt::json::{ + JsonDocument, JsonOptions, JsonPath, JsonSelector, JsonSubtreeKey, Segment, +}; use stack_encrypt::plan::{Laid, ValuePlan}; use stack_encrypt::target::{Json, Layout}; use stack_encrypt::{op, NonEmpty, Plan}; @@ -588,65 +590,77 @@ fn postgres_extracts_walks_and_orders_by_path() { .collect(); assert_eq!(tags, vec![json!("x"), json!("y")]); - // The subtree under `$.nested`, from a token derived on the client from - // the path alone: six entries in row 1, reassembling the sub-document; - // two in row 2. + // The read of `$.nested`, from a token derived on the client from the + // path alone: nested itself and its six entries in row 1, reassembling + // the sub-document; nested and two in row 2; nothing in row 3. let nested = subtree_key("$.nested"); - assert!(nested.gates().is_empty(), "a chain head's token"); + assert_eq!(nested.tokens().len(), 1, "one token for one node"); + let read = |db: &mut postgres::Client, + key: &JsonSubtreeKey, + id: i32, + budget: i32| + -> Vec { + db.query( + "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_subtree(doc, $1::text[], $2::int) e WHERE id = $3", + &[&key.tokens(), &budget, &id], + ) + .unwrap() + .iter() + .map(entry) + .collect() + }; for (id, expected) in [ - (1, fixture()["nested"].clone()), - (2, rows[1].1["nested"].clone()), + (1, Some(fixture()["nested"].clone())), + (2, Some(rows[1].1["nested"].clone())), + (3, None), ] { - let walked: Vec = db - .query( - "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_subtree(doc, $1::text) e WHERE id = $2", - &[&nested.token(), &id], - ) - .unwrap() - .iter() - .map(entry) - .collect(); - assert_eq!(walked.len(), if id == 1 { 6 } else { 2 }); + let walked = read(&mut db, &nested, id, 0); + assert_eq!(walked.len(), [0, 7, 3, 0][id as usize], "row {id}"); assert!(walked .iter() - .all(|e| e.d.is_some() && e.j.is_some() && e.h.is_some())); - let value: Value = wait( + .all(|e| e.d.is_some() && e.j.is_some() && e.h.is_some() && e.v.is_some())); + let value: Option = wait( &runtime, cipher.decrypt_as(JsonTreeSubtree::from_key(&nested, walked), column()), ) .unwrap(); assert_eq!(value, expected, "row {id}"); } - let garbage: i64 = db + // The default budget, as a caller who names none gets it. + let defaulted: i64 = db .query_one( - "SELECT count(*) FROM json_tree_rows, eql_v4.json_tree_subtree(doc, $1::text) e", - &[&"00".repeat(16)], + "SELECT count(*) FROM json_tree_rows, eql_v4.json_tree_subtree(doc, $1::text[]) e WHERE id = 1", + &[&nested.tokens()], ) .unwrap() .get(0); - assert_eq!(garbage, 0, "a wrong token unlocks nothing in any row"); - - // The budget: past it the walk returns the sealed root alone, at depth - // 0, and the rows open to the value at the walked path; at it, or - // with the budget off, the walk is complete; a wrong token still - // yields nothing under any budget. - let walked = |db: &mut postgres::Client, budget: i32| -> Vec { - db.query( - "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_subtree(doc, $1::text, $2::int) e WHERE id = 1", - &[&nested.token(), &budget], + assert_eq!(defaulted, 7, "under the default budget of 64"); + let garbage: i64 = db + .query_one( + "SELECT count(*) FROM json_tree_rows, eql_v4.json_tree_subtree(doc, $1::text[]) e", + &[&vec!["AAAAAAAAAAAAAAAAAAAAAA==".to_owned()]], ) .unwrap() - .iter() - .map(entry) - .collect() - }; - assert_eq!(walked(&mut db, 6).len(), 6, "at the budget, the subtree"); - assert_eq!(walked(&mut db, 0).len(), 6, "budget 0 never falls back"); - let root = walked(&mut db, 5); + .get(0); + assert_eq!(garbage, 0, "a wrong token finds nothing in any row"); + + // The budget counts rows, the node a token names included: past it the + // read returns the sealed root alone, at depth 0, and the rows open to + // the value at the path; at it, or with the budget off, the read is + // complete; a wrong token still yields nothing under any budget. + assert_eq!( + read(&mut db, &nested, 1, 7).len(), + 7, + "at the budget, the subtree" + ); + assert_eq!( + read(&mut db, &nested, 1, 0).len(), + 7, + "budget 0 never falls back" + ); + let root = read(&mut db, &nested, 1, 6); assert_eq!(root.len(), 1, "past the budget, the root alone"); assert_eq!((root[0].j, root[0].d), (Some(0), Some(0))); - // The key that keyed the walk builds the rows, so the path the - // fallback is opened at is always there. let value: Value = wait( &runtime, cipher.decrypt_as(JsonTreeSubtree::from_key(&nested, root), column()), @@ -655,181 +669,101 @@ fn postgres_extracts_walks_and_orders_by_path() { assert_eq!(value, fixture()["nested"]); let garbage: i64 = db .query_one( - "SELECT count(*) FROM json_tree_rows, eql_v4.json_tree_subtree(doc, $1::text, 1) e", - &[&"00".repeat(16)], + "SELECT count(*) FROM json_tree_rows, eql_v4.json_tree_subtree(doc, $1::text[], 1) e", + &[&vec!["AAAAAAAAAAAAAAAAAAAAAA==".to_owned()]], ) .unwrap() .get(0); assert_eq!(garbage, 0); - // The fallback opens to what the walk opens to, as a list too, so a - // caller's result does not change shape when a document grows past the - // budget: `$.nested`'s member values from `json_tree_subtree`, whose - // rows are the head's descendants (its key has no gates), and the list - // itself from the gated walk to `$.nested.list`, whose rows are the - // node the gates reach. The key says which. - let sorted = |mut values: Vec| { - values.sort_by_key(ToString::to_string); - values - }; - let open_list = |rows: Vec, key: &JsonSubtreeKey| -> Vec { - wait( - &runtime, - cipher.decrypt_as(JsonTreeSubtree::from_key(key, rows), column()), - ) - .unwrap() - }; - let walked_members = walked(&mut db, 0); - assert_eq!(walked_members.len(), 6); - let fallen_back = walked(&mut db, 5); - assert_eq!(fallen_back.len(), 1); - let members = sorted(vec![json!("constant"), json!([{"x": 1}, {"x": 2}])]); - assert_eq!(sorted(open_list(walked_members, &nested)), members); - assert_eq!(sorted(open_list(fallen_back, &nested)), members); - let list = subtree_key("$.nested.list"); - let list_rows = |db: &mut postgres::Client, budget: i32| -> Vec { - db.query( - "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_subtree_at(doc, $1::text, $2::text[], $3::int) e WHERE id = 1", - &[&list.token(), &list.gates(), &budget], - ) - .unwrap() - .iter() - .map(entry) - .collect() - }; - let walked_list = list_rows(&mut db, 0); - assert_eq!( - walked_list.len(), - 5, - "the list, its two elements and their x" - ); - let fallen_back = list_rows(&mut db, 3); - assert_eq!(fallen_back.len(), 1); - assert_eq!( - open_list(walked_list, &list), - vec![fixture()["nested"]["list"].clone()] - ); - assert_eq!( - open_list(fallen_back, &list), - vec![fixture()["nested"]["list"].clone()] - ); // The budget is gated on the header's sealed-root flag, the bit // `JsonHeader` writes: with the flags byte cleared, the same document - // has no root to return and is walked out under any budget. + // has no root to return and is read in full under any budget. let unflagged: i64 = db .query_one( "SELECT count(*) FROM json_tree_rows, eql_v4.json_tree_subtree( jsonb_set(doc::jsonb, '{h}', to_jsonb(encode(set_byte(decode(doc::jsonb ->> 'h', 'base64'), 1, 0), 'base64')))::public.eql_v4_json_tree, - $1::text, 1) e + $1::text[], 1) e WHERE id = 1", - &[&nested.token()], + &[&nested.tokens()], ) .unwrap() .get(0); - assert_eq!(unflagged, 6, "an unsealed root is walked out"); + assert_eq!(unflagged, 7, "an unsealed root is read in full"); - // A path below the head walks from the head's token through one gate - // per step: `$.nested.list[*].x` is the list's selector, then every - // element's (the wildcard's), then x's; each `x` comes back as a - // complete subtree. + // A wildcard: the client expands `$.nested.list[*].x` to one token per + // position below the bound and a probe at it; each `x` comes back as a + // node of its own. Past the budget the read falls back the same way, + // and the concrete paths are evaluated over the whole document. let xs = subtree_key("$.nested.list[*].x"); - assert_eq!(xs.token(), nested.token(), "anchored at the head"); - assert_eq!(xs.gates().len(), 3); - let walked: Vec = db - .query( - "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_subtree_at(doc, $1::text, $2::text[]) e WHERE id = 1", - &[&xs.token(), &xs.gates()], - ) - .unwrap() - .iter() - .map(entry) - .collect(); - let values: Vec = wait( - &runtime, - cipher.decrypt_as(JsonTreeSubtree::from_key(&xs, walked), column()), - ) - .unwrap(); - assert_eq!(values, vec![json!(1), json!(2)]); - // Past the budget, the gated walk falls back the same way, and the - // wildcard path selects every subtree over the whole document. - let root: Vec = db - .query( - "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_subtree_at(doc, $1::text, $2::text[], 1) e WHERE id = 1", - &[&xs.token(), &xs.gates()], + assert_eq!(xs.tokens().len(), 65, "64 positions and the probe"); + for budget in [0, 1, 64] { + let rows = read(&mut db, &xs, 1, budget); + let values: Vec = wait( + &runtime, + cipher.decrypt_as(JsonTreeSubtree::from_key(&xs, rows), column()), ) - .unwrap() - .iter() - .map(entry) - .collect(); - assert_eq!(root.len(), 1); - assert_eq!(root[0].d, Some(0)); - let values: Vec = wait( - &runtime, - cipher.decrypt_as(JsonTreeSubtree::from_key(&xs, root), column()), - ) - .unwrap(); - assert_eq!(values, vec![json!(1), json!(2)]); + .unwrap(); + assert_eq!(values, vec![json!(1), json!(2)], "budget {budget}"); + } - // A positional path below a key step: the second element's subtree, - // through the list's selector and the position `#1`, with the selector - // every element of the list carries. + // A position below a key step: the second element's own token. let second = subtree_key("$.nested.list[1]"); - assert_eq!( - second.gates()[1], - format!("#1:{}", selector("$.nested.list[*]")) - ); - let walked: Vec = db - .query( - "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_subtree_at(doc, $1::text, $2::text[]) e WHERE id = 1", - &[&second.token(), &second.gates()], - ) - .unwrap() - .iter() - .map(entry) - .collect(); let value: Value = wait( &runtime, - cipher.decrypt_as(JsonTreeSubtree::from_key(&second, walked), column()), + cipher.decrypt_as( + JsonTreeSubtree::from_key(&second, read(&mut db, &second, 1, 0)), + column(), + ), ) .unwrap(); assert_eq!(value, json!({"x": 2})); - // The array functions, by the head's token and the list's gate. + // The array functions, by the list's own token: a count per row, NULL + // where the row has no list. let list = subtree_key("$.nested.list"); - assert_eq!(list.gates().len(), 1); - let counts: Vec<(i32, i32)> = db + let counts: Vec<(i32, Option)> = db .query( - "SELECT id, eql_v4.json_tree_child_count(doc, $1::text, $2::text[]) FROM json_tree_rows ORDER BY id", - &[&list.token(), &list.gates()], + "SELECT id, eql_v4.json_tree_child_count(doc, $1::text) FROM json_tree_rows ORDER BY id", + &[&list.tokens()[0]], ) .unwrap() .iter() .map(|r| (r.get(0), r.get(1))) .collect(); - assert_eq!(counts, [(1, 2), (2, 0), (3, 0)]); + assert_eq!(counts, [(1, Some(2)), (2, Some(0)), (3, None)]); let elements: Vec = db .query( - "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_children(doc, $1::text, $2::text[]) e WHERE id = 1", - &[&list.token(), &list.gates()], + "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_children(doc, $1::text) e WHERE id = 1", + &[&list.tokens()[0]], ) .unwrap() .iter() .map(entry) .collect(); - assert_eq!(elements.len(), 2); assert_eq!( elements.iter().map(|e| (e.j, e.d)).collect::>(), [(Some(5), Some(1)), (Some(7), Some(1))] ); - // The head's own children, with no gate: deep and list. let members: i32 = db .query_one( "SELECT eql_v4.json_tree_child_count(doc, $1::text) FROM json_tree_rows WHERE id = 1", - &[&nested.token()], + &[&nested.tokens()[0]], ) .unwrap() .get(0); - assert_eq!(members, 2); + assert_eq!(members, 2, "nested's own members: deep and list"); + // The sealed root's members chain from nothing, so its token cannot + // count them: NULL, not a count of zero. + let root_key = subtree_key("$"); + let root_count: Option = db + .query_one( + "SELECT eql_v4.json_tree_child_count(doc, $1::text) FROM json_tree_rows WHERE id = 1", + &[&root_key.tokens()[0]], + ) + .unwrap() + .get(0); + assert_eq!(root_count, None); // Orderings at a path: the rows whose `$.number` is in range, and // sorted by the leaf's term. @@ -1256,15 +1190,584 @@ fn postgres_the_planner_picks_the_indexes_without_help() { ); } +// ============================================================================= +// Subtree reads against jsonb +// ============================================================================= + +/// `path` in PostgreSQL's SQL/JSON path language, strict: a key quoted as a +/// JSON string, so the keys that stress the path encoding (`.`, `[`, `*`, +/// `$`, quotes, backslashes, the empty key) name exactly one member. Strict, +/// because a lax path reads `$.a[0]` on an object as `$.a`, and `#>` reads a +/// position as a key; the index reads a position as an element only. +fn pg_path(path: &JsonPath) -> String { + let mut out = String::from("strict $"); + for segment in path.segments() { + match segment { + Segment::Key(key) => { + out.push('.'); + out.push_str(&serde_json::to_string(key).unwrap()); + } + Segment::Index(index) => out.push_str(&format!("[{index}]")), + Segment::Wildcard => out.push_str("[*]"), + } + } + out +} + +/// Every node's path in `value`, positionally, the root first. +fn node_paths(value: &Value) -> Vec { + fn visit(value: &Value, at: &str, out: &mut Vec) { + out.push(at.to_owned()); + match value { + Value::Object(map) => { + for (key, child) in map { + visit( + child, + &format!("{at}[{}]", serde_json::to_string(key).unwrap()), + out, + ); + } + } + Value::Array(items) => { + for (i, child) in items.iter().enumerate() { + visit(child, &format!("{at}[{i}]"), out); + } + } + _ => {} + } + } + let mut out = Vec::new(); + visit(value, "$", &mut out); + out.iter() + .map(|text| JsonPath::parse(text).unwrap()) + .collect() +} + +/// A temp table of `documents` sealed under `options`, beside their +/// plaintexts, ids from 1. +fn store( + db: &mut postgres::Client, + runtime: &tokio::runtime::Runtime, + cipher: &Cipher, + table: &str, + options: &JsonOptions, + documents: &[Value], +) { + db.batch_execute(&format!( + "CREATE TEMP TABLE {table} (id int PRIMARY KEY, plain jsonb NOT NULL, doc public.eql_v4_json_tree NOT NULL)" + )) + .unwrap(); + for (id, plain) in (1..).zip(documents) { + let stored = wait( + runtime, + cipher.default_keyset().run( + JsonTree::new(options.clone()).encryption(), + plain, + column(), + ), + ) + .unwrap(); + db.execute( + &format!("INSERT INTO {table} VALUES ($1, $2::text::jsonb, $3::text::jsonb::public.eql_v4_json_tree)"), + &[&id, &plain.to_string(), &serde_json::to_string(&stored).unwrap()], + ) + .unwrap(); + } +} + +/// The rows `json_tree_subtree(doc, key.tokens(), budget)` returns in row `id`. +fn subtree_rows( + db: &mut postgres::Client, + table: &str, + id: i32, + key: &JsonSubtreeKey, + budget: i32, +) -> Vec { + db.query( + &format!("SELECT e::jsonb::text FROM {table}, eql_v4.json_tree_subtree(doc, $1::text[], $2::int) e WHERE id = $3"), + &[&key.tokens(), &budget, &id], + ) + .unwrap_or_else(|error| panic!("{table} {}: {error}", key.path())) + .iter() + .map(|r| serde_json::from_str(&r.get::<_, String>(0)).unwrap()) + .collect() +} + +/// What `jsonb_path_query_first(plain, 'strict ')` holds in row `id`. +fn jsonb_first(db: &mut postgres::Client, table: &str, id: i32, path: &JsonPath) -> Option { + db.query_one( + &format!("SELECT jsonb_path_query_first(plain, $1::text::jsonpath, '{{}}', true)::text FROM {table} WHERE id = $2"), + &[&pg_path(path), &id], + ) + .unwrap() + .get::<_, Option>(0) + .map(|text| serde_json::from_str(&text).unwrap()) +} + +/// What `jsonb_path_query(plain, 'strict ')` returns in row `id`, +/// one concrete path at a time: the wildcards are expanded over the +/// plaintext's own arrays, and each concrete path is asked of `jsonb` with +/// [`jsonb_first`]. Strict mode aborts the whole query on the first element +/// a step does not fit (`.x` on a number), where the index answers for each +/// element on its own, so asking each concrete path keeps strict mode's +/// answer for every element without its abort. In document order. +fn jsonb_all( + db: &mut postgres::Client, + table: &str, + id: i32, + plain: &Value, + path: &JsonPath, +) -> Vec { + fn expand(value: Option<&Value>, at: JsonPath, rest: &[Segment], out: &mut Vec) { + let Some((step, rest)) = rest.split_first() else { + out.push(at); + return; + }; + match step { + Segment::Wildcard => { + let len = value.and_then(Value::as_array).map_or(0, Vec::len); + for i in 0..len { + let next = value.and_then(|v| v.as_array()).and_then(|a| a.get(i)); + expand(next, append(&at, &Segment::Index(i as u64)), rest, out); + } + } + Segment::Key(key) => { + let next = value.and_then(Value::as_object).and_then(|o| o.get(key)); + expand(next, append(&at, step), rest, out); + } + Segment::Index(i) => { + let next = value + .and_then(Value::as_array) + .and_then(|a| a.get(*i as usize)); + expand(next, append(&at, step), rest, out); + } + } + } + let mut concrete = Vec::new(); + expand( + Some(plain), + JsonPath::root(), + path.segments(), + &mut concrete, + ); + concrete + .iter() + .filter_map(|q| jsonb_first(db, table, id, q)) + .collect() +} + +/// `at` with one more step. +fn append(at: &JsonPath, step: &Segment) -> JsonPath { + let text = match step { + Segment::Key(key) => format!("[{}]", serde_json::to_string(key).unwrap()), + Segment::Index(i) => format!("[{i}]"), + Segment::Wildcard => "[*]".to_owned(), + }; + JsonPath::parse(&format!("{at}{text}")).unwrap() +} + +/// The child count jsonb gives the node at `path` in row `id`: an object's +/// keys, an array's length, `0` for a scalar (where jsonb raises), NULL +/// where there is no node. +fn jsonb_child_count( + db: &mut postgres::Client, + table: &str, + id: i32, + path: &JsonPath, +) -> Option { + db.query_one( + &format!( + "SELECT CASE WHEN v IS NULL THEN NULL + WHEN jsonb_typeof(v) = 'object' THEN (SELECT count(*)::int FROM jsonb_object_keys(v)) + WHEN jsonb_typeof(v) = 'array' THEN jsonb_array_length(v) + ELSE 0 END + FROM (SELECT jsonb_path_query_first(plain, $1::text::jsonpath, '{{}}', true) AS v FROM {table} WHERE id = $2) AS q" + ), + &[&pg_path(path), &id], + ) + .unwrap() + .get(0) +} + +/// The documents the read oracle stores: nested empties, degenerate roots, +/// the keys that stress the path encoding, numbers that `jsonb` spells +/// differently, a `null` leaf beside an absent key, a wide object, and the +/// shapes #1356 found. +fn read_documents() -> Vec { + let wide: Map = (0..300).map(|i| (format!("k{i}"), json!(i))).collect(); + let mut deep = json!(1); + for _ in 0..40 { + deep = json!({"d": [deep]}); + } + vec![ + json!({"p": {}, "l": [{}, [], "s", {"x": 1}], "h": {"p": {}, "l": [[], {}, null]}}), + json!({"p": [], "l": [[]], "h": {"p": []}}), + json!({"p": "text", "l": ["a", 0, false], "h": {"p": 7, "l": [true]}}), + json!({"q": 1, "h": {"q": null}}), + json!({"p": null, "l": [], "h": []}), + json!({"p": {"a": 1, "b": [2, {"c": null}]}, "l": [{"a": [1, 2]}, 5], "h": {"p": {"a": {"b": {}}}, "l": [{"a": 1}]}}), + json!({"p": [[], {}], "l": [[{}], [[]]], "h": {"l": [[{"x": []}]]}}), + json!({"a": {}}), + json!([[]]), + json!({"a": [{}]}), + json!([]), + json!({}), + json!("scalar"), + json!(7), + json!(null), + json!([{"x": 1}, {"y": 2}, 3]), + json!({ + "": 1, + "a.b": {"[*]": [{"$": 2}]}, + "\"\\": null, + "é✓": ["unicode"], + "0": {"x": "the key 0"}, + "arr": [{"x": "the index 0"}], + "*": {"[": {"]": true}}, + }), + json!({"n": [2, 2.0, 1e30, -0.0, -9223372036854775808i64, 18446744073709551615u64, 1.5, 1e-7]}), + json!({"null": null, "nested": {"null": null}}), + json!({"wide": wide}), + json!({"deep": deep}), + ] +} + +/// Needs an EQL v4 installation. Every read of a path with no wildcard +/// returns the node the path names with everything beneath it, so it answers +/// as `jsonb` does: an empty object, an empty array, a scalar (`null` +/// included) and a node the document does not have are four different +/// answers, held here to `jsonb_path_query_first(plain, 'strict ')` +/// over the plaintexts, as `Option`, `Value` and `Vec`. Every +/// node of every document is read, at a chain head, under positions and +/// deep, plus paths no document has, with the root sealed and open, read +/// in full (budget 0), past a budget of 1 (where every read of more than +/// one row falls back to the sealed root) and under the default. The same +/// pass holds `json_tree_child_count` and `json_tree_children` to `jsonb`'s +/// counts, and `->` to the read for every scalar at a path with no +/// position. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_every_subtree_read_answers_as_jsonb_does() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + let documents = read_documents(); + // Every node's path, but of the wide object's 300 members only the + // first, the last and one past it: each member is the same read. + let mut paths: Vec = documents + .iter() + .flat_map(node_paths) + .filter(|p| { + !matches!(p.segments(), [Segment::Key(w), Segment::Key(k)] + if w == "wide" && !["k0", "k299"].contains(&k.as_str())) + }) + .collect(); + for extra in [ + "$.q", + "$.p.missing", + "$[0]", + "$[1]", + "$.l[9]", + "$.h.l[0][0]", + r#"$["0"]"#, + "$.arr[1]", + r#"$["a.b"]["[*]"][0]["$"]"#, + "$.n[7]", + "$.wide.k300", + ] { + paths.push(JsonPath::parse(extra).unwrap()); + } + paths.sort(); + paths.dedup(); + let mut checked = [0usize; 4]; + for sealed_root in [true, false] { + let options = JsonOptions { + sealed_root, + ..JsonOptions::default() + }; + let table = if sealed_root { + "json_tree_reads_sealed" + } else { + "json_tree_reads_open" + }; + store(&mut db, &runtime, &cipher, table, &options, &documents); + for path in &paths { + let key = wait(&runtime, keyset.json_subtree_key(path, &options, column())).unwrap(); + assert_eq!(key.tokens().len(), 1, "{path}: one token"); + let selector = wait(&runtime, keyset.json_selector(path, column())).unwrap(); + for id in 1..=documents.len() as i32 { + let expected = jsonb_first(&mut db, table, id, path); + checked[match &expected { + None => 0, + Some(Value::Object(o)) if o.is_empty() => 1, + Some(Value::Array(a)) if a.is_empty() => 2, + Some(_) => 3, + }] += 1; + let at = format!("{table} {path}, row {id}"); + for budget in [0, 1, 64] { + let rows = subtree_rows(&mut db, table, id, &key, budget); + let fell_back = sealed_root && rows.len() == 1 && rows[0].j == Some(0); + if expected.is_none() { + assert!(rows.is_empty(), "{at}: no node, no row"); + } else if budget == 0 { + assert!( + !fell_back || path.segments().is_empty(), + "{at}: read in full" + ); + } + let opened: Option = wait( + &runtime, + cipher.decrypt_as(JsonTreeSubtree::from_key(&key, rows.clone()), column()), + ) + .unwrap_or_else(|error| panic!("{at}, budget {budget}: {error}")); + assert_eq!(opened, expected, "{at}, budget {budget}, as Option"); + let list: Vec = wait( + &runtime, + cipher.decrypt_as(JsonTreeSubtree::from_key(&key, rows.clone()), column()), + ) + .unwrap(); + assert_eq!( + list, + expected.clone().into_iter().collect::>(), + "{at}, as Vec" + ); + let one: Result = wait( + &runtime, + cipher.decrypt_as(JsonTreeSubtree::from_key(&key, rows), column()), + ); + match (&expected, one) { + (Some(expected), Ok(one)) => assert_eq!(&one, expected, "{at}, as Value"), + (None, Err(stack_encrypt::Error::OpenedType { expected })) => { + assert_eq!(expected, "a node at the walked path", "{at}") + } + (expected, one) => panic!("{at}: {expected:?} opened as {one:?}"), + } + } + + // The children and their count, by the node's own token. + let count: Option = db + .query_one( + &format!("SELECT eql_v4.json_tree_child_count(doc, $1::text) FROM {table} WHERE id = $2"), + &[&key.tokens()[0], &id], + ) + .unwrap() + .get(0); + let children: i64 = db + .query_one( + &format!("SELECT count(*) FROM {table}, eql_v4.json_tree_children(doc, $1::text) e WHERE id = $2"), + &[&key.tokens()[0], &id], + ) + .unwrap() + .get(0); + if sealed_root && path.segments().is_empty() { + assert_eq!( + count, None, + "{at}: the sealed root's members chain from nothing" + ); + assert_eq!(children, 0, "{at}"); + } else { + assert_eq!( + count, + jsonb_child_count(&mut db, table, id, path), + "{at}: child_count" + ); + assert_eq!(children, i64::from(count.unwrap_or(0)), "{at}: children"); + } + + // `->` returns the first entry at the path's selector, which + // reads a position as the wildcard: for a scalar at a path + // with no position it is the read's node. + if !path + .segments() + .iter() + .any(|s| matches!(s, Segment::Index(_))) + && expected + .as_ref() + .is_some_and(|v| !v.is_object() && !v.is_array()) + { + let arrow: Option = db + .query_one( + &format!( + "SELECT (doc -> $1::text)::jsonb::text FROM {table} WHERE id = $2" + ), + &[&selector.to_string(), &id], + ) + .unwrap() + .get(0); + let entry: JsonTreeEntry = serde_json::from_str(&arrow.unwrap()).unwrap(); + let value: Value = wait( + &runtime, + cipher.decrypt_as( + JsonTreeExtracted::from_selector(&selector, entry), + column(), + ), + ) + .unwrap(); + assert_eq!(Some(value), expected, "{at}: `->` and the read agree"); + } + } + } + } + assert!( + checked.iter().all(|n| *n > 0), + "the corpus reaches no node, `{{}}`, `[]` and other values: {checked:?}" + ); +} + +/// Whether a read of `pattern` at `bound` over `value` finds an array longer +/// than the bound at a wildcard level it reaches, which is when one of its +/// probes names a node. +fn truncated(value: &Value, pattern: &[Segment], bound: usize) -> bool { + let Some((step, rest)) = pattern.split_first() else { + return false; + }; + match (step, value) { + (Segment::Key(key), Value::Object(map)) => { + map.get(key).is_some_and(|v| truncated(v, rest, bound)) + } + (Segment::Index(i), Value::Array(items)) => items + .get(*i as usize) + .is_some_and(|v| truncated(v, rest, bound)), + (Segment::Wildcard, Value::Array(items)) => { + items.len() > bound || items.iter().any(|v| truncated(v, rest, bound)) + } + _ => false, + } +} + +/// Needs an EQL v4 installation. A wildcard read answers as +/// `jsonb_path_query(plain, 'strict ')` when every array it expands is +/// at most the bound long, in document order, and fails with the bound +/// otherwise, never with part of the list: at the bound minus one, at it +/// and one past it, under one wildcard and two, with the root sealed and +/// open, read in full, past a budget of 1 (the sealed-root fallback, which +/// evaluates the same probes) and under the default. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_wildcard_reads_answer_as_jsonb_path_query() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + let row = |n: usize, inner: usize| -> Value { + let items: Vec = (0..n) + .map(|i| json!({"x": i, "y": (0..inner).map(|k| json!({"z": k * 10 + i})).collect::>()})) + .collect(); + json!({"items": items, "top": (0..n).collect::>(), "obj": {"items": [{"x": "o"}]}}) + }; + let mut cases: Vec<(u32, Vec)> = Vec::new(); + for bound in [3u32, 64] { + let n = bound as usize; + cases.push(( + bound, + vec![ + row(n - 1, n - 1), + row(n, n), + row(n + 1, 1), + row(1, n + 1), + row(0, 0), + json!({"items": {"0": {"x": 1}}, "top": {}}), + json!([{"x": 1}, {"x": 2}]), + json!({"items": [1, [2], {"x": null}]}), + ], + )); + } + let patterns = [ + "$.items[*].x", + "$.items[*]", + "$.top[*]", + "$.items[*].y[*].z", + "$.items[*].y[*]", + "$.obj.items[*].x", + "$[*].x", + "$.items[0].y[*].z", + "$.missing[*]", + ]; + let mut outcomes = [0usize; 2]; + for (bound, documents) in &cases { + for sealed_root in [true, false] { + let options = JsonOptions { + sealed_root, + ..JsonOptions::default() + }; + let table = format!( + "json_tree_wild_{bound}_{}", + if sealed_root { "sealed" } else { "open" } + ); + store(&mut db, &runtime, &cipher, &table, &options, documents); + for pattern in patterns { + let path = JsonPath::parse(pattern).unwrap(); + let key = wait( + &runtime, + keyset.json_subtree_key_with_bound(&path, &options, *bound, column()), + ) + .unwrap(); + for (id, plain) in (1..).zip(documents) { + let expected = jsonb_all(&mut db, &table, id, plain, &path); + let too_long = truncated(plain, path.segments(), *bound as usize); + outcomes[usize::from(too_long)] += 1; + for budget in [0, 1, 64] { + let at = format!("{table} {pattern}, row {id}, budget {budget}"); + let rows = subtree_rows(&mut db, &table, id, &key, budget); + let list: Result, stack_encrypt::Error> = wait( + &runtime, + cipher.decrypt_as( + JsonTreeSubtree::from_key(&key, rows.clone()), + column(), + ), + ); + let one: Result, stack_encrypt::Error> = wait( + &runtime, + cipher.decrypt_as(JsonTreeSubtree::from_key(&key, rows), column()), + ); + if too_long { + for result in [list.map(|_| ()), one.map(|_| ())] { + match result { + Err(stack_encrypt::Error::JsonWildcardTruncated { + bound: b, + }) => { + assert_eq!(b, *bound, "{at}: the error names the bound") + } + other => panic!( + "{at}: an array past the bound must fail, got {other:?}" + ), + } + } + } else { + assert_eq!(list.unwrap(), expected, "{at}: as jsonb_path_query"); + match (one, expected.len()) { + (Ok(found), 0 | 1) => { + assert_eq!(found, expected.first().cloned(), "{at}") + } + (Err(stack_encrypt::Error::OpenedType { .. }), _) => {} + (other, n) => panic!("{at}: {n} nodes opened as {other:?}"), + } + } + } + } + } + } + } + assert!( + outcomes.iter().all(|n| *n > 0), + "both answers reached: {outcomes:?}" + ); +} + /// Needs an EQL v4 installation. A position names an array element and /// nothing else, in every row of the column, as a strict SQL/JSON path does -/// (`jsonb_path_query_first(plain, 'strict $.a[1]')`): where a -/// row's node at the path is an object, an object member ranked at that -/// ordinal is not returned, whether the client derived the position's -/// token (`$.a[1]`, directly after the head) or the walk took it as a gate -/// (`$.h.a[0]`, below a key step). Each walk's rows open to the plaintext's -/// value at the strict path, or come back empty where it has none. (`#>` -/// would not do as the oracle: it reads `1` as a key of an object.) +/// (`jsonb_path_query_first(plain, 'strict $.a[1]')`): where a row's node at +/// the path is an object, a member is not returned, the member named `"1"` +/// included, whether the position follows a head (`$.a[1]`) or a key step +/// (`$.h.a[0]`). A member's token and an element's are different hashes, +/// so no entry there carries the element's self tag. #[test] #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_a_position_reaches_array_elements_only() { @@ -1275,106 +1778,70 @@ fn postgres_a_position_reaches_array_elements_only() { let (cipher, _) = wait(&runtime, common::cipher()); let keyset = cipher.default_keyset(); let mut db = connect(); - db.batch_execute( - "CREATE TEMP TABLE json_tree_positions (id int PRIMARY KEY, plain jsonb NOT NULL, doc public.eql_v4_json_tree NOT NULL)", - ) - .unwrap(); - let rows = [ + let documents = [ json!({"a": [{"x": 1}, {"x": 2}], "h": {"a": [{"x": 3}, {"x": 4}]}}), json!({"a": {"k": {"x": 9}, "m": {"x": 6}}, "h": {"a": {"k": {"x": 8}, "m": {"x": 5}}}}), json!({"a": {"0": {"x": 7}, "1": {"x": 7}}, "h": {"a": {"0": {"x": 7}}}}), ]; - for (id, plain) in (1..).zip(&rows) { - let stored = wait(&runtime, seal(&cipher, plain)); - let stored = serde_json::to_string(&stored).unwrap(); - db.execute( - "INSERT INTO json_tree_positions VALUES ($1, $2::text::jsonb, $3::text::jsonb::public.eql_v4_json_tree)", - &[&id, &plain.to_string(), &stored], - ) - .unwrap(); - } + store( + &mut db, + &runtime, + &cipher, + "json_tree_positions", + &options(), + &documents, + ); for path in ["$.a[1]", "$.h.a[0]"] { + let path = JsonPath::parse(path).unwrap(); let key = wait( &runtime, - keyset.json_subtree_key(&JsonPath::parse(path).unwrap(), &options(), column()), + keyset.json_subtree_key(&path, &options(), column()), ) .unwrap(); for id in 1..=3 { - let walked: Vec = db - .query( - "SELECT e::jsonb::text FROM json_tree_positions, eql_v4.json_tree_subtree_at(doc, $1::text, $2::text[], 0) e WHERE id = $3", - &[&key.token(), &key.gates(), &id], - ) - .unwrap() - .iter() - .map(|r| serde_json::from_str(&r.get::<_, String>(0)).unwrap()) - .collect(); - let expected: Option = db - .query_one( - "SELECT jsonb_path_query_first(plain, ('strict ' || $1::text)::jsonpath, '{}', true)::text FROM json_tree_positions WHERE id = $2", - &[&path, &id], - ) - .unwrap() - .get(0); - match expected { - None => assert!(walked.is_empty(), "{path}, row {id}: jsonb has no node"), - Some(expected) => { - let value: Value = wait( - &runtime, - cipher.decrypt_as(JsonTreeSubtree::from_key(&key, walked), column()), - ) - .unwrap(); - assert_eq!( - value, - serde_json::from_str::(&expected).unwrap(), - "{path}, row {id}" - ); - } + let walked = subtree_rows(&mut db, "json_tree_positions", id, &key, 0); + let expected = jsonb_first(&mut db, "json_tree_positions", id, &path); + if expected.is_none() { + assert!(walked.is_empty(), "{path}, row {id}: jsonb has no node"); } + let value: Option = wait( + &runtime, + cipher.decrypt_as(JsonTreeSubtree::from_key(&key, walked), column()), + ) + .unwrap(); + assert_eq!(value, expected, "{path}, row {id}"); } + let counts: Vec<(i32, Option)> = db + .query( + "SELECT id, eql_v4.json_tree_child_count(doc, $1::text) FROM json_tree_positions ORDER BY id", + &[&key.tokens()[0]], + ) + .unwrap() + .iter() + .map(|r| (r.get(0), r.get(1))) + .collect(); + assert_eq!( + counts, + [(1, Some(1)), (2, None), (3, None)], + "{path}: x, in row 1 only" + ); } - // A position gate that does not name the elements' selector admits - // nothing: an ordinal alone cannot tell an element from a member. - let key = wait( - &runtime, - keyset.json_subtree_key(&JsonPath::parse("$.h.a[0]").unwrap(), &options(), column()), - ) - .unwrap(); - let bare = [key.gates()[0].clone(), "#0".to_owned()]; - let none: i64 = db - .query_one( - "SELECT count(*) FROM json_tree_positions, eql_v4.json_tree_subtree_at(doc, $1::text, $2::text[], 0) e", - &[&key.token(), &bare.as_slice()], - ) - .unwrap() - .get(0); - assert_eq!(none, 0); - let counts: Vec<(i32, i32)> = db - .query( - "SELECT id, eql_v4.json_tree_child_count(doc, $1::text, $2::text[]) FROM json_tree_positions ORDER BY id", - &[&key.token(), &key.gates()], - ) - .unwrap() - .iter() - .map(|r| (r.get(0), r.get(1))) - .collect(); - assert_eq!( - counts, - [(1, 1), (2, 0), (3, 0)], - "one member, x, under $.h.a[0] in row 1 only" - ); } -/// Needs an EQL v4 installation. What `->` and the walks return is checked +/// Needs an EQL v4 installation. What `->` and the reads return is checked /// by the client to be what it asked for: every entry of a row is genuine /// and opens under its own selector and position, so an answer made of the /// wrong entries would open unless the client refuses it. Real rows come /// back from PostgreSQL here, from a database answering the wrong question -/// (another path's entry, the sealed root, another position's subtree) and -/// from stored rows tampered with (a parent tag rewritten so the walk takes -/// a top-level field for a member, and two entries swapped in `sv`). Each -/// is refused: the wrong entry as `JsonWrongEntry`, a moved entry as -/// `Aead`, since its ciphertext is bound to the position it was sealed at. +/// (another path's entry, the sealed root, another position's subtree with +/// its self tag forged) and from stored rows tampered with (a self tag +/// forged so the read finds the wrong node, a top-level field moved under +/// another node, two entries swapped in `sv`). Each is refused: the wrong +/// entry as `JsonWrongEntry`, a moved entry as `Aead`, since its ciphertext +/// is bound to the position it was sealed at. Two answers are allowed by +/// design and shown to open: another row's node at the path (ADR 0010), +/// and the node at the same selector under another element, which no field +/// a row binds tells apart. #[test] #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_hostile_answers_are_refused_by_the_client() { @@ -1387,18 +1854,16 @@ fn postgres_hostile_answers_are_refused_by_the_client() { let (cipher, _) = wait(&runtime, common::cipher()); let keyset = cipher.default_keyset(); let mut db = connect(); - db.batch_execute( - "CREATE TEMP TABLE json_tree_hostile (id int PRIMARY KEY, doc public.eql_v4_json_tree NOT NULL)", - ) - .unwrap(); - for id in [1, 2] { - let stored = serde_json::to_string(&wait(&runtime, seal(&cipher, &fixture()))).unwrap(); - db.execute( - "INSERT INTO json_tree_hostile VALUES ($1, $2::text::jsonb::public.eql_v4_json_tree)", - &[&id, &stored], - ) - .unwrap(); - } + let other = + json!({"hello": "world-2", "nested": {"deep": "other", "list": [{"x": 3}]}, "number": 2}); + store( + &mut db, + &runtime, + &cipher, + "json_tree_hostile", + &JsonOptions::default(), + &[fixture(), fixture(), other.clone()], + ); let selector_of = |path: &str| -> JsonSelector { wait( &runtime, @@ -1417,7 +1882,6 @@ fn postgres_hostile_answers_are_refused_by_the_client() { ) .unwrap() }; - // `doc -> selector` in row `id`. let arrow = |db: &mut postgres::Client, id: i32, selector: &JsonSelector| -> JsonTreeEntry { let row = db .query_one( @@ -1427,16 +1891,8 @@ fn postgres_hostile_answers_are_refused_by_the_client() { .unwrap(); serde_json::from_str(&row.get::<_, String>(0)).unwrap() }; - // The walk `key` keys in row `id`, walked out (budget 0). - let walked = |db: &mut postgres::Client, id: i32, key: &JsonSubtreeKey| -> Vec { - db.query( - "SELECT e::jsonb::text FROM json_tree_hostile, eql_v4.json_tree_subtree_at(doc, $1::text, $2::text[], 0) e WHERE id = $3", - &[&key.token(), &key.gates(), &id], - ) - .unwrap() - .iter() - .map(|r| serde_json::from_str(&r.get::<_, String>(0)).unwrap()) - .collect() + let read = |db: &mut postgres::Client, id: i32, key: &JsonSubtreeKey| { + subtree_rows(db, "json_tree_hostile", id, key, 0) }; let extracted = |asked: &JsonSelector, entry: JsonTreeEntry| -> Result { wait( @@ -1456,6 +1912,15 @@ fn postgres_hostile_answers_are_refused_by_the_client() { "{what}: {result:?}" ); }; + // The self tag a read of `key` finds in row `id`. + let x_of = |db: &mut postgres::Client, id: i32, key: &JsonSubtreeKey| -> String { + db.query_one( + "SELECT eql_v4_internal.json_tree_self_tag(eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), decode($1::text, 'base64')) FROM json_tree_hostile WHERE id = $2", + &[&key.tokens()[0], &id], + ) + .unwrap() + .get(0) + }; // `->` on `$.hello` answered with the row's `$.number`, and with the // sealed root, which opens to the whole document. @@ -1473,51 +1938,87 @@ fn postgres_hostile_answers_are_refused_by_the_client() { assert_eq!( extracted(&selector_of("$"), root).unwrap(), fixture(), - "the root asked for at `$`" + "`$`" ); - // The walk to `$.nested.list[1]` answered with the rows of the walk to - // `$.nested.list[0]`: the same selectors, another position. + // The read of `$.nested.list[1]` answered with `$.nested.list[0]`'s + // rows, its self tag forged to the second element's: the same selector, + // and the position in its label tells them apart. let second = subtree_key("$.nested.list[1]"); assert_eq!( - opened(&second, walked(&mut db, 1, &second)).unwrap(), + opened(&second, read(&mut db, 1, &second)).unwrap(), json!({"x": 2}) ); - wrong( - opened( - &second, - walked(&mut db, 1, &subtree_key("$.nested.list[0]")), - ), - "another position's subtree", + let mut forged = read(&mut db, 1, &subtree_key("$.nested.list[0]")); + forged[0].x = serde_json::from_value(json!(x_of(&mut db, 1, &second))).unwrap(); + wrong(opened(&second, forged), "another position's subtree"); + + // Allowed: `$.nested.list[0].x` for `$.nested.list[1].x`, self tag + // forged; the same selector, the same label. + let second_x = subtree_key("$.nested.list[1].x"); + let mut forged = read(&mut db, 1, &subtree_key("$.nested.list[0].x")); + forged[0].x = serde_json::from_value(json!(x_of(&mut db, 1, &second_x))).unwrap(); + assert_eq!( + opened(&second_x, forged).unwrap(), + json!(1), + "another element's x, by design" ); - - // A stored row whose `$.number` (entry 9) carries a parent tag under - // `$.nested`'s token, at the ordinal after its two members: the walk - // takes it for nested's third member, and returns it right after - // nested's subtree, where it sits in `sv`. + // Allowed: another row's node at the path, with its own header. let nested = subtree_key("$.nested"); - assert!(nested.gates().is_empty()); assert_eq!( - opened(&nested, walked(&mut db, 1, &nested)).unwrap(), - fixture()["nested"] + opened(&nested, read(&mut db, 3, &nested)).unwrap(), + other["nested"], + "ADR 0010" + ); + + // A truncated read and a reordered one. + let mut truncated = read(&mut db, 1, &nested); + let _ = truncated.pop(); + assert!(opened(&nested, truncated).is_err(), "a truncated read"); + let mut reordered = read(&mut db, 1, &nested); + reordered.swap(2, 3); + assert_eq!( + opened(&nested, reordered).unwrap(), + fixture()["nested"], + "the rows are placed by their positions, not the order they came in" ); + + // A stored row whose `$.number` (entry 9) is moved under `$.nested`: + // a parent tag at the ordinal after its two members and a member's self + // tag. The read takes it for nested's third member and returns it after + // nested's subtree; nested's child count says two, and the row is + // refused. db.execute( - "UPDATE json_tree_hostile SET doc = jsonb_set(doc::jsonb, '{sv,9,p}', + "UPDATE json_tree_hostile SET doc = jsonb_set(jsonb_set(doc::jsonb, '{sv,9,p}', to_jsonb(eql_v4_internal.json_tree_parent_tag( - eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), decode($1::text, 'base64'), 2)))::public.eql_v4_json_tree + eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), decode($1::text, 'base64'), 2))), + '{sv,9,x}', + to_jsonb(eql_v4_internal.json_tree_self_tag( + eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), + eql_v4_internal.json_tree_member_token(decode($1::text, 'base64'), decode(doc::jsonb -> 'sv' -> 9 ->> 's', 'base64')))))::public.eql_v4_json_tree WHERE id = 1", - &[&nested.token()], + &[&nested.tokens()[0]], ) .unwrap(); - let rows = walked(&mut db, 1, &nested); - assert_eq!(rows.len(), 7, "the walk took number for a member"); - assert_eq!( - rows.last().map(|e| (e.j, e.d)), - Some((Some(9), Some(1))), - "number, one level below nested" - ); + let rows = read(&mut db, 1, &nested); + assert_eq!(rows.len(), 8, "the read took number for a member"); + assert_eq!(rows.last().map(|e| (e.j, e.d)), Some((Some(9), Some(1)))); wrong(opened(&nested, rows), "a top-level field as a member"); + // A stored row whose `$.hello` (1) carries `$.number`'s self tag: the + // read of `$.number` finds hello, whose selector is not number's. + let number = subtree_key("$.number"); + db.execute( + "UPDATE json_tree_hostile SET doc = jsonb_set(jsonb_set(doc::jsonb, '{sv,1,x}', doc::jsonb -> 'sv' -> 9 -> 'x'), + '{sv,9,x}', '\"AAAAAAAAAAA=\"')::public.eql_v4_json_tree WHERE id = 2", + &[], + ) + .unwrap(); + wrong( + opened(&number, read(&mut db, 2, &number)), + "a forged self tag", + ); + // A stored row whose `deep` (3) and `number` (9) swapped places: each // entry is found where it now sits, and fails to open there. db.execute( @@ -1530,59 +2031,317 @@ fn postgres_hostile_answers_are_refused_by_the_client() { let deep = selector_of("$.nested.deep"); let moved = extracted(&deep, arrow(&mut db, 2, &deep)); assert!(matches!(moved, Err(Error::Aead)), "`->`: {moved:?}"); - let moved = opened(&nested, walked(&mut db, 2, &nested)); - assert!(matches!(moved, Err(Error::Aead)), "the walk: {moved:?}"); + let moved = opened(&nested, read(&mut db, 2, &nested)); + assert!(matches!(moved, Err(Error::Aead)), "the read: {moved:?}"); +} + +/// A small deterministic generator of documents with arrays of objects, so +/// that a selector has several elements under it. +fn tamper_document(rng: &mut Lcg, depth: u32) -> Value { + match rng.below(if depth == 0 { 3 } else { 6 }) { + 0 => json!(rng.below(4)), + 1 => json!(["s", "t", "u"][rng.below(3) as usize]), + 2 => Value::Null, + 3 | 4 => Value::Array( + (0..rng.below(4)) + .map(|_| tamper_document(rng, depth - 1)) + .collect(), + ), + _ => { + let mut map = Map::new(); + for _ in 0..rng.below(4) { + let key = ["a", "b", "c", "0"][rng.below(4) as usize].to_owned(); + let _ = map.insert(key, tamper_document(rng, depth - 1)); + } + Value::Object(map) + } + } +} + +/// Every answer the read check cannot tell from the right one, by design: +/// the value at every concrete path that shares `path`'s selector and last +/// step, under another element of an array before it. +fn same_selector_values(plain: &Value, path: &JsonPath) -> Vec { + let segments = path.segments(); + let pattern: Vec = segments + .iter() + .enumerate() + .map(|(i, segment)| match segment { + Segment::Key(key) => format!("[{}]", serde_json::to_string(key).unwrap()), + Segment::Index(n) if i + 1 == segments.len() => format!("[{n}]"), + _ => "[*]".to_owned(), + }) + .collect(); + JsonPath::parse(&format!("${}", pattern.concat())) + .unwrap() + .select(plain) + .into_iter() + .cloned() + .collect() +} + +/// Needs an EQL v4 installation. No tampering with a stored row makes a +/// read answer wrongly without saying so. Random documents are stored, one +/// random change is made to each stored row (a self tag, parent tag or +/// selector rewritten, two entries swapped, an entry dropped or +/// duplicated, a node moved under another parent, an entry swapped with +/// the same position of another row, a read's rows truncated), and every +/// path of the untampered document is then read. The client either opens +/// exactly the `jsonb` answer over the untampered plaintext, or refuses, or +/// opens one of the answers it allows by design, each modelled here: no +/// node (`None`, the database withholding the rows, which no row can prove +/// it should have returned), and the node at the same selector and last +/// step under another element. Nothing else is accepted. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_reads_never_answer_wrongly_under_tampering() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + db.batch_execute( + "CREATE TEMP TABLE json_tree_tamper (id int PRIMARY KEY, plain jsonb NOT NULL, doc public.eql_v4_json_tree NOT NULL)", + ) + .unwrap(); + let mut rng = Lcg(0x5eed_7a3e); + let mut outcomes = std::collections::BTreeMap::<&str, usize>::new(); + const MUTATIONS: u64 = 9; + for round in 0..120i32 { + let id = round; + let mut plain = Value::Object(Map::new()); + while plain.as_object().is_some_and(Map::is_empty) { + plain = json!({"a": tamper_document(&mut rng, 3), "b": [tamper_document(&mut rng, 2), tamper_document(&mut rng, 2)]}); + } + let stored = serde_json::to_value(wait(&runtime, seal(&cipher, &plain))).unwrap(); + let other = serde_json::to_value(wait(&runtime, seal(&cipher, &plain))).unwrap(); + let sv = stored["sv"].as_array().unwrap().clone(); + let n = sv.len(); + let pick = |rng: &mut Lcg| rng.below(n as u64) as usize; + let mut tampered = stored.clone(); + let mutation = rng.below(MUTATIONS); + let mut truncate_rows = false; + let what = match mutation { + 0 => { + let (i, k) = (pick(&mut rng), pick(&mut rng)); + tampered["sv"][i]["x"] = sv[k]["x"].clone(); + "a self tag copied from another entry" + } + 1 => { + let (i, k) = (pick(&mut rng), pick(&mut rng)); + match sv[k].get("p") { + Some(p) => tampered["sv"][i]["p"] = p.clone(), + None => { + let _ = tampered["sv"][i].as_object_mut().unwrap().remove("p"); + } + } + "a parent tag copied, or removed" + } + 2 => { + let (i, k) = (pick(&mut rng), pick(&mut rng)); + tampered["sv"][i]["s"] = sv[k]["s"].clone(); + "a selector rewritten" + } + 3 => { + let (i, k) = (pick(&mut rng), pick(&mut rng)); + tampered["sv"].as_array_mut().unwrap().swap(i, k); + "two entries swapped" + } + 4 => { + let i = pick(&mut rng); + if n > 1 { + let _ = tampered["sv"].as_array_mut().unwrap().remove(i); + } + "an entry dropped" + } + 5 => { + let (i, k) = (pick(&mut rng), pick(&mut rng)); + tampered["sv"] + .as_array_mut() + .unwrap() + .insert(k, sv[i].clone()); + "an entry duplicated" + } + 6 => { + let i = pick(&mut rng); + tampered["sv"][i] = other["sv"][i].clone(); + "an entry swapped with another row's at its position" + } + 7 => { + truncate_rows = true; + "a read's rows truncated" + } + _ => "a node moved under another parent", + }; + let inserted = db.execute( + "INSERT INTO json_tree_tamper VALUES ($1, $2::text::jsonb, $3::text::jsonb::public.eql_v4_json_tree)", + &[&id, &plain.to_string(), &tampered.to_string()], + ); + if inserted.is_err() { + *outcomes.entry("refused by the domain CHECK").or_default() += 1; + continue; + } + let paths = node_paths(&plain); + if mutation == 8 && paths.len() > 2 { + // Move a random non-root node under a random container (not + // itself or beneath it) as that container's next child. + let from = &paths[1 + pick(&mut rng) % (paths.len() - 1)]; + let containers: Vec<&JsonPath> = paths + .iter() + .filter(|p| { + !p.segments().starts_with(from.segments()) + && p.select(&plain) + .first() + .is_some_and(|v| v.is_object() || v.is_array()) + && !p.segments().is_empty() + }) + .collect(); + if let Some(to) = containers.get(pick(&mut rng) % containers.len().max(1)) { + let to_key = wait( + &runtime, + keyset.json_subtree_key(to, &JsonOptions::default(), column()), + ) + .unwrap(); + let from_key = wait( + &runtime, + keyset.json_subtree_key(from, &JsonOptions::default(), column()), + ) + .unwrap(); + let width = match to.select(&plain)[0] { + Value::Object(m) => m.len(), + Value::Array(a) => a.len(), + _ => 0, + } as i32; + let member = matches!(from.segments().last(), Some(Segment::Key(_))); + db.execute( + "UPDATE json_tree_tamper SET doc = ( + SELECT jsonb_set(jsonb_set(doc::jsonb, ARRAY['sv', j::text, 'p'], + to_jsonb(eql_v4_internal.json_tree_parent_tag(iv, t, $3))), + ARRAY['sv', j::text, 'x'], + to_jsonb(eql_v4_internal.json_tree_self_tag(iv, CASE WHEN $4 + THEN eql_v4_internal.json_tree_member_token(t, s) + ELSE eql_v4_internal.json_tree_element_token(t, s, $3) END))) + FROM (SELECT eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h') AS iv, + decode($2::text, 'base64') AS t, + ((SELECT nodes FROM eql_v4_internal.json_tree_index(doc::jsonb)) + ->> eql_v4_internal.json_tree_self_tag(eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), decode($1::text, 'base64'))) AS j, + decode(doc::jsonb -> 'sv' -> ((SELECT nodes FROM eql_v4_internal.json_tree_index(doc::jsonb)) + ->> eql_v4_internal.json_tree_self_tag(eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), decode($1::text, 'base64')))::integer ->> 's', 'base64') AS s) AS q + )::public.eql_v4_json_tree WHERE id = $5", + &[&from_key.tokens()[0], &to_key.tokens()[0], &width, &member, &id], + ) + .unwrap(); + } + } + for path in &paths { + let key = wait( + &runtime, + keyset.json_subtree_key(path, &JsonOptions::default(), column()), + ) + .unwrap(); + let expected = path.select(&plain).first().cloned().cloned(); + let allowed = same_selector_values(&plain, path); + for budget in [0, 1] { + let mut rows = subtree_rows(&mut db, "json_tree_tamper", id, &key, budget); + if truncate_rows && rows.len() > 1 { + let _ = rows.pop(); + } + let opened: Result, stack_encrypt::Error> = wait( + &runtime, + cipher.decrypt_as(JsonTreeSubtree::from_key(&key, rows), column()), + ); + let outcome = match opened { + Err(_) => "refused", + Ok(found) if found == expected => "the right answer", + Ok(None) => "withheld", + Ok(Some(found)) if allowed.contains(&found) => "another element at the selector", + Ok(found) => panic!( + "round {round}, {what}: {path} at budget {budget} opened to {found:?}, where jsonb holds {expected:?}" + ), + }; + *outcomes.entry(outcome).or_default() += 1; + } + } + } + assert!( + outcomes.get("refused").copied().unwrap_or(0) > 0, + "{outcomes:?}" + ); + assert!( + outcomes.get("the right answer").copied().unwrap_or(0) > 0, + "{outcomes:?}" + ); + eprintln!("tampering outcomes: {outcomes:?}"); +} + +// ============================================================================= +// Hostile documents +// ============================================================================= + +/// A document built in SQL: a root, then `levels` levels each holding +/// `copies` entries, every one a member (selector all zero) of the level +/// above with that member's self tag, at the ordinals `0 … copies − 1` when +/// `spread`, else all at ordinal 0 (one parent tag, repeated). `flags` is +/// the header's flags byte. Built from the SQL chain primitives, as whoever +/// holds the all-zero token could build one. +fn forged_chain(flags: i32, levels: i64, copies: i64, spread: bool) -> String { + let ordinal = if spread { "copy.c" } else { "0" }; + format!( + "WITH RECURSIVE k AS ( + SELECT set_byte(set_byte(decode(repeat('00', 40), 'hex'), 0, 1), 1, {flags}) AS h, + decode(repeat('00', 16), 'hex') AS iv + ), link(n, t) AS ( + SELECT 0, decode(repeat('00', 16), 'hex') + UNION ALL + SELECT n + 1, eql_v4_internal.json_tree_member_token(t, decode(repeat('00', 16), 'hex')) FROM link WHERE n < {last} + ), copy(c) AS (SELECT generate_series(0, {copies} - 1)) + SELECT (jsonb_build_object( + 'v', 4, 'i', jsonb_build_object('t', 'users', 'c', 'profile'), + 'h', (SELECT encode(h, 'base64') FROM k), + 'sv', jsonb_build_array(jsonb_build_object('s', 'AAAAAAAAAAAAAAAAAAAAAA==', 'c', '00', + 'x', (SELECT eql_v4_internal.json_tree_self_tag(k.iv, decode(repeat('00', 16), 'hex')) FROM k))) + || (SELECT jsonb_agg(jsonb_build_object( + 's', 'AAAAAAAAAAAAAAAAAAAAAA==', 'c', to_hex(copy.c), + 'x', eql_v4_internal.json_tree_self_tag(k.iv, eql_v4_internal.json_tree_member_token(link.t, decode(repeat('00', 16), 'hex'))), + 'p', eql_v4_internal.json_tree_parent_tag(k.iv, link.t, {ordinal})) ORDER BY n, copy.c) + FROM link, k, copy), + 'q', '{{\"sv\": [], \"el\": []}}'::jsonb))::public.eql_v4_json_tree", + last = levels - 1 + ) } /// Needs an EQL v4 installation. A stored document is untrusted input: its -/// parent tags are not authenticated, so whoever holds a token can write a -/// chain of them deeper than any sealed document. A child's token is a -/// hash of its parent's, so the chain cannot loop back, and the walk, which -/// carries no depth bound, still ends after one row per entry: here a chain -/// of 300, past the 127 levels a sealed document can nest. -/// `statement_timeout` turns a walk that would not end into a failure -/// instead of a hung suite. +/// self tags and parent tags are not authenticated, so whoever holds a +/// token can write a chain of them deeper than any sealed document. The +/// read, which carries no depth bound, still ends: here a chain of 300, +/// past the 127 levels a sealed document can nest, under the sealed root +/// and not, in full and past a budget. `statement_timeout` turns a read +/// that would not end into a failure instead of a hung suite. #[test] #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_walks_end_on_hostile_documents() { let mut db = connect(); db.batch_execute("SET statement_timeout = '10s'").unwrap(); - // A root, then 300 entries each the only child of the one before, under - // the all-zero token. `flags` is the header's flags byte, so the same - // document is built with its root sealed and not. - let chain = |flags: i32| { - format!( - "WITH RECURSIVE k AS ( - SELECT set_byte(set_byte(decode(repeat('00', 40), 'hex'), 0, 1), 1, {flags}) AS h, - decode(repeat('00', 16), 'hex') AS iv - ), link(n, t) AS ( - SELECT 0, decode(repeat('00', 16), 'hex') - UNION ALL - SELECT n + 1, eql_v4_internal.json_tree_child_token(t, decode(repeat('00', 16), 'hex'), 0) FROM link WHERE n < 299 - ) - SELECT (jsonb_build_object( - 'v', 4, 'i', jsonb_build_object('t', 'users', 'c', 'profile'), - 'h', (SELECT encode(h, 'base64') FROM k), - 'sv', jsonb_build_array(jsonb_build_object('s', 'AAAAAAAAAAAAAAAAAAAAAA==', 'c', '00')) - || (SELECT jsonb_agg(jsonb_build_object('s', 'AAAAAAAAAAAAAAAAAAAAAA==', 'c', '00', 'p', eql_v4_internal.json_tree_parent_tag(k.iv, link.t, 0)) ORDER BY n) FROM link, k), - 'q', '{{\"sv\": [], \"el\": []}}'::jsonb))::public.eql_v4_json_tree" - ) - }; - let token = "AAAAAAAAAAAAAAAAAAAAAA==".to_owned(); - for (flags, budget, expected) in [(0, 0, 300), (0, 64, 300), (1, 0, 300), (1, 64, 1)] { - let doc = chain(flags); + let token = vec!["AAAAAAAAAAAAAAAAAAAAAA==".to_owned()]; + for (flags, budget, expected) in [(0, 0, 301), (0, 64, 301), (1, 0, 301), (1, 64, 1)] { + let doc = forged_chain(flags, 300, 1, false); let walked: i64 = db .query_one( - &format!("SELECT count(*) FROM ({doc}) AS d(doc), eql_v4.json_tree_subtree(d.doc, $1::text, {budget}) e"), + &format!("SELECT count(*) FROM ({doc}) AS d(doc), eql_v4.json_tree_subtree(d.doc, $1::text[], {budget}) e"), &[&token], ) .unwrap_or_else(|error| panic!("flags {flags}, budget {budget}: {error}")) .get(0); - assert_eq!(walked, expected, "flags {flags}, budget {budget}"); + assert_eq!( + walked, expected, + "flags {flags}, budget {budget}: the root and its chain" + ); let children: i64 = db .query_one( &format!("SELECT count(*) FROM ({doc}) AS d(doc), eql_v4.json_tree_children(d.doc, $1::text) e"), - &[&token], + &[&token[0]], ) .unwrap() .get(0); @@ -1591,48 +2350,28 @@ fn postgres_walks_end_on_hostile_documents() { } /// Needs an EQL v4 installation. Entries that share a parent tag do not -/// multiply a walk. Here every level of a 60-level chain is written twice: -/// two entries with the same `p`, so the same parent and ordinal, and the -/// same `s`, told apart only by `c`. Walked once per copy, the rows would -/// double at every level, 2^60 in all; the walk keeps one entry per tag and -/// reads its recursion through the entry count, so it returns one row per -/// level, under no budget (`0`), under an open root, and under a budget -/// above the entry count, and still falls back past a budget below it. +/// multiply a read. Here every level of a 60-level chain is written twice: +/// two entries with the same `p`, so the same parent and ordinal, the same +/// `s` and the same self tag, told apart only by `c`. The map keeps one +/// entry per tag, so the read returns one row per level and the root, under +/// no budget (`0`), under an open root, and under a budget above the entry +/// count, and still falls back past a budget below it. #[test] #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_walks_do_not_multiply_on_duplicated_entries() { let mut db = connect(); db.batch_execute("SET statement_timeout = '10s'").unwrap(); const LEVELS: i64 = 60; - let doc = |flags: i32| { - format!( - "WITH RECURSIVE k AS ( - SELECT set_byte(set_byte(decode(repeat('00', 40), 'hex'), 0, 1), 1, {flags}) AS h, - decode(repeat('00', 16), 'hex') AS iv - ), link(n, t) AS ( - SELECT 0, decode(repeat('00', 16), 'hex') - UNION ALL - SELECT n + 1, eql_v4_internal.json_tree_child_token(t, decode(repeat('00', 16), 'hex'), 0) FROM link WHERE n < {last} - ), copy(c) AS (VALUES ('00'), ('01')) - SELECT (jsonb_build_object( - 'v', 4, 'i', jsonb_build_object('t', 'users', 'c', 'profile'), - 'h', (SELECT encode(h, 'base64') FROM k), - 'sv', jsonb_build_array(jsonb_build_object('s', 'AAAAAAAAAAAAAAAAAAAAAA==', 'c', '00')) - || (SELECT jsonb_agg(jsonb_build_object('s', 'AAAAAAAAAAAAAAAAAAAAAA==', 'c', copy.c, 'p', eql_v4_internal.json_tree_parent_tag(k.iv, link.t, 0)) ORDER BY n, copy.c) FROM link, k, copy), - 'q', '{{\"sv\": [], \"el\": []}}'::jsonb))::public.eql_v4_json_tree", - last = LEVELS - 1 - ) - }; - let token = "AAAAAAAAAAAAAAAAAAAAAA==".to_owned(); + let token = vec!["AAAAAAAAAAAAAAAAAAAAAA==".to_owned()]; let entries = 1 + 2 * LEVELS; for (flags, budget, expected) in [ - (0, 0, LEVELS), - (1, 0, LEVELS), - (0, 10, LEVELS), - (1, 200, LEVELS), + (0, 0, LEVELS + 1), + (1, 0, LEVELS + 1), + (0, 10, LEVELS + 1), + (1, 200, LEVELS + 1), (1, 10, 1), ] { - let doc = doc(flags); + let doc = forged_chain(flags, LEVELS, 2, false); let stored: i64 = db .query_one( &format!("SELECT jsonb_array_length(d.doc::jsonb -> 'sv')::bigint FROM ({doc}) AS d(doc)"), @@ -1643,7 +2382,7 @@ fn postgres_walks_do_not_multiply_on_duplicated_entries() { assert_eq!(stored, entries); let walked: i64 = db .query_one( - &format!("SELECT count(*) FROM ({doc}) AS d(doc), eql_v4.json_tree_subtree(d.doc, $1::text, {budget}) e"), + &format!("SELECT count(*) FROM ({doc}) AS d(doc), eql_v4.json_tree_subtree(d.doc, $1::text[], {budget}) e"), &[&token], ) .unwrap_or_else(|error| panic!("flags {flags}, budget {budget}: {error}")) @@ -1653,10 +2392,53 @@ fn postgres_walks_do_not_multiply_on_duplicated_entries() { let children: i64 = db .query_one( &format!("SELECT count(*) FROM ({doc}) AS d(doc), eql_v4.json_tree_children(d.doc, $1::text) e"), - &[&token], + &[&token[0]], ) .unwrap() .get(0); assert_eq!(children, 1, "flags {flags}"); } } + +/// Needs an EQL v4 installation. A member's token is a hash of its +/// parent's and its selector, not of the probe that found it, so two +/// entries forged under one parent, at ordinals 0 and 1, with one selector +/// and the member rule's self tag, derive one token: the read visits the +/// subtree beneath it once for each, and level after level the rows would +/// double, 2^60 for 60 levels. The read takes its recursion through `LIMIT` +/// the entry count times the number of tokens, so it ends at that, and +/// returns no more rows than that under any budget. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_walks_do_not_multiply_on_forged_member_tags() { + let mut db = connect(); + db.batch_execute("SET statement_timeout = '10s'").unwrap(); + const LEVELS: i64 = 60; + let token = vec!["AAAAAAAAAAAAAAAAAAAAAA==".to_owned()]; + let entries = 1 + 2 * LEVELS; + for (flags, budget) in [(0, 0), (1, 0), (1, 64), (1, 1000)] { + let doc = forged_chain(flags, LEVELS, 2, true); + let walked: i64 = db + .query_one( + &format!("SELECT count(*) FROM ({doc}) AS d(doc), eql_v4.json_tree_subtree(d.doc, $1::text[], {budget}) e"), + &[&token], + ) + .unwrap_or_else(|error| panic!("flags {flags}, budget {budget}: {error}")) + .get(0); + assert!( + walked <= entries, + "flags {flags}, budget {budget}: {walked} rows" + ); + if flags == 1 && budget == 64 { + assert_eq!(walked, 1, "past the budget, the sealed root"); + } + let children: i64 = db + .query_one( + &format!("SELECT count(*) FROM ({doc}) AS d(doc), eql_v4.json_tree_children(d.doc, $1::text) e"), + &[&token[0]], + ) + .unwrap() + .get(0); + assert_eq!(children, 2, "flags {flags}: both forged children"); + } +} diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index 409199918..60d85af6e 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -12,8 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **A JSON index, `target::Json`** (`docs/plans/2026-10-17-json-tree-index.md`, ADR-0009). An `Index` whose stored form, `json::JsonDocument`, is one entry per node under one data key per - document: a path selector, the node's own ciphertext, a chain of tokens - derived by ordinal that the server walks from one client-derived token, + document: a path selector, the node's own ciphertext, a self tag and a + parent tag by which the server finds the node a client-derived token + names and walks beneath it, every token derivable from its node's path + alone, a stored containment projection that agrees with `jsonb @>` across array elements, and, at the paths `JsonOptions::ordered` names, an ordering term on number and string leaves. It is what EQL v4's `json_tree` family @@ -29,45 +31,62 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 a `JsonExtracted`, which refuses an entry that does not carry the selector (another path's, or the sealed root in place of a node) before any key is retrieved; `JsonSubtreeKey::open(rows)` builds a - `JsonSubtree`, which refuses a row that is not where the walk's rows must - be: a reached node without the path's selector or position, a row - without the selector of its parent's path and its own label (the key - holds the keyset's index-key PRF and the column's context to derive - it), and a row that does not follow the one before it in the document - or whose parent is not among the rows. Each refusal is the new - `Error::JsonWrongEntry`. Not checked, by design, is which row of the - column an entry comes from (ADR 0010). `JsonOptions::sealed_root`, - on by default, makes the root entry the whole document so no token for - the root exists. A document is at most 127 levels deep. New `TermError` - variants: `JsonPath`, `JsonTooDeep`, `JsonTooWide`, `JsonSealedRoot`, - `JsonLeafNotOrdered`, `JsonPathNotOrdered`, `JsonOrderingAtPosition` - and `JsonValueNotScalar`. `JsonTooWide` also refuses a subtree key for a - path naming an array position past 2^31 - 1, PostgreSQL's `integer`, - which the SQL walk casts a position gate to. `JsonOptions::ordering([..])` names the + `JsonSubtree`, which refuses a row that is not where the read's rows + must be: a node a token names without the path's selector or last step + (its key, its position, or the root), a row without the selector of its + parent's path and its own label (the key holds the keyset's index-key + PRF and the column's context to derive it), a row that does not follow + the one before it in the document or whose parent is not among the + rows, a row whose self tag is not the one the client recomputes + (unauthenticated, so a consistency check), an entry returned twice, and + rows of two documents. Each refusal is the new `Error::JsonWrongEntry`. + Not checked, by design: which row of the column + an entry comes from (ADR 0010), which element a node at a selector sits + under, and whether the database withheld rows. `JsonOptions::sealed_root`, + on by default, makes the root entry the whole document and chains the + top-level fields from nothing, so the root's token finds the root entry + and nothing beneath it. A document is at most 127 levels deep. New + `TermError` variants: `JsonPath`, `JsonTooDeep`, `JsonTooWide`, + `JsonWildcardTooWide`, `JsonLeafNotOrdered`, `JsonPathNotOrdered`, + `JsonOrderingAtPosition` and `JsonValueNotScalar`. `JsonTooWide` also + refuses a subtree key for a path naming an array position, or a + wildcard bound, past 2^31 - 1, PostgreSQL's `integer`: no stored + document has an element there. `JsonOptions::ordering([..])` names the ordered paths, `JsonPath::covers` is the rule that matches them, and a path naming an array position is refused there and in an ordering query, since the comparison extracts by selector and reads a position as the wildcard. A data plan reads and writes the index (`"json"`, or `{"json": {"sealed_root": bool, "ordered": [path, …]}}`) and refuses to run it with `Reason::JsonIndexPending`. `JsonPath` implements `Serialize` - and `Deserialize` as its text, read back through `JsonPath::parse`. A subtree key is a token and - `JsonSubtreeKey::gates()`, one gate per path step below it, a selector - or `#n:` for an array position. A walk past its budget returns - the sealed root in place of the subtree. The key carries the path it - was derived from (`JsonSubtreeKey::path`) and builds the rows' wrapper, - `key.open(rows) -> JsonSubtree`, the wrapper's only constructor; from - its gates it knows whether the rows are below the path's node - (`json_tree_subtree`) or at the nodes it reaches (`json_tree_subtree_at`), - so the rows open to the same value either way, over `JsonPath::select`, - which evaluates a path over a value. On the wire every field the SQL reads (`s`, the pairs - of `q`, `p`, `op`, `h`, tokens and gates) is padded standard base64, and + and `Deserialize` as its text, read back through `JsonPath::parse`. A subtree key + (`KeysetCipher::json_subtree_key`, or `json_subtree_key_with_bound`) is + one token per concrete path its path names, in document order: + exactly one for a path with no wildcard, `$` included, and for each + wildcard level the positions below its bound (64 by default, + `JsonSubtreeKey::DEFAULT_BOUND`) and a probe at the bound + (`JsonSubtreeKey::tokens()`, `probes()`, `bound()`), which + `json_tree_subtree(doc, tokens, budget)` takes. A read whose probe finds + an element, an array longer than the bound, fails with the new + `Error::JsonWildcardTruncated { bound }` rather than return part of the + answer, and a read that would send more than 8,192 tokens is refused + as `JsonWildcardTooWide`. The rows open as a `Value` (the node at the + path, `{}`, `[]` and scalars included), an `Option` (`None` where + the row has no node there) or a `Vec` (one value per concrete + path reached). A read past its budget returns the sealed root in place + of the rows; the key carries the path it was derived from + (`JsonSubtreeKey::path`) and builds the rows' wrapper, + `key.open(rows) -> JsonSubtree`, the wrapper's only constructor, and + evaluates its concrete paths and probes over the whole document, so the + rows open to the same answer either way, the truncation included, over + `JsonPath::select`, which evaluates a path over a value. On the wire every field the SQL reads (`s`, the pairs + of `q`, `x`, `p`, `op`, `h` and the tokens) is padded standard base64, and the ciphertext `c` is RFC 1924 base85 (`base85-simd`). The document's plaintext is held under `Protected` throughout, as the scalar paths hold theirs: each node's plaintext is encoded straight from the caller's value into a `Protected>` for the cipher, and what an entry opens to (a key, a scalar, a sealed root's document) stays wrapped until it is handed to the caller. The sealed-root fallback moves the nodes at the - walked path out of the whole document rather than copying them, so their + read's paths out of the whole document rather than copying them, so their plaintext exists once, and the rest of the document is wiped. - **A layout chosen by value, `target::Layout`, and diff --git a/packages/stack-encrypt/Cargo.toml b/packages/stack-encrypt/Cargo.toml index 5f02bb657..55d812128 100644 --- a/packages/stack-encrypt/Cargo.toml +++ b/packages/stack-encrypt/Cargo.toml @@ -117,9 +117,11 @@ zerokms-protocol = { workspace = true } tokio = { workspace = true, features = ["rt", "macros"] } # Compile-fail tests for the derive diagnostics (`tests/ui`). trybuild = "1" -# The generated containment property in tests/json_index.rs. Default features -# off, as stack-guest-abi takes it: `std` carries the failure persistence and -# the ChaCha RNG the run is seeded with, and nothing else is needed. +# Property tests of the JSON index's derivations and reads +# (tests/json_properties.rs), and the generated containment property in +# tests/json_index.rs. Default features off, as stack-guest-abi takes it: +# `std` carries the failure persistence and the ChaCha RNG the runs are +# seeded with, and nothing else is needed. proptest = { version = "1.7", default-features = false, features = ["std"] } # The derive diagnostics are recorded with every feature on, as `test:unit` diff --git a/packages/stack-encrypt/fuzz/corpus/json_wire_decode/valid-document-open-root b/packages/stack-encrypt/fuzz/corpus/json_wire_decode/valid-document-open-root index 7f6fdbc5a..3c4f8af14 100644 --- a/packages/stack-encrypt/fuzz/corpus/json_wire_decode/valid-document-open-root +++ b/packages/stack-encrypt/fuzz/corpus/json_wire_decode/valid-document-open-root @@ -1 +1 @@ -{"h":"AQAAAAAAAAAAAAAAAAAAAAAAAAAAANIkbh1SlApdiok/mH5A0y0gAOQA5EoyZTbrzBqVUIdV4xg/x1IurKpjtYcdkC3075ES","sv":[{"s":"GUG6+MOjeHGk8vJS5CJOxA==","c":"0z_z&4uZ@=$H@nfJ;TLc2!WvRB^)e4)~BCS@KxODY~VN"},{"s":"ExJSKX+WLXlRo47xPs+Pjw==","c":"0-Hyc){m-585eT>A%Ic${%f*gb&pQecVdC5C2t5g>oDx}hfnAi#J5%","p":"E2rwHzdyogU="},{"s":"ShAx/UGS67/ex/VpZHT5YA==","c":"0=HCNaNC0;x_5~{kj59-YDV^ypfcBUOTx8%H-x$IaWhH==A5ix","p":"RQv+PlaQGuI="},{"s":"YZINpBKKE2Pe4VuD40e6pQ==","c":"0vIUgnZMq7p)P+V%6nYK5j|htqqfK&yaub>9$on1oIAs7`y%6)7AB!D","p":"bBWX/L/rx/E="},{"s":"YZINpBKKE2Pe4VuD40e6pQ==","c":"0PC$Ka-Q6uX)YZMONS)XDKjmy~5JVD#o8)FslVOm8","p":"geMZYPBpVSU="},{"s":"ExJSKX+WLXlRo47xPs+Pjw==","c":"0@^HM$R|__1yG(mL4%}WLIoAol$|)kXNI=t9AAC","p":"nAFCVRKssP4="},{"s":"ExJSKX+WLXlRo47xPs+Pjw==","c":"0)zmeW|-=APNVqgRac|F_*IiJiWhC8bJ!7eQ0C`p_2t4_2mB3%w7sZ9","p":"rZKzWt2q2JA="}],"q":{"sv":[{"s":"GUG6+MOjeHGk8vJS5CJOxA==","vs":"zAI3S2R1ohpJhfgWmnRmxg=="},{"s":"ExJSKX+WLXlRo47xPs+Pjw==","vs":"eGqsMOSc2W0Bx4C7O8SneQ=="},{"s":"ExJSKX+WLXlRo47xPs+Pjw==","vs":"MvrShvASjRgE5vrU2PBgoA=="}],"el":[{"sv":[{"s":"ExJSKX+WLXlRo47xPs+Pjw==","vs":"O1k8IHoig1PAQJ3gPeclfQ=="},{"s":"ShAx/UGS67/ex/VpZHT5YA==","vs":"hUbQp5DbczWtT2O/9ubZtA=="},{"s":"YZINpBKKE2Pe4VuD40e6pQ==","vs":"Lh/z3wdOvqoTt0lQUcDDjw=="},{"s":"YZINpBKKE2Pe4VuD40e6pQ==","vs":"aGBns3vFOnXOivJuSRJDXQ=="}],"el":[]}]}} \ No newline at end of file +{"h":"AQAAAAAAAAAAAAAAAAAAAAAAAAAAABKjoDxWY2N1Hn+HQDu2vWAgAOfubJwCcrCwOjaq3vCLfA29jYljGGly556KMcRQeiun","sv":[{"s":"GUG6+MOjeHGk8vJS5CJOxA==","c":"0_uyytg+^^jIUMQ*P8>p;IEu=ZQ9Xw^{KVz1D3w7347i","x":"l9EsIL2s6tY="},{"s":"Tm5hoSb1R2y2K1ileozKQQ==","c":"0+kkq8e1{-XL@2MZDXUvlAv%WSTS}nF}E=+?%AZD$#wvdqOul(D`mF$yXi","x":"8B7I/IC2QHU=","p":"Gyb+szf69+k="},{"s":"lihJXUx9+YPMvJXQ97UpgA==","c":"0_0C__Q-7s#U|I6Xc(p@VGtf1<`j6+O$Zt18^xYwfWMc|_@{dBhGnZ0sQ","x":"EHsM/TDQzWY=","p":"4aak1I1XiHg="},{"s":"mwxTGtOaM0Q8dwFeVMXzBA==","c":"0*lamD&%#@u`*^LmtgKR$)#(Y)HnAKR#rkc?G=u3fT8C`zZiAfW%U-ybg@O(*+*&","x":"2dRytj3AHyo=","p":"pFq7aRktl8Y="},{"s":"9nWXlzomXDcM6TjK3prsEQ==","c":"0z}IN8BoRShf%PrQQI|uSX{dlGx5@=@U19tQ39|xgvfXfxbu?QkCHS","x":"Y7UNwlgne9Y=","p":"dptU/G6I8Os="},{"s":"x6fT5bkR9wVtuRnupRk3lg==","c":"0#0Aii;>yAW@~)Nzd-DsfniQDQ@Tnds2&Tl*xJNl5he=@3kGwc0X%{","x":"mSvdRQdQ58I=","p":"xgDrbEjem+U="},{"s":"2LHfpdQzfmw63dqQ9eP0Ww==","c":"0!1jIn&@=xa-xi;N{G<`#{BPgwjw41-P1Agg~#%hxbGTxv;Lk?q4GQFAX-X_D3&rHlapcvHuy0;PjR2tC{RZ(jCmqdqUH0_@","x":"uBM0XX3Lfug=","p":"SBbX+WYtUWM="},{"s":"2LHfpdQzfmw63dqQ9eP0Ww==","c":"0>FGSl`2Tkbx=6y4gLqw$K^nAYXT^fd!B8N!rZJ)rnk$pnD8>JN&","x":"YsBUhhzSpWI=","p":"gSinQifR5cY="},{"s":"guOaofIleFSaqgIbtgP9BA==","c":"0&?wi&QAzT4_JoEcrns4DtiFO4)PT&RZ!28}Hi;-w&S1-hb4@^;EEm$(Toj4284Jqu=>ag&T4>|x$U)_I?2^3vyd#JRP+T_3`zeL)SU#bj#f23ew$<;Z?-DqE)nv=R"},{"s":"Tm5hoSb1R2y2K1ileozKQQ==","c":"0$d}`UG9!!2zc|70dbbXQT*l>3XCn|avEj^q13yc$n~FYb6~5qH}s~Nu-QViqt(#"},{"s":"lihJXUx9+YPMvJXQ97UpgA==","c":"0zBNN!Rd^LxAecmwyi?Z8on|q_JlQjGl^PqrDNSannTJ7b}&!hH=h1RwE"},{"s":"mwxTGtOaM0Q8dwFeVMXzBA==","c":"0%?9A501BMDh5#JJ&PCWXKCCTdaSCMvk2kXf+}{^^L>^llxYVWG7(BPYkJSk90s@","p":"t980GLGUA+w="},{"s":"9nWXlzomXDcM6TjK3prsEQ==","c":"0+jHnFB2n_4P!_S^`TcBs)r{(Hr3>p?pqz&cZ8{nlHZA=e;o*DM-Y+","p":"T1Z1ZgBOtes="},{"s":"x6fT5bkR9wVtuRnupRk3lg==","c":"0$Xju@DoU4*%E4&sMnf`M&e0()eH`sd#7NAG4;zEU5D0sX~T=>y#6%","p":"a7mxfLHzS9k="},{"s":"2LHfpdQzfmw63dqQ9eP0Ww==","c":"0$r1?vQ(9$?($NHXdP^l`0eN!qO;;zMU4Ar$so8V;UrYt1`{M0%>cN9N3SSFSS)1D","p":"xHMEagMG00I="},{"s":"2LHfpdQzfmw63dqQ9eP0Ww==","c":"0uPz!h(9!yqN>{O#@uXGMJb<0xKyaqrdk1{&tku@_rE?}HohCNVg","p":"eriAuFTQ3sk="},{"s":"guOaofIleFSaqgIbtgP9BA==","c":"0>bADC+pE&wsaPW)#?6&TQzE}Ce;ANg8^g`Bhgkpxv0rM%d&Cz@!7Qt(Z~","op":"AXCYcmjGlPA9o88wtKHqszfNGZu5WS/7jkkQP0S/ijn14DbmuAaN9OyLuYmFzBvH/sbekttZj1Rq+SleI2caoiM="}],"q":{"sv":[{"s":"GUG6+MOjeHGk8vJS5CJOxA==","vs":"MUinEHjmPaxhTO0p+J6gNw=="},{"s":"Tm5hoSb1R2y2K1ileozKQQ==","vs":"1nnluVkGLHmNyEBbXNpLFw=="},{"s":"lihJXUx9+YPMvJXQ97UpgA==","vs":"o2ktqHlsWUKHWNjBs+cWBQ=="},{"s":"mwxTGtOaM0Q8dwFeVMXzBA==","vs":"0k1uvXj0q1y00BoSeO5uZA=="},{"s":"9nWXlzomXDcM6TjK3prsEQ==","vs":"LUR7qeHmg+k3cJgbmnUJnQ=="},{"s":"guOaofIleFSaqgIbtgP9BA==","vs":"KJQFi9ZscpCAnh7U2HOgLQ=="}],"el":[{"sv":[{"s":"x6fT5bkR9wVtuRnupRk3lg==","vs":"vzVi6dl1WYdpruf8GpBrVg=="},{"s":"2LHfpdQzfmw63dqQ9eP0Ww==","vs":"baRZIwaJ+CSTK/GflygMAg=="}],"el":[]},{"sv":[{"s":"x6fT5bkR9wVtuRnupRk3lg==","vs":"vzVi6dl1WYdpruf8GpBrVg=="},{"s":"2LHfpdQzfmw63dqQ9eP0Ww==","vs":"wYWQ5BRfSrEVO+HQeX/bxQ=="}],"el":[]}]}} \ No newline at end of file +{"h":"AQEAAAAAAAAAAAAAAAAAAAAAAAAAANsC3aoZQva91PovkvEWZaggAJwRJBgQdsDmNJ3XHovNeuV70W23BIlDF4Qu+cuTgRqH","sv":[{"s":"GUG6+MOjeHGk8vJS5CJOxA==","c":"0tmz}(4kIV*bT-Y5MTF{sUq(k=Re!jP7&Fi@0Pezt8>$SgeG~!U;=9miSK=#DOlF{S$ndLq39e+^@L|{u=5e>_NN4o_@2X^vj3!*6mG3b*wqQK-XrdkSe;6tGH6zWL2}3ln&{g%m|vh%PG|","x":"nMWzM77hqBA="},{"s":"Tm5hoSb1R2y2K1ileozKQQ==","c":"0yr=$24{O_hPt{$c(8#!8LRwdsXq9jo%A;S1i!`97AL=sPbA$+K^_9;?FV=4@Vo^","x":"HQ5OhDze0Hk="},{"s":"lihJXUx9+YPMvJXQ97UpgA==","c":"0$b2o5(E~qm0B1P7oXnlEMV4FlH?-vUc`CrMCb1<%8ve~yQp{nEOkBQq5","x":"GRz/GLLp6ks="},{"s":"mwxTGtOaM0Q8dwFeVMXzBA==","c":"0?A%>-*%=ham^fHsgxWZ1o;&yoG`?wMOEYmnSV5Yr<2-s)v1K0BI}pXl)e$nSl=4","x":"4FBgYlmPudM=","p":"l3USBrHwdH4="},{"s":"9nWXlzomXDcM6TjK3prsEQ==","c":"0wcKBzViBysC%aNMJnP^_ae-|gK|fncW@0&yso0wh^j~*lYKl!DC&s","x":"dRGJGHgaF7Y=","p":"PfzPJ0gCP/I="},{"s":"x6fT5bkR9wVtuRnupRk3lg==","c":"0^T~C2;@ZE#qZ=^%qj^OI7ifPjMYoqkTNFlCinE$R{=Nu2`XSvs@DS","x":"2UZoUrHXJ7w=","p":"oiEVu7HN/RU="},{"s":"2LHfpdQzfmw63dqQ9eP0Ww==","c":"0^R1qx3$FKImLK(gOeT{?TY34nSYCM*w)LrV?8agIV6Jldgext2m","x":"G0HU3UlPcoE=","p":"s1dLrpV4k1s="},{"s":"x6fT5bkR9wVtuRnupRk3lg==","c":"0;o0JjNE7=#Z20ZOQ>j`&n;Sh+&-bQqm2`wG4vfu>u;u!-KPUwT7c>","x":"JHDhxF9YqJ0=","p":"SSwUlW8t1wE="},{"s":"2LHfpdQzfmw63dqQ9eP0Ww==","c":"0u0WgruL?(nV>ObADC+pE&wsaPW)#?6&TQzE}Ce;ANg8^g`Bhgkpxv0rM%d&Cz@!7Qt(Z~","op":"AXCYcmjGlPA9o88wtKHqszfNGZu5WS/7jkkQP0S/ijn14DbmuAaN9OyLuYmFzBvH/sbekttZj1Rq+SleI2caoiM=","h":"AQEAAAAAAAAAAAAAAAAAAAAAAAAAAModWByqFwzHFmXW6zBOgBIgALdJJWr10FWvJW0c+enHwFzJNgKih/2xZckLO87P5bPA","j":9} \ No newline at end of file +{"c":"0(@$Cy~O#=NgO}P$S@8CV{XlV@;gK2F;ov-k&tO0AQ(;bMPz==Gf@e7VI}","d":1,"h":"AQEAAAAAAAAAAAAAAAAAAAAAAAAAANsC3aoZQva91PovkvEWZaggAJwRJBgQdsDmNJ3XHovNeuV70W23BIlDF4Qu+cuTgRqH","j":9,"op":"AXgV4faXQqbkymeu1HNoChFnQWm3lHYeHaTTIOgMnLD2Vt+1uMVUJZLzGtBBitIcghHqEdkgogrT4dDA4a6o3zg=","s":"guOaofIleFSaqgIbtgP9BA==","x":"9wyVuZ1FELc="} \ No newline at end of file diff --git a/packages/stack-encrypt/fuzz/fuzz_targets/json_wire_decode.rs b/packages/stack-encrypt/fuzz/fuzz_targets/json_wire_decode.rs index eb5c0ea67..ae4fbd1bc 100644 --- a/packages/stack-encrypt/fuzz/fuzz_targets/json_wire_decode.rs +++ b/packages/stack-encrypt/fuzz/fuzz_targets/json_wire_decode.rs @@ -6,7 +6,7 @@ use stack_encrypt::json::{JsonDocument, JsonHeader, JsonLeafQuery, JsonNode, Jso // Fuzz the JSON index's wire decoders. A stored `json_tree` document, and // every entry `->` or a walk returns, come back from the database through // these before anything is authenticated, so the text is attacker-controlled: -// the base64 of the selectors, parent tags, ordering terms and header, the +// the base64 of the selectors, self and parent tags, ordering terms and header, the // base85 of the ciphertexts (decoded with SIMD code on aarch64 and x86_64), // and the header's frozen byte layout. Decoding must never panic, only // return `Err`. diff --git a/packages/stack-encrypt/src/cipher.rs b/packages/stack-encrypt/src/cipher.rs index 327da2785..6c469e8ed 100644 --- a/packages/stack-encrypt/src/cipher.rs +++ b/packages/stack-encrypt/src/cipher.rs @@ -178,8 +178,9 @@ pub enum Error { /// or the sealed root in place of a node, or a subtree walk returned a /// row that is not where the walk's key says its rows are (a node of /// another path, a row out of its place in the document, a row whose - /// parent is not among the rows, or rows that stop before a node's - /// subtree is complete). Refused by the client, which + /// parent is not among the rows, rows that stop before a node's + /// subtree is complete, one entry returned twice, or rows of two + /// documents). Refused by the client, which /// derived the path's selector itself; see /// [`JsonSelector::open`](crate::json::JsonSelector::open) and /// [`JsonSubtreeKey::open`](crate::json::JsonSubtreeKey::open). @@ -202,6 +203,20 @@ pub enum Error { help("The entry opened, but it is not the one the query asked for. A database that answers the query it was sent never returns one: check that the rows came from the query the selector or subtree key was derived for, and treat them as tampered with if they did.") )] JsonWrongEntry { expected: &'static str }, + /// A JSON subtree read with a wildcard found an array longer than the + /// bound the wildcard was expanded to: the probe the read sent for the + /// element at `bound` named a node. The read fails rather than return + /// part of the answer; see + /// [`KeysetCipher::json_subtree_key_with_bound`](crate::KeysetCipher::json_subtree_key_with_bound). + /// + /// Names the bound and nothing of the path, whose key names can be + /// customer data. + #[error("a JSON array at the wildcard is longer than the read's bound of {bound}")] + #[diagnostic( + code(stack_encrypt::json_wildcard_truncated), + help("A wildcard read expands to the positions below its bound and fails when an array is longer, so it never returns part of the answer. Derive the key with a larger bound (`json_subtree_key_with_bound`), or read the array itself.") + )] + JsonWildcardTruncated { bound: u32 }, /// ZeroKMS returned a different number of keys than were requested. #[error("expected {expected} data keys from ZeroKMS but received {received}")] #[diagnostic(code(stack_encrypt::key_count_mismatch))] @@ -375,6 +390,7 @@ impl crate::ErrorPayload for Error { Self::OpenedType { expected } | Self::JsonWrongEntry { expected } => { payload([("expected", (*expected).into())]) } + Self::JsonWildcardTruncated { bound } => payload([("bound", (*bound).into())]), Self::DescriptorTooLong { len } => payload([ ("len", (*len).into()), ("limit", Descriptor::MAX_LEN.into()), diff --git a/packages/stack-encrypt/src/codes.rs b/packages/stack-encrypt/src/codes.rs index 0d3836fc9..9a581fb28 100644 --- a/packages/stack-encrypt/src/codes.rs +++ b/packages/stack-encrypt/src/codes.rs @@ -52,6 +52,7 @@ mod tests { Error::JsonWrongEntry { .. } => Error::JsonWrongEntry { expected: "the entry at the path asked for", }, + Error::JsonWildcardTruncated { .. } => Error::JsonWildcardTruncated { bound: 64 }, Error::KeyCountMismatch { .. } => Error::KeyCountMismatch { expected: 2, received: 1, @@ -98,7 +99,7 @@ mod tests { }), TermError::JsonTooDeep => TermError::JsonTooDeep, TermError::JsonTooWide => TermError::JsonTooWide, - TermError::JsonSealedRoot => TermError::JsonSealedRoot, + TermError::JsonWildcardTooWide => TermError::JsonWildcardTooWide, TermError::JsonLeafNotOrdered => TermError::JsonLeafNotOrdered, TermError::JsonPathNotOrdered => TermError::JsonPathNotOrdered, TermError::JsonOrderingAtPosition => TermError::JsonOrderingAtPosition, @@ -406,6 +407,10 @@ mod tests { }), json!({ "expected": "the entry at the path asked for" }), ), + ( + Box::new(Error::JsonWildcardTruncated { bound: 64 }), + json!({ "bound": 64 }), + ), ( Box::new(LeafBytesError::UnknownVersion(9)), json!({ "version": 9 }), diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index d50ab1f5f..1b4753cd3 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -24,7 +24,7 @@ use super::plain::{ insert_member, protected_array, protected_object, push_element, read_protected, take_protected, PlainJson, }; -use super::query::{Gate, JsonSelector, JsonSubtreeKey}; +use super::query::{JsonSelector, JsonSubtreeKey, Read}; use super::{JsonDocument, JsonEntry, JsonExtracted, JsonHeader, JsonOptions, JsonSubtree, Tag}; use crate::keyset::KeysetState; use crate::sem::TermError; @@ -51,23 +51,26 @@ const RANK_DOMAIN: &[u8] = b"stack-encrypt/json/rank/v1"; /// The bound is what reading the document back needs, not what the walk /// could manage. A sealed root's plaintext is the document as JSON, which /// `serde_json` parses with a nesting limit of 128 containers, so a -/// document whose innermost value sits 128 levels down seals and then never -/// opens. Refusing it here moves that failure from the read, where the -/// value is lost, to the write, where the caller still holds it. The bound -/// is the same under either root option, so the option never changes which -/// documents the index accepts. +/// document whose innermost value sits 128 levels down, or whose innermost +/// container is the 128th (an empty `[]` or `{}` at depth 127), seals and +/// then never opens. Refusing both here moves that failure from the read, +/// where the value is lost, to the write, where the caller still holds it. +/// The bound is the same under either root option, so the option never +/// changes which documents the index accepts. pub(crate) const MAX_DEPTH: usize = 127; /// The largest number the index gives an array element's position, a /// node's ordinal or a node's child count: PostgreSQL's `integer`, which is -/// what the SQL walk casts a position gate's ordinal to and what -/// `json_tree_child_count` returns. +/// what the SQL walk's ordinals are and what `json_tree_child_count` +/// returns. /// /// A document with a wider array or object is refused when it is sealed, /// and a path naming a position past this is refused when its subtree key /// is derived. No stored document has a node there, so the refusal loses no -/// answer, and without it the position would reach the server as a gate -/// that PostgreSQL refuses to cast (error 22003) instead of finding nothing. +/// answer: a position reaches the server only inside a token, so without +/// the refusal the read would find nothing, where the caller asked for +/// something no document can hold. A wildcard's bound is not held to this: +/// the token cap ([`MAX_READ_TOKENS`]) is far lower and refuses first. const MAX_ORDINAL: u32 = i32::MAX.unsigned_abs(); /// `n` as an ordinal, or [`TermError::JsonTooWide`] past [`MAX_ORDINAL`]. @@ -176,12 +179,15 @@ pub(crate) struct TermNode { pub(crate) positional: JsonPath, pub(crate) s: Tag, pub(crate) vs: Tag, - /// The node's token: every node has one except the sealed root. - pub(crate) token: Option, + /// The node's token, derivable from its path alone: every node has + /// one, the root and the chain's heads included. + pub(crate) token: Tag, /// The leaf ordering term, on a number or string. pub(crate) op: Option>, pub(crate) parent: Option, - /// The node's ordinal under its parent, when the parent has a token. + /// The node's ordinal under its parent, when it chains from its parent + /// (everything below the chain's heads): its index for an array + /// element, its keyed rank among its siblings for an object member. pub(crate) ordinal: Option, /// The nearest enclosing array element that is a container. pub(crate) element: Option, @@ -352,22 +358,62 @@ fn h16(parts: &[&[u8]]) -> Tag { truncate(&hasher.finalize()) } -/// `H("t" ‖ t(parent) ‖ s(child) ‖ LE32(ordinal))`: the token of the child -/// at `ordinal` under a node, whose selector is `selector`. Column-wide like -/// every token (no document salt), so the client derives an array -/// element's token from the path alone, and the server derives every hit's -/// from its parent's and the hit's stored `s` as it walks; no token is ever -/// stored. +/// `H16("t" ‖ t(parent) ‖ s(member))`: the token of the object member whose +/// selector is `selector`, under the object whose token is `parent`. +/// Column-wide like every token (no document salt), and free of the +/// member's ordinal, which is a rank among its siblings' keys: so the +/// client derives a member's token from the path alone, whatever else the +/// object holds in a row, and the server derives a hit's from its parent's +/// and the hit's stored `s` as it walks. No token is ever stored. /// -/// The selector is what keeps an element apart from an object member. An -/// ordinal alone does not: a member's is its rank among its siblings, and -/// rows of one column need not agree on a node's kind, so without it the -/// token of `$.a[0]` would be the token of whichever member of an object -/// `$.a` ranks first, in every row whose `$.a` is an object. An element's -/// selector is the wildcard's and a member's names its key, so the two -/// never share a token. -pub(super) fn child_token(parent: &Tag, selector: &Tag, ordinal: u32) -> Tag { - h16(&[b"t", parent, selector, &ordinal.to_le_bytes()]) +/// Object keys are unique, so the selector alone tells a member from its +/// siblings. It also keeps a member apart from an array element: an +/// element's selector is the wildcard path's and a member's names its key, +/// and an element's input is four bytes longer ([`element_token`]), so the +/// two never share a token. +pub(super) fn member_token(parent: &Tag, selector: &Tag) -> Tag { + h16(&[b"t", parent, selector]) +} + +/// `H16("t" ‖ t(array) ‖ s(element) ‖ LE32(index))`: the token of the +/// element at `index` of the array whose token is `array`, `selector` being +/// the selector every element of that array shares. An element's ordinal is +/// its index, so this is derivable from the path alone too. +pub(super) fn element_token(array: &Tag, selector: &Tag, index: u32) -> Tag { + h16(&[b"t", array, selector, &index.to_le_bytes()]) +} + +/// The first eight bytes of `H("x" ‖ iv ‖ t(node))`: the self tag every +/// entry carries, by which the server finds the node a token names. Salted +/// by the document like the parent tag, so it is unrelated bytes in every +/// row until a query supplies the token. Eight bytes, as the parent tag +/// ([`SelfTag`](super::SelfTag)). +pub(super) fn self_tag(iv: &[u8], token: &Tag) -> super::SelfTag { + let full = h16(&[b"x", iv, token]); + let mut tag = super::SelfTag::default(); + tag.copy_from_slice(&full[..8]); + tag +} + +/// The token the server derives for `entry`, found at `ordinal` under the +/// node whose token is `parent`: the member rule's if the entry's self tag +/// is that token's, else the element rule's if it is that one's, else none. +/// Nothing stored says whether a node is a member or an element (its kind +/// is inside `c`), so the self tag is what chooses. An entry that matches +/// neither is still a child, and is returned, but the walk does not descend +/// into it. +pub(super) fn hit_token( + iv: &[u8], + parent: &Tag, + entry: &super::JsonEntry, + ordinal: u32, +) -> Option { + let member = member_token(parent, &entry.s); + if self_tag(iv, &member) == entry.x { + return Some(member); + } + let element = element_token(parent, &entry.s, ordinal); + (self_tag(iv, &element) == entry.x).then_some(element) } /// The first eight bytes of `H("p" ‖ iv ‖ t(parent) ‖ LE32(ordinal))`: @@ -384,11 +430,13 @@ pub(super) fn parent_tag(iv: &[u8], parent: &Tag, ordinal: u32) -> super::Parent /// `PRF(json/rank ‖ column, PAE(t(parent), key))`: what an object's members /// are ranked by for their ordinals, so an ordinal does not rank key names. -/// Keyed with the index key (the design's decision 11). The chain below a -/// token is public to whoever holds it (the server derives each child's -/// token from its ordinal), so an unkeyed rank would let a token's holder +/// Keyed with the index key (the design's decision 11). The ordinals below +/// a token are public to whoever holds it (the server enumerates a node's +/// children by probing them), so an unkeyed rank would let a token's holder /// rank guessed key names and compare the order with the selector it finds -/// at each ordinal, in every row. Only the client ranks, when it seals. +/// at each ordinal, in every row. Only the client ranks, when it seals. A +/// member's token does not depend on its rank ([`member_token`]); its parent +/// tag does. fn rank( prf: &HmacSha256Prf, context: &Context<'_>, @@ -519,7 +567,6 @@ fn walk( None, None, None, - None, 0, )?; let alt = if Kind::of(value).is_container() { @@ -540,6 +587,10 @@ fn walk( } impl Walker<'_> { + /// Visit `value`, the node at `positional`. `chain` is its parent's + /// token and its ordinal under it when it chains from its parent, and + /// `None` for the root and, under the sealed root, for the chain's + /// heads, whose tokens are PRFs of their paths. #[allow(clippy::too_many_arguments)] fn visit( &mut self, @@ -548,15 +599,17 @@ impl Walker<'_> { wild: &JsonPath, label: LabelRef<'_>, parent: Option, - parent_token: Option, - ordinal: Option, + chain: Option<(Tag, u32)>, element: Option, depth: usize, ) -> Result { - if depth > MAX_DEPTH { + let kind = Kind::of(value); + // A container at MAX_DEPTH, empty or not, would be the 128th nested + // container, which serde_json refuses to read back; only a scalar + // may sit that deep. + if depth > MAX_DEPTH || (depth == MAX_DEPTH && kind.is_container()) { return Err(TermError::JsonTooDeep); } - let kind = Kind::of(value); let s = prf16( self.prf, SELECTOR_DOMAIN, @@ -564,18 +617,20 @@ impl Walker<'_> { &wild.selector_bytes(), )?; let vs = value_selector(self.prf, self.context, &s, value)?; - let token = match (depth, self.options.sealed_root) { - (0, true) => None, - (0, false) | (1, true) => Some(prf16( + let token = match (chain, label) { + (Some((parent, _)), LabelRef::Key(_)) => member_token(&parent, &s), + (Some((parent, o)), LabelRef::Index(_)) => element_token(&parent, &s, o), + _ => prf16( self.prf, TOKEN_DOMAIN, self.context, &positional.token_bytes().unwrap_or_default(), - )?), - _ => parent_token - .zip(ordinal) - .map(|(p, o)| child_token(&p, &s, o)), + )?, }; + let ordinal = chain.map(|(_, o)| o); + // Under the sealed root nothing chains from the root: its members + // are the chain's heads. + let chains = depth > 0 || !self.options.sealed_root; let op = if self.options.orders(positional) { leaf_op(self.prf, self.context, value)? } else { @@ -608,19 +663,21 @@ impl Walker<'_> { }; match value { Value::Object(map) => { - let ordinals: Option> = token - .map(|t| shuffled_ordinals(self.prf, self.context, &t, map)) + let ordinals: Option> = chains + .then(|| shuffled_ordinals(self.prf, self.context, &token, map)) .transpose()?; for (key, child) in map { - let ordinal = ordinals.as_ref().and_then(|o| o.get(key).copied()); + let chain = ordinals + .as_ref() + .and_then(|o| o.get(key).copied()) + .map(|o| (token, o)); let _ = self.visit( child, &positional.then(Segment::Key(key.clone())), &wild.then(Segment::Key(key.clone())), LabelRef::Key(key), Some(index), - token, - ordinal, + chain, child_element, depth + 1, )?; @@ -629,15 +686,17 @@ impl Walker<'_> { Value::Array(items) => { for (i, child) in items.iter().enumerate() { let i = u64::try_from(i).map_err(|_| TermError::JsonTooWide)?; - let ordinal = token.map(|_| fit_ordinal(i)).transpose()?; + let chain = chains + .then(|| fit_ordinal(i)) + .transpose()? + .map(|o| (token, o)); let _ = self.visit( child, &positional.then(Segment::Index(i)), &wild.then(Segment::Wildcard), LabelRef::Index(i), Some(index), - token, - ordinal, + chain, child_element, depth + 1, )?; @@ -864,13 +923,14 @@ fn seal( ) .map_err(|_| Error::Aead)?; let c = c.as_ref().to_vec(); - let parent_token = node.parent.and_then(|p| nodes[p].token); - let p = parent_token + let p = node + .parent .zip(node.ordinal) - .map(|(parent, o)| parent_tag(&iv, &parent, o)); + .map(|(parent, o)| parent_tag(&iv, &nodes[parent].token, o)); entries.push(JsonEntry { s: node.s, c, + x: self_tag(&iv, &node.token), p, op: node.op.clone(), }); @@ -1176,172 +1236,127 @@ impl DecryptInto for JsonExtracted { } } -/// Open the rows of a walk: each complete subtree the rows hold, as a -/// value, in order. What `json_tree_subtree_at` returns is one subtree per -/// node its gates reached, each rooted at depth `0`; what -/// `json_tree_subtree` returns is the children of one node, which -/// [`DecryptInto`] assembles into that node's value, or this opens -/// into its children's values. The sealed root a walk fell back to opens -/// whole, and the key's path selects over it to the same values the walked -/// rows open to: one per node the path reaches, for a key with gates -/// (`json_tree_subtree_at`); the children of the one node at the path, for -/// a key without (`json_tree_subtree`). An object's members come in an -/// order neither form fixes, and without their keys; open it as a -/// [`Value`] to keep them. Every row is checked against the key first -/// ([`JsonSubtreeKey::open`]); a row that does not fit is -/// [`Error::JsonWrongEntry`]. +/// Open the rows of a read into the values of the nodes its tokens named, +/// in the key's order, each still under its `Protected`. Shared by the +/// three openings below, so they agree on every row and on the fallback: +/// the sealed root a read fell back to opens whole, and the key's concrete +/// paths and probes are evaluated over it to the same values the walked +/// rows open to, the truncation included. +fn open_reads<'a, K>( + subtree: JsonSubtree, + cipher: &'a StackCipher, + context: AeadContext, +) -> Pending<'a, Vec>, K> { + if subtree.is_sealed_root() { + open_fallback(subtree, cipher, context) + .try_map(|(key, mut whole)| select_reads(&key, &mut whole)) + } else { + open_subtree(subtree, cipher, context) + } +} + +/// The one value a read reached, or none; more than one (a wildcard that +/// reached several nodes) is [`Error::OpenedType`]. +fn one_reached( + mut values: Vec>, +) -> Result>, Error> { + match values.len() { + 0 | 1 => Ok(values.pop()), + _ => Err(shape("one node at the walked path")), + } +} + +/// Open the rows of a read into a list: one value per concrete path the +/// key's tokens named that the row has, in the key's order, which is +/// document order. For a path with no wildcard that is the node at the path +/// or nothing; for a wildcard, every element it reaches below the bound. +/// A probe that named a node (an array longer than the bound) is +/// [`Error::JsonWildcardTruncated`], never a partial list. Every row is +/// checked against the key first ([`JsonSubtreeKey::open`]); a row that +/// does not fit is [`Error::JsonWrongEntry`]. impl DecryptInto> for JsonSubtree { type Context = AeadContext; fn decryption(self, context: Self::Context) -> Decryption, K> { Decryption::from_open(move |cipher| { - if self.is_sealed_root() { - let below = self.below(); - return open_fallback(self, cipher, context).try_map(move |(path, mut whole)| { - // The selected nodes move out of the document's - // `Protected` into one of their own, leaving `null` - // behind, so their plaintext is never copied; what is - // left of the document is wiped when it drops at the - // end of this closure. - let selected = take_protected(&mut whole, |doc: &mut PlainJson| { - let mut found = path.select_mut(doc.value_mut()); - let selected = if below { - match found.as_mut_slice() { - [node] => take_children(node), - _ => return None, - } - } else { - found.into_iter().map(std::mem::take).collect() - }; - Some(PlainJson::from_value(Value::Array(selected))) - }) - .ok_or_else(|| shape("one node at the walked path"))?; - Ok(match hand_over(selected) { - Value::Array(items) => items, - other => vec![other], - }) - }); - } - open_subtree(self, cipher, context) - .map(|forest| forest.into_iter().map(|(_, v)| hand_over(v)).collect()) + open_reads(self, cipher, context) + .map(|values| values.into_iter().map(hand_over).collect()) }) } } -/// Open the rows of a walk into the one value they hold. For -/// `json_tree_subtree(doc, token)`, the rows are the descendants of the -/// token's node and the value is that node's: an array when the rows are -/// elements, an object when they are members. For a gated -/// `json_tree_subtree_at`, the rows are the node the gates reached, at -/// depth `0`, and its descendants, and the value is that node's; more than -/// one reached node (a wildcard) is [`Error::OpenedType`], and opens -/// through [`DecryptInto>`] instead. No rows is no value -/// ([`Error::OpenedType`]): an empty container and a scalar both have no -/// descendants, and `->` on the node's selector returns the node itself. -/// The sealed root a walk fell back to opens whole, and the key's path -/// ([`JsonSubtreeKey::open`]) selects the one value under it; a path that -/// reaches no node, or more than one, is [`Error::OpenedType`]. Every row -/// is checked against the key first ([`JsonSubtreeKey::open`]); a row that -/// does not fit is [`Error::JsonWrongEntry`]. +/// Open the rows of a read into the value at the path, or `None` where the +/// row has no node there: the four answers `jsonb #>` gives, an object, an +/// array and a scalar (`null` included) as `Some`, and SQL `NULL` as +/// `None`, the way `->` returns SQL `NULL` for a path the row lacks. More +/// than one node (a wildcard that reached several) is +/// [`Error::OpenedType`]; a probe that named a node is +/// [`Error::JsonWildcardTruncated`]. +impl DecryptInto> for JsonSubtree { + type Context = AeadContext; + fn decryption(self, context: Self::Context) -> Decryption, K> { + Decryption::from_open(move |cipher| { + open_reads(self, cipher, context) + .try_map(|values| Ok(one_reached(values)?.map(hand_over))) + }) + } +} + +/// Open the rows of a read into the value at the path: an object, an array +/// or a scalar, empty containers included, as `jsonb` holds them there. No +/// node at the path is [`Error::OpenedType`]; open as `Option` to +/// read that as `None`. More than one node is [`Error::OpenedType`] too, +/// and opens through [`DecryptInto>`] instead; a probe that +/// named a node is [`Error::JsonWildcardTruncated`]. impl DecryptInto for JsonSubtree { type Context = AeadContext; fn decryption(self, context: Self::Context) -> Decryption { Decryption::from_open(move |cipher| { - if self.is_sealed_root() { - return open_fallback(self, cipher, context).try_map(|(path, mut whole)| { - // The one node at the path moves out of the document's - // `Protected` into one of its own, leaving `null` - // behind, so its plaintext is never copied; what is - // left of the document is wiped when it drops at the - // end of this closure. - let mut reached = 0; - let selected = take_protected(&mut whole, |doc: &mut PlainJson| { - let mut found = path.select_mut(doc.value_mut()); - reached = found.len(); - match found.as_mut_slice() { - [node] => Some(PlainJson::from_value(std::mem::take(&mut **node))), - _ => None, - } - }); - match (selected, reached) { - (Some(value), _) => Ok(hand_over(value)), - (None, 0) => Err(shape("a node at the walked path")), - (None, _) => Err(shape("one node at the walked path")), - } - }); - } - let below = self.below(); - open_subtree(self, cipher, context).try_map(move |forest| { - if !below { - let mut values: Vec> = - forest.into_iter().map(|(_, v)| v).collect(); - return match values.len() { - 1 => Ok(hand_over(values.remove(0))), - _ => Err(shape("one node at the walked path")), - }; - } - let mut members = None; - let mut elements: Vec<(u64, Protected)> = Vec::new(); - for (label, value) in forest { - match label { - Label::Key(key) => { - insert_member(members.get_or_insert_with(protected_object), key, value) - } - Label::Index(i) => elements.push((i, value)), - Label::Root => return Err(shape("a subtree below the root")), - } - } - match (members, elements.is_empty()) { - (Some(members), true) => Ok(hand_over(members)), - (None, false) => { - elements.sort_by_key(|(i, _)| *i); - let mut array = protected_array(elements.len()); - for (_, element) in elements { - push_element(&mut array, element); - } - Ok(hand_over(array)) - } - (None, true) => Err(shape("a node with children")), - (Some(_), false) => Err(shape("the children of one node")), - } + open_reads(self, cipher, context).try_map(|values| { + one_reached(values)? + .map(hand_over) + .ok_or_else(|| shape("a node at the walked path")) }) }) } } -/// One row of a walk, opened: the node, and the selector, position and -/// depth the server returned it with. The depth is the server's report and -/// optional on the wire; the rows are placed by their positions and the -/// opened nodes' child counts, and a depth that disagrees with the place -/// a row is put in is refused. +/// One row of a read, opened: the node, and the selector, self tag, +/// position and depth the server returned it with. The depth is the +/// server's report and optional on the wire; the rows are placed by their +/// positions and the opened nodes' child counts, and a depth that disagrees +/// with the place a row is put in is refused. struct Row { node: Opened, s: Tag, + x: super::SelfTag, j: u64, d: Option, } -/// Open every row and rebuild each complete subtree, labelled, each still -/// under its `Protected`, checking every row against the key as it goes -/// ([`place`]). +/// Open every row and place each node a token named, checking every row +/// against the key as it goes ([`place`]). fn open_subtree<'a, K>( subtree: JsonSubtree, cipher: &'a StackCipher, context: AeadContext, -) -> Pending<'a, Vec<(Label, Protected)>, K> { +) -> Pending<'a, Vec>, K> { let (mut nodes, key) = subtree.into_parts(); nodes.sort_by_key(|n| n.j); - // A walk returns each entry once. Two rows at one ordinal are the + // A read returns each entry once. Two rows at one ordinal are the // server repeating an entry, which would let it line up one container - // as many times as it likes. + // as many times as it likes: an honest database never returns one, so + // it is the wrong entry, not a shape the target refuses. if nodes.windows(2).any(|pair| pair[0].j == pair[1].j) { - return Pending::failed(cipher, shape("rows of distinct entries")); + return Pending::failed(cipher, wrong("rows of distinct entries")); } let Some(first) = nodes.first() else { return Pending::ready(cipher, Ok(Vec::new())); }; let header = first.header.clone(); + // A read is of one row, so its rows carry one header. Rows of two + // documents are not the answer to it. if nodes.iter().any(|n| n.header != header) { - return Pending::failed(cipher, shape("rows of one document")); + return Pending::failed(cipher, wrong("rows of one document")); } let (request, aad) = match retrieval(&header, context) { Ok(parts) => parts, @@ -1356,98 +1371,110 @@ fn open_subtree<'a, K>( Ok(Row { node: open_entry(&aes, &header, &aad, &n.entry, n.j)?, s: n.entry.s, + x: n.entry.x, j: n.j, d: n.depth, }) }) .collect::, Error>>()?; - place(rows, &key) + place(rows, &key, &header.iv) }) } -/// Put the opened rows of a walk back into the subtrees they hold, in -/// document order, refusing any row that is not where the key says the -/// walk's rows are ([`Error::JsonWrongEntry`]). +/// What [`rebuild_walked`] checks a row's derived fields with: the selectors +/// of the key's column, and the document's `iv`, which salts every self tag. +struct Check<'k> { + selectors: &'k Selectors, + iv: &'k [u8], +} + +/// Put the opened rows of a read back into the nodes its tokens named, in +/// the key's order, refusing any row that is not where the key says the +/// read's rows are ([`Error::JsonWrongEntry`]), and the whole read with +/// [`Error::JsonWildcardTruncated`] if a probe named a node. /// /// Each entry opened under its own selector `s` and position `j`, which its /// ciphertext is bound to, so each is a genuine node of the row; what the -/// check adds is that it is the node the walk asked for. A subtree's rows -/// in a document are its root and then every node beneath it, at -/// consecutive positions, each node's children in order; so the rows are -/// placed one by one, each the next position after the one before, one -/// level below its parent, and carrying the selector of its parent's path -/// extended by its own label, which the key derives from the decrypted -/// label. With gates, each subtree's root is a node the path reaches: the -/// path's selector, at depth `0`, at the position the path names if its -/// last step is one. Without gates, the rows are the children of the -/// path's node and their subtrees, side by side: each child carries the -/// selector of the path extended by its label, at depth `1`, right after -/// the subtree before it, and an element at the position after the -/// elements before it, from `0`. +/// check adds is that it is the node the read asked for. A read's rows are, +/// for each token that names a node, that node at depth `0` and then every +/// node beneath it at consecutive positions, each node's children in order, +/// and the key's tokens are in document order. So the rows are placed one +/// by one. A node at depth `0` must carry the self tag of a token of the +/// key not yet matched, and that token's path's selector and last step (its +/// key, its position, or the root). Every row beneath it must be the next +/// position after the one before, one level below its parent, carry the +/// selector of its parent's path extended by its own label, and carry the +/// self tag of the token its label derives under its parent's. +/// +/// The self tags are not authenticated, and the database holds every token +/// the read sent and every row's `iv`, so matching them is a consistency +/// check, which catches a corrupted row or a server that walked wrongly, +/// not a dishonest database. The selector, the position and the label, +/// which the AAD and the ciphertext cover, are what bind a row to its +/// place. Still allowed by design: an entry from another row of the +/// column, returned with that row's header (ADR 0010), and a node at the +/// same selector under another element (`$.items[0].price` for +/// `$.items[1].price`), which no bound field tells apart. fn place( rows: Vec, key: &JsonSubtreeKey, -) -> Result)>, Error> { - let below = key.gates.is_empty(); - let at = Protected::new(key.path.selector_bytes()); - let reached = key.selectors.of(at.clone())?; + iv: &[u8], +) -> Result>, Error> { + let check = Check { + selectors: &key.selectors, + iv, + }; + let refused = || wrong("a node the walked path reaches"); let mut rest = rows.into_iter(); - let mut forest = Vec::new(); - // Without gates, the position the next child of the path's node must - // have: right after the subtree of the one before. - let mut next: Option = None; + // The first token of the key not yet matched to a row. + let mut next = 0; + let mut reached = Vec::new(); + let mut truncated = false; while let Some(top) = rest.next() { - let (bytes, depth) = if below { - let bytes = child_selector_bytes(&at, &top.node.label) - .ok_or_else(|| wrong("a child of the walked node"))?; - // The walk does not return the node it starts from, so no row - // bounds where the children begin; an element's position does. - // The children of an array are its elements from `0`, so the - // `n`th row at depth 1 must be element `n`: a database that - // leaves out the first element would otherwise move every - // element down a position. A member's place is a keyed rank - // the client does not hold, so a member left out, and the last - // children of either, are still the database's word. - let out_of_place = - matches!(top.node.label, Label::Index(i) if i != forest.len() as u64); - if top.d.is_some_and(|d| d != 1) - || next.is_some_and(|j| j != top.j) - || out_of_place - || key.selectors.of(bytes.clone())? != top.s - { - return Err(wrong("a child of the walked node")); - } - (bytes, 1) + if top.d.is_some_and(|d| d != 0) { + return Err(refused()); + } + let at = key.reads[next..] + .iter() + .position(|read| self_tag(iv, &read.token) == top.x) + .ok_or_else(refused)? + + next; + next = at + 1; + let read = &key.reads[at]; + let bytes = Protected::new(read.path.selector_bytes()); + if key.selectors.of(bytes.clone())? != top.s || !names(&read.path, &top.node.label) { + return Err(refused()); + } + let mut cursor = top.j.checked_add(1).ok_or_else(refused)?; + let value = rebuild_walked( + top.node, + &bytes, + &read.token, + 0, + &mut rest, + &mut cursor, + &check, + )?; + if read.probe { + truncated = true; } else { - if top.d.is_some_and(|d| d != 0) - || top.s != reached - || !reaches(&key.path, &top.node.label) - { - return Err(wrong("a node the walked path reaches")); - } - (at.clone(), 0) - }; - let mut cursor = top - .j - .checked_add(1) - .ok_or_else(|| wrong("a node of the walk"))?; - let (label, node) = take_label(top.node); - let value = rebuild_walked(node, &bytes, depth, &mut rest, &mut cursor, &key.selectors)?; - next = Some(cursor); - forest.push((label, value)); + reached.push(value); + } } - Ok(forest) + if truncated { + return Err(Error::JsonWildcardTruncated { bound: key.bound }); + } + Ok(reached) } -/// Whether a node with `label` is one `path` reaches by its last step, -/// beyond what the selector says: a position names the element at that -/// position, where the selector reads every position as the wildcard. -fn reaches(path: &JsonPath, label: &Label) -> bool { - match path.segments().last() { - Some(Segment::Index(n)) => matches!(label, Label::Index(at) if at == n), - Some(Segment::Wildcard) => matches!(label, Label::Index(_)), - Some(Segment::Key(_)) => matches!(label, Label::Key(_)), - None => matches!(label, Label::Root), +/// Whether a node with `label` is the one the concrete `path` names by its +/// last step: the key, the position, or the root for `$`. +fn names(path: &JsonPath, label: &Label) -> bool { + match (path.segments().last(), label) { + (None, Label::Root) => true, + (Some(Segment::Key(want)), Label::Key(got)) => got.risky_ref() == want, + (Some(Segment::Index(want)), Label::Index(got)) => got == want, + _ => false, } } @@ -1478,14 +1505,26 @@ fn child_selector_bytes(parent: &Protected>, label: &Label) -> Option Option { + match label { + Label::Key(_) => Some(member_token(parent, s)), + Label::Index(i) => u32::try_from(*i).ok().map(|i| element_token(parent, s, i)), + Label::Root => None, + } +} + +/// [`rebuild`] for a read's rows, checking each child as it is taken: that +/// it is at `cursor`, the position after the row before it; that it is one +/// level below `node`, which sits `d` levels below the node a token named; +/// that it carries the selector of `node`'s path, whose selector bytes are +/// `bytes`, extended by its own label; and that it carries the self tag of +/// the token its label derives under `token`, `node`'s. A sealed root +/// among the rows is not a node of any walk. Rows that end before `node`'s +/// authenticated child count is met are refused like a gap: a read returns +/// a node's whole subtree, so the database left the next node out. /// /// The rows are consecutive entries of one sealed document, placed by its /// authenticated child counts, so they nest no deeper than the document @@ -1494,10 +1533,11 @@ fn child_selector_bytes(parent: &Protected>, label: &Label) -> Option>, + token: &Tag, d: u32, rest: &mut impl Iterator, cursor: &mut u64, - selectors: &Selectors, + check: &Check<'_>, ) -> Result, Error> { if node.whole.is_some() { return Err(wrong("a node of the walked subtree")); @@ -1505,37 +1545,45 @@ fn rebuild_walked( if d as usize > MAX_DEPTH { return Err(shape("a JSON document no deeper than the index seals")); } + let next = || wrong("the next node of the walked subtree"); let child = |rest: &mut dyn Iterator, cursor: &mut u64| { - // The walk returns a node's whole subtree and stops at no budget + // A read returns a node's whole subtree and stops at no budget // short of it (the SQL falls back to the sealed root instead), so // rows that run out before a child count is met are a row left out // at the end, the same refusal as one left out in the middle. - let row = rest - .next() - .ok_or_else(|| wrong("the next node of the walked subtree"))?; - let child_bytes = child_selector_bytes(bytes, &row.node.label) - .ok_or_else(|| wrong("the next node of the walked subtree"))?; + let row = rest.next().ok_or_else(next)?; + let child_bytes = child_selector_bytes(bytes, &row.node.label).ok_or_else(next)?; if row.j != *cursor || row.d.is_some_and(|got| Some(got) != d.checked_add(1)) - || selectors.of(child_bytes.clone())? != row.s + || check.selectors.of(child_bytes.clone())? != row.s { - return Err(wrong("the next node of the walked subtree")); + return Err(next()); } - *cursor = cursor - .checked_add(1) - .ok_or_else(|| wrong("the next node of the walked subtree"))?; - Ok::<_, Error>((row.node, child_bytes)) + let child_token = labelled_token(token, &row.s, &row.node.label).ok_or_else(next)?; + if self_tag(check.iv, &child_token) != row.x { + return Err(next()); + } + *cursor = cursor.checked_add(1).ok_or_else(next)?; + Ok::<_, Error>((row.node, child_bytes, child_token)) }; match node.kind { Kind::Object => { let mut object = protected_object(); for _ in 0..node.children { - let (member, member_bytes) = child(rest, cursor)?; + let (member, member_bytes, member_token) = child(rest, cursor)?; let (label, member) = take_label(member); let Label::Key(key) = label else { return Err(shape("an object member")); }; - let value = rebuild_walked(member, &member_bytes, d + 1, rest, cursor, selectors)?; + let value = rebuild_walked( + member, + &member_bytes, + &member_token, + d + 1, + rest, + cursor, + check, + )?; insert_member(&mut object, key, value); } Ok(object) @@ -1543,13 +1591,21 @@ fn rebuild_walked( Kind::Array => { let mut array = protected_array(node.children as usize); for i in 0..node.children { - let (element, element_bytes) = child(rest, cursor)?; + let (element, element_bytes, element_token) = child(rest, cursor)?; if !matches!(element.label, Label::Index(at) if at == u64::from(i)) { return Err(shape("an array element in order")); } push_element( &mut array, - rebuild_walked(element, &element_bytes, d + 1, rest, cursor, selectors)?, + rebuild_walked( + element, + &element_bytes, + &element_token, + d + 1, + rest, + cursor, + check, + )?, ); } Ok(array) @@ -1558,31 +1614,52 @@ fn rebuild_walked( } } -/// The values of a node's children, in the node's order, moved out of it: -/// an array's elements, an object's member values (each leaving `null` -/// under its key, which stays in the document to be wiped with it), and -/// none for a scalar. What a `json_tree_subtree` walk's rows open to as a -/// [`Vec`], taken out of the whole document when the walk fell back to the -/// sealed root. -fn take_children(node: &mut Value) -> Vec { - match node { - Value::Array(items) => std::mem::take(items), - Value::Object(members) => members.values_mut().map(std::mem::take).collect(), - _ => Vec::new(), - } +/// The key's reads evaluated over the whole document a read fell back to, +/// to what the walked rows open to: each concrete path's node, moved out of +/// the document's `Protected` into one of its own (leaving `null` behind, +/// so its plaintext is never copied), in the key's order, and the same +/// [`Error::JsonWildcardTruncated`] when a probe's path names a node. The +/// key's paths are disjoint, so moving one node out never touches another. +/// What is left of the document is wiped when it drops. +fn select_reads( + key: &JsonSubtreeKey, + whole: &mut Protected, +) -> Result>, Error> { + let truncated = { + let doc = whole.risky_ref().value(); + key.reads + .iter() + .any(|read| read.probe && !read.path.select(doc).is_empty()) + }; + if truncated { + return Err(Error::JsonWildcardTruncated { bound: key.bound }); + } + Ok(key + .reads + .iter() + .filter(|read| !read.probe) + .filter_map(|read| { + take_protected(whole, |doc: &mut PlainJson| { + read.path + .select_mut(doc.value_mut()) + .into_iter() + .next() + .map(|node| PlainJson::from_value(std::mem::take(node))) + }) + }) + .collect()) } -/// Open the sealed root a walk fell back to: the whole document, under its -/// `Protected`, with the path the rows were keyed from, which the caller -/// selects over. The key that built the rows attached the path, so it is -/// always there. +/// Open the sealed root a read fell back to: the whole document, under its +/// `Protected`, with the key that keyed the read, whose reads the caller +/// evaluates over it. fn open_fallback<'a, K>( subtree: JsonSubtree, cipher: &'a StackCipher, context: AeadContext, -) -> Pending<'a, (JsonPath, Protected), K> { - let path = subtree.path().clone(); - let Some(root) = subtree.nodes().first().cloned() else { +) -> Pending<'a, (JsonSubtreeKey, Protected), K> { + let (nodes, key) = subtree.into_parts(); + let Some(root) = nodes.into_iter().next() else { return Pending::failed(cipher, shape("the sealed root")); }; let (request, aad) = match retrieval(&root.header, context) { @@ -1590,11 +1667,11 @@ fn open_fallback<'a, K>( Err(error) => return Pending::failed(cipher, error), }; Pending::request(cipher, vec![request], move |responses| { - let key = responses.next_retrieved_key()?; - let aes = leaf_cipher(&key)?; + let retrieved = responses.next_retrieved_key()?; + let aes = leaf_cipher(&retrieved)?; let opened = open_entry(&aes, &root.header, &aad, &root.entry, root.j)?; let whole = opened.whole.ok_or_else(|| shape("a sealed root"))?; - Ok((path, whole)) + Ok((key, whole)) }) } @@ -1602,79 +1679,192 @@ fn open_fallback<'a, K>( // Extraction keys, on the keyset cipher // ============================================================================= -/// The key that walks `path`: the token of the deepest node on it the -/// client can derive, and one gate per step below that. +/// The most tokens one subtree read sends. A wildcard expands to `N + 1` +/// tokens under each node it sits below, so wildcards nested `w` deep send +/// `N^w + … + N + 1`: two levels at the default bound send 4,161, three +/// would send 266,305. Past this a read is refused +/// ([`TermError::JsonWildcardTooWide`]) rather than built. 8,192 admits +/// every one-level wildcard up to 8,191 elements and two levels up to a +/// bound of 90, the default included, and no third level: the server looks +/// every token up in every row before it knows whether the budget is +/// blown, so a wider read costs its tokens even when the whole document +/// comes back instead, and the largest read measured sent 4,161. +pub(crate) const MAX_READ_TOKENS: u64 = 1 << 13; + +/// How a path expands into the concrete paths a read sends tokens for. +struct Expansion<'p> { + segments: &'p [Segment], + /// The selector of each step's path, from the first step down: every + /// element of an array shares one, so it is the same for every concrete + /// path the read expands to. + steps: &'p [Tag], + bound: u32, + reads: Vec, +} + +impl Expansion<'_> { + /// Expand the steps from `k` on below the node at the concrete path + /// `at`, whose token is `token`, in document order: a key by the member + /// rule, a position by the element rule, and a wildcard into the + /// positions `0 … N − 1`, each expanded in turn, then the probe at `N`. + /// The recursion is as deep as the path, which the caller bounds. + fn descend(&mut self, at: JsonPath, token: Tag, k: usize) -> Result<(), TermError> { + let Some(segment) = self.segments.get(k) else { + self.reads.push(Read { + token, + path: at, + probe: false, + }); + return Ok(()); + }; + let s = self.steps[k]; + match segment { + Segment::Key(key) => self.descend( + at.then(Segment::Key(key.clone())), + member_token(&token, &s), + k + 1, + ), + Segment::Index(index) => { + let i = fit_ordinal(*index)?; + self.descend( + at.then(Segment::Index(*index)), + element_token(&token, &s, i), + k + 1, + ) + } + Segment::Wildcard => { + for i in 0..self.bound { + self.descend( + at.then(Segment::Index(u64::from(i))), + element_token(&token, &s, i), + k + 1, + )?; + } + self.reads.push(Read { + token: element_token(&token, &s, self.bound), + path: at.then(Segment::Index(u64::from(self.bound))), + probe: true, + }); + Ok(()) + } + } + } +} + +/// How many tokens a read of a path with `wildcards` wildcard steps sends +/// at `bound`: `bound^w + … + bound + 1`, or `None` past +/// [`MAX_READ_TOKENS`]. +fn read_tokens(wildcards: usize, bound: u32) -> Option { + let mut count: u64 = 1; + for _ in 0..wildcards { + count = count + .checked_mul(u64::from(bound))? + .checked_add(1) + .filter(|count| *count <= MAX_READ_TOKENS)?; + } + Some(count) +} + +/// The key that reads `path`: one token per concrete path the path names, +/// each derived from the path alone, in document order, with a probe for +/// the element at `bound` of each array a wildcard expands. /// -/// A chain head's token is a PRF of its path. An array element's is one -/// hash under its parent's, by its position and the element selector, so -/// the client follows array steps as far as they run from the head. An -/// object member's ordinal is a rank among its siblings' keys, which the -/// client does not know, so from the first key step on the client cannot -/// derive tokens and the server walks instead, through gates: the step's -/// selector for a key or a wildcard (every element's selector is the -/// wildcard's, so a wildcard admits them all), and for an array step the -/// position with the element selector, so a member that happens to sit at -/// that ordinal under an object is not taken for an element. +/// Every node's token derives from its path: the root's and, under the +/// sealed root, each chain head's are PRFs of their paths; below them a +/// member's is one hash under its parent's with its selector, and an +/// element's one hash with its selector and its index. So a path with no +/// wildcard sends exactly one token, its node's. A wildcard names no one +/// node, and the client expands it to the positions `0 … bound − 1`, plus +/// the probe at `bound`: the element itself, not the path below it, since +/// an element past the bound need not have what the path names. fn subtree_key_for( selectors: Selectors, options: &JsonOptions, path: &JsonPath, + bound: u32, ) -> Result { + // The bound is not checked against PostgreSQL's integer: a path with no + // wildcard never uses it, and with one the token cap refuses any bound + // of MAX_READ_TOKENS or more first, naming the bound in its help. + let segments = path.segments(); + // No sealed document has a node deeper than this, and the expansion + // recurses once per step. + if segments.len() > MAX_DEPTH { + return Err(TermError::JsonTooDeep); + } + for segment in segments { + if let Segment::Index(index) = segment { + let _ = fit_ordinal(*index)?; + } + } + let wildcards = segments + .iter() + .filter(|segment| matches!(segment, Segment::Wildcard)) + .count(); + let count = read_tokens(wildcards, bound).ok_or(TermError::JsonWildcardTooWide)?; let prf = &selectors.keyset.prf; let context = &selectors.context; - let segments = path.segments(); - let (head, rest) = if options.sealed_root { - let Some((first, rest)) = segments.split_first() else { - return Err(TermError::JsonSealedRoot); - }; - let head = JsonPath::root().then(first.clone()); - (head, rest) - } else { - (JsonPath::root(), segments) - }; - // A wildcard head names no one node. - let head_bytes = head.token_bytes().ok_or(TermError::JsonSealedRoot)?; - let mut token = prf16(prf, TOKEN_DOMAIN, context, &head_bytes)?; - let mut prefix = head; - let mut rest = rest.iter(); - let mut gates = Vec::new(); - for segment in rest.by_ref() { + let mut steps = Vec::with_capacity(segments.len()); + let mut prefix = JsonPath::root(); + for segment in segments { prefix = prefix.then(segment.clone()); - match segment { - Segment::Index(index) => { - let o = fit_ordinal(*index)?; - let element = prf16(prf, SELECTOR_DOMAIN, context, &prefix.selector_bytes())?; - token = child_token(&token, &element, o); - } - Segment::Key(_) | Segment::Wildcard => { - gates.push(Gate::Selector(prf16( - prf, - SELECTOR_DOMAIN, - context, - &prefix.selector_bytes(), - )?)); - break; + steps.push(prf16( + prf, + SELECTOR_DOMAIN, + context, + &prefix.selector_bytes(), + )?); + } + let mut expansion = Expansion { + segments, + steps: &steps, + bound, + reads: Vec::with_capacity(usize::try_from(count).unwrap_or_default()), + }; + let head = |at: &JsonPath| { + prf16( + prf, + TOKEN_DOMAIN, + context, + &at.token_bytes().unwrap_or_default(), + ) + }; + match segments.first() { + // Under the sealed root the chain starts at the heads, each a PRF + // of its path; a wildcard head is each position's head in turn. + Some(first) if options.sealed_root => { + let heads: Vec<(Segment, bool)> = match first { + Segment::Wildcard => (0..bound) + .map(|i| (Segment::Index(u64::from(i)), false)) + .chain(std::iter::once((Segment::Index(u64::from(bound)), true))) + .collect(), + step => vec![(step.clone(), false)], + }; + for (step, probe) in heads { + let at = JsonPath::root().then(step); + let token = head(&at)?; + if probe { + expansion.reads.push(Read { + token, + path: at, + probe, + }); + } else { + expansion.descend(at, token, 1)?; + } } } - } - for segment in rest { - prefix = prefix.then(segment.clone()); - gates.push(match segment { - Segment::Index(index) => Gate::Slot( - fit_ordinal(*index)?, - prf16(prf, SELECTOR_DOMAIN, context, &prefix.selector_bytes())?, - ), - Segment::Key(_) | Segment::Wildcard => Gate::Selector(prf16( - prf, - SELECTOR_DOMAIN, - context, - &prefix.selector_bytes(), - )?), - }); + // `$` itself, under either root, or any path from an open root: + // the root's token, and every step below it by rule. + _ => { + let root = JsonPath::root(); + let token = head(&root)?; + expansion.descend(root, token, 0)?; + } } Ok(JsonSubtreeKey { - token, - gates, + reads: expansion.reads, + bound, path: path.clone(), selectors, }) @@ -1702,36 +1892,63 @@ impl KeysetCipher<'_, K> { Pending::ready(self, selector) } - /// The key that walks the subtree under `path`, under the field - /// `descriptor` and the index's `options`: the token of the deepest - /// node on the path the client can derive (a chain head, then array - /// positions), and one gate per step below it, which the server walks - /// through. What `json_tree_subtree` and `json_tree_subtree_at` take. + /// The key that reads the node at `path`, under the field `descriptor` + /// and the index's `options`, with wildcards expanded to + /// [`JsonSubtreeKey::DEFAULT_BOUND`] elements: one token per concrete + /// path, each derived from the path alone, which is what + /// `json_tree_subtree(doc, key.tokens(), budget)` takes. See + /// [`json_subtree_key_with_bound`](Self::json_subtree_key_with_bound). /// /// # Errors /// - /// [`TermError::JsonSealedRoot`] for the root, or for a path whose - /// first step is a wildcard, under the sealed root: no token exists for - /// the root, and the document opens whole instead. - /// [`TermError::JsonTooWide`] for a path naming an array position past - /// 2³¹ − 1, PostgreSQL's `integer`: no stored document has an element - /// there. + /// As [`json_subtree_key_with_bound`](Self::json_subtree_key_with_bound). pub fn json_subtree_key<'c>( &self, path: &JsonPath, options: &JsonOptions, descriptor: NonEmpty>, + ) -> Pending<'_, JsonSubtreeKey, K> { + self.json_subtree_key_with_bound(path, options, JsonSubtreeKey::DEFAULT_BOUND, descriptor) + } + + /// The key that reads the node at `path`, under the field `descriptor` + /// and the index's `options`, with each wildcard step expanded to the + /// positions `0 … bound − 1`. A path with no wildcard sends exactly one + /// token, its node's, and nothing that covers any node above it; `$` + /// is a read too, of the root entry, which under the sealed root opens + /// to the whole document. A path with a wildcard sends one token per + /// concrete path and, for each array a wildcard expands, a probe for + /// the element at `bound`; a row where a probe names a node has an + /// array longer than `bound` there, and its rows open to + /// [`Error::JsonWildcardTruncated`] rather than to part of the answer. + /// Raise `bound`, or read the array itself, for longer arrays; lower it + /// for a nested wildcard over many rows, which sends `bound² + bound + + /// 1` tokens. Derived locally; the pending carries no request. + /// + /// # Errors + /// + /// [`TermError::JsonTooWide`] for a path naming an array position past + /// 2³¹ − 1: no stored document has an element there. + /// [`TermError::JsonTooDeep`] for a path deeper than any document the + /// index seals. [`TermError::JsonWildcardTooWide`] for a path whose + /// wildcards would expand past 8,192 tokens at `bound`; a path with no + /// wildcard takes any `bound`, since it uses none. + pub fn json_subtree_key_with_bound<'c>( + &self, + path: &JsonPath, + options: &JsonOptions, + bound: u32, + descriptor: NonEmpty>, ) -> Pending<'_, JsonSubtreeKey, K> { let context = descriptor.into_prf_context().into_owned(); let selectors = Selectors::new(Arc::clone(self.state()), context); - let key = subtree_key_for(selectors, options, path); + let key = subtree_key_for(selectors, options, path, bound); Pending::ready(self, key.map_err(Error::from)) } } #[cfg(test)] mod tests { - use super::super::base64; use super::*; use serde_json::json; use vitaminc_prf::PrfKeyInit; @@ -1797,7 +2014,6 @@ mod tests { &json!({"alice@example.com": {"ssn": "123"}}), &JsonOptions::default(), ); - assert!(t.nodes()[1].token.is_some(), "the terms hold a token"); let printed = format!("{t:?}"); assert_eq!(printed, "JsonTerms { nodes: 3, .. }"); assert!(!printed.contains("alice")); @@ -1815,119 +2031,418 @@ mod tests { } #[test] - fn the_sealed_root_has_no_token_and_its_fields_are_heads() { + fn every_node_has_a_token_and_only_what_chains_has_an_ordinal() { let sealed = terms(&json!({"a": {"b": 1}}), &JsonOptions::default()); - assert!(sealed.nodes()[0].token.is_none()); - assert!(sealed.nodes()[1].token.is_some()); - assert!(sealed.nodes()[1].ordinal.is_none(), "a head has no ordinal"); - assert!(sealed.nodes()[2].ordinal.is_some()); let open = terms(&json!({"a": {"b": 1}}), &open_root()); - assert!(open.nodes()[0].token.is_some()); - assert!(open.nodes()[1].ordinal.is_some()); - // The chain below a head is the same hash chain either way, but - // the heads differ: one is a PRF of the path, the other a hash - // under the root's token. - assert_ne!(sealed.nodes()[1].token, open.nodes()[1].token); + let root = prf16( + &prf(), + TOKEN_DOMAIN, + &Context::from_encoded(b"users/profile"), + &[], + ) + .unwrap(); + assert_eq!( + sealed.nodes()[0].token, + root, + "the sealed root's token is the PRF of `$`" + ); + assert_eq!(open.nodes()[0].token, root, "and so is the open root's"); + assert!( + sealed.nodes()[1].ordinal.is_none(), + "a head chains from nothing under the sealed root" + ); + assert!( + sealed.nodes()[2].ordinal.is_some(), + "below a head, everything chains" + ); + assert!( + open.nodes()[1].ordinal.is_some(), + "under an open root, a top-level field chains" + ); + assert_eq!( + open.nodes()[1].token, + member_token(&root, &open.nodes()[1].s), + "a top-level field under an open root is a member of the root" + ); + // The chain below a head is the same rule either way, but the heads + // differ: one is a PRF of the path, the other a hash under the + // root's token. + assert_ne!( + sealed.nodes()[1].token, + open.nodes()[1].token, + "the heads differ" + ); + assert_eq!( + sealed.nodes()[2].token, + member_token(&sealed.nodes()[1].token, &sealed.nodes()[2].s), + "a member's token is one hash under its parent's" + ); + } + + fn key_for(options: &JsonOptions, path: &str, bound: u32) -> Result { + subtree_key_for( + Selectors::for_tests(), + options, + &JsonPath::parse(path).unwrap(), + bound, + ) + } + + fn node<'t>(t: &'t JsonTerms, path: &str) -> &'t TermNode { + t.nodes() + .iter() + .find(|n| n.positional == JsonPath::parse(path).unwrap()) + .unwrap_or_else(|| panic!("the document has {path}")) + } + + mod given_a_path_with_no_wildcard { + use super::*; + + #[test] + fn the_key_sends_exactly_the_token_of_the_node_at_the_path() { + let value = json!({"a": {"b": [1, {"c": 2}], "": {"$": [[3]]}}, "l": [[0], 1]}); + for options in [JsonOptions::default(), open_root()] { + let t = terms(&value, &options); + for path in [ + "$", + "$.a", + "$.a.b", + "$.a.b[0]", + "$.a.b[1]", + "$.a.b[1].c", + "$.l", + "$.l[0]", + "$.l[0][0]", + "$.l[1]", + r#"$.a[""]"#, + r#"$.a[""]["$"][0][0]"#, + ] { + let key = key_for(&options, path, 64).unwrap(); + assert_eq!( + key.token_bytes(), + [node(&t, path).token], + "{path} under {options:?}: one token, the node's own" + ); + assert!( + key.probes().is_empty(), + "{path}: no probe without a wildcard" + ); + } + } + } + + #[test] + fn the_root_is_a_read_under_the_sealed_root_too() { + let key = key_for(&JsonOptions::default(), "$", 64).unwrap(); + assert_eq!(key.reads.len(), 1, "one token"); + assert_eq!(key.reads[0].path, JsonPath::root(), "the root's"); + } + + #[test] + fn a_position_past_postgres_integer_is_refused_not_sent() { + for path in [ + "$.a.b[2147483648]", + "$.a[2147483648]", + "$[2147483648]", + "$.a[4294967296]", + ] { + assert!( + matches!( + key_for(&JsonOptions::default(), path, 64), + Err(TermError::JsonTooWide) + ), + "{path}: no stored document has an element there" + ); + } + let last = key_for(&JsonOptions::default(), "$.a.b[2147483647]", 64).unwrap(); + assert_eq!( + last.reads.len(), + 1, + "the last position PostgreSQL's integer holds" + ); + assert_eq!(fit_ordinal(i32::MAX.unsigned_abs()).unwrap(), 2_147_483_647); + assert!(matches!( + fit_ordinal(2_147_483_648_u64), + Err(TermError::JsonTooWide) + )); + } + + #[test] + fn a_path_deeper_than_any_document_is_refused() { + let deepest = format!("${}", "[0]".repeat(MAX_DEPTH)); + assert!( + key_for(&JsonOptions::default(), &deepest, 64).is_ok(), + "the deepest path a sealed document has" + ); + let deeper = format!("${}", "[0]".repeat(MAX_DEPTH + 1)); + assert!( + matches!( + key_for(&JsonOptions::default(), &deeper, 64), + Err(TermError::JsonTooDeep) + ), + "one step deeper names no node of any document" + ); + } + } + + mod given_a_path_with_a_wildcard { + use super::*; + + #[test] + fn the_key_sends_each_position_below_the_bound_and_a_probe_at_it() { + let t = terms( + &json!({"a": [{"b": 1}, {"b": 2}, {"b": 3}]}), + &JsonOptions::default(), + ); + let key = key_for(&JsonOptions::default(), "$.a[*].b", 2).unwrap(); + let paths: Vec = key.reads.iter().map(|r| r.path.to_string()).collect(); + assert_eq!( + paths, + ["$.a[0].b", "$.a[1].b", "$.a[2]"], + "N + 1, in document order" + ); + assert_eq!(key.probes(), [2], "the probe last, the element itself"); + assert_eq!(key.bound(), 2); + for read in &key.reads { + assert_eq!( + read.token, + node(&t, &read.path.to_string()).token, + "{}: the token the stored node carries", + read.path + ); + } + } + + #[test] + fn nested_wildcards_multiply() { + for bound in [0, 1, 3, 64] { + let key = key_for(&JsonOptions::default(), "$.a[*].b[*]", bound).unwrap(); + let n = bound as usize; + assert_eq!(key.reads.len(), n * n + n + 1, "N² + N + 1 at N = {bound}"); + assert_eq!(key.probes().len(), n + 1, "N inner probes and one outer"); + assert_eq!( + key.reads.last().map(|r| r.path.to_string()), + Some(format!("$.a[{bound}]")), + "the outer probe last" + ); + let mut tokens: Vec = key.token_bytes(); + tokens.sort_unstable(); + tokens.dedup(); + assert_eq!(tokens.len(), key.reads.len(), "every token distinct"); + } + } + + #[test] + fn a_wildcard_head_expands_under_the_sealed_root() { + let t = terms(&json!([{"x": 1}, {"x": 2}]), &JsonOptions::default()); + let key = key_for(&JsonOptions::default(), "$[*].x", 2).unwrap(); + let paths: Vec = key.reads.iter().map(|r| r.path.to_string()).collect(); + assert_eq!(paths, ["$[0].x", "$[1].x", "$[2]"]); + assert_eq!(key.reads[0].token, node(&t, "$[0].x").token, "below a head"); + assert_eq!( + key.reads[2].token, + prf16( + &prf(), + TOKEN_DOMAIN, + &Context::from_encoded(b"users/profile"), + &JsonPath::parse("$[2]").unwrap().token_bytes().unwrap(), + ) + .unwrap(), + "the probe is the head at the bound" + ); + } + + #[test] + fn an_expansion_past_the_token_limit_is_refused() { + assert_eq!( + read_tokens(2, 64), + Some(4_161), + "two levels at the default bound" + ); + assert_eq!(read_tokens(2, 90), Some(8_191), "two levels at 90"); + assert_eq!(read_tokens(2, 91), None, "two levels at 91 is past 8,192"); + assert_eq!( + read_tokens(3, 64), + None, + "three levels at the default bound" + ); + assert_eq!( + read_tokens(0, u32::MAX), + Some(1), + "no wildcard is one token" + ); + assert_eq!(read_tokens(1, 8_191), Some(8_192), "the limit itself"); + assert_eq!(read_tokens(1, 8_192), None, "one past the limit"); + assert!( + matches!( + key_for(&JsonOptions::default(), "$.a[*].b[*].c[*]", 64), + Err(TermError::JsonWildcardTooWide) + ), + "refused before any token is derived" + ); + assert!( + matches!( + key_for(&JsonOptions::default(), "$.a[*]", 2_147_483_648), + Err(TermError::JsonWildcardTooWide) + ), + "a bound past PostgreSQL's integer is past the token cap first" + ); + let unused = key_for(&JsonOptions::default(), "$.a.b", u32::MAX).unwrap(); + assert_eq!( + unused.reads.len(), + 1, + "a path with no wildcard uses no bound, so any bound is taken" + ); + } } #[test] - fn the_subtree_key_derives_what_it_can_and_gates_the_rest() { - let value = json!({"a": {"b": [1, {"c": 2}]}, "l": [[0], 1]}); - let key = |options: &JsonOptions, path: &str| { - subtree_key_for( - Selectors::for_tests(), - options, - &JsonPath::parse(path).unwrap(), - ) - }; - let at = |t: &JsonTerms, path: &str| { - t.nodes() - .iter() - .find(|n| n.positional == JsonPath::parse(path).unwrap()) - .unwrap() - .clone() + fn a_hit_takes_the_rule_its_self_tag_matches() { + let iv = [3; 16]; + let parent = [5; 16]; + let s = [6; 16]; + let entry = |x: crate::json::SelfTag| JsonEntry { + s, + c: vec![], + x, + p: None, + op: None, }; - let options = JsonOptions::default(); - let t = terms(&value, &options); - // A head's token, and the positions under it, derive exactly. - for path in ["$.a", "$.l", "$.l[0]", "$.l[0][0]", "$.l[1]"] { - let k = key(&options, path).unwrap(); - assert_eq!(Some(*k.token_bytes()), at(&t, path).token, "{path}"); - assert!(k.gates().is_empty(), "{path}"); - } - // The server derives an object member's token the same way, from - // its parent's, its selector and its ordinal. - let a = at(&t, "$.a"); - let b = at(&t, "$.a.b"); + let member = member_token(&parent, &s); + let element = element_token(&parent, &s, 2); assert_eq!( - b.token, - Some(child_token(&a.token.unwrap(), &b.s, b.ordinal.unwrap())) + hit_token(&iv, &parent, &entry(self_tag(&iv, &member)), 2), + Some(member) ); - // The selector is bound: the same parent and ordinal under another - // selector (an element where the row holds a member) is another - // token. - assert_ne!( - b.token, - Some(child_token(&a.token.unwrap(), &a.s, b.ordinal.unwrap())) + assert_eq!( + hit_token(&iv, &parent, &entry(self_tag(&iv, &element)), 2), + Some(element) ); - // Below a key step, gates: the step's selector, a position, the - // wildcard's selector (every element's). - let k = key(&options, "$.a.b[1].c").unwrap(); - assert_eq!(Some(*k.token_bytes()), a.token); assert_eq!( - k.gates(), - [ - base64(&b.s), - format!("#1:{}", base64(&at(&t, "$.a.b[1]").s)), - base64(&at(&t, "$.a.b[1].c").s) - ] + hit_token(&iv, &parent, &entry(self_tag(&iv, &element)), 3), + None, + "the element rule takes the hit's own ordinal" ); - let k = key(&options, "$.a.b[*].c").unwrap(); assert_eq!( - k.gates(), - [ - base64(&b.s), - base64(&at(&t, "$.a.b[0]").s), - base64(&at(&t, "$.a.b[1].c").s) - ] + hit_token(&iv, &parent, &entry([0; 8]), 2), + None, + "neither rule" + ); + assert_ne!( + member, + element_token(&parent, &s, 0), + "a member is never an element" + ); + assert_ne!( + self_tag(&iv, &member), + self_tag(&[4; 16], &member), + "salted by the iv" ); - // Under the sealed root the root has no token; under an open root - // it has, and `$.a` is then one gate below it. - assert!(matches!( - key(&JsonOptions::default(), "$"), - Err(TermError::JsonSealedRoot) - )); - let open = open_root(); - let root = key(&open, "$").unwrap(); - assert!(root.gates().is_empty()); - let a = key(&open, "$.a").unwrap(); - assert_eq!(a.token_bytes(), root.token_bytes()); - assert_eq!(a.gates().len(), 1); } - #[test] - fn a_position_past_postgres_integer_is_refused_not_sent() { - let key = |path: &str| { - subtree_key_for( - Selectors::for_tests(), - &JsonOptions::default(), - &JsonPath::parse(path).unwrap(), - ) - }; - // The last position PostgreSQL's integer holds reaches the server - // as a gate it can cast, and finds nothing there. - let last = key("$.a.b[2147483647]").unwrap(); - assert!(last.gates()[1].starts_with("#2147483647:")); - // One past it fits u32 but not integer: refused here, as a gate and - // as a position the client chains itself. - for path in ["$.a.b[2147483648]", "$.a[2147483648]", "$.a[4294967296]"] { - assert!(matches!(key(path), Err(TermError::JsonTooWide)), "{path}"); + /// A member's token is the same in every row, whatever its siblings: + /// the property the change exists for, and what makes a read of + /// `$.a.b` send `$.a.b`'s token alone. + mod given_random_documents { + use super::*; + use proptest::prelude::*; + + fn key_name() -> impl Strategy { + prop_oneof![ + Just(String::new()), + Just("0".to_owned()), + Just("a.b".to_owned()), + Just("[*]".to_owned()), + Just("$".to_owned()), + Just("\"\\".to_owned()), + Just("é✓".to_owned()), + "[a-d]{1,3}", + ] + } + + fn siblings() -> impl Strategy> { + prop::collection::vec((key_name(), any::()), 0..12) + } + + proptest! { + #[test] + fn a_members_token_ignores_its_siblings( + left in siblings(), + right in siblings(), + sealed in any::(), + ) { + let options = JsonOptions { sealed_root: sealed, ..JsonOptions::new() }; + let row = |siblings: &[(String, i64)], b: Value| { + let mut a: Map = siblings + .iter() + .filter(|(k, _)| k != "b" && k != "c") + .map(|(k, v)| (k.clone(), json!(v))) + .collect(); + let _ = a.insert("b".into(), b); + terms(&json!({"a": a, "z": 1}), &options) + }; + let one = row(&left, json!({"c": 1})); + let two = row(&right, json!([true])); + prop_assert_eq!(node(&one, "$.a.b").token, node(&two, "$.a.b").token); + let key = key_for(&options, "$.a.b", 64).unwrap(); + prop_assert_eq!(key.token_bytes(), vec![node(&one, "$.a.b").token]); + prop_assert_ne!(node(&one, "$.a.b").token, node(&one, "$.a.b.c").token); + let c = key_for(&options, "$.a.c", 64).unwrap(); + prop_assert_ne!(c.token_bytes(), key.token_bytes(), "$.a.b and $.a.c differ"); + } + + #[test] + fn element_tokens_differ_by_index_and_from_every_member( + len in 1usize..20, + keys in prop::collection::btree_set(key_name(), 0..8), + ) { + let options = JsonOptions::default(); + let array = terms(&json!({"a": vec![0; len]}), &options); + let object: Map = keys.iter().map(|k| (k.clone(), json!(0))).collect(); + let object = terms(&json!({"a": object}), &options); + let mut elements: Vec = (0..len) + .map(|i| node(&array, &format!("$.a[{i}]")).token) + .collect(); + for member in object.nodes().iter().skip(2) { + prop_assert!(!elements.contains(&member.token), "a member is never an element"); + } + elements.sort_unstable(); + elements.dedup(); + prop_assert_eq!(elements.len(), len, "every index its own token"); + } + + #[test] + fn every_concrete_path_the_document_has_keys_its_own_token( + value in arbitrary_json(), + sealed in any::(), + ) { + let options = JsonOptions { sealed_root: sealed, ..JsonOptions::new() }; + let t = terms(&value, &options); + for n in t.nodes() { + let key = subtree_key_for(Selectors::for_tests(), &options, &n.positional, 4).unwrap(); + prop_assert_eq!(key.token_bytes(), vec![n.token], "{}", n.positional); + } + } + } + + /// Small documents over the keys that stress the path encoding. + pub(super) fn arbitrary_json() -> impl Strategy { + let leaf = prop_oneof![ + Just(Value::Null), + any::().prop_map(Value::Bool), + any::().prop_map(|n| json!(n)), + Just(json!(2.0)), + Just(json!(-0.0)), + "[a-z]{0,4}".prop_map(Value::String), + ]; + leaf.prop_recursive(4, 32, 5, |inner| { + prop_oneof![ + prop::collection::vec(inner.clone(), 0..5).prop_map(Value::Array), + prop::collection::btree_map(key_name(), inner, 0..5) + .prop_map(|m| Value::Object(m.into_iter().collect())), + ] + }) } - assert_eq!(fit_ordinal(i32::MAX.unsigned_abs()).unwrap(), 2_147_483_647); - assert!(matches!( - fit_ordinal(2_147_483_648_u64), - Err(TermError::JsonTooWide) - )); } #[test] @@ -2166,15 +2681,38 @@ mod tests { #[test] fn the_last_step_of_a_path_names_the_label_it_reaches() { - assert!(reaches(&path("$.a[2]"), &Label::Index(2)), "the position"); - assert!(!reaches(&path("$.a[2]"), &Label::Index(1)), "another one"); - assert!(!reaches(&path("$.a[2]"), &key("2")), "a member"); - assert!(reaches(&path("$.a[*]"), &Label::Index(9)), "any element"); - assert!(!reaches(&path("$.a[*]"), &key("x")), "not a member"); - assert!(reaches(&path("$.a.b"), &key("b")), "a member"); - assert!(!reaches(&path("$.a.b"), &Label::Index(0)), "not an element"); - assert!(reaches(&path("$"), &Label::Root), "the root"); - assert!(!reaches(&path("$"), &key("a")), "only the root"); + assert!(names(&path("$.a[2]"), &Label::Index(2)), "the position"); + assert!(!names(&path("$.a[2]"), &Label::Index(1)), "another one"); + assert!(!names(&path("$.a[2]"), &key("2")), "a member"); + assert!(names(&path("$.a.b"), &key("b")), "the member"); + assert!(!names(&path("$.a.b"), &key("c")), "another member"); + assert!(!names(&path("$.a.b"), &Label::Index(0)), "not an element"); + assert!(names(&path("$"), &Label::Root), "the root"); + assert!(!names(&path("$"), &key("a")), "only the root"); + assert!(!names(&path("$.a"), &Label::Root), "the root is no member"); + } + + #[test] + fn a_childs_token_follows_its_label() { + let (parent, s) = ([1; 16], [2; 16]); + assert_eq!( + labelled_token(&parent, &s, &key("k")), + Some(member_token(&parent, &s)) + ); + assert_eq!( + labelled_token(&parent, &s, &Label::Index(3)), + Some(element_token(&parent, &s, 3)) + ); + assert_eq!( + labelled_token(&parent, &s, &Label::Root), + None, + "the root is no child" + ); + assert_eq!( + labelled_token(&parent, &s, &Label::Index(u64::from(u32::MAX) + 1)), + None, + "no element sits past u32" + ); } #[test] diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index a4ee371c0..a145b913d 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -12,13 +12,23 @@ //! value, and for a container its child count. Nothing beneath a node is //! encrypted in that node's entry, so a document's ciphertext is linear //! in its size; +//! - the **self tag** `x`, the first eight bytes of `H("x" ‖ iv ‖ t)`, +//! `t` being the node's own **token**, on every entry: how the server +//! finds the node a client-derived token names. Every token derives from +//! its node's path alone: the root's (and, under the sealed root, each +//! top-level field's) is a PRF of its path, an object member's is +//! `H16("t" ‖ t(parent) ‖ s(member))` and an array element's +//! `H16("t" ‖ t(array) ‖ s(element) ‖ LE32(i))`, `i` its index. So the +//! client sends exactly the tokens of the nodes it reads; //! - the **parent tag** `p`, the first eight bytes of -//! `H("p" ‖ iv ‖ t(parent) ‖ LE32(o))`, which lets the server walk the subtree under a node from one client-derived -//! token and nothing else: the ordinals `o` under a node are dense from -//! zero, so a walk probes them in turn until the first miss, and derives -//! each hit's own token as `H("t" ‖ t(parent) ‖ s ‖ LE32(o))` from the -//! hit's own selector `s`, which keeps an array element's token apart -//! from an object member's at the same ordinal; +//! `H("p" ‖ iv ‖ t(parent) ‖ LE32(o))`, below the chain's heads, which +//! lets the server enumerate the subtree under a node from its token and +//! nothing else: the ordinals `o` under a node are dense from zero (an +//! element's is its index, a member's its keyed rank among its +//! siblings), so the server probes them in turn until the first miss. +//! Nothing stored says whether a hit is a member or an element, so the +//! server derives the member rule's token for it, keeps it if the hit's +//! self tag is that token's, and otherwise tries the element rule's; //! - a leaf ordering term `op` on the number and string leaves at the //! paths the index declares ordered ([`JsonOptions::ordered`]), the //! same OPE term the scalar [`Ope`](crate::target::Ope) index derives, @@ -31,10 +41,10 @@ //! exact containment. The value selectors live there and nowhere else. //! //! By default the root is **sealed** ([`JsonOptions::sealed_root`]): the root -//! entry's ciphertext is the whole document, the top-level fields are the -//! heads of the chain, and no token exists for the root, so no query can -//! hand the server a token that unlocks the shape of every row in the -//! column. +//! entry's ciphertext is the whole document and the top-level fields are +//! the heads of the chain, chained from nothing, so the root's token finds +//! the root entry and nothing beneath it, and no query can hand the server +//! a token that unlocks the shape of every row in the column. //! //! # What the types are //! @@ -71,17 +81,16 @@ //! `N² / 2^97`, negligible for any document. The ordinal still binds the //! entry to its place through the AAD, so a moved or reordered entry fails //! to open as the design requires. The cost is twelve bytes per entry. -//! - **Tokens chain by ordinal, not by label, and nothing stores a token.** -//! The design wraps each child's label-derived token under its parent's -//! (`w`) and stores the ordinal (`o`) so the server can find and unmask -//! it. Here a child's token is a hash of its parent's token, its own -//! selector and its ordinal, so the server derives it as it walks and -//! both fields go. The -//! ordinals under a node are dense from zero, so the walk needs no -//! fan-out bound either: it probes until the first miss. The price is -//! that the client cannot derive an object member's token (its ordinal -//! is a rank among sibling keys it does not know), so it walks from the -//! nearest token it can derive through one gate per step +//! - **Tokens chain by selector, and nothing stores a token.** The design +//! wraps each child's label-derived token under its parent's (`w`) and +//! stores the ordinal (`o`) so the server can find and unmask it. Here a +//! child's token is a hash of its parent's token and its own selector, +//! with its index for an array element, so the client derives any node's +//! token from the path and the server derives each child's as it walks, +//! and both fields go. The ordinals under a node are dense from zero, so +//! the walk needs no fan-out bound either: it probes until the first +//! miss. A wildcard names no one node, so the client expands it into +//! concrete paths up to a bound, with a probe past it //! ([`JsonSubtreeKey`]). //! //! # Byte encodings @@ -90,9 +99,9 @@ //! field the SQL compares, derives or decodes is standard base64 with //! padding, which is what PostgreSQL's `encode(x, 'base64')` writes and //! `decode(x, 'base64')` reads: the selectors `s` and the pairs of `q` -//! (sixteen bytes, 24 characters), the parent tag `p` (eight bytes, 12 -//! characters), the ordering term `op`, the tokens and gates a walk -//! takes, and the header `h`, which the SQL reads the `iv` and the +//! (sixteen bytes, 24 characters), the self tag `x` and the parent tag `p` +//! (eight bytes, 12 characters each), the ordering term `op`, the tokens a +//! read takes, and the header `h`, which the SQL reads the `iv` and the //! sealed-root flag from. The ciphertext `c`, which nothing in SQL //! decodes, is RFC 1924 base85 (`base85-simd`): five characters per four //! bytes against base64's eight per six, on the one field that is most of @@ -126,6 +135,7 @@ pub(crate) use derive::{seal_pending, terms_pending}; pub type Tag = [u8; 16]; /// An eight-byte parent tag: the first eight bytes of the parent-tag hash. +/// [`SelfTag`] has the same width for the same reason. /// A parent tag is a lookup key for the server's walk and nothing else (an /// entry's AAD binds its selector and ordinal, not its parent tag), so it /// needs only to be distinct among the entries of one document; eight @@ -133,6 +143,13 @@ pub type Tag = [u8; 16]; /// 10^11, and save eight bytes an entry over the full hash. pub type ParentTag = [u8; 8]; +/// An eight-byte self tag: the first eight bytes of `H("x" ‖ iv ‖ t)`, `t` +/// being the entry's own token. A lookup key for the server, by which it +/// finds the node a token names; like the parent tag, it is not bound by +/// the entry's AAD and needs only to be distinct among the entries of one +/// document. +pub type SelfTag = [u8; 8]; + /// How a JSON index lays a document out, and which of its leaves it /// orders. Part of the index's identity: a stored document and the query /// probing it must agree on it, so it rides in the data plan beside the @@ -363,13 +380,15 @@ impl JsonHeader { /// One entry of a stored document: one JSON node. /// -/// `s` and `c` are always present. `p` is present on a node whose parent -/// has a token (everything below the chain's heads); `op` on a number or -/// string leaf at a path the index orders ([`JsonOptions::ordered`]). +/// `s`, `c` and `x` are always present. `p` is present on a node that +/// chains from its parent (everything below the chain's heads); `op` on a +/// number or string leaf at a path the index orders +/// ([`JsonOptions::ordered`]). #[derive(Clone, Debug, PartialEq, Eq)] pub struct JsonEntry { s: Tag, c: Vec, + x: SelfTag, p: Option, op: Option>, } @@ -383,7 +402,11 @@ impl JsonEntry { pub fn ciphertext(&self) -> &[u8] { &self.c } - /// The parent tag, when the parent has a token. + /// The self tag, on every entry. + pub fn self_tag(&self) -> &SelfTag { + &self.x + } + /// The parent tag, when the node chains from its parent. pub fn parent_tag(&self) -> Option<&ParentTag> { self.p.as_ref() } @@ -431,48 +454,81 @@ impl JsonDocument { &self.projection } + /// The entry whose self tag is `token`'s, as the server finds the node + /// a token names: its ordinal and the entry. `None` for a token that + /// names no node of this document. + /// + /// Where several entries carry the tag, which no sealing produces, the + /// last of them, as the server's index keeps it + /// (`eql_v4_internal.json_tree_index` builds it with + /// `jsonb_object_agg`, which keeps the last value of a repeated key). + pub fn find(&self, token: &Tag) -> Option<(usize, &JsonEntry)> { + let x = derive::self_tag(&self.header.iv, token); + self.entries + .iter() + .enumerate() + .rev() + .find(|(_, e)| e.x == x) + } + /// The children of the node whose token is `token`, as the server - /// finds them: the entries at the ordinals `0, 1, 2, …` under it, - /// probed until the first miss, each with its own token derived. A - /// token that names no node yields nothing. + /// enumerates them: the entries at the ordinals `0, 1, 2, …` under it, + /// probed until the first miss, each with the token the server derives + /// for it, by the member rule or the element rule, whichever the entry's + /// self tag matches, and `None` when it matches neither. A token that + /// names no node yields nothing. /// /// What `json_tree_children(doc, token)` returns; one level of the /// walk [`subtree`](Self::subtree) runs to the bottom. - pub fn children(&self, token: &Tag) -> Vec<(usize, &JsonEntry, Tag)> { + pub fn children(&self, token: &Tag) -> Vec<(usize, &JsonEntry, Option)> { let iv = &self.header.iv; let mut out = Vec::new(); + if self.find(token).is_none() { + return out; + } // A node has at most as many children as the document has entries, // so the probe loop is bounded whatever the entries say. let most = u32::try_from(self.entries.len()).unwrap_or(u32::MAX); for o in 0..most { let tag = derive::parent_tag(iv, token, o); + // The last entry carrying the tag, as the server's index keeps + // it ([`find`](Self::find)). let Some((j, entry)) = self .entries .iter() .enumerate() + .rev() .find(|(_, e)| e.p.as_ref() == Some(&tag)) else { break; }; - out.push((j, entry, derive::child_token(token, &entry.s, o))); + out.push((j, entry, derive::hit_token(iv, token, entry, o))); } out } - /// Every entry under the node whose token is `token`, with its depth - /// below that node, as the server's recursive walk finds them: the node - /// itself is not included, since the walk addresses nodes by their - /// parent. In depth-first order, siblings by ordinal. + /// The rows the server's read returns for `tokens` with no budget: + /// for each token in order, the node it names at depth `0` and every + /// entry beneath it with its depth below that node, in document order. + /// A token that names no node yields nothing. What + /// `json_tree_subtree(doc, tokens, 0)` returns. /// /// A stored document is untrusted, and its tags need not form a tree: /// whoever holds a token can write a chain deeper than any document the - /// index seals, or a cycle. The walk stops descending below the deepest - /// level a sealed document has, and stops once it has returned as many - /// entries as the document holds, which a tree never exceeds, so it - /// ends, and in bounded stack, on any document. - pub fn subtree(&self, token: &Tag) -> Vec<(usize, &JsonEntry, u32)> { + /// index seals, or a cycle. Each token's walk stops descending below the + /// deepest level a sealed document has, and stops once it has returned + /// as many entries as the document holds, which a tree never exceeds, + /// so it ends, and in bounded stack, on any document. + pub fn subtree(&self, tokens: &[Tag]) -> Vec<(usize, &JsonEntry, u32)> { let mut out = Vec::new(); - self.walk(token, 1, &mut out); + for token in tokens { + let Some((j, entry)) = self.find(token) else { + continue; + }; + let mut rows = vec![(j, entry, 0)]; + self.walk(token, 1, &mut rows); + out.extend(rows); + } out } @@ -490,7 +546,9 @@ impl JsonDocument { return; } out.push((j, entry, depth)); - self.walk(&child, depth + 1, out); + if let Some(child) = child { + self.walk(&child, depth + 1, out); + } } } @@ -617,29 +675,27 @@ where } } -/// The entries the server's walk returned for one key, ready to open: -/// what `json_tree_subtree(doc, token)` or `json_tree_subtree_at(doc, -/// token, gates)` yields, collected, and what -/// [`DecryptInto`](crate::DecryptInto) reassembles into the value -/// under the walked path. Built only by the key that keyed the walk -/// ([`JsonSubtreeKey::open`]), so it always carries the walked path, and -/// what checks every row against it when the rows are opened (see +/// The entries the server returned for one subtree read, ready to open: +/// what `json_tree_subtree(doc, key.tokens(), budget)` yields, collected, +/// and what [`DecryptInto`](crate::DecryptInto), +/// `DecryptInto>` and `DecryptInto>` open into the +/// value at the read's path. Built only by the key that keyed the read +/// ([`JsonSubtreeKey::open`]), so it always carries the key, and what +/// checks every row against it when the rows are opened (see /// [`JsonSubtreeKey::open`] for the checks). /// -/// The ungated walk returns the node's descendants and not the node -/// itself, so the node's kind is read from its children's labels: elements -/// make an array, members an object. A node with no descendants, an empty -/// container or a scalar, yields no rows and so no value; `->` on its -/// selector returns the node itself. A gated walk also returns each node -/// its gates reach, at depth `0`, with that node's descendants beneath it. +/// The rows are, for each token of the key that names a node, the node at +/// depth `0` and everything beneath it, so the four answers `jsonb #>` +/// gives are four answers here: an empty object's entry, an empty array's +/// entry, a scalar's entry, and no rows for a node the row does not have. /// -/// Past its budget, a walk over a document with a sealed root returns the -/// root entry alone, at depth `0`, instead of a subtree it would be -/// cheaper to open whole: the client opens the document and takes the -/// walked path over it, to the same value the walked rows open to. Whether -/// the path names the node walked below (`json_tree_subtree`) or the nodes -/// reached (`json_tree_subtree_at`) is the key's to say, by whether it has -/// gates, so no caller attaches it. +/// Past its budget, a read of a document with a sealed root returns the +/// root entry alone, at depth `0`, instead of rows it would be cheaper to +/// open whole: the client opens the document and evaluates the key's +/// concrete paths and probes over it, to the same values the walked rows +/// open to, a probe that finds an element included. A read of `$` under +/// the sealed root is answered with the same one row, the root, which +/// opens to the whole document. #[derive(Clone, Debug, PartialEq, Eq)] pub struct JsonSubtree { nodes: Vec, @@ -647,9 +703,7 @@ pub struct JsonSubtree { } impl JsonSubtree { - /// The rows of a walk `key` keyed: below the path's node when the key - /// has no gates (`json_tree_subtree`), at the nodes it reaches - /// otherwise. + /// The rows of the read `key` keyed. pub(crate) fn keyed(nodes: Vec, key: JsonSubtreeKey) -> Self { Self { nodes, key } } @@ -659,24 +713,18 @@ impl JsonSubtree { &self.nodes } - /// The path the walk was keyed from. + /// The path the read was keyed from. pub fn path(&self) -> &JsonPath { &self.key.path } - /// Whether the rows are the descendants of the path's node - /// (`json_tree_subtree`, a key with no gates) rather than the nodes - /// the path reaches. - pub(crate) fn below(&self) -> bool { - self.key.gates.is_empty() - } - pub(crate) fn into_parts(self) -> (Vec, JsonSubtreeKey) { (self.nodes, self.key) } - /// Whether the rows are the sealed root the server returned in place - /// of a subtree past its budget: one row, the root, at depth `0`. + /// Whether the rows are the sealed root alone, at depth `0`: what the + /// server returns in place of a read past its budget, and for a read of + /// `$` under the sealed root. Either way it opens whole. pub fn is_sealed_root(&self) -> bool { matches!( self.nodes.as_slice(), @@ -750,13 +798,15 @@ struct DocumentWire { q: query::GroupWire, } -/// One entry, as stored: the tags and the ordering term in base64, the +/// One entry, as stored, `{s, c, x, p?, op?}`: the tags and the ordering +/// term in base64, the /// ciphertext in base85. #[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct EntryWire { s: String, c: String, + x: String, #[serde(default, skip_serializing_if = "Option::is_none")] p: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -768,6 +818,7 @@ impl From<&JsonEntry> for EntryWire { Self { s: base64(&e.s), c: base85(&e.c), + x: base64(&e.x), p: e.p.as_ref().map(|t| base64(t)), op: e.op.as_deref().map(base64), } @@ -781,6 +832,7 @@ impl TryFrom for JsonEntry { Ok(Self { s: unbase64_16(&w.s)?, c: unbase85(&w.c)?, + x: unbase64_8(&w.x)?, p: w.p.as_deref().map(unbase64_8).transpose()?, op: w.op.as_deref().map(unbase64).transpose()?, }) @@ -936,11 +988,13 @@ mod tests { } /// An entry at ordinal `o` under `parent`, as whoever holds `parent` can - /// write one: its parent tag is public given the token. + /// write one: its parent tag and its self tag as a member are public + /// given the token. fn chained(iv: &[u8], parent: &Tag, o: u32) -> JsonEntry { JsonEntry { s: [0; 16], c: o.to_le_bytes().to_vec(), + x: derive::self_tag(iv, &derive::member_token(parent, &[0; 16])), p: Some(derive::parent_tag(iv, parent, o)), op: None, } @@ -955,24 +1009,30 @@ mod tests { let mut entries = vec![JsonEntry { s: [0; 16], c: vec![0], + x: derive::self_tag(&iv, &token), p: None, op: None, }]; let mut parent = token; for _ in 0..1000 { entries.push(chained(&iv, &parent, 0)); - parent = derive::child_token(&parent, &[0; 16], 0); + parent = derive::member_token(&parent, &[0; 16]); } let chain = JsonDocument::new( JsonHeader::new(Uuid::from_u128(7), iv, vec![], true), entries, JsonProjection::default(), ); - let walked = chain.subtree(&token); - assert_eq!(walked.len(), derive::MAX_DEPTH, "cut at the deepest level"); + let walked = chain.subtree(&[token]); + assert_eq!( + walked.len(), + derive::MAX_DEPTH + 1, + "the token's node, then cut at the deepest level" + ); assert_eq!( walked.last().map(|(_, _, depth)| *depth as usize), - Some(derive::MAX_DEPTH) + Some(derive::MAX_DEPTH), + "no row below the deepest level a sealed document has" ); } diff --git a/packages/stack-encrypt/src/json/query.rs b/packages/stack-encrypt/src/json/query.rs index 8ca017004..c7d9d4075 100644 --- a/packages/stack-encrypt/src/json/query.rs +++ b/packages/stack-encrypt/src/json/query.rs @@ -346,37 +346,34 @@ impl Serialize for JsonSelector { } } -/// One gate of a subtree walk: what the entry at one depth below the -/// walk's token must be for the walk to descend through it. -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum Gate { - /// The selector of the path down to that depth: a key step, or a - /// wildcard, whose selector every element carries. - Selector(Tag), - /// An array position: the one ordinal probed at that depth, and the - /// selector of an element there (the wildcard's), which the entry at - /// that ordinal must carry. The ordinal alone would admit an object - /// member that ranks there in a row whose node at the path is an - /// object. - Slot(u32, Tag), -} - -impl Gate { - /// The gate as `json_tree_subtree_at` takes it: the selector's base64, or - /// `#n:` and the element selector's base64 for a position. - fn text(&self) -> String { - match self { - Gate::Selector(s) => base64(s), - Gate::Slot(n, s) => format!("#{n}:{}", base64(s)), - } - } +/// One token a subtree read sends: the token of the node at a concrete +/// path (no wildcard), the path, and whether it is a probe, the element at +/// the bound of a wildcard level, which the read sends to learn whether an +/// array there is longer than the bound and never answers with. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct Read { + pub(crate) token: Tag, + pub(crate) path: JsonPath, + pub(crate) probe: bool, } -/// The key of a subtree walk: the token of the deepest node on the path -/// the client can derive, one gate per step below it, and the path it was -/// derived from, which [`open`](Self::open) attaches to the rows the walk -/// returns. Serializes as `{"t": "…", "g": ["…", "#0:…", …]}`, `g` omitted -/// when empty; the path is not part of the wire. +/// The key of a subtree read: one token per concrete path the read's path +/// names, in document order, with a probe for each array a wildcard +/// expands, the bound the wildcards were expanded to, and the path it was +/// derived from, which +/// [`open`](Self::open) attaches to the rows the read returns. What +/// `json_tree_subtree(doc, tokens, budget)` takes is +/// [`tokens`](Self::tokens). Serializes as `{"t": ["…", …]}`; the paths +/// are not part of the wire. +/// +/// Every token derives from its node's path alone, so a path with no +/// wildcard sends exactly one token and nothing that covers a node above +/// it. A wildcard (`$.items[*].price`) names no one node: the key holds a +/// token for each position below the bound (`$.items[0].price` … +/// `$.items[N − 1].price`) and a probe for the element at the bound +/// (`$.items[N]`, the element itself). A row where a probe names a node +/// opens to [`Error::JsonWildcardTruncated`](crate::Error::JsonWildcardTruncated), +/// never to part of the answer. /// /// The key also holds what derives selectors under the column it was /// derived for (the keyset's index-key PRF, shared with the keyset handle, @@ -385,32 +382,47 @@ impl Gate { /// serialized. #[derive(Clone, PartialEq, Eq)] pub struct JsonSubtreeKey { - pub(crate) token: Tag, - pub(crate) gates: Vec, + pub(crate) reads: Vec, + pub(crate) bound: u32, pub(crate) path: JsonPath, pub(crate) selectors: Selectors, } impl JsonSubtreeKey { - /// The token, as the base64 the walk takes. - pub fn token(&self) -> String { - base64(&self.token) - } - - /// The token's bytes. A token is a capability over the structure of - /// one subtree in every row of the column, for as long as the column - /// lives: send it to the server for a query, and nowhere else. - pub fn token_bytes(&self) -> &Tag { - &self.token + /// The bound a wildcard expands to when the caller names none: 64 + /// elements per wildcard level, in the spirit of the walk's default + /// budget. + pub const DEFAULT_BOUND: u32 = 64; + + /// The tokens, as the base64 `json_tree_subtree` takes in its `tokens` + /// array, in the key's order, the probes among them. + pub fn tokens(&self) -> Vec { + self.reads.iter().map(|read| base64(&read.token)).collect() + } + + /// The tokens' bytes, in the key's order. A token is a capability over + /// the structure of its node's subtree in every row of the column, for + /// as long as the column lives: send it to the server for a query, and + /// nowhere else. + pub fn token_bytes(&self) -> Vec { + self.reads.iter().map(|read| read.token).collect() + } + + /// Which of [`tokens`](Self::tokens) are probes, by their places in it: + /// one per element a wildcard level sits below, last at its level. + /// Empty for a path with no wildcard. + pub fn probes(&self) -> Vec { + self.reads + .iter() + .enumerate() + .filter(|(_, read)| read.probe) + .map(|(i, _)| i) + .collect() } - /// The gates the walk descends through below the token, one per step, - /// as `json_tree_subtree_at` takes them: a selector's base64, or - /// `#n:` for an array position, the selector being the array - /// elements' own. Empty when the token names the walked node itself, - /// which `json_tree_subtree` takes. - pub fn gates(&self) -> Vec { - self.gates.iter().map(Gate::text).collect() + /// The bound each wildcard level was expanded to. + pub fn bound(&self) -> u32 { + self.bound } /// The path the key was derived from. @@ -418,47 +430,45 @@ impl JsonSubtreeKey { &self.path } - /// The rows the walk this key keyed returned, ready to open: the only - /// way to build a [`JsonSubtree`], so the rows always carry the path - /// a sealed-root fallback is opened at. With no gates the rows came - /// from `json_tree_subtree(doc, token)` and are the descendants of the - /// path's node; with gates they came from `json_tree_subtree_at(doc, - /// token, gates)` and are the nodes the path reaches, with theirs. The - /// key knows which, so the fallback opens to what the walked rows - /// open to either way. + /// The rows the read this key keyed returned, ready to open: the only + /// way to build a [`JsonSubtree`], so the rows always carry the key a + /// sealed-root fallback is evaluated under. /// /// Opening the rows checks each one against the key, and refuses the - /// rows with [`Error::JsonWrongEntry`] if one does not fit: a node the - /// path reaches must carry the path's selector, at depth `0`, with the - /// position the path names if it ends in one; every other row must - /// carry the selector of its parent's path extended by its own label - /// (its key, or the wildcard for an array element), sit one level - /// below its parent, and follow the row before it in the document with - /// no entry between, so that each node's rows are exactly its subtree. - /// A row whose parent is not among the rows is refused. - /// - /// With no gates the walk does not return the node it starts from, so - /// its children are checked against each other: they follow one - /// another with no gap, and an array's start at element `0`. + /// rows with [`Error::JsonWrongEntry`] if one does not fit: a node a + /// token names, at depth `0`, must carry the self tag of one of the + /// key's tokens (each matched once, in order) and that token's path's + /// selector and last step (its key, its position, or the root); every + /// row beneath it must carry the selector of its parent's path extended + /// by its own label, sit one level below its parent, follow the row + /// before it in the document with no entry between, and carry the self + /// tag of the token its label derives under its parent's. The self tags + /// are not authenticated: matching them catches a corrupted row or a + /// server that walked wrongly, and the selector, the position and the + /// label are what bind a row to its place. The rows must also be of + /// distinct entries of one document: an entry returned twice, or rows + /// carrying two headers, are refused the same way. /// - /// Not covered: which row of the column the entries come from (ADR - /// 0010), and, until the walk returns the node it starts from (#1356), - /// two things about a walk with no gates: which node at the path's - /// selector it descends from, and whether children were left out after - /// the last row, or members anywhere (a member's place is a keyed rank - /// the client does not hold). An array position before the last step - /// of a path reads as the wildcard to the selectors, so another - /// element's subtree at the same depth passes too. + /// Not covered, by design: which row of the column the entries come + /// from (ADR 0010), and which element a node at the path's selector + /// sits under, since every element of an array shares its selector: a + /// read of `$.items[1].price` can be answered with `$.items[0].price` + /// of the same row. Nor whether the database left whole nodes out: a + /// read answered with no rows opens as no node, and a wildcard read + /// with some elements' rows left out, its probe's included, opens to + /// the rest. pub fn open(&self, rows: Vec) -> JsonSubtree { JsonSubtree::keyed(rows, self.clone()) } } -/// The token is a capability: it prints as the key's shape, not its bytes. +/// The tokens are capabilities: the key prints as its shape, not its bytes. impl fmt::Debug for JsonSubtreeKey { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("JsonSubtreeKey") - .field("gates", &self.gates.len()) + .field("tokens", &self.reads.len()) + .field("probes", &self.reads.iter().filter(|r| r.probe).count()) + .field("bound", &self.bound) .finish_non_exhaustive() } } @@ -467,15 +477,9 @@ impl Serialize for JsonSubtreeKey { fn serialize(&self, s: S) -> Result { #[derive(Serialize)] struct Wire { - t: String, - #[serde(skip_serializing_if = "Vec::is_empty")] - g: Vec, - } - Wire { - t: base64(&self.token), - g: self.gates(), + t: Vec, } - .serialize(s) + Wire { t: self.tokens() }.serialize(s) } } @@ -831,36 +835,40 @@ mod tests { format!("JsonSelector({})", "q6urq6urq6urq6urq6urqw==") ); let key = JsonSubtreeKey { - token: [0xcd; 16], - gates: vec![Gate::Selector([0xab; 16]), Gate::Slot(3, [0xab; 16])], - path: JsonPath::parse("$.a.b[3]").unwrap(), + reads: vec![ + Read { + token: [0xcd; 16], + path: JsonPath::parse("$.a[0]").unwrap(), + probe: false, + }, + Read { + token: [0xab; 16], + path: JsonPath::parse("$.a[1]").unwrap(), + probe: true, + }, + ], + bound: 1, + path: JsonPath::parse("$.a[*]").unwrap(), selectors: Selectors::for_tests(), }; - assert_eq!(key.token(), "zc3Nzc3Nzc3Nzc3Nzc3NzQ=="); - assert_eq!(key.token_bytes(), &[0xcd; 16]); assert_eq!( - key.gates(), - ["q6urq6urq6urq6urq6urqw==", "#3:q6urq6urq6urq6urq6urqw=="] + key.tokens(), + ["zc3Nzc3Nzc3Nzc3Nzc3NzQ==", "q6urq6urq6urq6urq6urqw=="], + "every token, in order, the probe among them" ); + assert_eq!(key.token_bytes(), [[0xcd; 16], [0xab; 16]]); + assert_eq!(key.probes(), [1], "the probe is marked by its place"); + assert_eq!(key.bound(), 1); assert_eq!( serde_json::to_value(&key).unwrap(), - serde_json::json!({"t": "zc3Nzc3Nzc3Nzc3Nzc3NzQ==", "g": ["q6urq6urq6urq6urq6urqw==", "#3:q6urq6urq6urq6urq6urqw=="]}) + serde_json::json!({"t": ["zc3Nzc3Nzc3Nzc3Nzc3NzQ==", "q6urq6urq6urq6urq6urqw=="]}), + "the wire carries the tokens and nothing of the paths" ); assert_eq!( format!("{key:?}"), - "JsonSubtreeKey { gates: 2, .. }", + "JsonSubtreeKey { tokens: 2, probes: 1, bound: 1, .. }", "a token is a capability: it never prints" ); - let bare = JsonSubtreeKey { - token: [0xcd; 16], - gates: Vec::new(), - path: JsonPath::parse("$.a").unwrap(), - selectors: Selectors::for_tests(), - }; - assert_eq!( - serde_json::to_value(&bare).unwrap(), - serde_json::json!({"t": "zc3Nzc3Nzc3Nzc3Nzc3NzQ=="}) - ); } #[test] diff --git a/packages/stack-encrypt/src/sem/mod.rs b/packages/stack-encrypt/src/sem/mod.rs index 6e5d7bde7..35dcfce34 100644 --- a/packages/stack-encrypt/src/sem/mod.rs +++ b/packages/stack-encrypt/src/sem/mod.rs @@ -312,15 +312,15 @@ pub enum TermError { help("The JSON index numbers at most 2147483647 children under one node, so no stored document has an element past position 2147483647. Split the array, or ask for a position below that.") )] JsonTooWide, - /// A subtree key was asked for the root, or a path whose first step is - /// a wildcard, of a JSON index whose root is sealed: no token exists - /// for the root, by design, and the document opens whole instead. - #[error("the sealed root of a JSON document has no subtree token")] + /// A subtree read's wildcards would expand to more tokens than one read + /// sends: a wildcard expands to its bound plus one tokens under each node + /// it sits below, so nested wildcards multiply. + #[error("a JSON subtree read expands to too many tokens")] #[diagnostic( - code(stack_encrypt::json_sealed_root), - help("Under the sealed root (the default) the chain starts at the top-level fields: walk a subtree from `$.field` down, or open the whole document.") + code(stack_encrypt::json_wildcard_too_wide), + help("A read sends at most 8192 tokens. Lower the wildcard bound (`json_subtree_key_with_bound`), use fewer nested wildcards, or read the array itself.") )] - JsonSealedRoot, + JsonWildcardTooWide, /// An ordering at a JSON path was asked with a value that is neither a /// number nor a string, the only leaves the index orders. #[error("an ordering at a JSON path takes a number or a string")] @@ -374,7 +374,7 @@ impl crate::ErrorPayload for TermError { | Self::Normalize(_) | Self::JsonTooDeep | Self::JsonTooWide - | Self::JsonSealedRoot + | Self::JsonWildcardTooWide | Self::JsonLeafNotOrdered | Self::JsonPathNotOrdered | Self::JsonOrderingAtPosition diff --git a/packages/stack-encrypt/tests/fixtures/json_tree_vectors.json b/packages/stack-encrypt/tests/fixtures/json_tree_vectors.json new file mode 100644 index 000000000..2d14c339b --- /dev/null +++ b/packages/stack-encrypt/tests/fixtures/json_tree_vectors.json @@ -0,0 +1,134 @@ +{ + "_comment": "The JSON index's s, x and p per entry, and the tokens a read sends, under the deterministic key source. Read by tests/json_tree_vectors.rs; regenerate with STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS=1 only for an intended break. Do not edit by hand.", + "document": { + "a": { + "b": [ + 1, + { + "c": true + } + ] + }, + "d": "e" + }, + "open_root": { + "entries": [ + { + "p": null, + "s": "GUG6+MOjeHGk8vJS5CJOxA==", + "x": "PelL2/QWv0M=" + }, + { + "p": "8OU5UZoL5Hw=", + "s": "8OVeBxReEgPVrjL7qGYAbg==", + "x": "p7/wbsjaDjQ=" + }, + { + "p": "WN3/UZRAros=", + "s": "HlQhfaUoEj5O+JR/j9trVQ==", + "x": "W0aJLUxlVdA=" + }, + { + "p": "ZOSBKJZJ/NY=", + "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", + "x": "EVNlMjX0dyE=" + }, + { + "p": "ZHxAvDKdFsI=", + "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", + "x": "q22q1h6JGj0=" + }, + { + "p": "/EJ22V9WvdY=", + "s": "z1M7JTZ8C8WMd3dqhuZpfA==", + "x": "hMhl3B9cj/0=" + }, + { + "p": "sIWG0EdV0i8=", + "s": "Ie6Xm6ZxkV/9nBoIK7zXsQ==", + "x": "HRuqzbdL4ew=" + } + ], + "iv": "OtEqUO60j5CMut+3HQzYSA==", + "tokens": { + "$": [ + "tJdy239uYbuNLuRBLfJuNQ==" + ], + "$.a": [ + "ZDj2w3vE5Yb+ozBC9vU/xQ==" + ], + "$.a.b": [ + "/wnqVYhdW/Z2syRn3UT7BQ==" + ], + "$.a.b[0]": [ + "WDnRCSnfT++Ut0INNwLbJg==" + ], + "$.a.b[1]": [ + "Jmb/eVBOYiQpM4LoFylYJA==" + ], + "$.a.b[1].c": [ + "9o1SJ8daeeeWXi3Xn6duhw==" + ] + } + }, + "sealed_root": { + "entries": [ + { + "p": null, + "s": "GUG6+MOjeHGk8vJS5CJOxA==", + "x": "PelL2/QWv0M=" + }, + { + "p": null, + "s": "8OVeBxReEgPVrjL7qGYAbg==", + "x": "wl8ZJzvRwJ0=" + }, + { + "p": "Dy1WgDveXQU=", + "s": "HlQhfaUoEj5O+JR/j9trVQ==", + "x": "12lMeBeDMpo=" + }, + { + "p": "sJXI7nmR44s=", + "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", + "x": "/yUSe3uTE98=" + }, + { + "p": "d0RUZY7M9PI=", + "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", + "x": "7n2djhOoEXo=" + }, + { + "p": "4mV5CHfjZEg=", + "s": "z1M7JTZ8C8WMd3dqhuZpfA==", + "x": "A/QZeMRJngk=" + }, + { + "p": null, + "s": "Ie6Xm6ZxkV/9nBoIK7zXsQ==", + "x": "uGnxy4JLUhk=" + } + ], + "iv": "OtEqUO60j5CMut+3HQzYSA==", + "tokens": { + "$": [ + "tJdy239uYbuNLuRBLfJuNQ==" + ], + "$.a": [ + "eU6Mev5Z/qvIfCQXhnj0rg==" + ], + "$.a.b": [ + "0RFumQCVxgGHhXQQ9dxRLQ==" + ], + "$.a.b[0]": [ + "8hKJRkRA3odRkB9gErDAxg==" + ], + "$.a.b[1]": [ + "/SE/9hQBrjXTjmuZ7qmhNQ==" + ], + "$.a.b[1].c": [ + "AYlWSivEpmukX15V4GUrug==" + ] + } + } +} diff --git a/packages/stack-encrypt/tests/json_index.rs b/packages/stack-encrypt/tests/json_index.rs index 57059dc17..c058646b9 100644 --- a/packages/stack-encrypt/tests/json_index.rs +++ b/packages/stack-encrypt/tests/json_index.rs @@ -197,27 +197,9 @@ fn base64(bytes: &[u8]) -> String { Base64::encode_string(bytes) } -/// The ordinals the walk from `token` reaches. -fn subtree_ordinals(doc: &JsonDocument, token: &Tag) -> Vec { - doc.subtree(token).iter().map(|(j, _, _)| *j).collect() -} - -/// The nodes the walk from `token` reaches, as the server returns them. -fn subtree_nodes(doc: &JsonDocument, token: &Tag) -> Vec { - doc.subtree(token) - .iter() - .map(|(j, _, _)| doc.node(*j).unwrap()) - .collect() -} - -/// The token of the child at ordinal `j` under `token`, as the server -/// derives it. -fn child_token(doc: &JsonDocument, token: &Tag, j: usize) -> Tag { - doc.children(token) - .into_iter() - .find(|(ordinal, _, _)| *ordinal == j) - .map(|(_, _, t)| t) - .unwrap_or_else(|| panic!("entry {j} is a child of the token's node")) +/// The ordinals the read of `tokens` finds. +fn subtree_ordinals(doc: &JsonDocument, tokens: &[Tag]) -> Vec { + doc.subtree(tokens).iter().map(|(j, _, _)| *j).collect() } /// Node `j` as a walk returns it, `d` levels below where it started. @@ -227,50 +209,14 @@ fn with_depth(doc: &JsonDocument, j: usize, d: u32) -> JsonNode { serde_json::from_value(wire).unwrap() } -/// The rows `json_tree_subtree_at(doc, token, gates)` returns for `key`, -/// walked here as the SQL walks them: from the key's token, through each -/// gate (a selector the entry at that depth must carry, or `#n:`, -/// the child at ordinal `n` carrying it), then everything beneath each node -/// the gates reach, each row with its depth below that node. -fn gated_walk(doc: &JsonDocument, key: &JsonSubtreeKey) -> Vec { - let mut frontier = vec![(None, *key.token_bytes())]; - for gate in key.gates() { - let mut next = Vec::new(); - for (_, token) in &frontier { - let children = doc.children(token); - match gate.strip_prefix('#') { - Some(slot) => { - let (n, selector) = slot.split_once(':').unwrap(); - if let Some((j, entry, t)) = children.get(n.parse::().unwrap()) { - if base64(entry.selector()) == selector { - next.push((Some(*j), *t)); - } - } - } - None => next.extend( - children - .iter() - .filter(|(_, entry, _)| base64(entry.selector()) == gate) - .map(|(j, _, t)| (Some(*j), *t)), - ), - } - } - frontier = next; - } - let mut rows = Vec::new(); - for (reached, token) in frontier { - let below = u32::from(reached.is_some()); - if let Some(j) = reached { - rows.push(with_depth(doc, j, 0)); - } - rows.extend( - doc.subtree(&token) - .iter() - .map(|(j, _, d)| with_depth(doc, *j, d - 1 + below)), - ); - } - rows.sort_by_key(JsonNode::ordinal); - rows +/// The rows `json_tree_subtree(doc, key.tokens(), 0)` returns for `key`, +/// read here as the SQL reads them, each with its depth below the node its +/// token named. +fn read_rows(doc: &JsonDocument, key: &JsonSubtreeKey) -> Vec { + doc.subtree(&key.token_bytes()) + .iter() + .map(|(j, _, d)| with_depth(doc, *j, *d)) + .collect() } /// PostgreSQL's `jsonb @>`, from its documented rules: an object contains @@ -399,6 +345,31 @@ mod given_the_deepest_document { } } } + + /// An empty container at the deepest level is itself the 128th nested + /// container: `serde_json` refuses to read the document back, so the + /// index refuses to seal it, under either root option, where a scalar at + /// the same depth seals and opens. + #[tokio::test] + async fn an_empty_container_at_the_deepest_level_is_refused_when_sealed() { + let Scenario { cipher, .. } = scenario().await; + let arrays = (0..127).fold(json!([]), |inner, _| json!([inner])); + let objects = (0..127).fold(json!({}), |inner, _| json!({"k": inner})); + for deepest in [&arrays, &objects] { + assert!( + serde_json::from_str::(&deepest.to_string()).is_err(), + "128 nested containers: serde_json would not read the sealed root back" + ); + for options in [JsonOptions::default(), open_root()] { + let refused = try_seal(&cipher, deepest, &options).await; + assertions::refused_by_term( + &refused, + |e| matches!(e, TermError::JsonTooDeep), + &format!("an empty container 127 levels down under {options:?}"), + ); + } + } + } } // ============================================================================= @@ -452,11 +423,12 @@ mod given_the_fixture_document { let entry = entry.as_object().unwrap(); for key in entry.keys() { assert!( - ["s", "c", "p", "op"].contains(&key.as_str()), + ["s", "c", "x", "p", "op"].contains(&key.as_str()), "entry {j} carries an unknown key {key}" ); } - for (tag, chars, bytes) in [("s", 24, 16), ("p", 12, 8)] { + assert!(entry.contains_key("x"), "entry {j} carries its self tag"); + for (tag, chars, bytes) in [("s", 24, 16), ("x", 12, 8), ("p", 12, 8)] { if let Some(text) = entry.get(tag) { assert_eq!( text.as_str().unwrap().len(), @@ -499,6 +471,14 @@ mod given_the_fixture_document { j + 3 ); } + // Every entry carries a self tag, the root and the heads included, and + // every one is distinct, so its presence marks nothing. + let selves: std::collections::HashSet<_> = entries.iter().map(|e| *e.self_tag()).collect(); + assert_eq!( + selves.len(), + entries.len(), + "one distinct self tag per entry" + ); // Every parent tag is distinct: siblings are not linkable at rest. let tags: std::collections::HashSet<_> = entries[3..9] .iter() @@ -614,9 +594,9 @@ mod given_the_fixture_document { ); } - /// No token is stored: an entry below a head carries only the parent - /// tag, and no tag in the document is a token, or a parent's hash of one - /// without the document's salt. + /// No token is stored: every tag in the document is a self tag or a + /// parent tag, salted by the document, and no tag, nor a selector, is a + /// token or a prefix of one. #[tokio::test] async fn stores_no_token() { let Scenario { cipher, .. } = scenario().await; @@ -624,24 +604,25 @@ mod given_the_fixture_document { let nested = subtree_key(&cipher, "$.nested", JsonOptions::default()) .await .unwrap(); - let mut tokens = vec![*nested.token_bytes()]; - let mut frontier = vec![*nested.token_bytes()]; + let mut tokens = nested.token_bytes(); + let mut frontier = nested.token_bytes(); while let Some(token) = frontier.pop() { assert!( tokens.len() <= doc.entries().len(), "the chain is a tree: no token is reached twice" ); for (_, _, child) in doc.children(&token) { + let child = child.expect("every child of a sealed node takes a rule"); tokens.push(child); frontier.push(child); } } assert_eq!(tokens.len(), 7, "nested and its six descendants"); for (j, entry) in doc.entries().iter().enumerate() { - if let Some(p) = entry.parent_tag() { + for tag in std::iter::once(entry.self_tag()).chain(entry.parent_tag()) { assert!( - !tokens.iter().any(|t| t.starts_with(p)), - "entry {j}'s parent tag is not a prefix of a token" + !tokens.iter().any(|t| t.starts_with(tag)), + "entry {j}'s stored tag is not a prefix of a token" ); } assert!( @@ -650,8 +631,8 @@ mod given_the_fixture_document { ); } assert_eq!( - nested.token(), - base64(nested.token_bytes()), + nested.tokens(), + [base64(&nested.token_bytes()[0])], "a token's text form is its padded base64" ); } @@ -733,44 +714,48 @@ mod given_a_subtree_key { use super::*; #[tokio::test] - async fn unlocks_exactly_its_subtree_from_the_path_alone() { + async fn finds_exactly_its_node_from_the_path_alone() { let Scenario { cipher, .. } = scenario().await; let doc = seal(&cipher, &fixture()).await; let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) .await .unwrap(); - assert!( - key.gates().is_empty(), - "a chain head's token is derived, nothing to gate" + assert_eq!( + key.tokens().len(), + 1, + "a path with no wildcard sends one token" ); + assert!(key.probes().is_empty(), "and no probe"); assert_eq!( key.path(), &JsonPath::parse("$.nested").unwrap(), "the key carries the path it was derived from" ); - let under = doc.subtree(key.token_bytes()); + let under = doc.subtree(&key.token_bytes()); let found: Vec = under.iter().map(|(j, _, _)| *j).collect(); assert_eq!( found, - [3, 4, 5, 6, 7, 8], - "deep, list, both elements, both leaves" + [2, 3, 4, 5, 6, 7, 8], + "nested itself, then deep, list, both elements, both leaves" ); let depths: Vec = under.iter().map(|(_, _, d)| *d).collect(); assert_eq!( depths, - [1, 1, 2, 3, 2, 3], + [0, 1, 1, 2, 3, 2, 3], "each row's depth below $.nested" ); assert!( - doc.subtree(&[7; 16]).is_empty(), - "a wrong token unlocks nothing" + doc.subtree(&[[7; 16]]).is_empty(), + "a wrong token finds nothing" ); } - /// Below a head the client derives nothing but array positions: a key - /// step becomes a gate, and the walk from the head takes it. + /// Below a head the client derives every token itself: a member's from + /// its parent's and its selector, an element's with its index too. The + /// read of the second element sends that element's token, and nothing + /// that covers the list or nested. #[tokio::test] - async fn gates_each_key_step_below_its_head() { + async fn derives_every_token_below_its_head() { let Scenario { cipher, .. } = scenario().await; let doc = seal(&cipher, &fixture()).await; let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) @@ -779,26 +764,34 @@ mod given_a_subtree_key { let deeper = subtree_key(&cipher, "$.nested.list[1]", JsonOptions::default()) .await .unwrap(); + assert_eq!(deeper.tokens().len(), 1, "one token"); + assert_ne!(deeper.token_bytes(), key.token_bytes(), "not nested's"); assert_eq!( - deeper.token_bytes(), - key.token_bytes(), - "anchored at the head" + subtree_ordinals(&doc, &deeper.token_bytes()), + [7, 8], + "the second element and its x" ); - assert_eq!(deeper.gates().len(), 2, "list, then [1]"); + let list = subtree_key(&cipher, "$.nested.list", JsonOptions::default()) + .await + .unwrap(); + let (_, entry, list_token) = doc + .children(&key.token_bytes()[0]) + .into_iter() + .find(|(j, _, _)| *j == 4) + .expect("the list is a child of nested"); assert_eq!( - deeper.gates()[1], - format!("#1:{}", base64(doc.entries()[5].selector())), - "the position, and the selector every element of the list carries" + list_token, + Some(list.token_bytes()[0]), + "the server derives the token the client sends" ); - let list_token = child_token(&doc, key.token_bytes(), 4); - let second = child_token(&doc, &list_token, 7); assert_eq!( - subtree_ordinals(&doc, &second), - [8], - "the second element's x" + doc.find(&list.token_bytes()[0]).map(|(j, _)| j), + Some(4), + "the list's token finds the list" ); + assert_eq!(entry.selector(), doc.entries()[4].selector()); let children: Vec = doc - .children(&list_token) + .children(&list.token_bytes()[0]) .iter() .map(|(j, _, _)| *j) .collect(); @@ -807,63 +800,40 @@ mod given_a_subtree_key { [5, 7], "the children of a node, by ordinal: the list's two elements" ); - // A wildcard path anchors at the head and gates on each step's - // selector: the list's, then every element's (the wildcard's), then x's. - let gated = subtree_key(&cipher, "$.nested.list[*].x", JsonOptions::default()) - .await - .unwrap(); - assert_eq!( - gated.token_bytes(), - key.token_bytes(), - "a wildcard path anchors at the head" - ); - assert_eq!( - gated.gates(), - [ - base64(doc.entries()[4].selector()), - base64(doc.entries()[5].selector()), - base64(doc.entries()[6].selector()), - ], - "the list's selector, every element's, then x's" - ); + assert!(doc.children(&[7; 16]).is_empty(), "no node, no children"); } - /// The sealed root has no token; the open root's unlocks every node, and - /// under an open root the chain is derivable down to the first key step. + /// `$` is a read under either root. Under the sealed root its token + /// finds the root entry and nothing beneath it, since nothing chains + /// from it; under an open root it covers every node, and positions + /// derive below it. #[tokio::test] - async fn has_no_token_at_a_sealed_root_and_derives_from_an_open_one() { + async fn reads_the_root_under_either_option() { let Scenario { cipher, .. } = scenario().await; - for path in ["$", "$[*].x"] { - let refused = subtree_key(&cipher, path, JsonOptions::default()).await; - assertions::refused_by_term( - &refused, - |e| matches!(e, TermError::JsonSealedRoot), - &format!("{path} under the sealed root"), - ); - } - let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; - let root = subtree_key(&cipher, "$", open_root()).await.unwrap(); - assert_eq!( - open_doc.subtree(root.token_bytes()).len(), - 9, - "every entry but the root" - ); - // `$[0]` would be a position, `$.nested` is a gate. - let open_key = subtree_key(&cipher, "$.nested.list[1]", open_root()) + let doc = seal(&cipher, &fixture()).await; + let sealed_root = subtree_key(&cipher, "$", JsonOptions::default()) .await .unwrap(); assert_eq!( - open_key.token_bytes(), - root.token_bytes(), - "anchored at the open root" + doc.subtree(&sealed_root.token_bytes()) + .iter() + .map(|(j, _, d)| (*j, *d)) + .collect::>(), + [(0, 0)], + "the sealed root alone" + ); + let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; + let root = subtree_key(&cipher, "$", open_root()).await.unwrap(); + assert_eq!( + open_doc.subtree(&root.token_bytes()).len(), + 10, + "the root and every entry beneath it" ); - assert_eq!(open_key.gates().len(), 3, "nested, list, then [1]"); let array_doc = seal_with(&cipher, &json!([[1, 2], [3]]), &open_root()).await; let second_row = subtree_key(&cipher, "$[1]", open_root()).await.unwrap(); - assert!(second_row.gates().is_empty(), "positions derive"); assert_eq!( - subtree_ordinals(&array_doc, second_row.token_bytes()), - [5], + subtree_ordinals(&array_doc, &second_row.token_bytes()), + [4, 5], "$, [0], [0][0], [0][1], [1], [1][0]" ); } @@ -873,64 +843,40 @@ mod given_a_position { use super::*; /// A position names an array element and nothing else. A column's rows - /// need not agree on a node's kind, and an object member's ordinal is a - /// rank among its siblings, so a token derived for `$.a[0]` must not - /// open the rank-0 member of a row whose `$.a` is an object: `jsonb` has - /// no node at `{a,0}` there, and neither does the index. + /// need not agree on a node's kind, so a token derived for `$.a[0]` must + /// not find a member of a row whose `$.a` is an object, the member named + /// `"0"` included: `jsonb` has no node at `{a,0}` there under a strict + /// path, and neither does the index. A member's token and an element's + /// are different hashes, so no entry there carries the element's self + /// tag. The same holds below a key step (`$.h.a[0]`). #[tokio::test] async fn reaches_no_object_member() { let Scenario { cipher, .. } = scenario().await; - let first = subtree_key(&cipher, "$.a[0]", JsonOptions::default()) - .await - .unwrap(); - assert!( - first.gates().is_empty(), - "a position after the head derives" - ); - let array = seal(&cipher, &json!({"a": [{"x": 2}]})).await; - assert_eq!( - subtree_ordinals(&array, first.token_bytes()), - [3], - "$, $.a, $.a[0], $.a[0].x" - ); - for object in [ - json!({"a": {"k": {"x": 1}}}), - json!({"a": {"k": [1, 2]}}), - json!({"a": {"0": {"x": 1}}}), - ] { - let doc = seal(&cipher, &object).await; - assert!( - doc.subtree(first.token_bytes()).is_empty(), - "{object}: an object member is not an element" - ); + for path in ["$.a[0]", "$.h.a[0]"] { + let first = subtree_key(&cipher, path, JsonOptions::default()) + .await + .unwrap(); + let array = seal(&cipher, &json!({"a": [{"x": 2}], "h": {"a": [{"x": 2}]}})).await; + let found = subtree_ordinals(&array, &first.token_bytes()); + let at = array + .entries() + .iter() + .position(|e| e.self_tag() == array.entries()[found[0]].self_tag()) + .unwrap(); + assert_eq!(found, [at, at + 1], "{path}: the element and its x"); + for object in [ + json!({"a": {"k": {"x": 1}}, "h": {"a": {"k": {"x": 1}}}}), + json!({"a": {"k": [1, 2]}, "h": {"a": {"k": [1, 2]}}}), + json!({"a": {"0": {"x": 1}}, "h": {"a": {"0": {"x": 1}}}}), + ] { + let doc = seal(&cipher, &object).await; + assert!( + doc.subtree(&first.token_bytes()).is_empty(), + "{path} in {object}: an object member is not an element" + ); + } } } - - /// The same below a key step, where the position is a gate: the walk - /// from the head reaches `$.h.a` by its selector, and the gate at the - /// next step admits an element only. A member at ordinal 0 under an - /// object `$.h.a` carries its own selector, not the element's. - #[tokio::test] - async fn gates_on_the_elements_selector_below_a_key_step() { - let Scenario { cipher, .. } = scenario().await; - let gated = subtree_key(&cipher, "$.h.a[0]", JsonOptions::default()) - .await - .unwrap(); - assert_eq!(gated.gates().len(), 2, "a, then [0]"); - let element = cipher - .default_keyset() - .json_selector( - &JsonPath::parse("$.h.a[*]").unwrap(), - nonempty!("users").with("profile"), - ) - .await - .unwrap(); - assert_eq!( - gated.gates()[1], - format!("#0:{element}"), - "the position gate names the elements' selector" - ); - } } // ============================================================================= @@ -978,7 +924,7 @@ mod given_an_extracted_node { let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) .await .unwrap(); - let rows = subtree_nodes(&doc, key.token_bytes()); + let rows = read_rows(&doc, &key); let nested = cipher .decrypt_as::(key.open(rows.clone()), aead()) .await @@ -994,16 +940,15 @@ mod given_an_extracted_node { .unwrap(); assert_eq!( forest, - vec![json!("constant"), json!([{"x": 1}, {"x": 2}])], - "as a list, the walk opens to $.nested's members" + vec![fixture()["nested"].clone()], + "as a list, one value: the node at the path" ); - // The list is one gate below nested: the walk reaches it and returns - // it with its subtree. + // The list, below nested: its own token finds it and its subtree. let list = subtree_key(&cipher, "$.nested.list", JsonOptions::default()) .await .unwrap(); - let rows = gated_walk(&doc, &list); + let rows = read_rows(&doc, &list); let elements = cipher .decrypt_as::(list.open(rows), aead()) .await @@ -1066,86 +1011,90 @@ mod given_an_extracted_node { } } -mod given_gated_walk_rows { +mod given_wildcard_rows { use super::*; - /// The rows of a gated walk carry the node the gates reached at depth 0 - /// and its descendants beneath. Opened as a value, they are that node's - /// value; two reached nodes open as a list, or refuse to be one value. + /// A wildcard read's rows hold each element it reaches below the bound, + /// each at depth 0 with everything beneath it. As a list they open to + /// one value per element; as one value they refuse, unless exactly one + /// element is there. A probe that finds an element fails the read with + /// the bound. #[tokio::test] - async fn open_to_the_reached_nodes_value() { + async fn open_to_one_value_per_element_or_fail_with_the_bound() { let Scenario { cipher, .. } = scenario().await; let doc = seal(&cipher, &fixture()).await; - let nested = subtree_key(&cipher, "$.nested", JsonOptions::default()) + let xs = subtree_key(&cipher, "$.nested.list[*].x", JsonOptions::default()) .await .unwrap(); - let list = child_token(&doc, nested.token_bytes(), 4); - // What `json_tree_subtree_at(doc, t(nested), [s(list), #1])` returns: - // the second element at d = 0, its x at d = 1. - let rows_at = |reached: &[usize], below: &[(usize, u32)]| -> Vec { - reached - .iter() - .map(|j| (*j, 0u32)) - .chain(below.iter().copied()) - .map(|(j, d)| with_depth(&doc, j, d)) - .collect() - }; - let second = keyed(&cipher, "$.nested.list[1]", rows_at(&[7], &[(8, 1)])).await; - assert!( - !second.is_sealed_root(), - "a reached node is not the fallback" - ); - let value = cipher - .decrypt_as::(second.clone(), aead()) + assert_eq!(xs.tokens().len(), 65, "64 positions and the probe"); + assert_eq!(xs.probes(), [64]); + let rows = read_rows(&doc, &xs); + assert_eq!(rows.len(), 2, "both x"); + let values = cipher + .decrypt_as::, JsonSubtree>(xs.open(rows.clone()), aead()) .await .unwrap(); - assert_eq!(value, json!({"x": 2}), "the reached node's own value"); - let values = cipher - .decrypt_as::, JsonSubtree>(second, aead()) + assert_eq!(values, vec![json!(1), json!(2)], "one value per element"); + assertions::refused_as_opened_type( + &cipher + .decrypt_as::(xs.open(rows), aead()) + .await, + Some("one node at the walked path"), + "two elements are not one value", + ); + // At a bound of 1, the second element is the probe's: the read fails + // rather than return the first x alone, as a list and as one value. + let one = cipher + .default_keyset() + .json_subtree_key_with_bound( + &JsonPath::parse("$.nested.list[*].x").unwrap(), + &JsonOptions::default(), + 1, + nonempty!("users").with("profile"), + ) .await .unwrap(); + assert_eq!(one.tokens().len(), 2, "one position and the probe"); + let rows = read_rows(&doc, &one); assert_eq!( - values, - vec![json!({"x": 2})], - "as a list, the one reached node" - ); - // A scalar reached by a gate opens to the scalar, where the ungated - // walk of a scalar yields no rows and no value. - let x = cipher - .decrypt_as::( - keyed(&cipher, "$.nested.list[1].x", rows_at(&[8], &[])).await, - aead(), + rows.len(), + 3, + "the first x, and the second element with its x" + ); + for opened in [ + cipher + .decrypt_as::, JsonSubtree>(one.open(rows.clone()), aead()) + .await + .map(|_| ()), + cipher + .decrypt_as::, JsonSubtree>(one.open(rows.clone()), aead()) + .await + .map(|_| ()), + ] { + assert!( + matches!(opened, Err(Error::JsonWildcardTruncated { bound: 1 })), + "{opened:?}" + ); + } + // At a bound of 2, exactly as long: no probe finds anything. + let two = cipher + .default_keyset() + .json_subtree_key_with_bound( + &JsonPath::parse("$.nested.list[*].x").unwrap(), + &JsonOptions::default(), + 2, + nonempty!("users").with("profile"), ) .await .unwrap(); - assert_eq!(x, json!(2), "a scalar reached by a gate opens to it"); - // Two reached nodes (a wildcard): a list, never one value. - let both = keyed(&cipher, "$.nested.list[*].x", rows_at(&[6, 8], &[])).await; let values = cipher - .decrypt_as::, JsonSubtree>(both.clone(), aead()) + .decrypt_as::, JsonSubtree>(two.open(read_rows(&doc, &two)), aead()) .await .unwrap(); assert_eq!( values, vec![json!(1), json!(2)], - "two reached nodes open as a list" - ); - assertions::refused_as_opened_type( - &cipher.decrypt_as::(both, aead()).await, - Some("one node at the walked path"), - "two reached nodes are not one value", - ); - // The key says which walk the rows came from: the children of the - // list, which an ungated walk from the list's token would return, are - // not what the gated walk to the list returns, and are refused. - assertions::refused_as_wrong_entry( - &cipher - .decrypt_as::( - keyed(&cipher, "$.nested.list", subtree_nodes(&doc, &list)).await, - aead(), - ) - .await, - "the ungated walk's rows under the gated key", + "an array as long as the bound" ); } } @@ -1164,7 +1113,7 @@ mod given_a_derived_record { let list = subtree_key(&cipher, "$.nested.list", JsonOptions::default()) .await .unwrap(); - let rows = gated_walk(&doc, &list); + let rows = read_rows(&doc, &list); let subtree = list.open(rows.clone()); assert_eq!( subtree.nodes(), @@ -1315,53 +1264,45 @@ mod given_a_fallback_root { ); } - /// The sealed root a walk fell back to opens to exactly what the walked - /// rows open to, so a caller's result does not change shape when a - /// document grows past the budget, and no caller says which walk the - /// rows came from: the key does, by its gates. A key with none keyed - /// `json_tree_subtree`, whose rows are the descendants of the path's - /// node, so a `Vec` opens to that node's children; a key with gates - /// keyed `json_tree_subtree_at`, whose rows are the nodes the path - /// reaches. + /// The sealed root a read fell back to opens to exactly what the walked + /// rows open to, as a list, as an optional value and as one value, so a + /// caller's result does not change shape when a document grows past the + /// budget: the key's concrete paths and probes are evaluated over the + /// whole document. #[tokio::test] async fn opens_to_what_the_walk_opens_to() { let document = json!({ "list": [{"x": 1}, {"x": 2}], - "obj": {"a": "constant", "b": [1, 2], "c": {"d": [true, null]}}, + "obj": {"a": "constant", "b": [1, 2], "c": {"d": [true, null]}, "e": {}, "f": []}, "hello": "world" }); let Scenario { cipher, .. } = scenario().await; let doc = seal(&cipher, &document).await; let root = fallback(&doc); - let sorted = |mut values: Vec| { - values.sort_by_key(ToString::to_string); - values - }; - for (text, gated) in [ - ("$.list", false), - ("$.obj", false), - ("$.hello", false), - ("$.list[1]", false), - ("$.obj.b", true), - ("$.obj.c.d", true), - ("$.obj.b[0]", true), - ("$.list[*].x", true), - ("$.list[1].x", true), - ("$.obj.c", true), + for text in [ + "$", + "$.list", + "$.obj", + "$.hello", + "$.list[1]", + "$.obj.b", + "$.obj.c.d", + "$.obj.b[0]", + "$.list[*].x", + "$.list[1].x", + "$.obj.c", + "$.obj.e", + "$.obj.f", + "$.missing", + "$.obj.c.d[*]", + "$.list[*]", + "$[*]", + "$.obj.missing[*]", ] { let key = subtree_key(&cipher, text, JsonOptions::default()) .await .unwrap(); - assert_eq!( - !key.gates().is_empty(), - gated, - "{text}: whether the key gates" - ); - let walked = if gated { - gated_walk(&doc, &key) - } else { - subtree_nodes(&doc, key.token_bytes()) - }; + let walked = read_rows(&doc, &key); let fallen_back = key.open(vec![root.clone()]); assert!(fallen_back.is_sealed_root(), "{text}: the fallback"); let as_walked = cipher @@ -1372,49 +1313,23 @@ mod given_a_fallback_root { .decrypt_as::, JsonSubtree>(fallen_back.clone(), aead()) .await .unwrap(); - // An object's members come back in an order neither form fixes. - assert_eq!(sorted(as_fallen), sorted(as_walked), "{text}, as a list"); - // A walk with no rows (a scalar's, or an empty container's) is - // never past a budget, so it never falls back: nothing to compare. - if walked.is_empty() { - continue; - } + let expected: Vec = key.path().select(&document).into_iter().cloned().collect(); + assert_eq!(as_walked, expected, "{text}, walked, as a list"); + assert_eq!(as_fallen, expected, "{text}, fallen back, as a list"); let one_walked = cipher - .decrypt_as::(key.open(walked), aead()) + .decrypt_as::, JsonSubtree>(key.open(walked), aead()) .await; let one_fallen = cipher - .decrypt_as::(fallen_back, aead()) + .decrypt_as::, JsonSubtree>(fallen_back, aead()) .await; match (one_walked, one_fallen) { (Ok(a), Ok(b)) => assert_eq!(a, b, "{text}, as one value"), - (Err(_), Err(_)) => {} + (Err(Error::OpenedType { .. }), Err(Error::OpenedType { .. })) => { + assert!(expected.len() > 1, "{text}: refused only for several nodes") + } (a, b) => panic!("{text}: the walk opened to {a:?}, the fallback to {b:?}"), } } - // As one value an ungated walk opens to its node, either way. - let key = subtree_key(&cipher, "$.obj", JsonOptions::default()) - .await - .unwrap(); - assert_eq!( - cipher - .decrypt_as::(key.open(vec![root.clone()]), aead()) - .await - .unwrap(), - document["obj"], - "an ungated fallback opens to its node as one value" - ); - // An ungated key names one node: a fallback with nothing at the path - // is refused. - let missing = subtree_key(&cipher, "$.missing", JsonOptions::default()) - .await - .unwrap(); - assertions::refused_as_opened_type( - &cipher - .decrypt_as::, JsonSubtree>(missing.open(vec![root]), aead()) - .await, - Some("one node at the walked path"), - "an ungated fallback with nothing at the path", - ); } } @@ -1436,11 +1351,10 @@ mod given_rows_the_server_chose { let doc = seal(&cipher, &fixture()).await; let node = |j: usize| doc.node(j).unwrap(); let repeated = keyed(&cipher, "$.nested", vec![node(1), node(1)]).await; - assertions::refused_as_opened_type( + assertions::refused_as_wrong_entry( &cipher .decrypt_as::, JsonSubtree>(repeated, aead()) .await, - None, "a repeated row", ); diff --git a/packages/stack-encrypt/tests/json_properties.rs b/packages/stack-encrypt/tests/json_properties.rs new file mode 100644 index 000000000..c37d41749 --- /dev/null +++ b/packages/stack-encrypt/tests/json_properties.rs @@ -0,0 +1,557 @@ +//! Properties of the JSON index's reads over random documents, with no +//! database: the rows the server's read returns are modelled by +//! `JsonDocument::subtree`, the Rust twin of `json_tree_subtree`, and +//! opened through the key, as a caller opens what PostgreSQL returns. +//! +//! - **Metamorphic.** A read of `$.a` with `.b` applied on the client is the +//! read of `$.a.b`; a read of `$` (the open root walked, the sealed root +//! opened whole) with any path applied is the read of that path; `->` and +//! the read agree on every scalar; the reads of the top-level paths +//! reassemble the document; a node's children are its child count. +//! - **Independence.** Selectors, the entry layout and the tokens follow +//! from the shape and the keys alone, not from the values or the `iv`; +//! the stored tags and ciphertexts of two rows are all distinct. +//! - **What a read sends.** The tokens a key sends are exactly those of +//! the concrete paths its path names, below the bound, and the probes, and +//! no token of a node above them. +//! - **What a row stores.** `x` on every entry, `p` on every entry but the +//! root and the heads, under either root option. + +mod common; + +use std::collections::BTreeSet; + +use common::stack_cipher; +use proptest::prelude::*; +use serde_json::{json, Map, Value}; +use stack_encrypt::json::{ + JsonDocument, JsonExtracted, JsonNode, JsonOptions, JsonPath, JsonSubtree, Segment, +}; +use stack_encrypt::target::{AeadContext, CallerContext}; +use stack_encrypt::{nonempty, Error, StackCipher}; +use stack_kms::FakeDataKeySource; + +type Cipher = StackCipher; + +/// `proptest!` generates synchronous `#[test]` functions and runs each body +/// once per case through a plain closure, so `#[tokio::test]` cannot wrap a +/// property; each test builds one runtime and `block_on`s its cases. +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} + +fn aead() -> AeadContext { + AeadContext::from(nonempty!("users").with("profile")) +} + +fn options(sealed_root: bool) -> JsonOptions { + JsonOptions { + sealed_root, + ..JsonOptions::new() + } +} + +async fn seal(cipher: &Cipher, value: &Value, options: &JsonOptions) -> JsonDocument { + cipher + .default_keyset() + .run( + stack_encrypt::target::json_document::<_, stack_encrypt::target::Borrowed>( + options.clone(), + ), + value, + CallerContext::from(nonempty!("users").with("profile")), + ) + .await + .expect("seals") +} + +async fn key( + cipher: &Cipher, + path: &JsonPath, + options: &JsonOptions, + bound: u32, +) -> stack_encrypt::json::JsonSubtreeKey { + cipher + .default_keyset() + .json_subtree_key_with_bound(path, options, bound, nonempty!("users").with("profile")) + .await + .unwrap() +} + +/// The rows the server's read returns for `key`, in full. +fn rows(doc: &JsonDocument, key: &stack_encrypt::json::JsonSubtreeKey) -> Vec { + doc.subtree(&key.token_bytes()) + .iter() + .map(|(j, _, d)| with_depth(doc, *j, *d)) + .collect() +} + +fn with_depth(doc: &JsonDocument, j: usize, d: u32) -> JsonNode { + let mut wire = serde_json::to_value(doc.node(j).unwrap()).unwrap(); + wire["d"] = json!(d); + serde_json::from_value(wire).unwrap() +} + +/// The read of `path` in `doc`, as `Option`. +async fn read( + cipher: &Cipher, + doc: &JsonDocument, + path: &JsonPath, + options: &JsonOptions, +) -> Option { + let key = key(cipher, path, options, 4).await; + cipher + .decrypt_as::, JsonSubtree>(key.open(rows(doc, &key)), aead()) + .await + .unwrap_or_else(|error| panic!("{path}: {error}")) +} + +/// Every node's path in `value`, in the order the index stores the nodes: +/// depth first, the root first, members in key order. +fn node_paths(value: &Value) -> Vec { + fn visit(value: &Value, at: &JsonPath, out: &mut Vec) { + out.push(at.clone()); + match value { + Value::Object(map) => { + for (key, child) in map { + visit(child, &append(at, &Segment::Key(key.clone())), out); + } + } + Value::Array(items) => { + for (i, child) in items.iter().enumerate() { + visit(child, &append(at, &Segment::Index(i as u64)), out); + } + } + _ => {} + } + } + let mut out = Vec::new(); + visit(value, &JsonPath::root(), &mut out); + out +} + +fn append(at: &JsonPath, step: &Segment) -> JsonPath { + let text = match step { + Segment::Key(key) => format!("[{}]", serde_json::to_string(key).unwrap()), + Segment::Index(i) => format!("[{i}]"), + Segment::Wildcard => "[*]".to_owned(), + }; + JsonPath::parse(&format!("{at}{text}")).unwrap() +} + +/// Keys that stress the path encoding, beside ordinary ones: the empty key, +/// unicode, the characters a path spells steps with, quotes and +/// backslashes, and `"0"`, which must never meet the index 0. +fn key_name() -> impl Strategy { + prop_oneof![ + Just(String::new()), + Just("0".to_owned()), + Just("a.b".to_owned()), + Just("[*]".to_owned()), + Just("]".to_owned()), + Just("$".to_owned()), + Just("\"\\".to_owned()), + Just("é✓".to_owned()), + "[a-c]{1,2}", + ] +} + +fn leaf() -> impl Strategy { + prop_oneof![ + Just(Value::Null), + any::().prop_map(Value::Bool), + any::().prop_map(|n| json!(n)), + any::().prop_map(|n| json!(n)), + Just(json!(2.0)), + Just(json!(-0.0)), + Just(json!(1e30)), + // Quarters are exact in binary and in decimal. An arbitrary `f64` + // can come back one unit in the last place off: the stored text is + // parsed by serde_json without `float_roundtrip`, a defect of its + // own and not of the reads these properties are about. + (-4_000_000i32..4_000_000).prop_map(|n| json!(f64::from(n) / 4.0)), + "[a-z ]{0,6}".prop_map(Value::String), + ] +} + +fn document() -> impl Strategy { + leaf().prop_recursive(4, 40, 5, |inner| { + prop_oneof![ + prop::collection::vec(inner.clone(), 0..5).prop_map(Value::Array), + prop::collection::btree_map(key_name(), inner, 0..5) + .prop_map(|m| Value::Object(m.into_iter().collect::>())), + ] + }) +} + +/// `value` with every scalar replaced by another, the shape and the keys +/// kept. +fn revalue(value: &Value) -> Value { + match value { + Value::Object(map) => { + Value::Object(map.iter().map(|(k, v)| (k.clone(), revalue(v))).collect()) + } + Value::Array(items) => Value::Array(items.iter().map(revalue).collect()), + Value::String(s) => json!(format!("{s}~")), + Value::Number(_) => json!("a number once"), + Value::Bool(b) => json!(!b), + Value::Null => json!(0), + } +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 48, ..ProptestConfig::default() })] + + /// Every path's read is the value `jsonb` holds there (modelled by + /// `JsonPath::select`), and the reads of the root's children reassemble + /// the document. + #[test] + fn the_reads_of_every_path_reassemble_the_document(value in document(), sealed in any::()) { + let rt = runtime(); + rt.block_on(async { + let cipher = stack_cipher().await; + let options = options(sealed); + let doc = seal(&cipher, &value, &options).await; + for path in node_paths(&value) { + let expected = path.select(&value).first().cloned().cloned(); + prop_assert_eq!(read(&cipher, &doc, &path, &options).await, expected, "{}", path); + } + let rebuilt = match &value { + Value::Object(map) => { + let mut out = Map::new(); + for key in map.keys() { + let path = append(&JsonPath::root(), &Segment::Key(key.clone())); + let _ = out.insert(key.clone(), read(&cipher, &doc, &path, &options).await.unwrap()); + } + Value::Object(out) + } + Value::Array(items) => { + let mut out = Vec::new(); + for i in 0..items.len() { + let path = append(&JsonPath::root(), &Segment::Index(i as u64)); + out.push(read(&cipher, &doc, &path, &options).await.unwrap()); + } + Value::Array(out) + } + scalar => read(&cipher, &doc, &JsonPath::root(), &options).await.unwrap_or(scalar.clone()), + }; + prop_assert_eq!(rebuilt, value); + Ok(()) + })?; + } + + /// A read of a node with a step applied on the client is the read of the + /// longer path; a read of `$` with any path applied is the read of that + /// path. Under the sealed root `$` comes back as the root alone and + /// opens whole; under an open root it is walked. + #[test] + fn a_read_then_a_step_is_the_read_of_the_longer_path(value in document(), sealed in any::()) { + let rt = runtime(); + rt.block_on(async { + let cipher = stack_cipher().await; + let options = options(sealed); + let doc = seal(&cipher, &value, &options).await; + let whole = read(&cipher, &doc, &JsonPath::root(), &options).await; + prop_assert_eq!(whole.as_ref(), Some(&value), "`$` is the document"); + for path in node_paths(&value) { + let direct = read(&cipher, &doc, &path, &options).await; + let from_root = path.select(whole.as_ref().unwrap()).first().cloned().cloned(); + prop_assert_eq!(&direct, &from_root, "{} from `$`", path); + if let Some((last, parent)) = path.segments().split_last() { + let parent_path = parent.iter().fold(JsonPath::root(), |at, s| append(&at, s)); + let above = read(&cipher, &doc, &parent_path, &options).await; + let stepped = above.and_then(|v| append(&JsonPath::root(), last).select(&v).first().cloned().cloned()); + prop_assert_eq!(direct, stepped, "{} from its parent", path); + } + } + Ok(()) + })?; + } + + /// `->`, which returns the first entry at a selector, and the read agree + /// on every scalar at a path with no position; and the children the + /// server enumerates under a node are its child count, none under a + /// scalar. + #[test] + fn the_arrow_and_the_children_agree_with_the_read(value in document()) { + let rt = runtime(); + rt.block_on(async { + let cipher = stack_cipher().await; + let options = options(false); + let doc = seal(&cipher, &value, &options).await; + for path in node_paths(&value) { + let node = path.select(&value)[0].clone(); + let k = key(&cipher, &path, &options, 4).await; + let children = doc.children(&k.token_bytes()[0]).len(); + let count = match &node { + Value::Object(m) => m.len(), + Value::Array(a) => a.len(), + _ => 0, + }; + prop_assert_eq!(children, count, "{} children", path); + let positional = path.segments().iter().any(|s| matches!(s, Segment::Index(_))); + if positional || node.is_object() || node.is_array() { + continue; + } + let selector = cipher + .default_keyset() + .json_selector(&path, nonempty!("users").with("profile")) + .await + .unwrap(); + let j = doc.entries().iter().position(|e| e.selector() == selector.as_bytes()).unwrap(); + let arrow = cipher + .decrypt_as::(selector.open(doc.node(j).unwrap()), aead()) + .await + .unwrap(); + prop_assert_eq!(Some(arrow), read(&cipher, &doc, &path, &options).await, "{}", path); + } + Ok(()) + })?; + } + + /// Two rows of one shape and one set of keys, with different values and + /// different data keys: the same selectors in the same order, the same + /// nodes found by every path's token, and no self tag, parent tag or + /// ciphertext in common. + #[test] + fn selectors_and_tokens_follow_the_shape_alone(value in document(), sealed in any::()) { + let rt = runtime(); + rt.block_on(async { + let cipher = stack_cipher().await; + let options = options(sealed); + let one = seal(&cipher, &value, &options).await; + let two = seal(&cipher, &revalue(&value), &options).await; + prop_assert_ne!(one.header().iv(), two.header().iv()); + let s = |d: &JsonDocument| d.entries().iter().map(|e| *e.selector()).collect::>(); + prop_assert_eq!(s(&one), s(&two), "the same selectors, in the same order"); + for path in node_paths(&value) { + let k = key(&cipher, &path, &options, 4).await; + prop_assert_eq!( + one.find(&k.token_bytes()[0]).map(|(j, _)| j), + two.find(&k.token_bytes()[0]).map(|(j, _)| j), + "{} at one position in both rows", path + ); + } + let mut tags = BTreeSet::new(); + let mut total = 0; + for doc in [&one, &two] { + for e in doc.entries() { + total += 1 + usize::from(e.parent_tag().is_some()); + let _ = tags.insert(e.self_tag().to_vec()); + if let Some(p) = e.parent_tag() { + let _ = tags.insert(p.to_vec()); + } + } + } + prop_assert_eq!(tags.len(), total, "every stored tag distinct"); + let cs: BTreeSet> = one.entries().iter().chain(two.entries()).map(|e| e.ciphertext().to_vec()).collect(); + prop_assert_eq!(cs.len(), one.entries().len() * 2, "every ciphertext distinct"); + Ok(()) + })?; + } + + /// `x` on every entry, `p` on every entry but the root and the heads. + #[test] + fn every_entry_carries_a_self_tag_and_only_the_heads_lack_a_parent_tag(value in document(), sealed in any::()) { + let rt = runtime(); + rt.block_on(async { + let cipher = stack_cipher().await; + let doc = seal(&cipher, &value, &options(sealed)).await; + let wire = serde_json::to_value(&doc).unwrap(); + for (j, (entry, path)) in wire["sv"].as_array().unwrap().iter().zip(node_paths(&value)).enumerate() { + prop_assert!(entry.get("x").is_some(), "entry {} has x", j); + let head = path.segments().is_empty() || (sealed && path.segments().len() == 1); + prop_assert_eq!(entry.get("p").is_none(), head, "entry {} ({}): p absent only on a head", j, path); + } + Ok(()) + })?; + } + + /// A member's selector, and its token, are the same whatever its + /// siblings are. + #[test] + fn a_members_selector_and_token_ignore_its_siblings( + left in prop::collection::btree_map(key_name(), leaf(), 0..8), + right in prop::collection::btree_map(key_name(), leaf(), 0..8), + ) { + let rt = runtime(); + rt.block_on(async { + let cipher = stack_cipher().await; + let row = |siblings: &std::collections::BTreeMap| { + let mut a: Map = siblings.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + let _ = a.insert("member".into(), json!({"leaf": 1})); + json!({"a": a}) + }; + let path = JsonPath::parse("$.a.member.leaf").unwrap(); + let k = key(&cipher, &path, &options(true), 4).await; + let mut found = Vec::new(); + for siblings in [&left, &right] { + let doc = seal(&cipher, &row(siblings), &options(true)).await; + let (j, entry) = doc.find(&k.token_bytes()[0]).expect("found by its token"); + found.push(*entry.selector()); + prop_assert_eq!(read(&cipher, &doc, &path, &options(true)).await, Some(json!(1))); + prop_assert!(j > 0); + } + prop_assert_eq!(found[0], found[1]); + Ok(()) + })?; + } + + /// The tokens a read sends are exactly the tokens of the concrete paths + /// its path names below the bound and of its probes, in document order, + /// and none of a node above them; a path with no wildcard sends one. The + /// tokens are checked against a witness document holding every concrete + /// path and one element past the bound at every wildcard: each token + /// must find the node at its path there, by the stored self tag. + #[test] + fn a_key_sends_exactly_the_tokens_of_the_paths_it_names( + steps in prop::collection::vec(prop_oneof![ + key_name().prop_map(Segment::Key), + (0u64..3).prop_map(Segment::Index), + Just(Segment::Wildcard), + ], 0..4), + bound in 0u32..3, + sealed in any::(), + ) { + let rt = runtime(); + rt.block_on(async { + let cipher = stack_cipher().await; + let options = options(sealed); + let n = bound as usize; + fn witness(steps: &[Segment], n: usize) -> Value { + let Some((step, rest)) = steps.split_first() else { return json!(0) }; + match step { + Segment::Key(k) => json!({ k.clone(): witness(rest, n) }), + Segment::Index(i) => Value::Array((0..=*i).map(|_| witness(rest, n)).collect()), + Segment::Wildcard => Value::Array((0..=n).map(|_| witness(rest, n)).collect()), + } + } + /// Every concrete path the read names, in document order, and + /// whether it is a probe. + fn named(steps: &[Segment], at: JsonPath, n: usize, out: &mut Vec<(JsonPath, bool)>) { + let Some((step, rest)) = steps.split_first() else { out.push((at, false)); return }; + match step { + Segment::Wildcard => { + for i in 0..n { + named(rest, append(&at, &Segment::Index(i as u64)), n, out); + } + out.push((append(&at, &Segment::Index(n as u64)), true)); + } + other => named(rest, append(&at, other), n, out), + } + } + let value = witness(&steps, n); + let path = steps.iter().fold(JsonPath::root(), |at, s| append(&at, s)); + let doc = seal(&cipher, &value, &options).await; + let paths = node_paths(&value); + let k = key(&cipher, &path, &options, bound).await; + let mut expected = Vec::new(); + named(&steps, JsonPath::root(), n, &mut expected); + prop_assert_eq!(k.tokens().len(), expected.len(), "one token per concrete path"); + let probes: Vec = expected.iter().enumerate().filter(|(_, (_, p))| *p).map(|(i, _)| i).collect(); + prop_assert_eq!(k.probes(), probes); + if path.is_positional() { + prop_assert_eq!(k.tokens().len(), 1, "no wildcard, one token"); + } + let found: Vec = k + .token_bytes() + .iter() + .map(|t| doc.find(t).map(|(j, _)| j).expect("every token names a node of the witness")) + .collect(); + let at: Vec = expected + .iter() + .map(|(q, _)| paths.iter().position(|p| p == q).unwrap()) + .collect(); + prop_assert_eq!(&found, &at, "each token finds the node at its path"); + for (q, _) in &expected { + for len in 0..q.segments().len() { + let ancestor = q.segments()[..len].iter().fold(JsonPath::root(), |a, s| append(&a, s)); + let j = paths.iter().position(|p| *p == ancestor).unwrap(); + prop_assert!(!found.contains(&j), "no token of {}, above {}", ancestor, q); + } + } + // The parameter the SQL takes is the base64 of exactly these. + let sent: Value = serde_json::to_value(&k).unwrap(); + prop_assert_eq!(sent, json!({"t": k.tokens()})); + Ok(()) + })?; + } +} + +/// A wildcard read of a document whose array is at the bound minus one, +/// at it, or one past it: exact, exact, and the bound's error, never part +/// of the list, read in full and fallen back to the sealed root alike. +#[tokio::test] +async fn a_wildcard_read_fails_past_its_bound_and_only_there() { + let cipher = stack_cipher().await; + let path = JsonPath::parse("$.items[*].x").unwrap(); + for bound in [1u32, 3, 64] { + let k = key(&cipher, &path, &options(true), bound).await; + for len in [bound as usize - 1, bound as usize, bound as usize + 1] { + let value = json!({"items": (0..len).map(|i| json!({"x": i})).collect::>()}); + let doc = seal(&cipher, &value, &options(true)).await; + let root = with_depth(&doc, 0, 0); + for rows in [rows(&doc, &k), vec![root]] { + let opened = cipher + .decrypt_as::, JsonSubtree>(k.open(rows), aead()) + .await; + if len > bound as usize { + assert!( + matches!(opened, Err(Error::JsonWildcardTruncated { bound: b }) if b == bound), + "{len} past {bound}: {opened:?}" + ); + } else { + assert_eq!( + opened.unwrap(), + (0..len).map(|i| json!(i)).collect::>(), + "{len} within {bound}" + ); + } + } + } + } +} + +/// The stored size the self tag costs. Each entry gains `,"x":"…"`, twelve +/// characters of base64 and seven of framing, on entries of about two +/// hundred characters: about ten percent. The bounds are the measured +/// ratios on these two shapes with a margin, so a change that grows the +/// stored form further fails here before a benchmark notices. +#[tokio::test] +async fn the_self_tag_costs_about_a_tenth_of_the_stored_size() { + let cipher = stack_cipher().await; + let nested = json!({ + "number": 1, + "hello": "h1", + "orders": (0..12).map(|k| json!({ + "id": k, "total": k * 7, + "items": (0..5).map(|m| json!({"sku": format!("s{m}"), "qty": m, "gift": m == 0})).collect::>() + })).collect::>() + }); + let flat = json!({"number": 1, "hello": "h1", "flag": true}); + for (name, value, low, high) in [("nested", nested, 0.08, 0.12), ("flat", flat, 0.07, 0.12)] { + let doc = seal(&cipher, &value, &options(true)).await; + let wire = serde_json::to_value(&doc).unwrap(); + let with_x = wire.to_string().len(); + let mut without = wire.clone(); + for entry in without["sv"].as_array_mut().unwrap() { + let _ = entry.as_object_mut().unwrap().remove("x"); + } + let without_x = without.to_string().len(); + let growth = with_x as f64 / without_x as f64 - 1.0; + eprintln!( + "{name}: {without_x} -> {with_x} characters, +{:.1}%", + growth * 100.0 + ); + assert!( + (low..high).contains(&growth), + "{name}: the self tag adds {:.1}% where {:.0}% to {:.0}% was measured", + growth * 100.0, + low * 100.0, + high * 100.0 + ); + } +} diff --git a/packages/stack-encrypt/tests/json_read_checks.rs b/packages/stack-encrypt/tests/json_read_checks.rs index 53f0f626a..3c8f1fc85 100644 --- a/packages/stack-encrypt/tests/json_read_checks.rs +++ b/packages/stack-encrypt/tests/json_read_checks.rs @@ -1,7 +1,7 @@ //! What the client checks when it opens the rows a JSON index read //! returned, with no database: the entry `->` returns must carry the //! selector the client derived for the path, and every row of a subtree -//! walk must sit where the key that keyed the walk says its rows are. +//! read must sit where the key that keyed the read says its rows are. //! Each entry's ciphertext is bound to its own selector and position, so //! without these checks any entry of the row would open in place of the one //! asked for. @@ -117,53 +117,14 @@ fn row(doc: &JsonDocument, j: usize, d: Option) -> JsonNode { serde_json::from_value(wire).unwrap() } -/// The rows the SQL walk returns for `key`, walked as it walks: with no -/// gates, every descendant of the token's node from depth 1; with gates, -/// through each gate (a selector, or `#n:`), then each node the -/// gates reach at depth 0 with everything beneath it. +/// The rows `json_tree_subtree(doc, key.tokens(), 0)` returns for `key`, +/// read as the SQL reads them: for each token, the node it names at depth +/// 0 and everything beneath it. fn walk(doc: &JsonDocument, key: &JsonSubtreeKey) -> Vec { - let gates = key.gates(); - if gates.is_empty() { - return doc - .subtree(key.token_bytes()) - .iter() - .map(|(j, _, d)| row(doc, *j, Some(*d))) - .collect(); - } - let mut frontier = vec![(None, *key.token_bytes())]; - for gate in &gates { - let mut next = Vec::new(); - for (_, token) in &frontier { - let children = doc.children(token); - match gate.strip_prefix('#') { - Some(slot) => { - let (n, wanted) = slot.split_once(':').unwrap(); - if let Some((j, entry, t)) = children.get(n.parse::().unwrap()) { - if base64(entry.selector()) == wanted { - next.push((Some(*j), *t)); - } - } - } - None => next.extend( - children - .iter() - .filter(|(_, entry, _)| base64(entry.selector()) == *gate) - .map(|(j, _, t)| (Some(*j), *t)), - ), - } - } - frontier = next; - } - let mut rows = Vec::new(); - for (j, token) in frontier { - rows.push(row(doc, j.unwrap(), Some(0))); - rows.extend( - doc.subtree(&token) - .iter() - .map(|(j, _, d)| row(doc, *j, Some(*d))), - ); - } - rows + doc.subtree(&key.token_bytes()) + .iter() + .map(|(j, _, d)| row(doc, *j, Some(*d))) + .collect() } fn base64(bytes: &[u8]) -> String { @@ -291,10 +252,10 @@ mod given_the_entry_arrow_returned { } } -mod given_the_rows_of_a_walk_without_gates { +mod given_the_rows_of_a_read { use super::*; - const CHILD: &str = "a child of the walked node"; + const REACHED: &str = "a node the walked path reaches"; const NEXT: &str = "the next node of the walked subtree"; #[tokio::test] @@ -302,16 +263,15 @@ mod given_the_rows_of_a_walk_without_gates { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; - assert!(nested.gates().is_empty(), "a chain head's token"); let rows = walk(&doc, &nested); - assert_eq!(rows.len(), 6, "nested's descendants"); + assert_eq!(rows.len(), 7, "nested and its descendants"); let value = open_value(&cipher, nested.open(rows.clone())).await; assert_eq!(value.unwrap(), fixture()["nested"], "the sub-document"); let list = open_list(&cipher, nested.open(rows)).await; assert_eq!( list.unwrap(), - vec![json!("constant"), json!([{"x": 1}, {"x": 2}])], - "the members' values" + vec![fixture()["nested"].clone()], + "one node, one value" ); } @@ -321,10 +281,104 @@ mod given_the_rows_of_a_walk_without_gates { let doc = seal(&cipher, &json!({"a": {"x": 1}, "b": {"x": 2}})).await; let a = key(&cipher, "$.a").await; let b = key(&cipher, "$.b").await; - // The database answers `$.a` with `$.b`'s subtree: one genuine - // member, in its place, but under `$.b`'s path. - let refused = open_value(&cipher, a.open(walk(&doc, &b))).await; - assertions::wrong_entry(refused, CHILD); + // The database answers `$.a` with `$.b`'s subtree: genuine entries, + // in their places, under `$.b`'s path. + assertions::wrong_entry(open_value(&cipher, a.open(walk(&doc, &b))).await, REACHED); + // The same with `$.b`'s self tag rewritten to `$.a`'s token's: the + // self tag is the database's to write, and the selector still says + // which path the node is at. + let forged = with_x(walk(&doc, &b), 0, x_of(&doc, &a)); + assertions::wrong_entry(open_value(&cipher, a.open(forged)).await, REACHED); + } + + #[tokio::test] + async fn another_position_of_the_array_is_refused_by_its_label() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &fixture()).await; + let second = key(&cipher, "$.nested.list[1]").await; + let first = key(&cipher, "$.nested.list[0]").await; + // The first element carries the same selector as the second, and + // here the second's self tag: its decrypted position tells them + // apart. + let forged = with_x(walk(&doc, &first), 0, x_of(&doc, &second)); + assertions::wrong_entry(open_value(&cipher, second.open(forged)).await, REACHED); + } + + #[tokio::test] + async fn a_node_no_token_names_is_refused() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &fixture()).await; + let hello = key(&cipher, "$.hello").await; + // `$.hello`'s own entry with another self tag: the database walked + // from a token the read did not send. + let forged = with_x(walk(&doc, &hello), 0, "AAAAAAAAAAA=".into()); + assertions::wrong_entry(open_value(&cipher, hello.open(forged)).await, REACHED); + } + + #[tokio::test] + async fn one_token_answered_twice_is_refused() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &fixture()).await; + let xs = key(&cipher, "$.nested.list[*].x").await; + // Both x, the second carrying the first's self tag: each a genuine + // node at the selector, but the read sent each token once, and a + // row matches only a token after the one before it. + let rows = walk(&doc, &xs); + assert_eq!(rows.len(), 2, "both x"); + let first = serde_json::to_value(&rows[0]).unwrap()["x"] + .as_str() + .unwrap() + .to_owned(); + let forged = with_x(rows, 1, first); + assertions::wrong_entry(open_list(&cipher, xs.open(forged)).await, REACHED); + } + + #[tokio::test] + async fn an_entry_returned_twice_is_refused_before_any_key_is_retrieved() { + let (cipher, retrieved) = cipher().await; + let doc = seal(&cipher, &fixture()).await; + let nested = key(&cipher, "$.nested").await; + // nested's rows with its first member, `deep` (3), returned twice: + // a genuine entry at its own position, which would let the database + // line one container up as often as it likes. An honest database + // never returns one, so it is the wrong entry, not a shape. + let mut rows = walk(&doc, &nested); + rows.insert(2, rows[1].clone()); + assertions::wrong_entry( + open_value(&cipher, nested.open(rows)).await, + "rows of distinct entries", + ); + assert_eq!( + retrieved.load(Ordering::SeqCst), + 0, + "the refusal retrieved no key" + ); + } + + #[tokio::test] + async fn rows_of_two_documents_are_refused() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &fixture()).await; + let other = seal(&cipher, &fixture()).await; + let xs = key(&cipher, "$.nested.list[*].x").await; + // The first x (6) from this row and the second (8) from another, + // each with its own row's header. Either row's answer alone opens + // (ADR 0010), but a read is of one row, and its rows of two are not + // the answer to it. + let rows = vec![walk(&doc, &xs)[0].clone(), walk(&other, &xs)[1].clone()]; + assertions::wrong_entry( + open_list(&cipher, xs.open(rows)).await, + "rows of one document", + ); + } + + #[tokio::test] + async fn a_child_with_a_forged_self_tag_is_refused() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &fixture()).await; + let nested = key(&cipher, "$.nested").await; + let forged = with_x(walk(&doc, &nested), 2, "AAAAAAAAAAA=".into()); + assertions::wrong_entry(open_value(&cipher, nested.open(forged)).await, NEXT); } #[tokio::test] @@ -333,12 +387,14 @@ mod given_the_rows_of_a_walk_without_gates { let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; // `number` (9) sits right after nested's subtree in the document, - // so it follows the rows without a gap; its selector is not a - // member of nested's. + // so it follows the rows without a gap; no token the read sent + // names it. let mut rows = walk(&doc, &nested); rows.push(row(&doc, 9, Some(1))); - let refused = open_value(&cipher, nested.open(rows)).await; - assertions::wrong_entry(refused, CHILD); + assertions::wrong_entry(open_value(&cipher, nested.open(rows)).await, REACHED); + let mut rows = walk(&doc, &nested); + rows.push(row(&doc, 9, None)); + assertions::wrong_entry(open_value(&cipher, nested.open(rows)).await, REACHED); } #[tokio::test] @@ -352,46 +408,41 @@ mod given_the_rows_of_a_walk_without_gates { .into_iter() .filter(|n| n.ordinal() != 5) .collect(); - let refused = open_value(&cipher, nested.open(rows)).await; - assertions::wrong_entry(refused, NEXT); + assertions::wrong_entry(open_value(&cipher, nested.open(rows)).await, NEXT); } #[tokio::test] - async fn a_row_reported_at_the_wrong_depth_is_refused() { + async fn a_truncated_read_is_refused() { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; - let rows: Vec = walk(&doc, &nested) - .into_iter() - .map(|n| { - if n.ordinal() == 6 { - row(&doc, 6, Some(1)) - } else { - n - } - }) - .collect(); + let mut rows = walk(&doc, &nested); + // The last row is the second element's `x` (8): without it the + // element says it has a member the rows do not hold, the same + // dropped row as a gap, at the end of the rows. + assert_eq!(rows.pop().map(|n| n.ordinal()), Some(8), "x, the last row"); let refused = open_value(&cipher, nested.open(rows)).await; assertions::wrong_entry(refused, NEXT); } #[tokio::test] - async fn a_member_reported_at_the_wrong_depth_is_refused() { + async fn a_row_reported_at_the_wrong_depth_is_refused() { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; - let rows: Vec = walk(&doc, &nested) - .into_iter() - .map(|n| { - if n.ordinal() == 3 { - row(&doc, 3, Some(2)) - } else { - n - } - }) - .collect(); - let refused = open_value(&cipher, nested.open(rows)).await; - assertions::wrong_entry(refused, CHILD); + for (j, d, expected) in [(6, 1, NEXT), (3, 2, NEXT), (2, 1, REACHED)] { + let rows: Vec = walk(&doc, &nested) + .into_iter() + .map(|n| { + if n.ordinal() == j as u64 { + row(&doc, j, Some(d)) + } else { + n + } + }) + .collect(); + assertions::wrong_entry(open_value(&cipher, nested.open(rows)).await, expected); + } } #[tokio::test] @@ -399,11 +450,14 @@ mod given_the_rows_of_a_walk_without_gates { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; - // `deep` (3) and then the second element's `x` (8) alone: its - // parent and everything between are missing. - let rows = vec![row(&doc, 3, Some(1)), row(&doc, 8, Some(3))]; - let refused = open_value(&cipher, nested.open(rows)).await; - assertions::wrong_entry(refused, CHILD); + // nested, `deep` (3) and then the second element's `x` (8) alone: + // its parent and everything between are missing. + let rows = vec![ + row(&doc, 2, Some(0)), + row(&doc, 3, Some(1)), + row(&doc, 8, Some(3)), + ]; + assertions::wrong_entry(open_value(&cipher, nested.open(rows)).await, NEXT); } #[tokio::test] @@ -417,8 +471,48 @@ mod given_the_rows_of_a_walk_without_gates { .into_iter() .filter(|n| n.ordinal() != 3) .collect(); - let refused = open_value(&cipher, a.open(rows)).await; - assertions::wrong_entry(refused, CHILD); + assertions::wrong_entry(open_value(&cipher, a.open(rows)).await, NEXT); + } + + /// The one substitution only the consecutive-position rule catches + /// (design decision 8): a read of `$.nested.list[0]` answered with the + /// element itself (5) and then the other element's `x` (8) in place of + /// its own (6), carrying 6's self tag. Every element's `x` shares one + /// selector, the label is `x` either way, the depth and the parent's + /// child count agree, and the forged self tag matches; the row's + /// authenticated position, 8 where 6 is due, is the one thing that + /// refuses it. Opened, it would read `{"x": 2}` for `{"x": 1}`. + #[tokio::test] + async fn another_elements_child_in_place_of_the_nodes_own_is_refused_by_its_position() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &fixture()).await; + let first = key(&cipher, "$.nested.list[0]").await; + let honest = walk(&doc, &first); + assert_eq!( + honest.iter().map(|n| n.ordinal()).collect::>(), + [5, 6], + "the element and its x" + ); + assert_eq!( + open_value(&cipher, first.open(honest)).await.unwrap(), + json!({"x": 1}) + ); + let own_x = serde_json::to_value(doc.node(6).unwrap()).unwrap()["x"] + .as_str() + .unwrap() + .to_owned(); + let swapped = with_x(vec![row(&doc, 5, Some(0)), row(&doc, 8, Some(1))], 1, own_x); + assertions::wrong_entry(open_value(&cipher, first.open(swapped)).await, NEXT); + } + + #[tokio::test] + async fn the_sealed_root_among_read_rows_is_refused() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &fixture()).await; + let list = key(&cipher, "$.nested.list").await; + let mut rows = walk(&doc, &list); + rows.push(row(&doc, 0, Some(1))); + assertions::wrong_entry(open_value(&cipher, list.open(rows)).await, REACHED); } #[tokio::test] @@ -433,56 +527,37 @@ mod given_the_rows_of_a_walk_without_gates { let doc = seal_with(&cipher, &value, &open_root()).await; let root = key_with(&cipher, "$", &open_root()).await; let rows = walk(&doc, &root); - assert_eq!(rows.len(), 127, "every node below the root"); + assert_eq!(rows.len(), 128, "the root and every node below it"); let opened = open_value(&cipher, root.open(rows)).await; assert_eq!(opened.unwrap(), value, "the whole document"); + let deepest = key_with(&cipher, &format!("${}", "[0]".repeat(126)), &open_root()).await; + let opened = open_value(&cipher, deepest.open(walk(&doc, &deepest))).await; + assert_eq!( + opened.unwrap(), + json!([1]), + "126 positions down, by its own token" + ); } #[tokio::test] - async fn a_subtree_cut_short_is_refused_as_the_wrong_entry() { - let (cipher, _) = cipher().await; - let doc = seal(&cipher, &fixture()).await; - let nested = key(&cipher, "$.nested").await; - // The last row is the second element's `x` (8): without it the - // element says it has a member the rows do not hold. - let mut rows = walk(&doc, &nested); - assert_eq!(rows.pop().map(|n| n.ordinal()), Some(8), "x, the last row"); - let refused = open_value(&cipher, nested.open(rows)).await; - assertions::wrong_entry(refused, NEXT); - } - - #[tokio::test] - async fn a_leading_element_left_out_is_refused() { + async fn an_array_read_missing_its_first_or_last_element_is_refused() { let (cipher, _) = cipher().await; let doc = seal(&cipher, &json!({"arr": [10, 20, 30]})).await; let arr = key(&cipher, "$.arr").await; - assert!(arr.gates().is_empty(), "a chain head's token"); - // The walk does not return the node it starts from, so nothing - // before the first row bounds where the rows begin. Leaving out the - // first element would open as `[20, 30]`, every element moved down - // one position. + // The read returns the array's own node, whose child count bounds + // its elements: leaving out the first would have opened as + // `[20, 30]` when the walk began at the children (review of + // #1358), and leaving out the last as `[10, 20]`. let rows = walk(&doc, &arr); - let first = rows[0].ordinal(); - let rows: Vec = rows.into_iter().filter(|n| n.ordinal() != first).collect(); - assertions::wrong_entry(open_value(&cipher, arr.open(rows.clone())).await, CHILD); - assertions::wrong_entry(open_list(&cipher, arr.open(rows)).await, CHILD); - } - - #[tokio::test] - async fn trailing_children_left_out_are_not_detected_by_a_walk_without_gates() { - let (cipher, _) = cipher().await; - let doc = seal(&cipher, &json!({"arr": [10, 20, 30]})).await; - let arr = key(&cipher, "$.arr").await; - // What the design leaves open until the walk returns the node it - // starts from (#1356): the node's child count is in the node's own - // entry, so the rows cannot say how many children follow the last - // one returned. The database withholding the last element opens as - // the shorter array. A member left out of an object is the same, - // wherever it sits, since nothing the client holds ranks the keys. - let mut rows = walk(&doc, &arr); - rows.pop(); - let opened = open_value(&cipher, arr.open(rows)).await; - assert_eq!(opened.unwrap(), json!([10, 20]), "the elements before it"); + let (first, last) = (rows[1].ordinal(), rows[rows.len() - 1].ordinal()); + for dropped in [first, last] { + let rows: Vec = walk(&doc, &arr) + .into_iter() + .filter(|n| n.ordinal() != dropped) + .collect(); + assertions::wrong_entry(open_value(&cipher, arr.open(rows.clone())).await, NEXT); + assertions::wrong_entry(open_list(&cipher, arr.open(rows)).await, NEXT); + } } #[tokio::test] @@ -490,8 +565,6 @@ mod given_the_rows_of_a_walk_without_gates { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; - // The depth is the database's report; the rows' positions and the - // opened nodes place them, so rows without it open the same. let rows: Vec = walk(&doc, &nested) .iter() .map(|n| row(&doc, n.ordinal() as usize, None)) @@ -501,92 +574,135 @@ mod given_the_rows_of_a_walk_without_gates { } } -mod given_the_rows_of_a_gated_walk { +/// What the read check allows by design, and says it allows: no bound field +/// names an entry's row, or the element a node at a selector sits under. +mod given_answers_the_check_cannot_tell_apart { use super::*; - const REACHED: &str = "a node the walked path reaches"; - #[tokio::test] - async fn the_honest_rows_open_to_the_reached_nodes() { + async fn another_rows_node_at_the_same_path_opens() { let (cipher, _) = cipher().await; - let doc = seal(&cipher, &fixture()).await; - let second = key(&cipher, "$.nested.list[1]").await; - let value = open_value(&cipher, second.open(walk(&doc, &second))).await; - assert_eq!(value.unwrap(), json!({"x": 2}), "the second element"); - let xs = key(&cipher, "$.nested.list[*].x").await; - let values = open_list(&cipher, xs.open(walk(&doc, &xs))).await; - assert_eq!(values.unwrap(), vec![json!(1), json!(2)], "every x"); + let other = seal(&cipher, &json!({"nested": {"deep": "other", "list": []}})).await; + let nested = key(&cipher, "$.nested").await; + // Row provenance is the database's word (ADR 0010). + let opened = open_value(&cipher, nested.open(walk(&other, &nested))).await; + assert_eq!( + opened.unwrap(), + json!({"deep": "other", "list": []}), + "the other row's" + ); } #[tokio::test] - async fn another_position_of_the_array_is_refused() { + async fn the_same_selector_under_another_element_opens() { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; - let second = key(&cipher, "$.nested.list[1]").await; - // The first element carries the same selector as the second: the - // position in its label tells them apart. - let first = key(&cipher, "$.nested.list[0]").await; - let refused = open_value(&cipher, second.open(walk(&doc, &first))).await; - assertions::wrong_entry(refused, REACHED); + let second = key(&cipher, "$.nested.list[1].x").await; + // `$.nested.list[0].x` (6) with `$.nested.list[1].x`'s self tag: + // the same selector and the same label, which is all a row binds. + let forged = vec![{ + let mut wire = serde_json::to_value(row(&doc, 6, Some(0))).unwrap(); + wire["x"] = json!(x_of(&doc, &second)); + serde_json::from_value::(wire).unwrap() + }]; + let opened = open_value(&cipher, second.open(forged)).await; + assert_eq!( + opened.unwrap(), + json!(1), + "the first element's x, by design" + ); } +} - #[tokio::test] - async fn a_node_of_another_path_is_refused() { - let (cipher, _) = cipher().await; - let doc = seal(&cipher, &fixture()).await; - let xs = key(&cipher, "$.nested.list[*].x").await; - let refused = open_list(&cipher, xs.open(vec![row(&doc, 3, Some(0))])).await; - assertions::wrong_entry(refused, REACHED); - } +/// The four answers a read at a path with no wildcard gives, as `jsonb #>` +/// gives them: an empty object, an empty array, a scalar and no node, as a +/// `Value`, an `Option` and a `Vec`. +mod given_a_read_of_one_node { + use super::*; #[tokio::test] - async fn a_descendant_left_without_its_reached_node_is_refused() { + async fn each_answer_opens_as_jsonb_holds_it() { let (cipher, _) = cipher().await; - let doc = seal(&cipher, &fixture()).await; - let list = key(&cipher, "$.nested.list").await; - // The list's own row is dropped: what is left are its children, - // which the gated walk does not return on their own. - let rows: Vec = walk(&doc, &list) - .into_iter() - .filter(|n| n.ordinal() != 4) - .collect(); - let refused = open_value(&cipher, list.open(rows)).await; - assertions::wrong_entry(refused, REACHED); + let document = json!({"o": {}, "a": [], "s": "text", "n": null, "z": 0}); + let doc = seal(&cipher, &document).await; + for (path, expected) in [ + ("$.o", Some(json!({}))), + ("$.a", Some(json!([]))), + ("$.s", Some(json!("text"))), + ("$.n", Some(Value::Null)), + ("$.missing", None), + ("$.o.missing", None), + ("$.a[0]", None), + ] { + let read = key(&cipher, path).await; + let rows = walk(&doc, &read); + let option = cipher + .decrypt_as::, JsonSubtree>(read.open(rows.clone()), aead()) + .await + .unwrap(); + assert_eq!(option, expected, "{path} as Option"); + let list = open_list(&cipher, read.open(rows.clone())).await.unwrap(); + assert_eq!( + list, + expected.clone().into_iter().collect::>(), + "{path} as Vec" + ); + match (open_value(&cipher, read.open(rows)).await, expected) { + (Ok(value), Some(expected)) => assert_eq!(value, expected, "{path} as Value"), + (Err(Error::OpenedType { expected }), None) => { + assert_eq!(expected, "a node at the walked path", "{path}") + } + (got, expected) => panic!("{path}: {got:?}, expected {expected:?}"), + } + } } +} - #[tokio::test] - async fn a_reached_node_reported_below_depth_zero_is_refused() { - let (cipher, _) = cipher().await; - let doc = seal(&cipher, &fixture()).await; - let xs = key(&cipher, "$.nested.list[*].x").await; - let refused = open_list(&cipher, xs.open(vec![row(&doc, 6, Some(1))])).await; - assertions::wrong_entry(refused, REACHED); - } +/// The in-crate model of the SQL reads finds what the SQL finds where a +/// document repeats a tag, which no sealing produces: the server's index +/// keeps the last entry carrying it (`jsonb_object_agg` keeps the last +/// value of a repeated key), so the model's `find` and `children` do too. +#[tokio::test] +async fn the_model_keeps_the_last_entry_carrying_a_tag_as_the_sql_index_does() { + let (cipher, _) = cipher().await; + let doc = seal_with(&cipher, &fixture(), &open_root()).await; + let root = key_with(&cipher, "$", &open_root()).await; + let hello = key_with(&cipher, "$.hello", &open_root()).await; + // `hello` (1) appended again at the end: its self tag and parent tag + // now name two entries, 1 and 10. + let mut wire = serde_json::to_value(&doc).unwrap(); + let sv = wire["sv"].as_array_mut().unwrap(); + let copy = sv[1].clone(); + sv.push(copy); + let forged: JsonDocument = serde_json::from_value(wire).unwrap(); + assert_eq!( + forged.find(&hello.token_bytes()[0]).map(|(j, _)| j), + Some(10), + "the last entry with the self tag" + ); + let children: Vec = forged + .children(&root.token_bytes()[0]) + .iter() + .map(|(j, _, _)| *j) + .collect(); + assert_eq!(children.len(), 3, "the root's three members"); + assert!( + children.contains(&10) && !children.contains(&1), + "the last entry with the parent tag: {children:?}" + ); +} - #[tokio::test] - async fn a_subtree_cut_short_is_refused_as_the_wrong_entry() { - let (cipher, _) = cipher().await; - let doc = seal(&cipher, &fixture()).await; - let second = key(&cipher, "$.nested.list[1]").await; - // The element (7) says it has one member, and the database leaves - // that member (8) out: the same dropped row as a gap, at the end of - // the rows instead of between two of them. - let mut rows = walk(&doc, &second); - assert_eq!(rows.pop().map(|n| n.ordinal()), Some(8), "x, the last row"); - let refused = open_value(&cipher, second.open(rows)).await; - assertions::wrong_entry(refused, "the next node of the walked subtree"); - } +/// The self tag the stored node at `key`'s only token carries, as the +/// database could write it on any entry: it holds the token and the `iv`. +fn x_of(doc: &JsonDocument, key: &JsonSubtreeKey) -> String { + let (j, _) = doc.find(&key.token_bytes()[0]).expect("the node is there"); + base64(doc.entries()[j].self_tag()) +} - #[tokio::test] - async fn the_sealed_root_among_walked_rows_is_refused() { - let (cipher, _) = cipher().await; - let doc = seal(&cipher, &fixture()).await; - let list = key(&cipher, "$.nested.list").await; - // The root at depth 1 is not the fallback, and its selector is not - // the list's. - let mut rows = walk(&doc, &list); - rows.push(row(&doc, 0, Some(1))); - let refused = open_value(&cipher, list.open(rows)).await; - assertions::wrong_entry(refused, REACHED); - } +/// `rows` with the self tag of the row at `at` rewritten to `x`. +fn with_x(mut rows: Vec, at: usize, x: String) -> Vec { + let mut wire = serde_json::to_value(&rows[at]).unwrap(); + wire["x"] = json!(x); + rows[at] = serde_json::from_value(wire).unwrap(); + rows } diff --git a/packages/stack-encrypt/tests/json_tree_vectors.rs b/packages/stack-encrypt/tests/json_tree_vectors.rs new file mode 100644 index 000000000..47465614a --- /dev/null +++ b/packages/stack-encrypt/tests/json_tree_vectors.rs @@ -0,0 +1,148 @@ +//! The JSON index's derivations, pinned: the selector `s`, the self tag `x` +//! and the parent tag `p` of every entry of one document, and the tokens a +//! read sends for the root, a chain head, an object member and an array +//! element, under both root options. Sealed under the deterministic key +//! source, whose index key is fixed and whose `iv` follows from its seed, +//! so every one of them is the same in every run. The ciphertexts carry a +//! fresh nonce each and are not pinned. +//! +//! A stored document's `x` and `p` are derived from the tokens, and a read +//! finds nodes by them, so a change to any derivation here is a change to +//! the stored form: every stored row stops answering reads. Regenerate with +//! `STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS=1` (exactly `1`; any other value +//! regenerates nothing), deliberately not the crate's routine +//! `STACK_ENCRYPT_UPDATE_FIXTURES`, and only when the break is intended. + +mod common; + +use std::ffi::OsStr; +use std::path::PathBuf; + +use serde_json::{json, Value}; +use stack_encrypt::json::{JsonDocument, JsonOptions, JsonPath}; +use stack_encrypt::nonempty; +use stack_encrypt::target::CallerContext; + +const SEED: [u8; 32] = [0x4a; 32]; + +fn fixture_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/json_tree_vectors.json") +} + +/// Whether the fixture is to be rewritten: only for exactly +/// `STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS=1`. A rewrite hides a real +/// drift, so `0`, an empty value or any other spelling must not trigger one. +fn update_requested(value: Option<&OsStr>) -> bool { + value == Some(OsStr::new("1")) +} + +fn document() -> Value { + json!({"a": {"b": [1, {"c": true}]}, "d": "e"}) +} + +/// The paths whose tokens are pinned: the root, a head, a member below a +/// head, an array, an element, and a member below an element. +const READS: [&str; 6] = ["$", "$.a", "$.a.b", "$.a.b[0]", "$.a.b[1]", "$.a.b[1].c"]; + +fn base64(bytes: &[u8]) -> String { + use base64ct::{Base64, Encoding}; + Base64::encode_string(bytes) +} + +async fn vectors(options: &JsonOptions) -> Value { + let cipher = common::deterministic_cipher(SEED).await; + let doc: JsonDocument = cipher + .default_keyset() + .run( + stack_encrypt::target::json_document::<_, stack_encrypt::target::Borrowed>( + options.clone(), + ), + &document(), + CallerContext::from(nonempty!("users").with("profile")), + ) + .await + .expect("seals"); + let entries: Vec = doc + .entries() + .iter() + .map(|e| { + json!({ + "s": base64(e.selector()), + "x": base64(e.self_tag()), + "p": e.parent_tag().map(|p| base64(p)), + }) + }) + .collect(); + let mut tokens = serde_json::Map::new(); + for path in READS { + let key = cipher + .default_keyset() + .json_subtree_key( + &JsonPath::parse(path).unwrap(), + options, + nonempty!("users").with("profile"), + ) + .await + .unwrap(); + let _ = tokens.insert(path.to_owned(), json!(key.tokens())); + } + json!({ + "iv": base64(doc.header().iv()), + "entries": entries, + "tokens": tokens, + }) +} + +const DRIFT: &str = "\ +Under the same deterministic key source, the JSON index's selectors, self tags, \ +parent tags or read tokens no longer match tests/fixtures/json_tree_vectors.json. \ +A stored json_tree document's `x` and `p` are derived from these tokens, and a \ +read finds nodes by them, so a change here stops every stored row answering \ +reads: a breaking change to stored data, not a fixture refresh. Regenerate with \ +STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS=1 only when the break is intended, and \ +say so in the changeset."; + +#[tokio::test] +async fn the_derivations_are_pinned_under_the_deterministic_source() { + let actual = json!({ + "_comment": "The JSON index's s, x and p per entry, and the tokens a read sends, under the deterministic key source. Read by tests/json_tree_vectors.rs; regenerate with STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS=1 only for an intended break. Do not edit by hand.", + "document": document(), + "sealed_root": vectors(&JsonOptions::default()).await, + "open_root": vectors(&JsonOptions { sealed_root: false, ..JsonOptions::new() }).await, + }); + if update_requested(std::env::var_os("STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS").as_deref()) { + std::fs::write( + fixture_path(), + serde_json::to_string_pretty(&actual).unwrap() + "\n", + ) + .unwrap(); + } + let pinned: Value = + serde_json::from_str(&std::fs::read_to_string(fixture_path()).expect("the fixture exists")) + .unwrap(); + assert_eq!( + pinned["document"], actual["document"], + "the pinned document" + ); + assert_eq!( + pinned["sealed_root"]["iv"], actual["sealed_root"]["iv"], + "the deterministic source's iv moved: a test-infrastructure change in stack-kms, not a stored-data break" + ); + assert_eq!(pinned, actual, "{DRIFT}"); +} + +#[test] +fn only_exactly_one_requests_an_update() { + assert!(update_requested(Some(OsStr::new("1"))), "exactly 1"); + for other in ["0", "", "true", "yes", " 1"] { + assert!(!update_requested(Some(OsStr::new(other))), "{other:?}"); + } + assert!(!update_requested(None), "unset"); +} + +#[tokio::test] +async fn the_deterministic_source_seals_the_same_tags_twice() { + let one = vectors(&JsonOptions::default()).await; + let two = vectors(&JsonOptions::default()).await; + assert_eq!(one, two, "a pin needs a run that repeats"); +}