diff --git a/.changeset/eql-json-tree-planner-and-leakage.md b/.changeset/eql-json-tree-planner-and-leakage.md new file mode 100644 index 000000000..7479de1cc --- /dev/null +++ b/.changeset/eql-json-tree-planner-and-leakage.md @@ -0,0 +1,5 @@ +--- +"@cipherstash/eql": patch +--- + +EQL v4 `json_tree` (unreleased): the reference docs gain "What an encrypted JSON tree reveals" (`docs/reference/json-tree-leakage.md`), one page on what a stored row shows the database at rest (including the row's shape, which the stored depth-first entry order gives away), what each query type hands it, what is and is not authenticated, and the side-by-side with v3 `ste_vec`. The encryption test suite now holds the planner to the GIN and btree indexes with every setting at its default, on inline and TOASTed documents, for one-shot and prepared statements, and CI runs that suite on every supported PostgreSQL version. diff --git a/.github/workflows/test-eql.yml b/.github/workflows/test-eql.yml index e0b1f16be..438cfb3bf 100644 --- a/.github/workflows/test-eql.yml +++ b/.github/workflows/test-eql.yml @@ -100,7 +100,8 @@ on: - "packages/eql/docker/**" - "packages/eql/README.md" - "packages/eql/SUPABASE.md" - # The `rust-crates` job's encryption steps compile these by PATH + # The `rust-crates` and `rust-encryption-postgres` jobs' encryption + # steps compile these by PATH # (`eql-bindings`' `stack-encrypt` feature): stack-encrypt, its derive # and stack-kms through `path =`, and stack-auth and stack-profile # through `workspace = true` entries the root `Cargo.toml` resolves to @@ -449,6 +450,70 @@ jobs: run: | mise run test:sqlx:partition + # The Rust encryption suite against a live database, once per supported + # PostgreSQL version: the JSON tree family's oracle (containment over the + # encrypted column against native `jsonb @>` over the plaintexts), its + # walks and orderings, and the plan-shape test that holds the planner to + # the GIN and btree indexes with every setting at its default. Plan choice + # is a per-version property of the planner, which is why this is a matrix + # and not a step of `rust-crates`. PR and manual dispatch run PG 17 alone; + # a push or the schedule runs the full set, like the SQLx jobs (see + # `setup`). Listed in `ci-required`'s `needs`, like every job here: the + # step it replaced sat inside `rust-crates`, which is. + rust-encryption-postgres: + name: "Rust encryption with PostgreSQL ${{ matrix.postgres-version }}" + needs: [changes, setup] + # Exclusion, not an allowlist — see the note on `build-archive`. + if: >- + github.event_name != 'pull_request' + || needs.changes.outputs.relevant == 'true' + runs-on: blacksmith-16vcpu-ubuntu-2204 + strategy: + fail-fast: false + matrix: + postgres-version: ${{ fromJSON(needs.setup.outputs.pg-versions) }} + services: + encryption-postgres: + image: postgres:${{ matrix.postgres-version }} + env: + POSTGRES_PASSWORD: postgres + # Container port only: Docker picks the host port, and the test step + # reads it from `job.services`, so the number is written once. + ports: + - 5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + cache: true + working_directory: packages/eql + # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs + cache_key_prefix: mise-v1-ubuntu-2204 + # Restores what `rust-crates` saves (the stack-encrypt feature tree) + # and never saves itself: a matrix of fast jobs racing one key would + # overwrite it with whichever finished first. + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: packages/eql + shared-key: rust-crates + save-if: false + # The task takes the connection from EQL_TEST_DATABASE_URL alone, and + # the host port in it comes from the service, so the two cannot + # disagree. + - name: Test Rust encryption with PostgreSQL ${{ matrix.postgres-version }} + env: + EQL_TEST_DATABASE_URL: "host=localhost port=${{ job.services.encryption-postgres.ports['5432'] }} user=postgres password=postgres dbname=postgres" + run: mise run test:encryption:postgres + # docs:validate + Clean-DB v3 install smoke. Both are version-relevant, so # they follow the event's PG set (PG17 on PR; 14-17 in the queue). Moved out # of the old per-version test job so they run ONCE per version, not per shard. @@ -548,20 +613,6 @@ jobs: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' runs-on: blacksmith-16vcpu-ubuntu-2204 - services: - encryption-postgres: - image: postgres:17 - env: - POSTGRES_PASSWORD: postgres - # Container port only: Docker picks the host port, and the test step - # reads it from `job.services`, so the number is written once. - ports: - - 5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -603,20 +654,14 @@ jobs: - name: Check the timing harness's Cargo.lock run: cargo metadata --locked --format-version 1 --manifest-path tests/timing/ore_block_v2/ctgen/Cargo.toml > /dev/null - # The three encryption checks are mise tasks rather than raw cargo here, - # so `eql-suite-ci.test.mjs` counts them among the cargo tasks a root + # The encryption checks are mise tasks rather than raw cargo here, so + # `eql-suite-ci.test.mjs` counts them among the cargo tasks a root # workflow must reach, and `eql-workflow-filters.test.mjs` derives the - # paths they read. The database task takes the connection from - # EQL_TEST_DATABASE_URL alone, and the host port in it comes from the - # service, so the two cannot disagree. + # paths they read. The database half runs in `rust-encryption-postgres` + # below, once per supported PostgreSQL version. - name: Test Rust text equality run: mise run test:encryption - - name: Test Rust text equality with PostgreSQL - env: - EQL_TEST_DATABASE_URL: "host=localhost port=${{ job.services.encryption-postgres.ports['5432'] }} user=postgres password=postgres dbname=postgres" - run: mise run test:encryption:postgres - - name: Compile Rust text equality for WASI without HTTP run: mise run check:encryption:wasi @@ -1011,8 +1056,9 @@ jobs: ci-required: name: "ci-required" needs: [changes, setup, build-archive, test, validate, schema, rust-crates, - codegen, self-contained-v3, matrix-coverage, splinter, docs-static, - known-failures, doc-anchors, e2e] + rust-encryption-postgres, codegen, self-contained-v3, + matrix-coverage, splinter, docs-static, known-failures, + doc-anchors, e2e] if: always() runs-on: blacksmith-16vcpu-ubuntu-2204 # Same opt-out as `setup`, and this is the worse of the two to get wrong: diff --git a/docs/plans/2026-10-17-json-tree-index.md b/docs/plans/2026-10-17-json-tree-index.md index f7b38e340..7c78baf30 100644 --- a/docs/plans/2026-10-17-json-tree-index.md +++ b/docs/plans/2026-10-17-json-tree-index.md @@ -282,6 +282,8 @@ On a flat five-node document the proposal is about 8% larger: the two chain fiel > **2026-10-22:** parent tags are position-keyed (Phase 1), so siblings do not share a `p` at rest. Since Phase 3b there is no `g`: which leaves sit in the same array element is visible at rest through the nesting of the stored projection `q` instead. +> **2026-10-23 (review of #1336):** the "proposed" column of the shape-equality and parent-and-child rows is wrong for the format as built, and was wrong for the design as written. Neither the tags nor their salt are what decides it: the entries of `sv` (and the pairs of `q`) are stored in depth-first pre-order, an object's members in sorted key order, under selectors that are the same in every row. A row's sequence of selectors therefore fixes its shape exactly, the parent of every node is inferable by comparing rows (and is exact for the top-level fields under the sealed root, whose entries are the ones with no `p`), every array's length is the count of its element selector, and the relative order of sibling key names is visible across rows. `packages/eql/docs/reference/json-tree-leakage.md` states the format as built; the decision this leaves open is recorded under Phase 4. + ### 7. Performance - **Encrypt:** per node two PRFs (`s`, `vs`), three hashes (`t`, `p`, mask), one small AEAD. v3 does two MACs and one AEAD over the subtree bytes. Fewer bytes encrypted at every depth above the leaves. @@ -487,6 +489,15 @@ The rows were taken from the Phase 3c tables (one ordered path a corpus), re-enc The nested document is now about 14 times its plaintext on disk (18 at Phase 3c, 40 at Phase 3). The byte saving exceeds the character saving: pglz compressed hex better than it compresses base64 and base85, but not by enough to matter. The benchmark corpora are not re-ingested and no query is re-timed in this note; the walk, containment and ordering paths are covered by the oracle suite, which passed against the bundle built from the tree. Of the two levers left from the Phase 3 list, one is closed and one is open. **The derived nonce is rejected (2026-10-23).** It would save the twelve random bytes in every `c`, about 7% of a nested document, but it needs vitaminc to expose a caller-chosen nonce (a nonce strategy that reads the leaf's AAD, and a nonce-less leaf wire form), and a caller-chosen nonce is exactly what vitaminc's AEAD layer refuses to offer: its misuse resistance rests on never letting a caller repeat a (key, nonce) pair, and the safety argument here ("every seal mints a fresh data key") is an invariant of stack-encrypt that vitaminc could not check. Seven percent does not buy weakening that promise; the random nonce stays. Unpadded base64 (two characters a tag, about 3%) remains open and untaken. +## Implementation notes (2026-10-23, Phase 4: the planner held to its indexes, and the leakage page) + +Two of the three items that gate a release are closed here; the third, the final re-bench, follows them. + +- **Planner fragility (gate 2).** `eql_v4.json_tree_query` is plpgsql with `COST 1000` so that a bitmap scan over the GIN index beats a sequential scan even when the documents are TOASTed and the heap looks like a few pages. That was observed in the benches, at 10,000 and 100,000 rows, on one PostgreSQL version, and nothing held it. `postgres_the_planner_picks_the_indexes_without_help` in the encryption suite now does: on a flat table (1,000 four-node documents, every row inline) and a nested table (300 documents of about 220 nodes, every row TOASTed, both regimes pinned by `pg_column_size`), with every planner setting at its default and after `ANALYZE`, containment plans as a bitmap scan over the GIN index and a range at an ordered path as a scan of the functional btree on `ord_term(doc -> selector)`. A control sets the function's cost to 1 for one transaction and shows the planner taking the sequential scan on the nested table, so the assertion is known to be able to fail. The suite runs in `test-eql.yml` in a new `rust-encryption-postgres` job, a matrix over the supported PostgreSQL versions (17 on a pull request, 14 to 17 on a push, like the SQLx jobs); it was a step of the single-version `rust-crates` job. Measured locally on 14, 15, 16 and 17 before the commit. +- **Leakage (gate 3).** `packages/eql/docs/reference/json-tree-leakage.md` is the one page: what a stored row shows at rest, field by field; what each query type hands the database; what is authenticated; the side-by-side with v3 from section 6 above, brought up to date (no `g`, position-keyed `p`, ordering opt-in) and corrected where section 6 was wrong (the entry order; see the note under its table); and the two options that change the picture. It says what a token is and that it is column-wide and does not expire, which is the sentence a customer most needs before sending one. +- **Open for the owner: the stored entry order (2026-10-23, review of #1336).** `sv` and `q` hold the nodes in the walk's order, depth-first, an object's members in the sorted order of their keys (serde_json's `Map` without `preserve_order`; a build that unifies that feature on would store insertion order instead, which leaks the application's field order). Across rows this reveals each row's shape, its parent relation, every array's length and the alphabetical order of sibling key names, which partly undoes what the keyed rank (decision 11) hides from a token holder. The leakage page says so. Closing it is a format change, not a documentation one: a stored order that does not follow the tree (a document-keyed permutation, say, with `j` the permuted position), a rebuild that does not read the tree from the order, and the same for the pairs of `q`, whose nesting would still group array elements. Not taken here. +- **Open for the owner: an extracted entry opens under the selector the server returns with it.** `JsonNode` and `JsonSubtree` open each entry under the `s` and `j` that arrive beside it, and under the data key the header that arrives with it names. The entry's AAD (`entry_aad` in `packages/stack-encrypt/src/json/derive.rs`) binds the keyset, the column's context, the data key's tag, the selector and the ordinal, and nothing that identifies a row. So a decrypted value is always one sealed in that column, but not necessarily the one at the path asked for, nor one from the row the result puts it in: `->` on `$.salary` can be answered with the row's `$.bonus`, with the sealed root (which opens to the whole document), or with another row's `$.salary` returned together with that row's header. The last is by design: two values sealed under the same context are interchangeable between rows (ADR 0010 in `packages/stack-encrypt/docs/adr`), and row provenance is the database's word. The leakage page states this under "What is authenticated". Checking the returned `s` against the selector the client derived would close the path substitution for `->`, though not the row one; a walk's rows would need each child's selector checked against its parent's path. + ## Decisions to take before Phase 1 > **2026-10-22:** these are the open questions as they stood before Phase 1; the implementation notes above record how each was taken. Since then, decision 4's group tags exist only as the nesting of the stored projection `q`, decision 9 was settled by storing the projection (Phase 3), and decision 10's ordinal is not stored (Phase 3b). diff --git a/packages/eql/docs/README.md b/packages/eql/docs/README.md index 8b7a6c1d9..b5b9d7e84 100644 --- a/packages/eql/docs/README.md +++ b/packages/eql/docs/README.md @@ -14,6 +14,7 @@ This directory contains the documentation for the Encrypt Query Language (EQL). - [Writing fast queries against EQL columns](reference/query-performance.md) - Performance overview (points to Database Indexes) - [Adding a Scalar Encrypted-Domain Type](reference/adding-a-scalar-encrypted-domain-type.md) - How the `eql_v3.` domain families are generated - [EQL with JSON and JSONB](reference/json-support.md) +- [What an encrypted JSON tree reveals](reference/json-tree-leakage.md) - What `public.eql_v4_json_tree` (EQL v4) shows the database at rest and at query time, and what it does not - [EQL payload / wire format](../crates/eql-bindings/README.md) - Canonical wire types for the encrypted payload (envelope `v`/`i`/`c` and the `hm`/`op`/`ob`/`bf` index terms) - [Client-side index configuration](https://cipherstash.com/docs/stack/cipherstash/encryption/schema) - Configuring searchable encryption in CipherStash Stack / CipherStash Proxy diff --git a/packages/eql/docs/reference/json-tree-leakage.md b/packages/eql/docs/reference/json-tree-leakage.md new file mode 100644 index 000000000..bfb6ffab2 --- /dev/null +++ b/packages/eql/docs/reference/json-tree-leakage.md @@ -0,0 +1,67 @@ +# What an encrypted JSON tree reveals + +`public.eql_v4_json_tree` (EQL v4) stores a JSON document so that PostgreSQL can answer containment, path extraction, ordering at chosen paths and subtree reads without the key. Every value, key name and container sits inside an AES-256-GCM ciphertext under the document's data key, and nothing on this page shows the text of one. What the database can see is **structure**, and a great deal of it: which paths a row has and how many times, the order the nodes are stored in, which (path, value) pairs repeat across rows, and lengths. Each item below is what one query capability or one storage choice costs. Read this before you put a column on it, and decide the two options at the end with it in mind. + +## On this page + +- [What a stored row shows](#what-a-stored-row-shows) +- [What a query shows](#what-a-query-shows) +- [What is authenticated](#what-is-authenticated) +- [Compared with EQL v3 (`public.eql_v3_json_search`)](#compared-with-eql-v3-publiceql_v3_json_search) +- [The two options that change the picture](#the-two-options-that-change-the-picture) + +## What a stored row shows + +Someone who holds the column and no key, such as a database administrator, a backup, or an attacker with a copy of the table, learns the following from each row. "Across rows" means a comparison between rows of the same column. No selector, value selector or ordering term links one column to another; the header names the keyset, which is the same for every column it seals. + +| field | what it reveals | +|---|---| +| `s`, the path selector on every node | **Which paths the row has, and how many nodes sit at each.** A selector is a keyed hash of the path, the same in every row of the column, so two rows with a selector in common hold the same path. Key names are inside the hash and not readable. Array elements share their array's selector, so `$.tags[0]` and `$.tags[1]` look alike, but each element is its own entry: the number of entries carrying `$.tags[*]`'s selector is the length of `$.tags` (the total over every array at that path, when the path runs through another array). | +| the order of the entries in `sv` | **The row's shape.** Entries are stored in depth-first order, the root first, each node followed by everything beneath it, and an object's members in the sorted order of their key names. So a row's sequence of selectors fixes its shape exactly: two rows with the same sequence have the same tree with the same key names, and two rows that differ show where. Which selector's path sits under which is not written down, but it is the same in every row, so an observer comparing rows learns it, and with it each row's parent and child relations and the node count of every subtree. Siblings stored in sorted key order also show, across rows, **the alphabetical order of sibling key names** (for example that the key behind one selector sorts before the key behind another), which narrows guesses at what each selector names. | +| `q`, the containment projection: one `{s, vs}` pair per node, nested by array element, in the same order | **Whether two rows hold the same value at the same path**, through the value selector `vs`, a keyed hash of the path and the value (for an object or an array, of its kind alone). So at each path the database sees which rows agree and how often each value occurs: a frequency histogram of every path, and whether one row's document contains another's. Not the value itself, and not whether two different paths hold the same value. **Which nodes sit inside the same array element**, through the nesting, for every element that is an object or an array. That grouping is what makes containment exact for arrays of objects, and it is the one thing this format reveals that v3 did not. | +| `p`, the parent tag below the chain's heads | Nothing beyond the entry order on its own. It is salted by the document's `iv`, so the same position in two rows is unrelated bytes, and it does not say which entry is the parent. Which entries carry no `p` does show: with the sealed root on, those are the root's direct members, and since everything beneath a member follows it, **the node count of each top-level field's subtree** can be read off a single row exactly. With the sealed root off, only the root lacks one. | +| each ciphertext's length, and the number of entries | **The node count**, and for each node the length of its key name (for an object member) plus, for a scalar, the length of its value, exactly, less the AEAD's fixed overhead. A string's length in bytes and a number's length in digits are therefore visible per node. With the sealed root on, the root entry's length is the document's length as JSON. | +| `op`, the ordering term, only on number and string leaves at the paths the index orders | **The order of that leaf's value among every row that holds one, and which rows hold equal values.** The term is order-preserving encryption (CLLW OPE) under one key per column, so leaves at two ordered paths compare with each other too. It is deterministic and not a random value: two values whose encodings share leading bytes (for a string, its first characters) get terms that share leading bytes, so the term also shows how long that common prefix is. A term's presence at an ordered path also says the leaf is a number or a string. By default no path is ordered; see the options below. | +| `h`, the header | The keyset the data key was minted under, the data key's identifier (unique to the row), and whether the root is sealed. `v` and `i` are the version and the table and column identifier every EQL value carries. | + +Not revealed at rest: any value's or key name's text, and whether two different paths hold the same value. Revealed at rest, by the entry order, although no field stores it: the shape of every row, the parent of every node, every array's length, and the sorted order of sibling key names. + +## What a query shows + +A query hands the database a term derived from the plaintext it asks about, and the database's answer is visible to it. Each query type reveals the following, in addition to the rows it returns. + +- **Containment (`@>`, `<@`), and equality at a path, which is sent as a containment needle.** The needle's own projection: the selectors of its paths, the value selectors of its nodes, and its element grouping. So the database learns which paths and which (path, value) pairs you asked about, and which rows hold them. +- **Path extraction (`->`, `->>`, `json_tree_path_query`, `json_tree_path_exists`).** The selector of the path asked. A position in the path reads as the wildcard, so a position is not sent. The entry returned is a ciphertext. +- **Ordering at a path (`<`, `<=`, `>`, `>=`).** The path's selector and the bound's own ordering term, which compares with the stored terms exactly as they compare with each other: the database learns the bound's place in the order of the stored values at every ordered path, and whether it equals one. `ORDER BY ord_term(…)` sends no term and shows nothing the stored terms do not. +- **A subtree read (`json_tree_subtree`, `json_tree_subtree_at`, `json_tree_children`, `json_tree_child_count`).** A **token**, and one gate per step below it. The token is not the token of the node you asked for: it is the token of the deepest node on the path the client can derive, which is the top-level field the path starts with or, while the path goes on through array positions, the element they reach. Reading `$.profile.address.city` sends `$.profile`'s token. A gate is the step's selector or, for an array position, `#n:`, with the position `n` in the clear. A token is a capability over the structure of its node's subtree in *every row of the column*: whoever holds it can find each node under that field in any row, see how the nodes nest, link each one to its parent, and derive the tokens of the nodes beneath. It opens no value. Tokens are column-wide and do not expire, so a token sent once is useful to the database for as long as the column lives. Send a token to the database in a query and nowhere else. With the sealed root on (the default) the broadest token a query can send covers one top-level field's subtree, and no token covers a whole document. With it off, every subtree read whose path begins with a key sends the root's token, which covers every row's whole document. + +## What is authenticated + +Only the ciphertext `c`. Opening an entry verifies that it was sealed under this keyset, for this column, under the data key its row's header names (a fresh one for every row), with the selector `s` and the position `j` it is opened with; a ciphertext moved into another row or column, or opened under another selector or position, fails. Nothing in it names a row, so an entry that travels with its own row's header still opens; see below. When the client opens a whole document it takes each entry's position from its place in `sv`, so a moved, swapped or re-labelled entry fails to open. The selectors, the projection, the parent tags, the ordering terms, the header's sealed-root flag, and which entries a row holds are not authenticated, by the same rule every EQL stored value follows (ADR 0010 in `packages/stack-encrypt/docs/adr`). + +The consequence to design around: tampering with those fields changes **which rows a query returns, where a walk stops, and which entries of a row an extraction or a walk hands back**. An entry the database returns from `->` or a walk carries its own `s` and `j`, and the client opens it under those, and the header it carries names the data key, so a decrypted value is always one that was sealed in this column, but the client does not check today that it is the entry at the path it asked for, or that it comes from the row the result puts it in: a database that wants to can answer `$.salary` with the same row's `$.bonus`, or with another row's `$.salary`. A query's result is not integrity-protected; each decrypted value is genuine, but where it came from is the database's word. + +## Compared with EQL v3 (`public.eql_v3_json_search`) + +| observer sees | v3 (`ste_vec`) | v4 (`json_tree`) | +|---|---|---| +| which paths a row has | yes | yes | +| which (path, value) pairs a row has | yes | yes | +| node count | yes, two entries per node | yes, one entry per node | +| array lengths | yes | yes, from how many entries carry the element selector | +| size of every subtree | yes: each path entry's ciphertext is the subtree under it | no ciphertext spans a subtree, but its node count follows from the entry order, and its size is the sum of its entries' lengths | +| parent and child relation | inferable from subtree sizes and co-occurrence across rows | not stored, but inferable from the entry order and co-occurrence across rows; exact for top-level fields under the sealed root | +| whether two rows have the same shape | yes, selectors are column-deterministic | yes, selectors are column-deterministic and entries are stored in depth-first order | +| which leaves sit in the same array element | no | yes, through the projection's nesting: the one new disclosure, required for exact containment, bounded to array elements | +| equal values in sibling elements | ciphertexts equal | ciphertexts differ | +| ordering terms | on scalar leaves, with no per-path choice | only at the paths the index orders; none by default | +| a subtree read | reveals the selector asked | reveals a token for the top-level field the path starts with, and with it that field's shape in every row for the life of the column | + +## The two options that change the picture + +- **`sealed_root`** (default on). The root entry's ciphertext is the whole document, so a full read is one decryption and the broadest token a query can send is one top-level field's. The costs are that the root entry's length reveals the document's size, the row is larger, and the root's direct members are marked by their missing `p`, which puts each top-level field's node count in plain view. Off, there is no document-sized ciphertext, but every subtree read whose path begins with a key sends the root's token, the whole column's shape. +- **`ordered`** (default none). Each path listed carries an ordering term and takes on the `op` row above; a path not listed cannot be range-queried or sorted, and a query that tries is refused by the client before any key loads. Order only the paths your queries sort or range over. + +Two disclosures above are properties of the stored format that no option changes today: the depth-first entry order, with members in sorted key order, and the client opening an extracted entry under the selector the database returns with it. Both are open decisions recorded in the design document (`docs/plans/2026-10-17-json-tree-index.md` at the repository root, Phase 4 notes). + +To report a concern with any of this, see `SECURITY.md` at the repository root. diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index cce5e3fb6..e1c4a97aa 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -880,6 +880,330 @@ fn postgres_extracts_walks_and_orders_by_path() { } } +/// A document of the flat shape: four nodes, under PostgreSQL's TOAST +/// threshold, so the row sits inline in the heap. +fn flat_document(i: i64) -> Value { + json!({"number": i, "hello": format!("h{}", i % 17), "flag": i % 2 == 0}) +} + +/// A document of the nested shape: about 220 nodes, well over the TOAST +/// threshold, so the heap holds a pointer and the document sits in the +/// TOAST relation. +fn nested_document(i: i64) -> Value { + let orders: Vec = (0..12) + .map(|k| { + let items: Vec = (0..5) + .map(|m| json!({"sku": format!("s{}", (i + k + m) % 23), "qty": m + 1, "gift": m == 0})) + .collect(); + json!({"id": i * 100 + k, "total": (i + k) % 997, "items": items}) + }) + .collect(); + json!({"number": i, "hello": format!("h{}", i % 17), "orders": orders}) +} + +/// Needs an EQL v4 installation. The planner chooses the indexes on its +/// own, with every setting at its default, on both shapes a stored +/// document takes: inline (a small document, under the TOAST threshold) +/// and TOASTed (a large one, where the heap holds a pointer and a +/// sequential scan looks cheap by page count). `json_tree_query` is +/// plpgsql with `COST 1000` for exactly that reason: the planner cannot +/// see that evaluating it detoasts the row, so its cost has to say so. +/// The oracle test above shows the GIN index with sequential scans turned +/// off; this one shows it with them on, which is what a customer's +/// database runs. The ordering path is held the same way: a functional +/// btree on `ord_term(doc -> selector)` serves a range at a path. Both +/// are asked three times: as one-shot statements, planned with their +/// operands in view; as a prepared statement executed past its fifth run, +/// where `plan_cache_mode = auto` may plan once for any operand; and as +/// the same statement under `plan_cache_mode = force_generic_plan`, since +/// `auto` keeps a custom plan whenever it costs less than the generic one +/// and would otherwise leave the generic plan untested. +/// +/// The last block is a control: with the function's cost set to the +/// minimum for one transaction, the planner prefers the sequential scan +/// on the TOASTed table, which is what the assertion above guards +/// against. Read it as "the assertion can fail", not as a target. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_the_planner_picks_the_indexes_without_help() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let plan = plan(); + let mut db = connect(); + + let version: i32 = db + .query_one("SHOW server_version_num", &[]) + .unwrap() + .get::<_, String>(0) + .parse() + .unwrap(); + + // The operands: a containment needle per shape, and one range bound at + // `$.number`, whose selector the functional index is built on. + let bound = wait( + &runtime, + cipher + .query(op::AtPath(op::Gte), &("$.number", 1)) + .on(&plan), + ) + .unwrap(); + let number = bound.selector().to_string(); + let leaf = |value: i64| { + let query = wait( + &runtime, + cipher + .query(op::AtPath(op::Gte), &("$.number", value)) + .on(&plan), + ) + .unwrap(); + serde_json::to_string(&JsonTreeLeafQuery::try_from(&query).unwrap()).unwrap() + }; + let needle = |value: Value| { + let query = wait(&runtime, cipher.query(op::Contains, &value).on(&plan)).unwrap(); + serde_json::to_string(&JsonTreeQuery::try_from(&query).unwrap()).unwrap() + }; + + let explain = + |db: &mut postgres::Client, sql: &str, params: &[&(dyn postgres::types::ToSql + Sync)]| { + db.query(&format!("EXPLAIN {sql}"), params) + .unwrap_or_else(|e| panic!("{sql}: {e}")) + .iter() + .map(|r| r.get::<_, String>(0)) + .collect::>() + .join("\n") + }; + + struct Shape { + name: &'static str, + rows: i64, + document: fn(i64) -> Value, + needle: Value, + inline: bool, + } + let shapes = [ + Shape { + name: "flat", + rows: 1000, + document: flat_document, + needle: json!({"hello": "h3"}), + inline: true, + }, + Shape { + name: "nested", + rows: 300, + document: nested_document, + needle: json!({"orders": [{"items": [{"sku": "s7", "gift": true}]}]}), + inline: false, + }, + ]; + + for shape in &shapes { + let table = format!("json_tree_plan_{}", shape.name); + db.batch_execute(&format!( + "CREATE TEMP TABLE {table} (id int PRIMARY KEY, doc public.eql_v4_json_tree NOT NULL); + CREATE INDEX {table}_gin ON {table} USING gin ((eql_v4.json_tree_query(doc)) jsonb_path_ops); + CREATE INDEX {table}_ord ON {table} (eql_v4.ord_term(doc -> '{number}'::text));" + )) + .unwrap(); + for i in 0..shape.rows { + let stored = wait(&runtime, seal(&cipher, &(shape.document)(i))); + let stored = serde_json::to_string(&stored).unwrap(); + db.execute( + &format!( + "INSERT INTO {table} VALUES ($1, $2::text::jsonb::public.eql_v4_json_tree)" + ), + &[&(i as i32), &stored], + ) + .unwrap(); + } + db.batch_execute(&format!("ANALYZE {table}")).unwrap(); + + // The two storage regimes the shapes stand for, pinned, so the test + // keeps meaning when a document's size changes. Read from the TOAST + // relation itself: a value stored out of line is one `chunk_id` + // there, so the count of distinct chunk ids is the count of + // TOASTed rows, on every supported version (`pg_column_toast_chunk_id` + // is PostgreSQL 17 only, and `pg_column_size` reports the stored + // size, which says nothing about where it is stored). + let (column, toasted): (i32, i64) = { + let column: i32 = db + .query_one( + &format!("SELECT avg(pg_column_size(doc))::int FROM {table}"), + &[], + ) + .unwrap() + .get(0); + let toast: String = db + .query_one( + "SELECT reltoastrelid::regclass::text FROM pg_class WHERE oid = $1::text::regclass", + &[&table], + ) + .unwrap() + .get(0); + let toasted: i64 = db + .query_one( + &format!("SELECT count(DISTINCT chunk_id) FROM {toast}"), + &[], + ) + .unwrap() + .get(0); + (column, toasted) + }; + if shape.inline { + assert_eq!( + toasted, 0, + "{}: every row inline, {column} bytes on average", + shape.name + ); + } else { + assert_eq!( + toasted, shape.rows, + "{}: every row TOASTed, {column} bytes on average", + shape.name + ); + } + + // Containment: a bitmap scan over the GIN index, on its own. + let operand = needle(shape.needle.clone()); + let text = explain( + &mut db, + &format!("SELECT id FROM {table} WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree"), + &[&operand], + ); + assert!( + text.contains(&format!("Bitmap Index Scan on {table}_gin")), + "{} on PostgreSQL {version}: containment should use the GIN index without help:\n{text}", + shape.name + ); + let matched: i64 = db + .query_one( + &format!("SELECT count(*) FROM {table} WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree"), + &[&operand], + ) + .unwrap() + .get(0); + assert!( + matched > 0, + "{}: the needle should match some rows", + shape.name + ); + + // A range at the ordered path: the functional btree. + let operand = leaf(shape.rows - 5); + let text = explain( + &mut db, + &format!("SELECT id FROM {table} WHERE (doc -> $1::text) >= $2::text::jsonb::eql_v4.query_json_tree_leaf"), + &[&number, &operand], + ); + assert!( + text.contains(&format!("Index Scan using {table}_ord")) + || text.contains(&format!("Bitmap Index Scan on {table}_ord")), + "{} on PostgreSQL {version}: a range at an ordered path should use the btree without help:\n{text}", + shape.name + ); + let matched: i64 = db + .query_one( + &format!("SELECT count(*) FROM {table} WHERE (doc -> $1::text) >= $2::text::jsonb::eql_v4.query_json_tree_leaf"), + &[&number, &operand], + ) + .unwrap() + .get(0); + assert_eq!(matched, 5, "{}: the top five numbers", shape.name); + + // The same two queries as a pooled driver runs them: one prepared + // statement, executed past the five custom plans after which + // `plan_cache_mode = auto` may switch to a generic plan, whose + // operand is a parameter the planner cannot see. The selector stays + // a literal, as it has to for the functional index to match. + // `EXPLAIN EXECUTE` shows the plan the cache would run next. Under + // `auto` that may still be a custom plan (the cache keeps one while + // it costs less than the generic plan), so each statement is then + // explained again under `force_generic_plan`, which is the plan a + // pooled driver can reach whatever the costs say. + let containment = needle(shape.needle.clone()); + let range = leaf(shape.rows - 5); + for literal in [&containment, &range] { + assert!(!literal.contains('\''), "an operand embeds as a literal"); + } + db.batch_execute(&format!( + "PREPARE {table}_contains(text) AS + SELECT id FROM {table} WHERE doc @> $1::jsonb::eql_v4.query_json_tree; + PREPARE {table}_range(text) AS + SELECT id FROM {table} WHERE (doc -> '{number}'::text) >= $1::jsonb::eql_v4.query_json_tree_leaf;" + )) + .unwrap(); + for (statement, operand, index, what) in [ + ("contains", &containment, "_gin", "containment"), + ("range", &range, "_ord", "a range at an ordered path"), + ] { + for _ in 0..6 { + db.batch_execute(&format!("EXECUTE {table}_{statement}('{operand}')")) + .unwrap(); + } + let explain_execute = |db: &mut postgres::Client| { + db.simple_query(&format!("EXPLAIN EXECUTE {table}_{statement}('{operand}')")) + .unwrap() + .into_iter() + .filter_map(|message| match message { + postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_string), + _ => None, + }) + .collect::>() + .join("\n") + }; + let uses_index = |text: &str| { + text.contains(&format!("Index Scan using {table}{index}")) + || text.contains(&format!("Bitmap Index Scan on {table}{index}")) + }; + let text = explain_execute(&mut db); + assert!( + uses_index(&text), + "{} on PostgreSQL {version}: {what} from a prepared statement past its fifth execution should use the index:\n{text}", + shape.name + ); + db.batch_execute("SET plan_cache_mode = force_generic_plan") + .unwrap(); + let text = explain_execute(&mut db); + db.batch_execute("RESET plan_cache_mode").unwrap(); + assert!( + text.contains("$1"), + "{} on PostgreSQL {version}: {what} under force_generic_plan should be planned with its operand as a parameter:\n{text}", + shape.name + ); + assert!( + uses_index(&text), + "{} on PostgreSQL {version}: {what} from a prepared statement's generic plan should use the index:\n{text}", + shape.name + ); + } + } + + // The control: with `json_tree_query` priced as if it were free, the + // planner takes the sequential scan over the TOASTed table. + let mut tx = db.transaction().unwrap(); + tx.batch_execute("ALTER FUNCTION eql_v4.json_tree_query(public.eql_v4_json_tree) COST 1") + .unwrap(); + let operand = needle(shapes[1].needle.clone()); + let text = tx + .query( + "EXPLAIN SELECT id FROM json_tree_plan_nested WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree", + &[&operand], + ) + .unwrap() + .iter() + .map(|r| r.get::<_, String>(0)) + .collect::>() + .join("\n"); + tx.rollback().unwrap(); + assert!( + text.contains("Seq Scan on json_tree_plan_nested"), + "on PostgreSQL {version}, with the function priced as free the planner should take the sequential scan, which is what the cost guards against:\n{text}" + ); +} + /// Needs an EQL v4 installation. A position names an array element and /// nothing else, in every row of the column, as a strict SQL/JSON path does /// (`jsonb_path_query_first(plain, 'strict $.a[1]')`): where a diff --git a/scripts/__tests__/eql-suite-ci.test.mjs b/scripts/__tests__/eql-suite-ci.test.mjs index ee00032a8..1c9dc60fe 100644 --- a/scripts/__tests__/eql-suite-ci.test.mjs +++ b/scripts/__tests__/eql-suite-ci.test.mjs @@ -200,6 +200,34 @@ describe('a root workflow runs the EQL SQLx suite', () => { }) }) +describe('`ci-required` aggregates every job in the EQL workflow', () => { + // `ci-required` is the one check meant to gate a merge, and it reads + // `needs.*.result`, so a job missing from its `needs:` can fail without + // turning it red. Silent when wrong: the failing job still shows in the + // checks list, and the aggregate beside it is green. The PostgreSQL half of + // the Rust encryption suite left `rust-crates` (which is listed) for a job + // of its own and was missing from the list until the review of #1336. + const AGGREGATOR = 'ci-required' + + it('lists every other job in its `needs:`', () => { + const jobs = readWorkflow(EQL_WORKFLOW)?.jobs ?? {} + const names = Object.keys(jobs) + // Guards the guard: an empty job map passes the comparison below. + expect(names.length).toBeGreaterThan(1) + expect(names, `${EQL_WORKFLOW} has no \`${AGGREGATOR}\` job`).toContain( + AGGREGATOR, + ) + const needs = [jobs[AGGREGATOR]?.needs ?? []].flat() + const missing = names.filter( + (name) => name !== AGGREGATOR && !needs.includes(name), + ) + expect( + missing, + `These jobs in ${EQL_WORKFLOW} are not in \`${AGGREGATOR}\`'s \`needs:\`, so a failure in them leaves the required check green:\n${missing.map((name) => ` ${name}`).join('\n')}`, + ).toEqual([]) + }) +}) + describe('the relevance filter still selects the imported tree', () => { const wf = readWorkflow(EQL_WORKFLOW)