diff --git a/Cargo.lock b/Cargo.lock index 271420eac..a77b94a88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3419,6 +3419,7 @@ dependencies = [ "tokio", "trybuild", "uuid", + "vitaminc 0.8.0", "vitaminc-aead 0.8.0", "vitaminc-aead-frame", "vitaminc-encrypt 0.8.0", diff --git a/languages/golang/encrypt/guest/Cargo.lock b/languages/golang/encrypt/guest/Cargo.lock index 34e8e7007..fa249ed7d 100644 --- a/languages/golang/encrypt/guest/Cargo.lock +++ b/languages/golang/encrypt/guest/Cargo.lock @@ -2174,6 +2174,7 @@ dependencies = [ "stack-kms", "thiserror 1.0.69", "uuid", + "vitaminc 0.8.0", "vitaminc-aead 0.8.0", "vitaminc-aead-frame", "vitaminc-encrypt 0.8.0", diff --git a/packages/eql/Cargo.lock b/packages/eql/Cargo.lock index 06e657798..ba95bebae 100644 --- a/packages/eql/Cargo.lock +++ b/packages/eql/Cargo.lock @@ -4279,6 +4279,7 @@ dependencies = [ "stack-kms", "thiserror 1.0.69", "uuid", + "vitaminc 0.8.0", "vitaminc-aead 0.8.0", "vitaminc-aead-frame", "vitaminc-encrypt 0.8.0", diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index fe4ea4f19..b0ca380d5 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -1906,9 +1906,12 @@ fn postgres_hostile_answers_are_refused_by_the_client() { cipher.decrypt_as(JsonTreeSubtree::from_key(key, rows), column()), ) }; + // Refused: as the wrong entry, or at the cipher. Under the cascading + // tags (spike) a row the chain misplaces fails to authenticate, since + // its parent's AAD covers its tag. let wrong = |result: Result, what: &str| { assert!( - matches!(result, Err(Error::JsonWrongEntry { .. })), + matches!(result, Err(Error::JsonWrongEntry { .. }) | Err(Error::Aead)), "{what}: {result:?}" ); }; diff --git a/packages/stack-encrypt/Cargo.toml b/packages/stack-encrypt/Cargo.toml index 7ed76d784..da36c8eeb 100644 --- a/packages/stack-encrypt/Cargo.toml +++ b/packages/stack-encrypt/Cargo.toml @@ -42,6 +42,8 @@ vitaminc-normalize = { workspace = true } vitaminc-ore = { workspace = true } vitaminc-prf = { workspace = true } vitaminc-protected = { workspace = true } +# Spike (#1357): the CSPRNG that shuffles a document's stored entry order. +vitaminc = { workspace = true } # `dynamic` only: serde over the transport frame, which reads a binding's # control data (statistics, options, a plan) into typed messages. vitaminc-aead-frame = { workspace = true, optional = true, features = ["serde"] } diff --git a/packages/stack-encrypt/src/cipher.rs b/packages/stack-encrypt/src/cipher.rs index 6c469e8ed..e41aed689 100644 --- a/packages/stack-encrypt/src/cipher.rs +++ b/packages/stack-encrypt/src/cipher.rs @@ -592,6 +592,16 @@ impl StackCipher { &self.kms } + /// The loaded state of the keyset with `id`, if this cipher has loaded + /// it: what opening a JSON document with its root open needs, since the + /// client walks the document's chain itself (spike, cascading tags). + pub(crate) fn loaded_keyset(&self, id: Uuid) -> Option> { + match self.keysets().get(&IdentifiedBy::Uuid(id)) { + Lookup::Hit(state) => Some(state), + Lookup::Stale(_) | Lookup::Miss(_) => None, + } + } + fn keysets(&self) -> std::sync::MutexGuard<'_, KeysetCache> { // The cache holds no invariant a panic mid-update could break (an // insert is two map writes, and a stale name entry only points at diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index 1b4753cd3..ae871a9a9 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -39,7 +39,10 @@ const VALUE_DOMAIN: &[u8] = b"stack-encrypt/json/value/v1"; /// under the sealed root. const TOKEN_DOMAIN: &[u8] = b"stack-encrypt/json/token/v1"; /// PAE domain of an entry's AAD. -const ENTRY_AAD_DOMAIN: &[u8] = b"stack-encrypt/json/entry/v1"; +/// `v2` (spike, cascading tags): the AAD binds a digest of the children's +/// AEAD tags beside the selector and the position, so a container's +/// ciphertext commits to exactly the children it was sealed over. +const ENTRY_AAD_DOMAIN: &[u8] = b"stack-encrypt/json/entry/v2"; /// PAE domain of an object member's rank under its parent's token, which /// orders the members' ordinals. Keyed, so a token does not let its holder /// test key names. @@ -790,8 +793,19 @@ pub(crate) fn seal_pending<'a, K: 'static>( } /// An entry's AAD: the domain, the keyset, the column's AAD, the key tag, -/// the entry's selector and its ordinal. -fn entry_aad(keyset_id: Uuid, aad: &Context<'_>, tag: &[u8], s: &Tag, j: u64) -> Context<'static> { +/// the entry's selector, its ordinal, and the digest of its children's +/// AEAD tags ([`children_digest`]). The last is what cascades: a child's +/// tag covers that child's own children, so a node's ciphertext commits +/// to its whole subtree, and the client verifies a subtree from the +/// leaves up to the node it read, whatever order the rows arrive in. +fn entry_aad( + keyset_id: Uuid, + aad: &Context<'_>, + tag: &[u8], + s: &Tag, + j: u64, + children: &ChildrenDigest, +) -> Context<'static> { Context::pae(&[ ENTRY_AAD_DOMAIN, keyset_id.as_bytes(), @@ -799,9 +813,42 @@ fn entry_aad(keyset_id: Uuid, aad: &Context<'_>, tag: &[u8], s: &Tag, j: u64) -> tag, s, &j.to_le_bytes(), + children, ]) } +/// SHA-256 over the AEAD tags of a node's children, in ordinal order (the +/// order the parent tags enumerate them: a member's keyed rank, an +/// element's index); for the chain's heads under the sealed root, which +/// have no ordinal, in stored order. Over no tags for a leaf and for an +/// empty container. +type ChildrenDigest = [u8; 32]; + +/// The AEAD tag of a sealed entry: the last sixteen bytes of its +/// ciphertext, as vitaminc seals a leaf (`version ‖ nonce ‖ ciphertext ‖ +/// tag`). Public, like the ciphertext it ends. +fn aead_tag(c: &[u8]) -> &[u8] { + &c[c.len().saturating_sub(16)..] +} + +fn children_digest<'c>(children: impl IntoIterator) -> ChildrenDigest { + let mut hasher = Sha256::new(); + hasher.update(b"k"); + for c in children { + hasher.update(aead_tag(c)); + } + hasher.finalize().into() +} + +/// The digest a leaf, an empty container, and the sealed root are opened +/// with. The sealed root's ciphertext is the whole document, which no +/// read beneath it is verified against (its heads chain from nothing, and +/// the budget fallback returns the root alone), so it commits to no +/// children; see the write-up for the root manifest that would. +fn leaf_digest() -> ChildrenDigest { + children_digest(std::iter::empty()) +} + fn leaf_cipher(key: &stack_kms::DataKey) -> Result { Aes256Cipher::new(&AesKey::from(*key.key())).map_err(|_| Error::Aead) } @@ -892,6 +939,29 @@ fn sealed_root_plaintext(value: &Value) -> Result>, serde_json }) } +/// Spike (#1357): whether documents are sealed with their entries in a +/// random stored order, `STACK_ENCRYPT_JSON_SHUFFLE=1`. A process-wide +/// switch so every existing test and the PostgreSQL oracle run under it +/// unchanged; a real change would be a layout option recorded like +/// `sealed_root`, or simply the format. +pub(crate) fn shuffled() -> bool { + static SHUFFLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *SHUFFLED + .get_or_init(|| std::env::var_os("STACK_ENCRYPT_JSON_SHUFFLE").is_some_and(|v| v == "1")) +} + +/// The stored slot of each of `n` nodes, by walk index: the identity, or +/// under `rng` a uniform permutation of the slots `1..n` with the root +/// kept at `0`, where the sealed-root fallback and a whole-document open +/// look for it (every document has a root, so its slot says nothing). +fn slots(n: usize, rng: Option<&mut vitaminc::random::SafeRand>) -> Vec { + let mut slots: Vec = (0..n).collect(); + if let Some(rng) = rng { + super::query::fisher_yates(&mut slots[1..], &mut |m| rng.next_below(m)); + } + slots +} + fn seal( walked: Walked, mut whole: Option>>, @@ -910,24 +980,65 @@ fn seal( let tag = key.tag; let Walked { terms, plain } = walked; let nodes = &terms.nodes; - let mut entries = Vec::with_capacity(nodes.len()); - for (j, (node, plain)) in nodes.iter().zip(plain).enumerate() { - // The root's plaintext, the first, is the whole document when the - // root is sealed (`whole` is taken there, and is `None` after); the + // Each node's children, in the order their digest takes them: by + // ordinal where they chain from the node, else (the heads under the + // sealed root) in stored order. The walk visits a node's children in + // stored order, so a stable sort by ordinal gives both. + let mut children: Vec> = vec![Vec::new(); nodes.len()]; + for (index, node) in nodes.iter().enumerate() { + if let Some(parent) = node.parent { + children[parent].push(index); + } + } + for list in &mut children { + list.sort_by_key(|&i| nodes[i].ordinal.unwrap_or(0)); + } + // Sealed from the leaves up: a node's AAD takes its children's tags, + // so the children are sealed first. The walk is pre-order, so every + // descendant of a node has a larger index, and the reverse order seals + // children before parents. + let mut plain: Vec>>> = plain.into_iter().map(Some).collect(); + let mut ciphertexts: Vec>> = vec![None; nodes.len()]; + // Where each node is stored: its walk index, or (spike, #1357) a random + // slot, the root staying at 0. The AAD binds the slot. + let mut rng = shuffled() + .then(vitaminc::random::SafeRand::from_entropy) + .transpose() + .map_err(|_| Error::Other("no entropy for the stored order".into()))?; + let slots = slots(nodes.len(), rng.as_mut()); + for j in (0..nodes.len()).rev() { + let node = &nodes[j]; + let plaintext = plain[j].take().ok_or(Error::Aead)?; + // The root's plaintext is the whole document when the root is + // sealed, and then commits to no children (`leaf_digest`); the // node encoding it replaces is wiped as it drops. - let plaintext = whole.take().unwrap_or(plain); + let (plaintext, digest) = match (j == 0).then(|| whole.take()).flatten() { + Some(whole) => (whole, leaf_digest()), + None => ( + plaintext, + children_digest( + children[j] + .iter() + .map(|&i| ciphertexts[i].as_deref().unwrap_or_default()), + ), + ), + }; let c = cipher .seal( plaintext, - &entry_aad(keyset_id, aad, &tag, &node.s, j as u64), + &entry_aad(keyset_id, aad, &tag, &node.s, slots[j] as u64, &digest), ) .map_err(|_| Error::Aead)?; - let c = c.as_ref().to_vec(); + ciphertexts[j] = Some(c.as_ref().to_vec()); + } + let mut entries: Vec> = vec![None; nodes.len()]; + for (j, node) in nodes.iter().enumerate() { + let c = ciphertexts[j].take().ok_or(Error::Aead)?; let p = node .parent .zip(node.ordinal) .map(|(parent, o)| parent_tag(&iv, &nodes[parent].token, o)); - entries.push(JsonEntry { + entries[slots[j]] = Some(JsonEntry { s: node.s, c, x: self_tag(&iv, &node.token), @@ -935,7 +1046,14 @@ fn seal( op: node.op.clone(), }); } - let projection = super::query::group_of(nodes, None); + let entries = entries + .into_iter() + .map(|entry| entry.ok_or(Error::Aead)) + .collect::, _>>()?; + let mut projection = super::query::group_of(nodes, None); + if let Some(rng) = rng.as_mut() { + projection.shuffle(&mut |n| rng.next_below(n)); + } Ok(JsonDocument::new( JsonHeader::new(keyset_id, iv, tag, sealed_root), entries, @@ -1057,11 +1175,12 @@ fn open_entry( aad: &Context<'_>, entry: &JsonEntry, j: u64, + children: &ChildrenDigest, ) -> Result { let plaintext = cipher .open( LocalCipherText::from(entry.c.clone()), - &entry_aad(header.keyset_id, aad, &header.tag, &entry.s, j), + &entry_aad(header.keyset_id, aad, &header.tag, &entry.s, j, children), ) .map_err(|_| Error::Aead)?; decode_node(&plaintext) @@ -1072,61 +1191,6 @@ fn take_label(mut node: Opened) -> (Label, Opened) { (std::mem::replace(&mut node.label, Label::Root), node) } -/// Rebuild the value of `node`, taking its descendants from `rest`, the -/// pre-order sequence after it. Every child is taken with `next`, so each -/// step consumes an entry and the rebuild ends on any sequence. The value -/// is built under `Protected`: each member and element is moved in from -/// its own, so a rebuild that fails part-way wipes what it had. -/// -/// `depth` is `node`'s depth below the rebuild's start. Each entry is -/// authentic, but which entries the server returned, and so how deep the -/// containers it lines up nest, is not: rows that repeat one container, or -/// list only the containers of a wide document, would otherwise recurse as -/// deep as there are rows. Nothing the index seals nests deeper than -/// [`MAX_DEPTH`], so deeper is refused as a shape mismatch. -fn rebuild( - node: Opened, - rest: &mut impl Iterator, - depth: usize, -) -> Result, Error> { - if let Some(whole) = node.whole { - return Ok(whole); - } - if depth > MAX_DEPTH { - return Err(shape("a JSON document no deeper than the index seals")); - } - match node.kind { - Kind::Object => { - let mut object = protected_object(); - for _ in 0..node.children { - let (label, member) = take_label(next_child(rest)?); - let Label::Key(key) = label else { - return Err(shape("an object member")); - }; - let value = rebuild(member, rest, depth + 1)?; - insert_member(&mut object, key, value); - } - Ok(object) - } - Kind::Array => { - let mut array = protected_array(node.children as usize); - for i in 0..node.children { - let element = next_child(rest)?; - if !matches!(element.label, Label::Index(at) if at == u64::from(i)) { - return Err(shape("an array element in order")); - } - push_element(&mut array, rebuild(element, rest, depth + 1)?); - } - Ok(array) - } - _ => node.scalar.ok_or_else(|| shape("a JSON scalar")), - } -} - -fn next_child(rest: &mut impl Iterator) -> Result { - rest.next().ok_or_else(|| shape("a complete JSON subtree")) -} - /// The value the caller asked for, out of the `Protected` that held it: /// the one place an opened JSON plaintext is unwrapped. fn hand_over(value: Protected) -> Value { @@ -1139,7 +1203,13 @@ fn retrieval( header: &JsonHeader, context: AeadContext, ) -> Result<(Request, Context<'static>), Error> { - let piece = context.into_context(); + retrieval_of(header, context.into_context()) +} + +fn retrieval_of( + header: &JsonHeader, + piece: vitaminc_aead::ContextPiece<'static>, +) -> Result<(Request, Context<'static>), Error> { let descriptor = Descriptor::from_piece(&piece); descriptor.check()?; Ok(( @@ -1149,16 +1219,40 @@ fn retrieval( } /// Open the document: one key retrieval, then the sealed root alone, or -/// every entry rebuilt into the value. +/// every entry placed by the chain and opened under the cascade. +/// +/// With the root open the client walks the chain itself, as the server +/// would for a read of `$`: it derives the root's token under the index +/// key, finds each node's children by their parent tags and opens each +/// node under the digest of its children's tags. So it needs the keyset +/// the document names, loaded (spike: a keyset this cipher has not loaded +/// is refused rather than fetched). fn open_document<'a, K>( doc: JsonDocument, cipher: &'a StackCipher, context: AeadContext, ) -> Pending<'a, Value, K> { - let (request, aad) = match retrieval(&doc.header, context) { + let piece = context.into_context(); + let prf_context = piece.clone().into_prf_context().into_owned(); + let (request, aad) = match retrieval_of(&doc.header, piece) { Ok(parts) => parts, Err(error) => return Pending::failed(cipher, error), }; + let keyset = if doc.header.sealed_root { + None + } else { + match cipher.loaded_keyset(doc.header.keyset_id) { + Some(state) => Some(state), + None => { + return Pending::failed( + cipher, + Error::Other( + "opening a JSON document with its root open needs its keyset loaded".into(), + ), + ) + } + } + }; Pending::request(cipher, vec![request], move |responses| { let key = responses.next_retrieved_key()?; let aes = leaf_cipher(&key)?; @@ -1167,26 +1261,52 @@ fn open_document<'a, K>( .entries .first() .ok_or_else(|| shape("a document with a root entry"))?; - return open_entry(&aes, &doc.header, &aad, root, 0)? + return open_entry(&aes, &doc.header, &aad, root, 0, &leaf_digest())? .whole .map(hand_over) .ok_or_else(|| shape("a sealed root")); } - let opened = doc + let keyset = keyset.ok_or_else(|| shape("a loaded keyset"))?; + let root = JsonPath::root(); + let token = prf16( + &keyset.prf, + TOKEN_DOMAIN, + &prf_context, + &root.token_bytes().unwrap_or_default(), + )?; + // A document whose chain the root's token does not enter is not + // this column's (or its tags are not a chain): opening its first + // entry under this context says so as the cipher does, rather + // than as rows no read reached. + if doc.find(&token).is_none() { + let first = doc + .entries + .first() + .ok_or_else(|| shape("a document with a root entry"))?; + let _ = open_entry(&aes, &doc.header, &aad, first, 0, &leaf_digest())?; + return Err(shape("a document whose chain reaches its root")); + } + let selectors = Selectors::new(keyset, prf_context); + let reads = [Read { + token, + path: root, + probe: false, + }]; + let rows = doc .entries .iter() .enumerate() - .map(|(j, entry)| open_entry(&aes, &doc.header, &aad, entry, j as u64)) - .collect::, _>>()?; - let mut rest = opened.into_iter(); - let root = rest - .next() - .ok_or_else(|| shape("a document with a root entry"))?; - let value = rebuild(root, &mut rest, 0)?; - if rest.next().is_some() { - return Err(shape("a document with no entries past its root")); + .map(|(j, entry)| Row { + entry: entry.clone(), + j: j as u64, + d: None, + }) + .collect(); + let mut values = place(rows, &reads, &selectors, None, &doc.header, &aad, &aes)?; + match (values.pop(), values.is_empty()) { + (Some(value), true) => Ok(hand_over(value)), + _ => Err(shape("a document whose chain reaches its root")), } - Ok(hand_over(value)) }) } @@ -1225,7 +1345,18 @@ impl DecryptInto for JsonExtracted { Pending::request(cipher, vec![request], move |responses| { let key = responses.next_retrieved_key()?; let aes = leaf_cipher(&key)?; - let opened = open_entry(&aes, &node.header, &aad, &node.entry, node.j)?; + // A scalar, an empty container and the sealed root commit + // to no children. A container with children does, and its + // entry alone cannot open: under the cascade it is refused + // at the cipher, where it was refused by kind before. + let opened = open_entry( + &aes, + &node.header, + &aad, + &node.entry, + node.j, + &leaf_digest(), + )?; opened .whole .or(opened.scalar) @@ -1320,15 +1451,12 @@ impl DecryptInto for JsonSubtree { } } -/// One row of a read, opened: the node, and the selector, self tag, -/// position and depth the server returned it with. The depth is the -/// server's report and optional on the wire; the rows are placed by their -/// positions and the opened nodes' child counts, and a depth that disagrees -/// with the place a row is put in is refused. +/// One row of a read as the server returned it, not yet opened: the entry, +/// the position `j` its ciphertext is bound to, and the depth the server +/// reported, which is optional on the wire and checked against the place +/// the row is put in. struct Row { - node: Opened, - s: Tag, - x: super::SelfTag, + entry: JsonEntry, j: u64, d: Option, } @@ -1366,107 +1494,238 @@ fn open_subtree<'a, K>( let key_bytes = responses.next_retrieved_key()?; let aes = leaf_cipher(&key_bytes)?; let rows = nodes - .iter() - .map(|n| { - Ok(Row { - node: open_entry(&aes, &header, &aad, &n.entry, n.j)?, - s: n.entry.s, - x: n.entry.x, - j: n.j, - d: n.depth, - }) + .into_iter() + .map(|n| Row { + entry: n.entry, + j: n.j, + d: n.depth, }) - .collect::, Error>>()?; - place(rows, &key, &header.iv) + .collect(); + place( + rows, + &key.reads, + &key.selectors, + Some(key.bound), + &header, + &aad, + &aes, + ) }) } -/// What [`rebuild_walked`] checks a row's derived fields with: the selectors -/// of the key's column, and the document's `iv`, which salts every self tag. -struct Check<'k> { - selectors: &'k Selectors, - iv: &'k [u8], +/// Where a node sits for the selector check: at the top of a read, named +/// by a concrete path, or under a node whose selector bytes are known. +enum Under<'p> { + Path(&'p JsonPath), + Node(&'p Protected>), } -/// Put the opened rows of a read back into the nodes its tokens named, in -/// the key's order, refusing any row that is not where the key says the -/// read's rows are ([`Error::JsonWrongEntry`]), and the whole read with -/// [`Error::JsonWildcardTruncated`] if a probe named a node. +/// The rows of a read, placed by the chain and opened under the cascade. /// -/// Each entry opened under its own selector `s` and position `j`, which its -/// ciphertext is bound to, so each is a genuine node of the row; what the -/// check adds is that it is the node the read asked for. A read's rows are, -/// for each token that names a node, that node at depth `0` and then every -/// node beneath it at consecutive positions, each node's children in order, -/// and the key's tokens are in document order. So the rows are placed one -/// by one. A node at depth `0` must carry the self tag of a token of the -/// key not yet matched, and that token's path's selector and last step (its -/// key, its position, or the root). Every row beneath it must be the next -/// position after the one before, one level below its parent, carry the -/// selector of its parent's path extended by its own label, and carry the -/// self tag of the token its label derives under its parent's. +/// The server's tags say where each row sits: the node a token names is +/// the row carrying its self tag, and a node's children are the rows +/// carrying its parent tags at the ordinals `0, 1, 2, …`, each with the +/// token the member or element rule derives for it. The tags are the +/// database's word, so they only say where to *try* each row. What +/// decides is the cipher: every node opens under the digest of its +/// children's AEAD tags, so a node whose children were swapped, dropped, +/// added or reordered does not open, and neither does a node put under +/// the wrong parent, since its parent's digest covers it. A row's +/// selector and decrypted label are then checked against the path the +/// chain gives it, and the depth the server reported against its place. /// -/// The self tags are not authenticated, and the database holds every token -/// the read sent and every row's `iv`, so matching them is a consistency -/// check, which catches a corrupted row or a server that walked wrongly, -/// not a dishonest database. The selector, the position and the label, -/// which the AAD and the ciphertext cover, are what bind a row to its -/// place. Still allowed by design: an entry from another row of the -/// column, returned with that row's header (ADR 0010), and a node at the -/// same selector under another element (`$.items[0].price` for -/// `$.items[1].price`), which no bound field tells apart. +/// Rows arrive in any order: nothing here reads the position `j` as +/// document order, so a stored order that says nothing about the tree +/// (#1357) places the same way. +struct Placing<'k> { + rows: Vec, + used: Vec, + by_x: std::collections::HashMap, + by_p: std::collections::HashMap, + selectors: &'k Selectors, + header: &'k JsonHeader, + aad: &'k Context<'k>, + aes: &'k Aes256Cipher, +} + +#[allow(clippy::too_many_arguments)] fn place( rows: Vec, - key: &JsonSubtreeKey, - iv: &[u8], + reads: &[Read], + selectors: &Selectors, + bound: Option, + header: &JsonHeader, + aad: &Context<'_>, + aes: &Aes256Cipher, ) -> Result>, Error> { - let check = Check { - selectors: &key.selectors, - iv, + // The server's index keeps the last entry carrying a repeated tag + // (`jsonb_object_agg`); so does this, so the client places rows the way + // the server found them. + let mut by_x = std::collections::HashMap::with_capacity(rows.len()); + let mut by_p = std::collections::HashMap::with_capacity(rows.len()); + for (i, row) in rows.iter().enumerate() { + let _ = by_x.insert(row.entry.x, i); + if let Some(p) = row.entry.p { + let _ = by_p.insert(p, i); + } + } + let mut placing = Placing { + used: vec![false; rows.len()], + rows, + by_x, + by_p, + selectors, + header, + aad, + aes, }; - let refused = || wrong("a node the walked path reaches"); - let mut rest = rows.into_iter(); - // The first token of the key not yet matched to a row. - let mut next = 0; + let iv = &header.iv; let mut reached = Vec::new(); let mut truncated = false; - while let Some(top) = rest.next() { - if top.d.is_some_and(|d| d != 0) { - return Err(refused()); - } - let at = key.reads[next..] - .iter() - .position(|read| self_tag(iv, &read.token) == top.x) - .ok_or_else(refused)? - + next; - next = at + 1; - let read = &key.reads[at]; - let bytes = Protected::new(read.path.selector_bytes()); - if key.selectors.of(bytes.clone())? != top.s || !names(&read.path, &top.node.label) { - return Err(refused()); - } - let mut cursor = top.j.checked_add(1).ok_or_else(refused)?; - let value = rebuild_walked( - top.node, - &bytes, - &read.token, - 0, - &mut rest, - &mut cursor, - &check, - )?; + for read in reads { + let Some(&at) = placing.by_x.get(&self_tag(iv, &read.token)) else { + continue; + }; + let (value, _) = placing.node(at, &read.token, Under::Path(&read.path), 0)?; if read.probe { truncated = true; } else { reached.push(value); } } + // Every row the server returned must be one the chain placed: a row + // no token's subtree reaches is not an answer to the read. + if placing.used.iter().any(|used| !used) { + return Err(wrong("rows the walked paths reach")); + } if truncated { - return Err(Error::JsonWildcardTruncated { bound: key.bound }); + return Err(Error::JsonWildcardTruncated { + bound: bound.unwrap_or(0), + }); } Ok(reached) } +impl Placing<'_> { + /// Place and open the row at `at`, the node whose token is `token`, + /// sitting `under` its parent at depth `d` below the node its read + /// named: its children first, by the chain, then the node under the + /// digest of their tags, then the checks. Returns the node's value and + /// its label, which its parent places it by. + fn node( + &mut self, + at: usize, + token: &Tag, + under: Under<'_>, + d: u32, + ) -> Result<(Protected, Label), Error> { + let refused = || wrong("a node the walked path reaches"); + if d as usize > MAX_DEPTH { + return Err(shape("a JSON document no deeper than the index seals")); + } + // A chain that reaches a row twice is the server lining one entry + // up in two places; an honest walk never does. + if std::mem::replace(&mut self.used[at], true) { + return Err(refused()); + } + if self.rows[at].d.is_some_and(|got| got != d) { + return Err(refused()); + } + let iv = &self.header.iv; + // The children, by the chain: the rows at the parent tags of the + // ordinals from zero until the first miss, each with the token its + // self tag says it has. + let mut children: Vec<(usize, Tag)> = Vec::new(); + for o in 0..u32::try_from(self.rows.len()).unwrap_or(u32::MAX) { + let Some(&child) = self.by_p.get(&parent_tag(iv, token, o)) else { + break; + }; + let child_token = + hit_token(iv, token, &self.rows[child].entry, o).ok_or_else(refused)?; + children.push((child, child_token)); + } + let digest = children_digest( + children + .iter() + .map(|&(i, _)| self.rows[i].entry.c.as_slice()), + ); + let row = &self.rows[at]; + let opened = open_entry(self.aes, self.header, self.aad, &row.entry, row.j, &digest)?; + if opened.whole.is_some() + && !matches!(under, Under::Path(path) if path.segments().is_empty()) + { + return Err(wrong("a node of the walked subtree")); + } + if opened.children as usize != children.len() { + return Err(shape("a container with the children it was sealed over")); + } + let (label, opened) = take_label(opened); + // The selector the chain gives this node: the read's path for the + // node it named, its parent's extended by its own label below. + let bytes = match under { + Under::Path(path) => { + if !names(path, &label) { + return Err(refused()); + } + Protected::new(path.selector_bytes()) + } + Under::Node(parent) => child_selector_bytes(parent, &label).ok_or_else(refused)?, + }; + if self.selectors.of(bytes.clone())? != row.entry.s { + return Err(refused()); + } + if let Some(whole) = opened.whole { + return Ok((whole, label)); + } + match opened.kind { + Kind::Object => { + let mut object = protected_object(); + for (child, child_token) in children { + let (value, child_label) = + self.member(child, token, &child_token, &bytes, d)?; + let Label::Key(key) = child_label else { + return Err(shape("an object member")); + }; + insert_member(&mut object, key, value); + } + Ok((object, label)) + } + Kind::Array => { + let mut array = protected_array(children.len()); + for (i, (child, child_token)) in children.into_iter().enumerate() { + let (value, child_label) = + self.member(child, token, &child_token, &bytes, d)?; + if !matches!(child_label, Label::Index(at) if at == i as u64) { + return Err(shape("an array element in order")); + } + push_element(&mut array, value); + } + Ok((array, label)) + } + _ => Ok((opened.scalar.ok_or_else(|| shape("a JSON scalar"))?, label)), + } + } + + /// Place and open a child, and check that the token the server's rule + /// gave it is the one its decrypted label derives under its parent's: + /// a member's by its key, an element's by its index. + fn member( + &mut self, + child: usize, + parent_token: &Tag, + child_token: &Tag, + parent_bytes: &Protected>, + d: u32, + ) -> Result<(Protected, Label), Error> { + let (value, label) = self.node(child, child_token, Under::Node(parent_bytes), d + 1)?; + let s = self.rows[child].entry.s; + if labelled_token(parent_token, &s, &label).as_ref() != Some(child_token) { + return Err(wrong("the next node of the walked subtree")); + } + Ok((value, label)) + } +} + /// Whether a node with `label` is the one the concrete `path` names by its /// last step: the key, the position, or the root for `$`. fn names(path: &JsonPath, label: &Label) -> bool { @@ -1516,104 +1775,6 @@ fn labelled_token(parent: &Tag, s: &Tag, label: &Label) -> Option { } } -/// [`rebuild`] for a read's rows, checking each child as it is taken: that -/// it is at `cursor`, the position after the row before it; that it is one -/// level below `node`, which sits `d` levels below the node a token named; -/// that it carries the selector of `node`'s path, whose selector bytes are -/// `bytes`, extended by its own label; and that it carries the self tag of -/// the token its label derives under `token`, `node`'s. A sealed root -/// among the rows is not a node of any walk. Rows that end before `node`'s -/// authenticated child count is met are refused like a gap: a read returns -/// a node's whole subtree, so the database left the next node out. -/// -/// The rows are consecutive entries of one sealed document, placed by its -/// authenticated child counts, so they nest no deeper than the document -/// does, which is never deeper than [`MAX_DEPTH`]; a document that does is -/// refused as a shape mismatch rather than recursed into. -fn rebuild_walked( - node: Opened, - bytes: &Protected>, - token: &Tag, - d: u32, - rest: &mut impl Iterator, - cursor: &mut u64, - check: &Check<'_>, -) -> Result, Error> { - if node.whole.is_some() { - return Err(wrong("a node of the walked subtree")); - } - if d as usize > MAX_DEPTH { - return Err(shape("a JSON document no deeper than the index seals")); - } - let next = || wrong("the next node of the walked subtree"); - let child = |rest: &mut dyn Iterator, cursor: &mut u64| { - // A read returns a node's whole subtree and stops at no budget - // short of it (the SQL falls back to the sealed root instead), so - // rows that run out before a child count is met are a row left out - // at the end, the same refusal as one left out in the middle. - let row = rest.next().ok_or_else(next)?; - let child_bytes = child_selector_bytes(bytes, &row.node.label).ok_or_else(next)?; - if row.j != *cursor - || row.d.is_some_and(|got| Some(got) != d.checked_add(1)) - || check.selectors.of(child_bytes.clone())? != row.s - { - return Err(next()); - } - let child_token = labelled_token(token, &row.s, &row.node.label).ok_or_else(next)?; - if self_tag(check.iv, &child_token) != row.x { - return Err(next()); - } - *cursor = cursor.checked_add(1).ok_or_else(next)?; - Ok::<_, Error>((row.node, child_bytes, child_token)) - }; - match node.kind { - Kind::Object => { - let mut object = protected_object(); - for _ in 0..node.children { - let (member, member_bytes, member_token) = child(rest, cursor)?; - let (label, member) = take_label(member); - let Label::Key(key) = label else { - return Err(shape("an object member")); - }; - let value = rebuild_walked( - member, - &member_bytes, - &member_token, - d + 1, - rest, - cursor, - check, - )?; - insert_member(&mut object, key, value); - } - Ok(object) - } - Kind::Array => { - let mut array = protected_array(node.children as usize); - for i in 0..node.children { - let (element, element_bytes, element_token) = child(rest, cursor)?; - if !matches!(element.label, Label::Index(at) if at == u64::from(i)) { - return Err(shape("an array element in order")); - } - push_element( - &mut array, - rebuild_walked( - element, - &element_bytes, - &element_token, - d + 1, - rest, - cursor, - check, - )?, - ); - } - Ok(array) - } - _ => node.scalar.ok_or_else(|| shape("a JSON scalar")), - } -} - /// The key's reads evaluated over the whole document a read fell back to, /// to what the walked rows open to: each concrete path's node, moved out of /// the document's `Protected` into one of its own (leaving `null` behind, @@ -1669,7 +1830,14 @@ fn open_fallback<'a, K>( Pending::request(cipher, vec![request], move |responses| { let retrieved = responses.next_retrieved_key()?; let aes = leaf_cipher(&retrieved)?; - let opened = open_entry(&aes, &root.header, &aad, &root.entry, root.j)?; + let opened = open_entry( + &aes, + &root.header, + &aad, + &root.entry, + root.j, + &leaf_digest(), + )?; let whole = opened.whole.ok_or_else(|| shape("a sealed root"))?; Ok((key, whole)) }) @@ -1950,6 +2118,42 @@ impl KeysetCipher<'_, K> { #[cfg(test)] mod tests { use super::*; + + #[test] + fn the_aead_tag_is_the_last_sixteen_bytes_of_the_ciphertext() { + let c: Vec = (0..40).collect(); + assert_eq!(aead_tag(&c), &c[24..]); + assert_eq!(aead_tag(&c).len(), 16); + // A ciphertext shorter than a tag is taken whole, so a truncated + // entry still contributes something its parent was sealed over. + assert_eq!(aead_tag(&[1, 2, 3]), &[1, 2, 3]); + assert_eq!(aead_tag(&[]), &[] as &[u8]); + } + + #[test] + fn the_children_digest_covers_each_tag_in_order() { + let a: Vec = (0..40).collect(); + let b: Vec = (100..140).collect(); + let ab = children_digest([a.as_slice(), b.as_slice()]); + let ba = children_digest([b.as_slice(), a.as_slice()]); + assert_ne!(ab, ba, "order is part of the digest"); + assert_ne!(ab, leaf_digest(), "a container is not a leaf"); + assert_eq!(leaf_digest(), children_digest(std::iter::empty())); + // Only the tag counts: two ciphertexts with the same last sixteen + // bytes digest the same. + let mut a2 = a.clone(); + a2[0] ^= 0xff; + assert_eq!( + children_digest([a.as_slice()]), + children_digest([a2.as_slice()]) + ); + let mut a3 = a.clone(); + a3[39] ^= 0xff; + assert_ne!( + children_digest([a.as_slice()]), + children_digest([a3.as_slice()]) + ); + } use serde_json::json; use vitaminc_prf::PrfKeyInit; diff --git a/packages/stack-encrypt/src/json/query.rs b/packages/stack-encrypt/src/json/query.rs index c7d9d4075..6c3f56065 100644 --- a/packages/stack-encrypt/src/json/query.rs +++ b/packages/stack-encrypt/src/json/query.rs @@ -42,6 +42,19 @@ pub struct JsonProjection { } impl JsonProjection { + /// Spike (#1357): put the pairs of every group, and the groups of every + /// element list, in a random order, `below(n)` drawing a uniform index + /// under `n`. Containment is `jsonb @>` over the projection, which reads + /// `sv` and `el` as sets, so the order changes no answer; it stops the + /// projection restating the document's depth-first order. + pub(crate) fn shuffle(&mut self, below: &mut impl FnMut(u32) -> u32) { + fisher_yates(&mut self.sv, below); + fisher_yates(&mut self.el, below); + for group in &mut self.el { + group.shuffle(below); + } + } + /// `jsonb @>` over two projections: every needle pair is some pair of /// this projection, and every needle element fits inside some element /// of it, recursively. What the server computes natively over the two @@ -793,6 +806,15 @@ impl Json { } } +/// A uniform shuffle of `items`, `below(n)` drawing a uniform index under +/// `n` (spike, #1357). +pub(crate) fn fisher_yates(items: &mut [T], below: &mut impl FnMut(u32) -> u32) { + for i in (1..items.len()).rev() { + let j = below(u32::try_from(i + 1).unwrap_or(u32::MAX)) as usize; + items.swap(i, j.min(i)); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/packages/stack-encrypt/tests/json_index.rs b/packages/stack-encrypt/tests/json_index.rs index c893bc0c9..c5f628ef3 100644 --- a/packages/stack-encrypt/tests/json_index.rs +++ b/packages/stack-encrypt/tests/json_index.rs @@ -915,9 +915,11 @@ mod given_an_extracted_node { fixture(), "the sealed root is the document" ); - assertions::refused_as_opened_type( + // Under the cascading tags (spike) a container's entry alone does + // not open: its AAD covers its children's tags, which `->` does not + // return. It was refused by kind after opening before. + assertions::refused_as_tampered( &scalar("$.nested", 2).await, - None, "a container node holds no value of its own", ); @@ -1442,7 +1444,9 @@ mod given_a_tampered_document { let truncated = tamper(wire.clone(), &|sv| { let _ = sv.pop(); }); - assertions::refused_as_opened_type(&open(&cipher, truncated).await, None, "truncated"); + // Under the cascading tags (spike) a document missing entries fails at + // its root's cipher, whose AAD covers its children's tags. + assertions::refused_as_tampered(&open(&cipher, truncated).await, "truncated"); let flipped = tamper(wire.clone(), &|sv| { let c = sv[1]["c"].as_str().unwrap().to_owned(); @@ -2157,7 +2161,7 @@ mod given_a_float_at_full_precision { let key = subtree_key(cipher, "$.list", JsonOptions::default()) .await .unwrap(); - let rows = subtree_nodes(&sealed, key.token_bytes()); + let rows = read_rows(&sealed, &key); let element = sealed.node(2).unwrap(); vec![ ( @@ -2171,7 +2175,10 @@ mod given_a_float_at_full_precision { ( "the extracted element, as `->` returns it", cipher - .decrypt_as::(element, aead()) + .decrypt_as::( + extracted(cipher, "$.list[*]", element).await, + aead(), + ) .await .unwrap(), ), @@ -2884,8 +2891,10 @@ mod assertions { /// `result` is the refusal of an entry that is genuine but is not the /// one asked for: another path's, or a row the walk did not reach. pub(super) fn refused_as_wrong_entry(result: &Result, what: &str) { + // Under the cascading tags (spike) a row the chain misplaces fails + // at the cipher, since its parent's AAD covers its tag. assert!( - matches!(result, Err(Error::JsonWrongEntry { .. })), + matches!(result, Err(Error::JsonWrongEntry { .. }) | Err(Error::Aead)), "{what}: should be refused as the wrong entry, got {result:?}" ); } diff --git a/packages/stack-encrypt/tests/json_read_checks.rs b/packages/stack-encrypt/tests/json_read_checks.rs index 3c8f1fc85..7634447f3 100644 --- a/packages/stack-encrypt/tests/json_read_checks.rs +++ b/packages/stack-encrypt/tests/json_read_checks.rs @@ -151,13 +151,15 @@ async fn open_list(cipher: &Cipher, rows: JsonSubtree) -> Result, Err mod assertions { use super::*; - /// The read was refused as the wrong entry, naming `expected`. + /// The read was refused: as the wrong entry, or at the cipher. Under + /// the cascading tags (spike) a row that is not where the chain says + /// fails to authenticate, since its parent's AAD covers its tag, so the + /// refusal that the position check once made is the cipher's now; the + /// `expected` text names the check that made it before, for the record. pub(crate) fn wrong_entry(result: Result, expected: &str) { match result { - Err(Error::JsonWrongEntry { expected: got }) => { - assert_eq!(got, expected, "the refusal names what the read expected") - } - other => panic!("expected JsonWrongEntry ({expected}), got {other:?}"), + Err(Error::JsonWrongEntry { .. }) | Err(Error::Aead) => {} + other => panic!("expected a refusal ({expected}), got {other:?}"), } } }