From f048ac3aefd990d1afe99cccf0068b65352da12a Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 18:30:35 +1100 Subject: [PATCH 1/4] feat(stack-encrypt): serde for the order terms, and an owning from_bytes EQL v4's bindings are to store stack-encrypt's own order terms, typed by scheme and plaintext, rather than v3's untyped hex newtypes. That needs the terms to be serde types. OreTerm, OpeTerm and OreBlockTerm now serialize as lowercase hex in a human-readable format and as raw bytes in a binary one, through serdect, whose hex (base16ct) is constant-time. Deserializing goes through from_bytes, so a term of another scheme or a malformed one is refused at the boundary, with an error that names a length, never a byte. from_bytes now takes the Vec it decodes and keeps it, so neither deserializing nor a caller that owns the bytes pays a copy; TryFrom<&[u8]> stays and copies once. OpeTerm also checks its leading carry byte is 0 or 1: clww-ore's comparator refuses any other, but its from_bytes checks only the length. Upstream serde for these types is proposed in cipherstash/ore.rs#137, and constant-time FromHex in clww-ore in cipherstash/ore.rs#138. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- Cargo.lock | 1 + Cargo.toml | 2 + languages/golang/encrypt/guest/Cargo.lock | 1 + packages/eql/Cargo.lock | 1 + packages/stack-encrypt/CHANGELOG.md | 14 ++ packages/stack-encrypt/Cargo.toml | 3 + packages/stack-encrypt/fuzz/Cargo.lock | 1 + .../fuzz/fuzz_targets/term_decode.rs | 6 +- packages/stack-encrypt/src/sem/mod.rs | 113 ++++++++++--- packages/stack-encrypt/tests/frozen_bytes.rs | 18 +-- packages/stack-encrypt/tests/ore_block.rs | 21 ++- packages/stack-encrypt/tests/term_serde.rs | 150 ++++++++++++++++++ 12 files changed, 290 insertions(+), 41 deletions(-) create mode 100644 packages/stack-encrypt/tests/term_serde.rs diff --git a/Cargo.lock b/Cargo.lock index 2509f8fea..a5c8a503c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3402,6 +3402,7 @@ dependencies = [ "rust_decimal", "serde", "serde_json", + "serdect", "stack-auth", "stack-encrypt-derive", "stack-kms", diff --git a/Cargo.toml b/Cargo.toml index 0d0ec9100..340cf5e3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,6 +120,8 @@ rustls-platform-verifier = "0.6" rustls-webpki = { version = "0.103", default-features = false, features = ["alloc"] } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0.132" } +# Constant-time hex (through base16ct) for serde over the order terms. +serdect = { version = "0.3", default-features = false, features = ["alloc"] } thiserror = "1.0.56" tokio = { version = "1.47.1", features = ["full"] } tracing = { version = "0.1", features = ["log"] } diff --git a/languages/golang/encrypt/guest/Cargo.lock b/languages/golang/encrypt/guest/Cargo.lock index 047be4155..13b5d089c 100644 --- a/languages/golang/encrypt/guest/Cargo.lock +++ b/languages/golang/encrypt/guest/Cargo.lock @@ -2160,6 +2160,7 @@ dependencies = [ "ore-rs", "serde", "serde_json", + "serdect", "stack-encrypt-derive", "stack-kms", "thiserror 1.0.69", diff --git a/packages/eql/Cargo.lock b/packages/eql/Cargo.lock index 1051159b6..333fc34d4 100644 --- a/packages/eql/Cargo.lock +++ b/packages/eql/Cargo.lock @@ -4263,6 +4263,7 @@ dependencies = [ "rust_decimal", "serde", "serde_json", + "serdect", "stack-encrypt-derive", "stack-kms", "thiserror 1.0.69", diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index 066aa9178..efa3ee790 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **The order terms are serde types.** `OreTerm`, `OpeTerm` and + `OreBlockTerm` serialize as the lowercase hex of their bytes in a + human-readable format such as JSON, and as the raw bytes in a binary one, + through `serdect`, whose hex is constant-time. Deserializing decodes + through `from_bytes`, so it refuses what `from_bytes` refuses, and no + error carries a byte of the term. EQL v4's bindings store them this way. + - **Dates, timestamps and decimals are ordered** (#1236). The new `value` module holds `Date`, `Timestamp` and `Decimal` (and their `RangeError`), moved here from eql-bindings, which re-exports them at @@ -265,6 +272,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Breaking +- **The order terms' `from_bytes` takes the buffer.** `OreTerm`, `OpeTerm` + and `OreBlockTerm::from_bytes` take a `Vec` and keep it, copying + nothing; each also implements `TryFrom>`. `TryFrom<&[u8]>` + remains, and copies once. `OpeTerm` now refuses a term whose leading + carry byte is not 0 or 1, which clww-ore's comparator already refuses + (`TermBytesError::MalformedCllwCiphertext`). + - **stack-encrypt requires the vitaminc 0.8 crates, clww-ore 0.6 and ore-rs 0.10.** A crate that names a vitaminc type stack-encrypt takes or returns moves to 0.8 with it. What changes for a caller: diff --git a/packages/stack-encrypt/Cargo.toml b/packages/stack-encrypt/Cargo.toml index 28682bc2f..075d033e8 100644 --- a/packages/stack-encrypt/Cargo.toml +++ b/packages/stack-encrypt/Cargo.toml @@ -46,6 +46,9 @@ vitaminc-protected = { workspace = true } # control data (statistics, options, a plan) into typed messages. vitaminc-aead-frame = { workspace = true, optional = true, features = ["serde"] } serde = { workspace = true } +# The order terms' serde: lowercase hex in JSON, raw bytes in a binary +# format, through serdect's constant-time hex. +serdect = { workspace = true } # `Descriptor`: the base64 rendering of a non-textual context for ZeroKMS. base64ct = { version = "1.7", features = ["alloc"] } diff --git a/packages/stack-encrypt/fuzz/Cargo.lock b/packages/stack-encrypt/fuzz/Cargo.lock index cde185181..7869b22eb 100644 --- a/packages/stack-encrypt/fuzz/Cargo.lock +++ b/packages/stack-encrypt/fuzz/Cargo.lock @@ -2077,6 +2077,7 @@ dependencies = [ "ore-rs", "serde", "serde_json", + "serdect", "stack-encrypt-derive", "stack-kms", "thiserror 1.0.69", diff --git a/packages/stack-encrypt/fuzz/fuzz_targets/term_decode.rs b/packages/stack-encrypt/fuzz/fuzz_targets/term_decode.rs index 7a5334331..f95b16f84 100644 --- a/packages/stack-encrypt/fuzz/fuzz_targets/term_decode.rs +++ b/packages/stack-encrypt/fuzz/fuzz_targets/term_decode.rs @@ -21,21 +21,21 @@ fuzz_target!(|bytes: &[u8]| { } let _ = MatchTerms::::from_bytes(bytes); // One fixed-width and one variable-width CLLW output each for ORE and OPE. - if let Ok(term) = OreTerm::::from_bytes(bytes) { + if let Ok(term) = OreTerm::::from_bytes(bytes.to_vec()) { assert_eq!( term.as_bytes(), bytes, "OreTerm decode is not lossless" ); } - if let Ok(term) = OreTerm::::from_bytes(bytes) { + if let Ok(term) = OreTerm::::from_bytes(bytes.to_vec()) { assert_eq!( term.as_bytes(), bytes, "OreTerm decode is not lossless" ); } - if let Ok(term) = OpeTerm::::from_bytes(bytes) { + if let Ok(term) = OpeTerm::::from_bytes(bytes.to_vec()) { assert_eq!( term.as_bytes(), bytes, diff --git a/packages/stack-encrypt/src/sem/mod.rs b/packages/stack-encrypt/src/sem/mod.rs index d10b9b402..968f098d3 100644 --- a/packages/stack-encrypt/src/sem/mod.rs +++ b/packages/stack-encrypt/src/sem/mod.rs @@ -94,12 +94,24 @@ //! * [`OreTerm`] / [`OpeTerm`] — the raw CLLW ciphertext bytes, unframed //! ([`as_bytes`](OreTerm::as_bytes) / [`to_bytes`](OreTerm::to_bytes) / //! [`from_bytes`](OreTerm::from_bytes)). +//! * [`OreBlockTerm`] — the serialised ore-rs ciphertext, header first +//! ([`as_bytes`](OreBlockTerm::as_bytes) / +//! [`from_bytes`](OreBlockTerm::from_bytes)). //! //! Every kind decodes through `TryFrom<&[u8]>` as well, failing with a //! [`TermBytesError`]; [`EqualityTerm`] additionally keeps an infallible -//! [`from_bytes`](EqualityTerm::from_bytes) over a `[u8; 32]`. `as_bytes` -//! exists only where the term *is* a contiguous buffer (equality, ORE, OPE); -//! a [`MatchTerms`] is canonically a position list, so it has none. +//! [`from_bytes`](EqualityTerm::from_bytes) over a `[u8; 32]`. The three +//! order terms keep the buffer they decode, so their `from_bytes` takes it +//! by value (`TryFrom>` too) and copies nothing; `TryFrom<&[u8]>` +//! copies once. `as_bytes` exists only where the term *is* a contiguous +//! buffer (equality and the order terms); a [`MatchTerms`] is canonically a +//! position list, so it has none. +//! +//! The order terms are also serde types: lowercase hex in a human-readable +//! format such as JSON, the raw bytes in a binary one, through +//! [`serdect`](https://docs.rs/serdect), whose hex is constant-time. +//! Deserializing decodes through `from_bytes`, so it refuses what +//! `from_bytes` refuses, and its errors carry no byte of the term. //! //! For equality and ORE/OPE the transport bytes are also the stored form, //! and they share the *shape* of the v1 / `cipherstash-client` encodings — a @@ -1288,7 +1300,7 @@ index_term!(OpeTerm, T: Orderable); index_term!(OreBlockTerm, T: Orderable); macro_rules! term_wrapper { - ($name:ident, $inner:ty) => { + ($name:ident, $inner:ty, $well_formed:expr) => { impl $name { /// Wrap an already-generated CLLW ciphertext. pub fn new(output: $inner) -> Self { @@ -1307,13 +1319,19 @@ macro_rules! term_wrapper { /// Decode a term from its frozen byte encoding — the raw CLLW /// ciphertext bytes, the inverse of [`as_bytes`](Self::as_bytes) - /// — for terms persisted server-side. Structural only (length - /// checks); the caller asserts the bytes were generated for this - /// source type `T` and under the same context. - pub fn from_bytes(bytes: &[u8]) -> Result { - <$inner>::try_from(bytes) + /// — for terms persisted server-side. Takes the buffer and keeps + /// it, copying nothing. Structural only (the length, and an OPE + /// term's leading carry); the caller asserts the bytes were + /// generated for this source type `T` and under the same + /// context. + pub fn from_bytes(bytes: Vec) -> Result { + let len = bytes.len(); + if !$well_formed(&bytes) { + return Err(TermBytesError::MalformedCllwCiphertext(len)); + } + <$inner>::from_bytes(bytes) .map(Self::new) - .map_err(|_| TermBytesError::MalformedCllwCiphertext(bytes.len())) + .map_err(|_| TermBytesError::MalformedCllwCiphertext(len)) } /// The frozen byte encoding: the raw CLLW ciphertext bytes, @@ -1331,11 +1349,20 @@ macro_rules! term_wrapper { } /// The inherent `from_bytes` as a std conversion — the same decoder. + impl TryFrom> for $name { + type Error = TermBytesError; + + fn try_from(bytes: Vec) -> Result { + Self::from_bytes(bytes) + } + } + + /// The inherent `from_bytes` over a copy of `bytes`. impl TryFrom<&[u8]> for $name { type Error = TermBytesError; fn try_from(bytes: &[u8]) -> Result { - Self::from_bytes(bytes) + Self::from_bytes(bytes.to_vec()) } } @@ -1347,6 +1374,8 @@ macro_rules! term_wrapper { } } + term_serde!($name); + impl fmt::Debug for $name { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple(stringify!($name)).field(&self.0).finish() @@ -1384,8 +1413,35 @@ macro_rules! term_wrapper { }; } -term_wrapper!(OreTerm, OreClww8VariableV1); -term_wrapper!(OpeTerm, OpeClww8VariableV1); +/// The serde impls of an order term `$name` with `as_bytes` and an +/// owning `from_bytes`: lowercase hex in a human-readable format, the raw +/// bytes in a binary one, both through `serdect` (constant-time hex). +/// Deserializing refuses what `from_bytes` refuses; its errors name a +/// length or the encoding, never a byte of the term. +macro_rules! term_serde { + ($name:ident) => { + impl serde::Serialize for $name { + fn serialize(&self, serializer: S) -> Result { + serdect::slice::serialize_hex_lower_or_bin(&self.as_bytes(), serializer) + } + } + + impl<'de, T: Orderable> serde::Deserialize<'de> for $name { + fn deserialize>(deserializer: D) -> Result { + let bytes = serdect::slice::deserialize_hex_or_bin_vec(deserializer)?; + Self::from_bytes(bytes).map_err(serde::de::Error::custom) + } + } + }; +} + +term_wrapper!(OreTerm, OreClww8VariableV1, |_: &[u8]| true); +// An OPE term leads with the carry out of its top byte, 0 or 1. clww-ore's +// comparator refuses any other, and its `from_bytes` does not check it. +term_wrapper!(OpeTerm, OpeClww8VariableV1, |bytes: &[u8]| matches!( + bytes.first(), + Some(0 | 1) +)); /// A block ORE index term for a source value of type `T`: ore-rs's 6-bit /// block ORE encryption of the value's `vitaminc-ore` order encoding at @@ -1439,15 +1495,17 @@ fn ore_block_compare(a: &ore_rs::Term, b: &ore_rs::Term) -> Option OreBlockTerm { /// Decode a term from its frozen byte encoding, the serialised ore-rs - /// ciphertext, for terms persisted server-side. Structural only: the - /// bytes must be a well-formed 6-bit or chained block ORE ciphertext - /// (header, block count, length, padding policy and symbols); the - /// caller asserts they were generated for this source type `T` and - /// under the same context. - pub fn from_bytes(bytes: &[u8]) -> Result { - let term = ore_rs::Term::from_bytes(bytes.to_vec()); + /// ciphertext, for terms persisted server-side. Takes the buffer and + /// keeps it, copying nothing. Structural only: the bytes must be a + /// well-formed 6-bit or chained block ORE ciphertext (header, block + /// count, length, padding policy and symbols); the caller asserts they + /// were generated for this source type `T` and under the same context. + pub fn from_bytes(bytes: Vec) -> Result { + let term = ore_rs::Term::from_bytes(bytes); if ore_block_compare(&term, &term).is_none() { - return Err(TermBytesError::MalformedOreBlockCiphertext(bytes.len())); + return Err(TermBytesError::MalformedOreBlockCiphertext( + term.as_bytes().len(), + )); } Ok(Self(term, PhantomData)) } @@ -1480,14 +1538,25 @@ impl OreBlockTerm { } /// The inherent `from_bytes` as a std conversion — the same decoder. +impl TryFrom> for OreBlockTerm { + type Error = TermBytesError; + + fn try_from(bytes: Vec) -> Result { + Self::from_bytes(bytes) + } +} + +/// The inherent `from_bytes` over a copy of `bytes`. impl TryFrom<&[u8]> for OreBlockTerm { type Error = TermBytesError; fn try_from(bytes: &[u8]) -> Result { - Self::from_bytes(bytes) + Self::from_bytes(bytes.to_vec()) } } +term_serde!(OreBlockTerm); + /// The frozen byte encoding — the same bytes as the inherent `as_bytes`. impl AsRef<[u8]> for OreBlockTerm { fn as_ref(&self) -> &[u8] { diff --git a/packages/stack-encrypt/tests/frozen_bytes.rs b/packages/stack-encrypt/tests/frozen_bytes.rs index d5f63e7f6..06605aa53 100644 --- a/packages/stack-encrypt/tests/frozen_bytes.rs +++ b/packages/stack-encrypt/tests/frozen_bytes.rs @@ -475,7 +475,7 @@ async fn ore_term_encoding_is_the_raw_cllw_bytes() { assert_eq!(term.to_bytes(), term.as_bytes()); assert_eq!(term.as_ref(), term.as_bytes()); assert_eq!( - OreTerm::::from_bytes(term.as_bytes()).expect("decode ore term"), + OreTerm::::from_bytes(term.to_bytes()).expect("decode ore term"), term ); // The std conversion is the same decoder. @@ -499,7 +499,7 @@ async fn ope_term_encoding_is_the_raw_cllw_bytes() { "00a77ed973ff1af2e5d5e3779471df246d17a515f0131eb87e35e02e7a80aa5292" ); assert_eq!( - OpeTerm::::from_bytes(term.as_bytes()).expect("decode ope term"), + OpeTerm::::from_bytes(term.to_bytes()).expect("decode ope term"), term ); assert_eq!( @@ -515,15 +515,15 @@ fn ore_term_from_bytes_rejects_wrong_length() { // that plus a leading carry byte. The length checks are structural: // the type parameter names the source, and nothing in the bytes does. assert_eq!( - OreTerm::::from_bytes(&[0u8; 31]), + OreTerm::::from_bytes(vec![0u8; 31]), Err(TermBytesError::MalformedCllwCiphertext(31)) ); assert_eq!( - OreTerm::::from_bytes(&[0u8; 33]), + OreTerm::::from_bytes(vec![0u8; 33]), Err(TermBytesError::MalformedCllwCiphertext(33)) ); assert_eq!( - OpeTerm::::from_bytes(&[0u8; 32]), + OpeTerm::::from_bytes(vec![0u8; 32]), Err(TermBytesError::MalformedCllwCiphertext(32)) ); } @@ -542,11 +542,11 @@ async fn variable_length_ore_and_ope_terms_decode() { .expect("ore term"); assert_eq!(ore.as_bytes().len(), 6 * 8); assert_eq!( - OreTerm::::from_bytes(ore.as_bytes()).expect("decode"), + OreTerm::::from_bytes(ore.to_bytes()).expect("decode"), ore ); assert_eq!( - OreTerm::::from_bytes(&ore.as_bytes()[1..]), + OreTerm::::from_bytes(ore.as_bytes()[1..].to_vec()), Err(TermBytesError::MalformedCllwCiphertext(6 * 8 - 1)) ); @@ -557,11 +557,11 @@ async fn variable_length_ore_and_ope_terms_decode() { .expect("ope term"); assert_eq!(ope.as_bytes().len(), 6 * 8 + 1); assert_eq!( - OpeTerm::::from_bytes(ope.as_bytes()).expect("decode"), + OpeTerm::::from_bytes(ope.to_bytes()).expect("decode"), ope ); assert_eq!( - OpeTerm::::from_bytes(&ope.as_bytes()[1..]), + OpeTerm::::from_bytes(ope.as_bytes()[1..].to_vec()), Err(TermBytesError::MalformedCllwCiphertext(6 * 8)) ); } diff --git a/packages/stack-encrypt/tests/ore_block.rs b/packages/stack-encrypt/tests/ore_block.rs index 6c0dc9db5..1b894daa2 100644 --- a/packages/stack-encrypt/tests/ore_block.rs +++ b/packages/stack-encrypt/tests/ore_block.rs @@ -356,8 +356,8 @@ async fn one_value_is_new_bytes_that_compare_equal() { assert_ne!(a, b, "a fresh nonce each time"); assert_eq!(left(&a), left(&b)); let (a, b) = ( - OreBlockTerm::::from_bytes(&a).unwrap(), - OreBlockTerm::::from_bytes(&b).unwrap(), + OreBlockTerm::::from_bytes(a.clone()).unwrap(), + OreBlockTerm::::from_bytes(b.clone()).unwrap(), ); assert_eq!(a.compare(&b), Some(Ordering::Equal)); assert!(a == b, "== is the scheme's equality"); @@ -497,8 +497,13 @@ async fn decoding_checks_the_shape() { let keyset = cipher.default_keyset(); let bit6 = term(&keyset, 42i64).await.into_bytes(); let chained = term(&keyset, "alice".to_string()).await.into_bytes(); - assert!(OreBlockTerm::::from_bytes(&bit6).is_ok()); - assert!(OreBlockTerm::::from_bytes(&chained).is_ok()); + assert!(OreBlockTerm::::from_bytes(bit6.clone()).is_ok()); + assert!(OreBlockTerm::::from_bytes(chained.clone()).is_ok()); + // The std conversions are the same decoder, by value and by copy. + let by_value = OreBlockTerm::::try_from(bit6.clone()).unwrap(); + assert_eq!(by_value.as_bytes(), &bit6[..]); + let by_copy = OreBlockTerm::::try_from(&chained[..]).unwrap(); + assert_eq!(by_copy.as_bytes(), &chained[..]); for bad in [ &bit6[..bit6.len() - 1], &chained[..10], @@ -506,13 +511,15 @@ async fn decoding_checks_the_shape() { &[0x02, 0x04, 0, 1][..], ] { assert_eq!( - OreBlockTerm::::from_bytes(bad).unwrap_err(), + OreBlockTerm::::from_bytes(bad.to_vec()).unwrap_err(), TermBytesError::MalformedOreBlockCiphertext(bad.len()) ); + assert!(OreBlockTerm::::try_from(bad).is_err()); + assert!(OreBlockTerm::::try_from(bad.to_vec()).is_err()); } // A 6-bit and a chained term have no order between them. - let a = OreBlockTerm::::from_bytes(&bit6).unwrap(); - let b = OreBlockTerm::::from_bytes(&chained).unwrap(); + let a = OreBlockTerm::::from_bytes(bit6).unwrap(); + let b = OreBlockTerm::::from_bytes(chained).unwrap(); assert_eq!(a.compare(&b), None); assert!(a != b); } diff --git a/packages/stack-encrypt/tests/term_serde.rs b/packages/stack-encrypt/tests/term_serde.rs new file mode 100644 index 000000000..4bafad099 --- /dev/null +++ b/packages/stack-encrypt/tests/term_serde.rs @@ -0,0 +1,150 @@ +//! The order terms as serde types: lowercase hex in JSON, decoded through +//! the same structural check as `from_bytes`, with errors that carry no +//! byte of the term. + +use serde_json::{json, Value}; +use stack_encrypt::nonempty; +use stack_encrypt::sem::{OpeTerm, Orderable, OreBlockTerm, OreTerm, TermBytesError}; +use stack_encrypt::target::EncryptInto; +use stack_encrypt::{KeysetCipher, StackCipher}; +use stack_kms::FakeDataKeySource; + +async fn cipher() -> StackCipher { + StackCipher::builder() + .kms(FakeDataKeySource::new()) + .init() + .await + .expect("build cipher") +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +async fn block( + keyset: &KeysetCipher<'_, FakeDataKeySource>, + value: T, +) -> OreBlockTerm { + keyset + .ore_block_term(value, nonempty!("users/n")) + .await + .expect("a block ORE term") +} + +/// Each order term serializes as the lowercase hex of its bytes, and reads +/// back as the same bytes, in either case of hex. +#[tokio::test] +async fn an_order_term_is_its_bytes_in_lowercase_hex() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let ore: OreTerm = 42u32 + .encrypt_into_with_context(&keyset, nonempty!("users/age")) + .await + .expect("ore term"); + let ope: OpeTerm = "alice" + .to_string() + .encrypt_into_with_context(&keyset, nonempty!("users/name")) + .await + .expect("ope term"); + let fixed = block(&keyset, 42i64).await; + let chained = block(&keyset, "alice".to_string()).await; + + let cases: [(&[u8], Value); 4] = [ + (ore.as_bytes(), serde_json::to_value(&ore).unwrap()), + (ope.as_bytes(), serde_json::to_value(&ope).unwrap()), + (fixed.as_bytes(), serde_json::to_value(&fixed).unwrap()), + (chained.as_bytes(), serde_json::to_value(&chained).unwrap()), + ]; + for (bytes, value) in &cases { + assert_eq!(value, &Value::String(hex(bytes))); + } + + assert_eq!( + serde_json::from_value::>(cases[0].1.clone()).unwrap(), + ore + ); + assert_eq!( + serde_json::from_value::>(cases[1].1.clone()).unwrap(), + ope + ); + let back: OreBlockTerm = serde_json::from_value(cases[2].1.clone()).unwrap(); + assert_eq!(back.as_bytes(), fixed.as_bytes()); + let upper = json!(hex(chained.as_bytes()).to_uppercase()); + let back: OreBlockTerm = serde_json::from_value(upper).unwrap(); + assert_eq!(back.as_bytes(), chained.as_bytes()); + assert_eq!( + serde_json::to_value(&back).unwrap(), + cases[3].1, + "written back in lowercase" + ); +} + +/// Deserializing refuses what `from_bytes` refuses: a term of the wrong +/// length or shape, an OPE carry other than 0 or 1, and anything that is +/// not hex. No error repeats the term. +#[tokio::test] +async fn deserializing_refuses_what_from_bytes_refuses_and_names_no_byte() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let fixed = hex(block(&keyset, 42i64).await.as_bytes()); + let truncated = &fixed[..fixed.len() - 2]; + + let block_error = serde_json::from_value::>(json!(truncated)) + .unwrap_err() + .to_string(); + assert_eq!( + block_error, + TermBytesError::MalformedOreBlockCiphertext(truncated.len() / 2).to_string() + ); + let ore_error = serde_json::from_value::>(json!("ab".repeat(31))) + .unwrap_err() + .to_string(); + assert_eq!( + ore_error, + TermBytesError::MalformedCllwCiphertext(31).to_string() + ); + + let carry_two = format!("02{}", "00".repeat(16)); + let ope_error = serde_json::from_value::>(json!(carry_two)) + .unwrap_err() + .to_string(); + assert_eq!( + ope_error, + TermBytesError::MalformedCllwCiphertext(17).to_string() + ); + + for not_a_term in [ + json!(format!("{fixed}0")), + json!(format!("zz{}", &fixed[2..])), + json!(42), + json!([fixed]), + ] { + let error = serde_json::from_value::>(not_a_term.clone()) + .unwrap_err() + .to_string(); + assert!(!error.contains(&fixed[4..]), "{not_a_term}: {error}"); + } +} + +/// An OPE term leads with a carry of 0 or 1, through every decoder. +#[test] +fn an_ope_carry_other_than_zero_or_one_is_refused() { + for carry in [0u8, 1] { + let mut bytes = vec![0u8; 17]; + bytes[0] = carry; + assert!(OpeTerm::::from_bytes(bytes.clone()).is_ok()); + assert!(OpeTerm::::try_from(&bytes[..]).is_ok()); + assert!(OpeTerm::::try_from(bytes).is_ok()); + } + for carry in [2u8, 0xff] { + let mut bytes = vec![0u8; 17]; + bytes[0] = carry; + assert_eq!( + OpeTerm::::from_bytes(bytes.clone()), + Err(TermBytesError::MalformedCllwCiphertext(17)) + ); + assert!(OpeTerm::::try_from(bytes).is_err()); + } + // An ORE term has no carry: any leading byte is a byte of the value. + assert!(OreTerm::::from_bytes(vec![0xff; 16]).is_ok()); +} From da783932bc0b0bed7a3e19c6b1644bfdde89d4fd Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 19:31:41 +1100 Subject: [PATCH 2/4] feat(eql-bindings, eql): EQL v4 order terms are Stack Encrypt's typed terms, stored as one hex string EQL v4 reused v3's term newtypes for its order terms: `ob` was `OreBlock256(Vec)`, a one-element array of hex, and `op` an `OpeCllw(String)`. Neither said which scheme or plaintext it held, and neither checked anything when read. A v4 `ob` field is now `stack_encrypt::sem::OreBlockTerm` and an `op` field `OpeTerm`, over the type's plaintext (`DateOrdOre::ob` is an `OreBlockTerm`). Each serializes as one lowercase hex string and is decoded through `from_bytes` when a payload is read, so an EQL v3 array, a term of the other scheme or a malformed term is refused there. `v4::terms` no longer re-exports v3's order terms. `OreBlockHex` and `OpeHex`, generated, describe the JSON to TypeScript and JSON Schema. One catalog entry, `eql_domains::Term::v4_format`, names each term's Stack Encrypt type, its JSON shape and the hex prefixes it starts with (block ORE `0202`/`0203`, OPE carry `00`/`01`). Codegen writes the Rust field, the JSON Schema pattern and the v4 domain CHECKs from it: every v4 domain carrying `ob` or `op`, stored, query or range bound, requires a JSON string with one of those prefixes. The CHECK reads only the prefix, so a write costs the same whatever the term's length; the comparator still checks the full structure. The v4 SQL reads `ob` as one hex string: `has_ore_block_v2` tests for a string, the extractor decodes it straight to the composite, which now holds one term rather than an array, and the one-element array overload of the comparator is gone. The v3 bundle is byte-for-byte unchanged. Exact per-type CHECKs (block count and length) wait on a key-free shape API in ore-rs and clww-ore, cipherstash/ore.rs#139. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- .../eql/crates/eql-bindings/src/encryption.rs | 97 +-------- .../eql/crates/eql-bindings/src/v4/bigint.rs | 43 +++- .../eql/crates/eql-bindings/src/v4/date.rs | 43 +++- .../eql/crates/eql-bindings/src/v4/double.rs | 43 +++- .../eql/crates/eql-bindings/src/v4/integer.rs | 43 +++- .../eql/crates/eql-bindings/src/v4/mod.rs | 37 +++- .../eql/crates/eql-bindings/src/v4/numeric.rs | 43 +++- .../eql/crates/eql-bindings/src/v4/real.rs | 43 +++- .../crates/eql-bindings/src/v4/smallint.rs | 43 +++- .../eql-bindings/src/v4/term_formats.rs | 40 ++++ .../eql/crates/eql-bindings/src/v4/text.rs | 63 ++++-- .../crates/eql-bindings/src/v4/timestamp.rs | 43 +++- .../eql/crates/eql-codegen/src/bindings.rs | 198 ++++++++++++++--- .../eql/crates/eql-codegen/src/context.rs | 43 ++++ .../eql/crates/eql-codegen/src/generate.rs | 25 ++- packages/eql/crates/eql-codegen/src/range.rs | 39 ++-- .../eql-codegen/templates/query_types.sql.j2 | 17 +- .../crates/eql-codegen/templates/types.sql.j2 | 12 + packages/eql/crates/eql-codegen/tests/cli.rs | 4 +- packages/eql/crates/eql-domains/src/lib.rs | 2 + packages/eql/crates/eql-domains/src/term.rs | 72 ++++++ packages/eql/crates/eql-domains/src/tests.rs | 19 ++ .../src/v3/scalars/bigint/bigint_types.sql | 14 ++ .../bigint/query_bigint_range_types.sql | 10 +- .../v3/scalars/bigint/query_bigint_types.sql | 14 ++ .../eql/src/v3/scalars/date/date_types.sql | 14 ++ .../scalars/date/query_date_range_types.sql | 10 +- .../src/v3/scalars/date/query_date_types.sql | 14 ++ .../src/v3/scalars/double/double_types.sql | 14 ++ .../double/query_double_range_types.sql | 10 +- .../v3/scalars/double/query_double_types.sql | 14 ++ .../src/v3/scalars/integer/integer_types.sql | 14 ++ .../integer/query_integer_range_types.sql | 10 +- .../scalars/integer/query_integer_types.sql | 14 ++ .../src/v3/scalars/numeric/numeric_types.sql | 14 ++ .../numeric/query_numeric_range_types.sql | 10 +- .../scalars/numeric/query_numeric_types.sql | 14 ++ .../scalars/real/query_real_range_types.sql | 10 +- .../src/v3/scalars/real/query_real_types.sql | 14 ++ .../eql/src/v3/scalars/real/real_types.sql | 14 ++ .../smallint/query_smallint_range_types.sql | 10 +- .../scalars/smallint/query_smallint_types.sql | 14 ++ .../v3/scalars/smallint/smallint_types.sql | 14 ++ .../scalars/text/query_text_range_types.sql | 10 +- .../src/v3/scalars/text/query_text_types.sql | 22 +- .../eql/src/v3/scalars/text/text_types.sql | 24 ++ .../timestamp/query_timestamp_range_types.sql | 10 +- .../timestamp/query_timestamp_types.sql | 14 ++ .../v3/scalars/timestamp/timestamp_types.sql | 14 ++ .../src/v3/sem/ore_block_256/functions.sql | 59 ++--- .../eql/src/v3/sem/ore_block_256/types.sql | 13 ++ .../tests/encryption/tests/v4_ore_block.rs | 206 +++++++++++++++--- .../eql/tests/encryption/tests/v4_types.rs | 13 +- 53 files changed, 1294 insertions(+), 369 deletions(-) create mode 100644 packages/eql/crates/eql-bindings/src/v4/term_formats.rs diff --git a/packages/eql/crates/eql-bindings/src/encryption.rs b/packages/eql/crates/eql-bindings/src/encryption.rs index 457ae3fb5..83a72338f 100644 --- a/packages/eql/crates/eql-bindings/src/encryption.rs +++ b/packages/eql/crates/eql-bindings/src/encryption.rs @@ -100,9 +100,7 @@ pub mod targets; use base64::{engine::general_purpose::STANDARD, Engine as _}; #[allow(deprecated)] use stack_encrypt::sem::MatchTerm; -use stack_encrypt::sem::{ - EqualityTerm, MatchConfig, MatchOptions, OpeTerm, Orderable, OreBlockTerm, TermError, Tokenizer, -}; +use stack_encrypt::sem::{EqualityTerm, MatchConfig, MatchOptions, TermError, Tokenizer}; use stack_encrypt::target::transcode::{Reader, Transcode, Visitor}; use stack_encrypt::target::{self, AeadContext, CallerContext, IndexSpec}; use stack_encrypt::{ @@ -112,7 +110,7 @@ use stack_encrypt::{ }; use crate::{ - v3::terms::{BloomFilter, Ciphertext, Hmac256, OpeCllw, OreBlock256}, + v3::terms::{BloomFilter, Ciphertext, Hmac256}, v4::terms::SearchBloomFilter, Identifier, }; @@ -450,97 +448,6 @@ impl DecryptField for SearchBloomFilter { } } -/// Builds EQL's hex OPE term from the native order-preserving term. -/// -/// The bytes are Stack Encrypt's CLLW OPE ciphertext of the value's -/// ADR-0002 order key, which `vitaminc-ore` encodes: an integer at its own -/// width, a float as the `f64` it widens to (`-0.0` ties `0.0`, every NaN is -/// one value after `+Infinity`), text in Unicode 16 NFC with case and accents -/// folded. EQL compares `op` as `bytea`, which is the lexicographic byte -/// order the scheme preserves, so `<` on the stored terms is `<` on the -/// order keys. Matching cipherstash-client's bytes is not a goal: -/// cipherstash-client never writes an `eql_v4` column. -/// -/// Text with no fold (a code point Unicode 16 does not assign) has no order -/// term and is refused, stored or queried. -#[doc(hidden)] -pub struct OpeVisitor; -impl Visitor for OpeVisitor { - type Value = OpeCllw; - fn ope(self, term: OpeTerm) -> Result { - Ok(OpeCllw(hex::encode(term.as_bytes()))) - } -} -impl Transcode for OpeCllw { - type Visitor = OpeVisitor; - fn visitor() -> Self::Visitor { - OpeVisitor - } -} -impl EncryptFrom for OpeCllw { - type Context = CallerContext; - fn encryption<'s, K: 'static>() -> Encryption<'s, S, Self, K, Self::Context> - where - S: 's, - { - target::ope().transcode() - } -} -impl Decryptable for OpeCllw { - const DECRYPTABLE: bool = false; -} -impl DecryptField for OpeCllw { - fn decryption_field(self, _: Ctx) -> Option> { - None - } -} - -/// Builds EQL v4's `ob` from the native block ORE term: one element, the -/// hex of the whole ore-rs ciphertext. -/// -/// The ciphertext is ore-rs's 6-bit block ORE of the value's ADR-0002 order -/// key (`OreBlockTerm`): the fixed-width scheme for integers and floats, the -/// chained one, padded to a power-of-two length bucket of at least 16 -/// bytes, for text. EQL v4 compares `ob` with the comparator for that -/// format, which takes exactly one element; EQL v3's `ob` holds 8-bit block -/// terms, up to six for text, that cipherstash-client writes. A query -/// carries the full ciphertext too: the comparator reads the right half of -/// the operand a btree index or `between` passes second. -/// -/// Text with no fold, text whose fold ends in NUL or is longer than 1023 -/// bytes has no block ORE term and is refused, stored or queried. -#[doc(hidden)] -pub struct OreBlockVisitor; -impl Visitor for OreBlockVisitor { - type Value = OreBlock256; - fn ore_block(self, term: OreBlockTerm) -> Result { - Ok(OreBlock256(vec![hex::encode(term.as_bytes())])) - } -} -impl Transcode for OreBlock256 { - type Visitor = OreBlockVisitor; - fn visitor() -> Self::Visitor { - OreBlockVisitor - } -} -impl EncryptFrom for OreBlock256 { - type Context = CallerContext; - fn encryption<'s, K: 'static>() -> Encryption<'s, S, Self, K, Self::Context> - where - S: 's, - { - target::ore_block().transcode() - } -} -impl Decryptable for OreBlock256 { - const DECRYPTABLE: bool = false; -} -impl DecryptField for OreBlock256 { - fn decryption_field(self, _: Ctx) -> Option> { - None - } -} - #[cfg(test)] mod tests { use stack_encrypt::{nonempty, Descriptor, Label}; diff --git a/packages/eql/crates/eql-bindings/src/v4/bigint.rs b/packages/eql/crates/eql-bindings/src/v4/bigint.rs index 49d1d961b..4a123b0ab 100644 --- a/packages/eql/crates/eql-bindings/src/v4/bigint.rs +++ b/packages/eql/crates/eql-bindings/src/v4/bigint.rs @@ -1,10 +1,11 @@ // @generated by eql-codegen from the eql-domains catalog — do not edit //! The `bigint` encrypted-domain family — generated from the eql-domains catalog. -use crate::v4::terms::{Ciphertext, Hmac256, OpeCllw, OreBlock256}; +use crate::v4::terms::{Ciphertext, Hmac256, OpeHex, OreBlockHex}; use crate::v4::{DomainType, SchemaVersion}; use crate::Identifier; use schemars::{schema_for, JsonSchema, Schema}; use serde::{Deserialize, Serialize}; +use stack_encrypt::sem::{OpeTerm, OreBlockTerm}; use ts_rs::TS; /// `public.eql_v4_bigint` — storage-only domain. /// @@ -129,7 +130,9 @@ pub struct BigintOrdOre { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for BigintOrdOre { fn sql_domain_static() -> &'static str { @@ -179,7 +182,9 @@ pub struct BigintOrd { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for BigintOrd { fn sql_domain_static() -> &'static str { @@ -229,7 +234,9 @@ pub struct BigintOrdOpe { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for BigintOrdOpe { fn sql_domain_static() -> &'static str { @@ -328,7 +335,9 @@ pub struct BigintOrdOreQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for BigintOrdOreQuery { fn sql_domain_static() -> &'static str { @@ -377,7 +386,9 @@ pub struct BigintOrdQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for BigintOrdQuery { fn sql_domain_static() -> &'static str { @@ -426,7 +437,9 @@ pub struct BigintOrdOpeQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for BigintOrdOpeQuery { fn sql_domain_static() -> &'static str { @@ -467,8 +480,12 @@ impl DomainType for BigintOrdOpeQuery { pub struct BigintRangeOreQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OreBlock256, - pub hi: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub lo: OreBlockTerm, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub hi: OreBlockTerm, } impl DomainType for BigintRangeOreQuery { fn sql_domain_static() -> &'static str { @@ -509,8 +526,12 @@ impl DomainType for BigintRangeOreQuery { pub struct BigintRangeOpeQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OpeCllw, - pub hi: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub lo: OpeTerm, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub hi: OpeTerm, } impl DomainType for BigintRangeOpeQuery { fn sql_domain_static() -> &'static str { diff --git a/packages/eql/crates/eql-bindings/src/v4/date.rs b/packages/eql/crates/eql-bindings/src/v4/date.rs index a47bf0ee0..a35417d9a 100644 --- a/packages/eql/crates/eql-bindings/src/v4/date.rs +++ b/packages/eql/crates/eql-bindings/src/v4/date.rs @@ -1,10 +1,11 @@ // @generated by eql-codegen from the eql-domains catalog — do not edit //! The `date` encrypted-domain family — generated from the eql-domains catalog. -use crate::v4::terms::{Ciphertext, Hmac256, OpeCllw, OreBlock256}; +use crate::v4::terms::{Ciphertext, Hmac256, OpeHex, OreBlockHex}; use crate::v4::{DomainType, SchemaVersion}; use crate::Identifier; use schemars::{schema_for, JsonSchema, Schema}; use serde::{Deserialize, Serialize}; +use stack_encrypt::sem::{OpeTerm, OreBlockTerm}; use ts_rs::TS; /// `public.eql_v4_date` — storage-only domain. /// @@ -129,7 +130,9 @@ pub struct DateOrdOre { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for DateOrdOre { fn sql_domain_static() -> &'static str { @@ -179,7 +182,9 @@ pub struct DateOrd { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for DateOrd { fn sql_domain_static() -> &'static str { @@ -229,7 +234,9 @@ pub struct DateOrdOpe { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for DateOrdOpe { fn sql_domain_static() -> &'static str { @@ -328,7 +335,9 @@ pub struct DateOrdOreQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for DateOrdOreQuery { fn sql_domain_static() -> &'static str { @@ -377,7 +386,9 @@ pub struct DateOrdQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for DateOrdQuery { fn sql_domain_static() -> &'static str { @@ -426,7 +437,9 @@ pub struct DateOrdOpeQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for DateOrdOpeQuery { fn sql_domain_static() -> &'static str { @@ -467,8 +480,12 @@ impl DomainType for DateOrdOpeQuery { pub struct DateRangeOreQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OreBlock256, - pub hi: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub lo: OreBlockTerm, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub hi: OreBlockTerm, } impl DomainType for DateRangeOreQuery { fn sql_domain_static() -> &'static str { @@ -509,8 +526,12 @@ impl DomainType for DateRangeOreQuery { pub struct DateRangeOpeQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OpeCllw, - pub hi: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub lo: OpeTerm, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub hi: OpeTerm, } impl DomainType for DateRangeOpeQuery { fn sql_domain_static() -> &'static str { diff --git a/packages/eql/crates/eql-bindings/src/v4/double.rs b/packages/eql/crates/eql-bindings/src/v4/double.rs index e5f10d4a8..ae77dd429 100644 --- a/packages/eql/crates/eql-bindings/src/v4/double.rs +++ b/packages/eql/crates/eql-bindings/src/v4/double.rs @@ -1,10 +1,11 @@ // @generated by eql-codegen from the eql-domains catalog — do not edit //! The `double` encrypted-domain family — generated from the eql-domains catalog. -use crate::v4::terms::{Ciphertext, Hmac256, OpeCllw, OreBlock256}; +use crate::v4::terms::{Ciphertext, Hmac256, OpeHex, OreBlockHex}; use crate::v4::{DomainType, SchemaVersion}; use crate::Identifier; use schemars::{schema_for, JsonSchema, Schema}; use serde::{Deserialize, Serialize}; +use stack_encrypt::sem::{OpeTerm, OreBlockTerm}; use ts_rs::TS; /// `public.eql_v4_double` — storage-only domain. /// @@ -129,7 +130,9 @@ pub struct DoubleOrdOre { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for DoubleOrdOre { fn sql_domain_static() -> &'static str { @@ -179,7 +182,9 @@ pub struct DoubleOrd { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for DoubleOrd { fn sql_domain_static() -> &'static str { @@ -229,7 +234,9 @@ pub struct DoubleOrdOpe { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for DoubleOrdOpe { fn sql_domain_static() -> &'static str { @@ -328,7 +335,9 @@ pub struct DoubleOrdOreQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for DoubleOrdOreQuery { fn sql_domain_static() -> &'static str { @@ -377,7 +386,9 @@ pub struct DoubleOrdQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for DoubleOrdQuery { fn sql_domain_static() -> &'static str { @@ -426,7 +437,9 @@ pub struct DoubleOrdOpeQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for DoubleOrdOpeQuery { fn sql_domain_static() -> &'static str { @@ -467,8 +480,12 @@ impl DomainType for DoubleOrdOpeQuery { pub struct DoubleRangeOreQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OreBlock256, - pub hi: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub lo: OreBlockTerm, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub hi: OreBlockTerm, } impl DomainType for DoubleRangeOreQuery { fn sql_domain_static() -> &'static str { @@ -509,8 +526,12 @@ impl DomainType for DoubleRangeOreQuery { pub struct DoubleRangeOpeQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OpeCllw, - pub hi: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub lo: OpeTerm, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub hi: OpeTerm, } impl DomainType for DoubleRangeOpeQuery { fn sql_domain_static() -> &'static str { diff --git a/packages/eql/crates/eql-bindings/src/v4/integer.rs b/packages/eql/crates/eql-bindings/src/v4/integer.rs index 9a4d49486..68873ed5a 100644 --- a/packages/eql/crates/eql-bindings/src/v4/integer.rs +++ b/packages/eql/crates/eql-bindings/src/v4/integer.rs @@ -1,10 +1,11 @@ // @generated by eql-codegen from the eql-domains catalog — do not edit //! The `integer` encrypted-domain family — generated from the eql-domains catalog. -use crate::v4::terms::{Ciphertext, Hmac256, OpeCllw, OreBlock256}; +use crate::v4::terms::{Ciphertext, Hmac256, OpeHex, OreBlockHex}; use crate::v4::{DomainType, SchemaVersion}; use crate::Identifier; use schemars::{schema_for, JsonSchema, Schema}; use serde::{Deserialize, Serialize}; +use stack_encrypt::sem::{OpeTerm, OreBlockTerm}; use ts_rs::TS; /// `public.eql_v4_integer` — storage-only domain. /// @@ -129,7 +130,9 @@ pub struct IntegerOrdOre { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for IntegerOrdOre { fn sql_domain_static() -> &'static str { @@ -179,7 +182,9 @@ pub struct IntegerOrd { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for IntegerOrd { fn sql_domain_static() -> &'static str { @@ -229,7 +234,9 @@ pub struct IntegerOrdOpe { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for IntegerOrdOpe { fn sql_domain_static() -> &'static str { @@ -328,7 +335,9 @@ pub struct IntegerOrdOreQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for IntegerOrdOreQuery { fn sql_domain_static() -> &'static str { @@ -377,7 +386,9 @@ pub struct IntegerOrdQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for IntegerOrdQuery { fn sql_domain_static() -> &'static str { @@ -426,7 +437,9 @@ pub struct IntegerOrdOpeQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for IntegerOrdOpeQuery { fn sql_domain_static() -> &'static str { @@ -467,8 +480,12 @@ impl DomainType for IntegerOrdOpeQuery { pub struct IntegerRangeOreQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OreBlock256, - pub hi: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub lo: OreBlockTerm, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub hi: OreBlockTerm, } impl DomainType for IntegerRangeOreQuery { fn sql_domain_static() -> &'static str { @@ -509,8 +526,12 @@ impl DomainType for IntegerRangeOreQuery { pub struct IntegerRangeOpeQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OpeCllw, - pub hi: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub lo: OpeTerm, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub hi: OpeTerm, } impl DomainType for IntegerRangeOpeQuery { fn sql_domain_static() -> &'static str { diff --git a/packages/eql/crates/eql-bindings/src/v4/mod.rs b/packages/eql/crates/eql-bindings/src/v4/mod.rs index 24955deb5..9dc950e60 100644 --- a/packages/eql/crates/eql-bindings/src/v4/mod.rs +++ b/packages/eql/crates/eql-bindings/src/v4/mod.rs @@ -20,13 +20,25 @@ //! operand of `eql_v4.between(column, range)`. EQL v3 has no range domains. //! //! v4 has no SteVec bindings, payload enums, TypeScript export or -//! published JSON Schema yet; those follow in `@cipherstash/eql` 4.0. The -//! term newtypes and the [`DomainType`] trait are shared with v3: a v4 -//! `Ciphertext` or `Hmac256` is v3's type, so a change to a term's validation -//! in `v3::terms` changes v4 too. The one v4 term v3 has no use for is -//! [`terms::SearchBloomFilter`], the stored filter of a search domain. What -//! separates the two on the wire is the envelope's [`SchemaVersion`], which -//! refuses the other generation's `v`. +//! published JSON Schema yet; those follow in `@cipherstash/eql` 4.0. +//! +//! The order terms are Stack Encrypt's own: an `ob` field is a +//! [`stack_encrypt::sem::OreBlockTerm`] and an `op` field a +//! [`stack_encrypt::sem::OpeTerm`], each over the type's plaintext +//! (`DateOrdOre::ob` is an `OreBlockTerm`). Each is one lowercase hex +//! string on the wire, the ore-rs or clww-ore ciphertext, and reading one +//! checks its structure, so an EQL v3 term, or a term of the other scheme, +//! is refused when the payload is read. [`terms::OreBlockHex`] and +//! [`terms::OpeHex`] describe that JSON to TypeScript and JSON Schema. The +//! domain CHECKs refuse a value that is not such a string. +//! +//! The other term newtypes and the [`DomainType`] trait are shared with v3: +//! a v4 `Ciphertext` or `Hmac256` is v3's type, so a change to its +//! validation in `v3::terms` changes v4 too. v3's order terms +//! (`OreBlock256`, `OpeCllw`) are not among them. The one v4 term v3 has no +//! use for is [`terms::SearchBloomFilter`], the stored filter of a search +//! domain. What separates the two on the wire is the envelope's +//! [`SchemaVersion`], which refuses the other generation's `v`. /// Generated: the `op::Answers` impls through which Stack Encrypt's typed /// query chain queries a producible type, each type's @@ -49,6 +61,7 @@ pub mod numeric; pub mod plaintext; pub mod real; pub mod smallint; +// Generated: the JSON of the order terms, re-exported from `terms`. /// Generated: the table of EQL types a Stack Encrypt data plan may name as /// a field target (`TARGETS`) and the by-name dispatch that runs a /// producible type's own plan. The descriptor type, the errors and the @@ -56,16 +69,20 @@ pub mod smallint; /// `targets` feature only. #[cfg(feature = "targets")] pub mod targets; +mod term_formats; pub mod text; pub mod timestamp; pub use crate::v3::domain_type::{DomainType, SQL_SCHEMA}; pub use inventory::{all, all_query, all_range_query}; -/// The wire-field newtypes of the v4 payloads: v3's, shared, and -/// [`SearchBloomFilter`](terms::SearchBloomFilter). +/// The wire-field types of the v4 payloads: v3's that are not order terms, +/// shared; [`SearchBloomFilter`](terms::SearchBloomFilter); and the JSON of +/// the order terms, [`OreBlockHex`](terms::OreBlockHex) and +/// [`OpeHex`](terms::OpeHex), whose fields hold Stack Encrypt's terms. pub mod terms { - pub use crate::v3::terms::*; + pub use super::term_formats::{OpeHex, OreBlockHex}; + pub use crate::v3::terms::{BloomFilter, Ciphertext, Hmac256}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; diff --git a/packages/eql/crates/eql-bindings/src/v4/numeric.rs b/packages/eql/crates/eql-bindings/src/v4/numeric.rs index 408ad6b1f..18a61159f 100644 --- a/packages/eql/crates/eql-bindings/src/v4/numeric.rs +++ b/packages/eql/crates/eql-bindings/src/v4/numeric.rs @@ -1,10 +1,11 @@ // @generated by eql-codegen from the eql-domains catalog — do not edit //! The `numeric` encrypted-domain family — generated from the eql-domains catalog. -use crate::v4::terms::{Ciphertext, Hmac256, OpeCllw, OreBlock256}; +use crate::v4::terms::{Ciphertext, Hmac256, OpeHex, OreBlockHex}; use crate::v4::{DomainType, SchemaVersion}; use crate::Identifier; use schemars::{schema_for, JsonSchema, Schema}; use serde::{Deserialize, Serialize}; +use stack_encrypt::sem::{OpeTerm, OreBlockTerm}; use ts_rs::TS; /// `public.eql_v4_numeric` — storage-only domain. /// @@ -129,7 +130,9 @@ pub struct NumericOrdOre { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for NumericOrdOre { fn sql_domain_static() -> &'static str { @@ -179,7 +182,9 @@ pub struct NumericOrd { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for NumericOrd { fn sql_domain_static() -> &'static str { @@ -229,7 +234,9 @@ pub struct NumericOrdOpe { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for NumericOrdOpe { fn sql_domain_static() -> &'static str { @@ -328,7 +335,9 @@ pub struct NumericOrdOreQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for NumericOrdOreQuery { fn sql_domain_static() -> &'static str { @@ -377,7 +386,9 @@ pub struct NumericOrdQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for NumericOrdQuery { fn sql_domain_static() -> &'static str { @@ -426,7 +437,9 @@ pub struct NumericOrdOpeQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for NumericOrdOpeQuery { fn sql_domain_static() -> &'static str { @@ -467,8 +480,12 @@ impl DomainType for NumericOrdOpeQuery { pub struct NumericRangeOreQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OreBlock256, - pub hi: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub lo: OreBlockTerm, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub hi: OreBlockTerm, } impl DomainType for NumericRangeOreQuery { fn sql_domain_static() -> &'static str { @@ -509,8 +526,12 @@ impl DomainType for NumericRangeOreQuery { pub struct NumericRangeOpeQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OpeCllw, - pub hi: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub lo: OpeTerm, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub hi: OpeTerm, } impl DomainType for NumericRangeOpeQuery { fn sql_domain_static() -> &'static str { diff --git a/packages/eql/crates/eql-bindings/src/v4/real.rs b/packages/eql/crates/eql-bindings/src/v4/real.rs index 8f668c900..3438e79c1 100644 --- a/packages/eql/crates/eql-bindings/src/v4/real.rs +++ b/packages/eql/crates/eql-bindings/src/v4/real.rs @@ -1,10 +1,11 @@ // @generated by eql-codegen from the eql-domains catalog — do not edit //! The `real` encrypted-domain family — generated from the eql-domains catalog. -use crate::v4::terms::{Ciphertext, Hmac256, OpeCllw, OreBlock256}; +use crate::v4::terms::{Ciphertext, Hmac256, OpeHex, OreBlockHex}; use crate::v4::{DomainType, SchemaVersion}; use crate::Identifier; use schemars::{schema_for, JsonSchema, Schema}; use serde::{Deserialize, Serialize}; +use stack_encrypt::sem::{OpeTerm, OreBlockTerm}; use ts_rs::TS; /// `public.eql_v4_real` — storage-only domain. /// @@ -129,7 +130,9 @@ pub struct RealOrdOre { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for RealOrdOre { fn sql_domain_static() -> &'static str { @@ -179,7 +182,9 @@ pub struct RealOrd { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for RealOrd { fn sql_domain_static() -> &'static str { @@ -229,7 +234,9 @@ pub struct RealOrdOpe { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for RealOrdOpe { fn sql_domain_static() -> &'static str { @@ -328,7 +335,9 @@ pub struct RealOrdOreQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for RealOrdOreQuery { fn sql_domain_static() -> &'static str { @@ -377,7 +386,9 @@ pub struct RealOrdQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for RealOrdQuery { fn sql_domain_static() -> &'static str { @@ -426,7 +437,9 @@ pub struct RealOrdOpeQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for RealOrdOpeQuery { fn sql_domain_static() -> &'static str { @@ -467,8 +480,12 @@ impl DomainType for RealOrdOpeQuery { pub struct RealRangeOreQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OreBlock256, - pub hi: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub lo: OreBlockTerm, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub hi: OreBlockTerm, } impl DomainType for RealRangeOreQuery { fn sql_domain_static() -> &'static str { @@ -509,8 +526,12 @@ impl DomainType for RealRangeOreQuery { pub struct RealRangeOpeQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OpeCllw, - pub hi: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub lo: OpeTerm, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub hi: OpeTerm, } impl DomainType for RealRangeOpeQuery { fn sql_domain_static() -> &'static str { diff --git a/packages/eql/crates/eql-bindings/src/v4/smallint.rs b/packages/eql/crates/eql-bindings/src/v4/smallint.rs index 6fe3c3ef8..880fc78d8 100644 --- a/packages/eql/crates/eql-bindings/src/v4/smallint.rs +++ b/packages/eql/crates/eql-bindings/src/v4/smallint.rs @@ -1,10 +1,11 @@ // @generated by eql-codegen from the eql-domains catalog — do not edit //! The `smallint` encrypted-domain family — generated from the eql-domains catalog. -use crate::v4::terms::{Ciphertext, Hmac256, OpeCllw, OreBlock256}; +use crate::v4::terms::{Ciphertext, Hmac256, OpeHex, OreBlockHex}; use crate::v4::{DomainType, SchemaVersion}; use crate::Identifier; use schemars::{schema_for, JsonSchema, Schema}; use serde::{Deserialize, Serialize}; +use stack_encrypt::sem::{OpeTerm, OreBlockTerm}; use ts_rs::TS; /// `public.eql_v4_smallint` — storage-only domain. /// @@ -129,7 +130,9 @@ pub struct SmallintOrdOre { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for SmallintOrdOre { fn sql_domain_static() -> &'static str { @@ -179,7 +182,9 @@ pub struct SmallintOrd { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for SmallintOrd { fn sql_domain_static() -> &'static str { @@ -229,7 +234,9 @@ pub struct SmallintOrdOpe { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for SmallintOrdOpe { fn sql_domain_static() -> &'static str { @@ -328,7 +335,9 @@ pub struct SmallintOrdOreQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for SmallintOrdOreQuery { fn sql_domain_static() -> &'static str { @@ -377,7 +386,9 @@ pub struct SmallintOrdQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for SmallintOrdQuery { fn sql_domain_static() -> &'static str { @@ -426,7 +437,9 @@ pub struct SmallintOrdOpeQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for SmallintOrdOpeQuery { fn sql_domain_static() -> &'static str { @@ -467,8 +480,12 @@ impl DomainType for SmallintOrdOpeQuery { pub struct SmallintRangeOreQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OreBlock256, - pub hi: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub lo: OreBlockTerm, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub hi: OreBlockTerm, } impl DomainType for SmallintRangeOreQuery { fn sql_domain_static() -> &'static str { @@ -509,8 +526,12 @@ impl DomainType for SmallintRangeOreQuery { pub struct SmallintRangeOpeQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OpeCllw, - pub hi: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub lo: OpeTerm, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub hi: OpeTerm, } impl DomainType for SmallintRangeOpeQuery { fn sql_domain_static() -> &'static str { diff --git a/packages/eql/crates/eql-bindings/src/v4/term_formats.rs b/packages/eql/crates/eql-bindings/src/v4/term_formats.rs new file mode 100644 index 000000000..1c985f5d0 --- /dev/null +++ b/packages/eql/crates/eql-bindings/src/v4/term_formats.rs @@ -0,0 +1,40 @@ +// @generated by eql-codegen from the eql-domains catalog — do not edit +//! Generated: the JSON of the terms EQL v4 stores in its own format, +//! for TypeScript and JSON Schema. See `eql_domains::V4TermFormat`. +use schemars::{json_schema, JsonSchema, Schema, SchemaGenerator}; +use std::borrow::Cow; +use ts_rs::TS; +/// The JSON of an EQL v4 `ob` term. An ore-rs 6-bit block ORE ciphertext (wire version 2), as lowercase hex. +/// +/// Names the JSON of a [`stack_encrypt::sem::OreBlockTerm`] field for TypeScript and JSON Schema, and holds no term itself. Its pattern, `^(0202|0203)([0-9a-f]{2})+$`, takes the prefixes the domain CHECKs require. +#[derive(Clone, Debug, TS)] +pub struct OreBlockHex(pub String); +impl JsonSchema for OreBlockHex { + fn schema_name() -> Cow<'static, str> { + "OreBlockHex".into() + } + fn json_schema(_: &mut SchemaGenerator) -> Schema { + json_schema!( + { "type" : "string", "pattern" : "^(0202|0203)([0-9a-f]{2})+$", "description" + : "An ore-rs 6-bit block ORE ciphertext (wire version 2), as lowercase hex.", + } + ) + } +} +/// The JSON of an EQL v4 `op` term. A clww-ore CLWW OPE ciphertext, as lowercase hex: a carry byte, then 8 bytes per byte of the value. +/// +/// Names the JSON of a [`stack_encrypt::sem::OpeTerm`] field for TypeScript and JSON Schema, and holds no term itself. Its pattern, `^(00|01)([0-9a-f]{16})+$`, takes the prefixes the domain CHECKs require. +#[derive(Clone, Debug, TS)] +pub struct OpeHex(pub String); +impl JsonSchema for OpeHex { + fn schema_name() -> Cow<'static, str> { + "OpeHex".into() + } + fn json_schema(_: &mut SchemaGenerator) -> Schema { + json_schema!( + { "type" : "string", "pattern" : "^(00|01)([0-9a-f]{16})+$", "description" : + "A clww-ore CLWW OPE ciphertext, as lowercase hex: a carry byte, then 8 bytes per byte of the value.", + } + ) + } +} diff --git a/packages/eql/crates/eql-bindings/src/v4/text.rs b/packages/eql/crates/eql-bindings/src/v4/text.rs index 3314d75ad..121ea00ce 100644 --- a/packages/eql/crates/eql-bindings/src/v4/text.rs +++ b/packages/eql/crates/eql-bindings/src/v4/text.rs @@ -1,10 +1,11 @@ // @generated by eql-codegen from the eql-domains catalog — do not edit //! The `text` encrypted-domain family — generated from the eql-domains catalog. -use crate::v4::terms::{BloomFilter, Ciphertext, Hmac256, OpeCllw, OreBlock256, SearchBloomFilter}; +use crate::v4::terms::{BloomFilter, Ciphertext, Hmac256, OpeHex, OreBlockHex, SearchBloomFilter}; use crate::v4::{DomainType, SchemaVersion}; use crate::Identifier; use schemars::{schema_for, JsonSchema, Schema}; use serde::{Deserialize, Serialize}; +use stack_encrypt::sem::{OpeTerm, OreBlockTerm}; use ts_rs::TS; /// `public.eql_v4_text` — storage-only domain. /// @@ -180,7 +181,9 @@ pub struct TextOrdOre { pub i: Identifier, pub c: Ciphertext, pub hm: Hmac256, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for TextOrdOre { fn sql_domain_static() -> &'static str { @@ -231,7 +234,9 @@ pub struct TextOrd { pub i: Identifier, pub c: Ciphertext, pub hm: Hmac256, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for TextOrd { fn sql_domain_static() -> &'static str { @@ -282,7 +287,9 @@ pub struct TextOrdOpe { pub i: Identifier, pub c: Ciphertext, pub hm: Hmac256, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for TextOrdOpe { fn sql_domain_static() -> &'static str { @@ -333,7 +340,9 @@ pub struct TextSearchOre { pub i: Identifier, pub c: Ciphertext, pub hm: Hmac256, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, pub bf: SearchBloomFilter, } impl DomainType for TextSearchOre { @@ -385,7 +394,9 @@ pub struct TextSearch { pub i: Identifier, pub c: Ciphertext, pub hm: Hmac256, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, pub bf: SearchBloomFilter, } impl DomainType for TextSearch { @@ -535,7 +546,9 @@ pub struct TextOrdOreQuery { #[stash(context_field)] pub i: Identifier, pub hm: Hmac256, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for TextOrdOreQuery { fn sql_domain_static() -> &'static str { @@ -585,7 +598,9 @@ pub struct TextOrdQuery { #[stash(context_field)] pub i: Identifier, pub hm: Hmac256, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for TextOrdQuery { fn sql_domain_static() -> &'static str { @@ -635,7 +650,9 @@ pub struct TextOrdOpeQuery { #[stash(context_field)] pub i: Identifier, pub hm: Hmac256, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for TextOrdOpeQuery { fn sql_domain_static() -> &'static str { @@ -681,7 +698,9 @@ pub struct TextSearchOreQuery { #[serde(default, skip_serializing_if = "Option::is_none")] pub hm: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub ob: Option, + #[ts(as = "Option")] + #[schemars(with = "Option")] + pub ob: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub bf: Option, } @@ -748,7 +767,7 @@ pub struct TextSearchOreOrderQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub ob: OreBlock256, + pub ob: OreBlockTerm, } impl From for TextSearchOreQuery { fn from(query: TextSearchOreOrderQuery) -> Self { @@ -804,7 +823,9 @@ pub struct TextSearchQuery { #[serde(default, skip_serializing_if = "Option::is_none")] pub hm: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub op: Option, + #[ts(as = "Option")] + #[schemars(with = "Option")] + pub op: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub bf: Option, } @@ -871,7 +892,7 @@ pub struct TextSearchOrderQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + pub op: OpeTerm, } impl From for TextSearchQuery { fn from(query: TextSearchOrderQuery) -> Self { @@ -922,8 +943,12 @@ impl From for TextSearchQuery { pub struct TextRangeOreQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OreBlock256, - pub hi: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub lo: OreBlockTerm, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub hi: OreBlockTerm, } impl DomainType for TextRangeOreQuery { fn sql_domain_static() -> &'static str { @@ -964,8 +989,12 @@ impl DomainType for TextRangeOreQuery { pub struct TextRangeOpeQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OpeCllw, - pub hi: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub lo: OpeTerm, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub hi: OpeTerm, } impl DomainType for TextRangeOpeQuery { fn sql_domain_static() -> &'static str { diff --git a/packages/eql/crates/eql-bindings/src/v4/timestamp.rs b/packages/eql/crates/eql-bindings/src/v4/timestamp.rs index 3f335a38a..422aca9e9 100644 --- a/packages/eql/crates/eql-bindings/src/v4/timestamp.rs +++ b/packages/eql/crates/eql-bindings/src/v4/timestamp.rs @@ -1,10 +1,11 @@ // @generated by eql-codegen from the eql-domains catalog — do not edit //! The `timestamp` encrypted-domain family — generated from the eql-domains catalog. -use crate::v4::terms::{Ciphertext, Hmac256, OpeCllw, OreBlock256}; +use crate::v4::terms::{Ciphertext, Hmac256, OpeHex, OreBlockHex}; use crate::v4::{DomainType, SchemaVersion}; use crate::Identifier; use schemars::{schema_for, JsonSchema, Schema}; use serde::{Deserialize, Serialize}; +use stack_encrypt::sem::{OpeTerm, OreBlockTerm}; use ts_rs::TS; /// `public.eql_v4_timestamp` — storage-only domain. /// @@ -129,7 +130,9 @@ pub struct TimestampOrdOre { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for TimestampOrdOre { fn sql_domain_static() -> &'static str { @@ -179,7 +182,9 @@ pub struct TimestampOrd { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for TimestampOrd { fn sql_domain_static() -> &'static str { @@ -229,7 +234,9 @@ pub struct TimestampOrdOpe { #[stash(context_field)] pub i: Identifier, pub c: Ciphertext, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for TimestampOrdOpe { fn sql_domain_static() -> &'static str { @@ -328,7 +335,9 @@ pub struct TimestampOrdOreQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub ob: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub ob: OreBlockTerm, } impl DomainType for TimestampOrdOreQuery { fn sql_domain_static() -> &'static str { @@ -377,7 +386,9 @@ pub struct TimestampOrdQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for TimestampOrdQuery { fn sql_domain_static() -> &'static str { @@ -426,7 +437,9 @@ pub struct TimestampOrdOpeQuery { pub v: SchemaVersion, #[stash(context_field)] pub i: Identifier, - pub op: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub op: OpeTerm, } impl DomainType for TimestampOrdOpeQuery { fn sql_domain_static() -> &'static str { @@ -467,8 +480,12 @@ impl DomainType for TimestampOrdOpeQuery { pub struct TimestampRangeOreQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OreBlock256, - pub hi: OreBlock256, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub lo: OreBlockTerm, + #[ts(as = "OreBlockHex")] + #[schemars(with = "OreBlockHex")] + pub hi: OreBlockTerm, } impl DomainType for TimestampRangeOreQuery { fn sql_domain_static() -> &'static str { @@ -509,8 +526,12 @@ impl DomainType for TimestampRangeOreQuery { pub struct TimestampRangeOpeQuery { pub v: SchemaVersion, pub i: Identifier, - pub lo: OpeCllw, - pub hi: OpeCllw, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub lo: OpeTerm, + #[ts(as = "OpeHex")] + #[schemars(with = "OpeHex")] + pub hi: OpeTerm, } impl DomainType for TimestampRangeOpeQuery { fn sql_domain_static() -> &'static str { diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index b1cfa1e5b..add9714c8 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -199,8 +199,9 @@ fn render_struct(eql: EqlVersion, family: &DomainFamily, domain: &Domain) -> Tok fields.extend(quote! { pub c: Ciphertext, }); for term in Term::payload_terms(domain.terms) { let fid = format_ident!("{}", term.json_key()); - let tid = format_ident!("{}", stored_term_newtype(eql, domain, term)); - fields.extend(quote! { pub #fid: #tid, }); + let ty = term_type(eql, family, term, stored_term_newtype(eql, domain, term)); + let attrs = term_field_attrs(eql, term, false); + fields.extend(quote! { #attrs pub #fid: #ty, }); } // The domain's required term keys, threaded through the trait so @@ -309,14 +310,16 @@ fn render_query_struct(eql: EqlVersion, family: &DomainFamily, domain: &Domain) fields.extend(quote! { #context pub i: Identifier, }); for term in Term::payload_terms(domain.terms) { let fid = format_ident!("{}", term.json_key()); - let tid = format_ident!("{}", term.binding_newtype()); + let ty = term_type(eql, family, term, term.binding_newtype()); + let attrs = term_field_attrs(eql, term, optional); fields.extend(if optional { quote! { #[serde(default, skip_serializing_if = "Option::is_none")] - pub #fid: Option<#tid>, + #attrs + pub #fid: Option<#ty>, } } else { - quote! { pub #fid: #tid, } + quote! { #attrs pub #fid: #ty, } }); } let term_keys = Term::term_json_keys(domain.terms); @@ -367,8 +370,9 @@ fn render_query_struct(eql: EqlVersion, family: &DomainFamily, domain: &Domain) /// carries a range, so [`render_family_bindings`] emits these for v4 alone. /// /// The fields are the envelope's `v` and `i`, then the bounds under -/// [`RANGE_BOUND_KEYS`], each the term newtype the family's single-bound query -/// struct for the same term holds (`OreBlock256` for `ob`, `OpeCllw` for `op`): +/// [`RANGE_BOUND_KEYS`], each the term type the family's single-bound query +/// struct for the same term holds (`OreBlockTerm` for an integer's `ob`, +/// `OpeTerm` for its `op`, [`term_type`]): /// the single-bound domain's `ob` or `op` value, in a `{v, i, lo, hi}` /// envelope. No struct derives Stack Encrypt's traits yet: the ordered types /// it ranges over are not producible, and the engine does not build the @@ -397,7 +401,8 @@ fn render_range_query_struct(family: &DomainFamily, term: Term) -> TokenStream { {columns}. Required keys: {keys_str}.", prefix = eql.prefix() ); - let tid = format_ident!("{}", term.binding_newtype()); + let ty = term_type(eql, family, term, term.binding_newtype()); + let attrs = term_field_attrs(eql, term, false); let [lo, hi] = RANGE_BOUND_KEYS.map(|k| format_ident!("{k}")); let bound_keys = RANGE_BOUND_KEYS; @@ -411,8 +416,10 @@ fn render_range_query_struct(family: &DomainFamily, term: Term) -> TokenStream { pub struct #ident { pub v: SchemaVersion, pub i: Identifier, - pub #lo: #tid, - pub #hi: #tid, + #attrs + pub #lo: #ty, + #attrs + pub #hi: #ty, } impl DomainType for #ident { @@ -455,19 +462,85 @@ pub fn stored_term_newtype(eql: EqlVersion, domain: &Domain, term: Term) -> &'st } } -/// The term newtypes a family module names: each stored struct's -/// ([`stored_term_newtype`]) and each query struct's -/// ([`Term::binding_newtype`]), after `Ciphertext`, in first-use order. -fn family_terms<'d>( +/// The Rust type of a struct field carrying `term`, in a struct of +/// `family`. An EQL v4 order term is Stack Encrypt's own term over the +/// family's plaintext, as [`Term::v4_format`] names it (`OreBlockTerm`, +/// `OpeTerm`): typed by its scheme and by what it orders, and checked +/// for its structure when it is read. Any other term is `newtype`, the +/// shared binding newtype the caller picked. +pub fn term_type( eql: EqlVersion, - domains: impl Iterator, -) -> Vec<&'static str> { - let mut used: Vec<&'static str> = vec!["Ciphertext"]; + family: &DomainFamily, + term: Term, + newtype: &'static str, +) -> TokenStream { + match (eql, term.v4_format()) { + (EqlVersion::V4, Some(format)) => { + let ty = format_ident!("{}", format.rust_type); + let (_, rust) = plaintext_of(family.name) + .expect("a family with an order term names its Rust plaintext"); + let plaintext: syn::Type = + syn::parse_str(rust).expect("a family's Rust plaintext is a type path"); + quote!(#ty<#plaintext>) + } + _ => { + let ty = format_ident!("{newtype}"); + quote!(#ty) + } + } +} + +/// The attributes a field of [`term_type`] needs: none for a binding +/// newtype, and for a Stack Encrypt term, its JSON shape for TypeScript and +/// JSON Schema (`OreBlockHex`, `OpeHex`), which describe the hex string the +/// term serializes as. `optional` for a field that holds an `Option`. +fn term_field_attrs(eql: EqlVersion, term: Term, optional: bool) -> TokenStream { + match (eql, term.v4_format()) { + (EqlVersion::V4, Some(format)) => { + let shape = if optional { + format!("Option<{}>", format.json_shape) + } else { + format.json_shape.to_string() + }; + quote! { + #[ts(as = #shape)] + #[schemars(with = #shape)] + } + } + _ => TokenStream::new(), + } +} + +/// What a family module imports for its terms, in first-use order: the +/// names it takes from its version's `terms` module, `Ciphertext` first +/// (each stored struct's newtype, [`stored_term_newtype`], each query +/// struct's, [`Term::binding_newtype`], and a v4 order term's JSON shape), +/// and the Stack Encrypt term types it names (`OreBlockTerm`, `OpeTerm`). +struct FamilyTerms { + terms: Vec<&'static str>, + sem: Vec<&'static str>, +} + +fn family_terms<'d>(eql: EqlVersion, domains: impl Iterator) -> FamilyTerms { + fn add(list: &mut Vec<&'static str>, name: &'static str) { + if !list.contains(&name) { + list.push(name); + } + } + let mut used = FamilyTerms { + terms: vec!["Ciphertext"], + sem: Vec::new(), + }; for d in domains { for term in Term::payload_terms(d.terms) { - for t in [stored_term_newtype(eql, d, term), term.binding_newtype()] { - if !used.contains(&t) { - used.push(t); + match (eql, term.v4_format()) { + (EqlVersion::V4, Some(format)) => { + add(&mut used.terms, format.json_shape); + add(&mut used.sem, format.rust_type); + } + _ => { + add(&mut used.terms, stored_term_newtype(eql, d, term)); + add(&mut used.terms, term.binding_newtype()); } } } @@ -518,7 +591,7 @@ fn render_query_projections( .map(|&term| { let ident = format_ident!("{}", query_projection_ident(family, domain, term)); let fid = format_ident!("{}", term.json_key()); - let tid = format_ident!("{}", term.binding_newtype()); + let tid = term_type(eql, family, term, term.binding_newtype()); let ops = Term::operators_for_terms(domain.terms) .into_iter() .filter(|op| Term::extractor_for_operator(domain.terms, op) == Some(term.extractor())) @@ -836,13 +909,18 @@ fn ts_export(version: EqlVersion) -> TokenStream { } } -/// The `use` header every family module shares: the term newtypes the -/// family names, the trait, and the version's own `SchemaVersion`. -fn family_imports(version: EqlVersion, used: &[&'static str]) -> TokenStream { - let used_idents: Vec<_> = used.iter().map(|t| format_ident!("{t}")).collect(); +/// The `use` header every family module shares: the term types the family +/// names, the trait, and the version's own `SchemaVersion`. +fn family_imports(version: EqlVersion, used: &FamilyTerms) -> TokenStream { + let used_idents: Vec<_> = used.terms.iter().map(|t| format_ident!("{t}")).collect(); + let sem = (!used.sem.is_empty()).then(|| { + let idents = used.sem.iter().map(|t| format_ident!("{t}")); + quote!(use stack_encrypt::sem::{ #(#idents),* };) + }); let module = format_ident!("{}", module_name(version)); quote! { use schemars::{schema_for, JsonSchema, Schema}; + #sem use crate::#module::terms::{ #(#used_idents),* }; use crate::#module::{DomainType, SchemaVersion}; @@ -1425,9 +1503,71 @@ fn render_v4_bindings(dir: &Path) -> Vec<(PathBuf, String)> { rendered.push((dir.join("targets.rs"), crate::targets::render_targets_rs())); rendered.push((dir.join("answers.rs"), crate::targets::render_answers_rs())); rendered.push((dir.join("inventory.rs"), render_v4_inventory_rs())); + rendered.push((dir.join("term_formats.rs"), render_v4_term_formats_rs())); rendered } +/// Render the generated `crates/eql-bindings/src/v4/term_formats.rs`: for +/// each term EQL v4 stores in its own format ([`Term::v4_format`]), the type +/// that names that term's JSON for TypeScript and JSON Schema (`OreBlockHex`, +/// `OpeHex`). A field holds Stack Encrypt's term; these say what its JSON +/// is, with the catalog's pattern, the one the domain CHECKs are made from. +pub fn render_v4_term_formats_rs() -> String { + let every = [Term::Hm, Term::Ore, Term::Bloom, Term::Ope]; + let shapes: TokenStream = every + .into_iter() + .filter_map(|term| term.v4_format().map(|format| (term, format))) + .map(|(term, format)| { + let ident = format_ident!("{}", format.json_shape); + let name = format.json_shape; + let pattern = format.pattern(); + let description = format.description; + let summary = format!( + " The JSON of an EQL v4 `{}` term. {description}", + term.json_key() + ); + let detail = format!( + " Names the JSON of a [`stack_encrypt::sem::{}`] field for TypeScript and JSON \ + Schema, and holds no term itself. Its pattern, `{pattern}`, takes the prefixes \ + the domain CHECKs require.", + format.rust_type + ); + quote! { + #[doc = #summary] + #[doc = ""] + #[doc = #detail] + #[derive(Clone, Debug, TS)] + pub struct #ident(pub String); + + impl JsonSchema for #ident { + fn schema_name() -> Cow<'static, str> { + #name.into() + } + + fn json_schema(_: &mut SchemaGenerator) -> Schema { + json_schema!({ + "type": "string", + "pattern": #pattern, + "description": #description, + }) + } + } + } + }) + .collect(); + format_rs(quote! { + //! Generated: the JSON of the terms EQL v4 stores in its own format, + //! for TypeScript and JSON Schema. See `eql_domains::V4TermFormat`. + + use std::borrow::Cow; + + use schemars::{json_schema, JsonSchema, Schema, SchemaGenerator}; + use ts_rs::TS; + + #shapes + }) +} + /// Regenerate every committed Rust binding file under `out_root`: one /// `.rs` per catalog family plus the `inventory.rs` `all()` list. /// Hand-written `terms.rs` / `domain_type.rs` / `mod.rs` carry no marker, so @@ -1754,10 +1894,14 @@ mod tests { let v4 = tmp.path().join("crates/eql-bindings/src/v4"); // v3: scalar families + json_storage + payload + query_payload + // inventory. v4: every family with a scalar domain + targets + - // inventory. + // answers + inventory + term_formats. let families = eql_domains::scalar_families().count(); let v4_families = eql_domains::families_with_scalar_domains().count(); - assert_eq!(written.len(), (families + 4) + (v4_families + 3)); + assert_eq!(written.len(), (families + 4) + (v4_families + 4)); + assert!(tmp + .path() + .join("crates/eql-bindings/src/v4/term_formats.rs") + .is_file()); assert!(dir.join("integer.rs").is_file()); assert!(dir.join("text.rs").is_file()); assert!(dir.join("json_storage.rs").is_file()); diff --git a/packages/eql/crates/eql-codegen/src/context.rs b/packages/eql/crates/eql-codegen/src/context.rs index 67c820214..e0a87dc90 100644 --- a/packages/eql/crates/eql-codegen/src/context.rs +++ b/packages/eql/crates/eql-codegen/src/context.rs @@ -161,6 +161,10 @@ pub struct DomainBlock { // its own version block, with a present `nonempty_array_keys` key still // a non-empty array. pub optional_term_keys: Vec, + // The terms EQL v4 stores as hex strings (`Term::v4_format`), each with + // the prefixes its CHECK requires. Rendered in a v4 block; the v3 block + // keeps `nonempty_array_keys`. + pub v4_hex_terms: Vec, // sql_str-escaped one-line human description rendered as `COMMENT ON DOMAIN`. // Capability text is derived from the domain's terms (via // `Term::operators_for_terms`), so it can't drift from the generated CHECK / @@ -169,6 +173,43 @@ pub struct DomainBlock { pub comment: String, } +/// The CHECK of one term EQL v4 stores as a hex string: the value is a JSON +/// string whose first `prefix_len` characters are one of `prefixes`. Two +/// constant-time string operations on top of the key lookup, so the cost of +/// a write does not grow with the term; the comparator checks the rest. +#[derive(serde::Serialize)] +pub struct HexTermCheck { + pub key: String, // sql_str-escaped JSON key: `ob`, `op` + pub prefix_len: usize, + pub prefixes: String, // SQL list of literals: `'0202', '0203'` +} + +/// The [`HexTermCheck`]s of `terms`, in key order. +pub fn v4_hex_terms(terms: &[Term]) -> Vec { + Term::payload_terms(terms) + .into_iter() + .filter_map(|term| term.v4_format().map(|format| (term, format))) + .map(|(term, format)| { + let prefix_len = format.prefixes[0].len(); + assert!( + format.prefixes.iter().all(|p| p.len() == prefix_len), + "the prefixes of `{}` have one length", + term.json_key() + ); + HexTermCheck { + key: sql_str(term.json_key()), + prefix_len, + prefixes: format + .prefixes + .iter() + .map(|p| format!("'{}'", sql_str(p))) + .collect::>() + .join(", "), + } + }) + .collect() +} + /// Concise capability phrase from a domain's operator set — `equality`, /// `ordering`, `containment` (joined), or `storage only` when term-less. Derived /// from `operators_for_terms` so it tracks the generated surface; kept short so @@ -243,6 +284,7 @@ pub fn domain_block(family_name: &str, domain: &Domain) -> DomainBlock { forbidden_keys: vec![], // A stored value carries every term in both versions. optional_term_keys: vec![], + v4_hex_terms: v4_hex_terms(domain.terms), comment: scalar_domain_comment(family_name, domain), } } @@ -278,6 +320,7 @@ pub fn query_domain_block(family_name: &str, domain: &Domain) -> DomainBlock { .collect(), forbidden_keys: vec![sql_str("c")], optional_term_keys, + v4_hex_terms: v4_hex_terms(domain.terms), comment: query_domain_comment(family_name, domain), } } diff --git a/packages/eql/crates/eql-codegen/src/generate.rs b/packages/eql/crates/eql-codegen/src/generate.rs index ce35b1cb7..30ea4fc2a 100644 --- a/packages/eql/crates/eql-codegen/src/generate.rs +++ b/packages/eql/crates/eql-codegen/src/generate.rs @@ -2574,7 +2574,8 @@ mod tests { search4.contains("AND VALUE ? 'v'") && search4.contains("NOT (VALUE ? 'c')"), "{search4}" ); - // The ORE twin: a present `ob` is still a non-empty array. + // The ORE twin: a present `ob` is a non-empty array in v3, and in v4 + // a string with a block ORE prefix. let ore3 = domain_check(&v3, "eql_v3", "query_text_search_ore"); assert!(ore3.contains("AND VALUE ? 'ob'"), "{ore3}"); assert!( @@ -2584,12 +2585,26 @@ mod tests { let ore4 = domain_check(&v4, "eql_v4", "query_text_search_ore"); assert!( ore4.contains("AND (VALUE ? 'hm' OR VALUE ? 'ob' OR VALUE ? 'bf')") - && ore4 - .contains("AND (NOT (VALUE ? 'ob') OR (jsonb_typeof(VALUE -> 'ob') = 'array'"), + && ore4.contains( + "AND (NOT (VALUE ? 'ob') OR (jsonb_typeof(VALUE -> 'ob') = 'string'\n \ + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203')))" + ), "{ore4}" ); - // Every other query operand is the same in both versions. - for other in ["query_text_eq", "query_text_match", "query_text_ord"] { + assert!(!ore4.contains("'array'"), "{ore4}"); + // An order term is a string with its scheme's prefix in v4 only. + let ord3 = domain_check(&v3, "eql_v3", "query_text_ord"); + let ord4 = domain_check(&v4, "eql_v4", "query_text_ord"); + assert!(!ord3.contains("left(VALUE"), "{ord3}"); + assert!( + ord4.contains( + "AND jsonb_typeof(VALUE -> 'op') = 'string'\n \ + AND left(VALUE ->> 'op', 2) IN ('00', '01')" + ), + "{ord4}" + ); + // Every query operand without an order term is the same in both versions. + for other in ["query_text_eq", "query_text_match"] { assert_eq!( domain_check(&v3, "eql_v3", other) .replace("eql_v3", "eql_vN") diff --git a/packages/eql/crates/eql-codegen/src/range.rs b/packages/eql/crates/eql-codegen/src/range.rs index a231c7c9e..62fcd9aa3 100644 --- a/packages/eql/crates/eql-codegen/src/range.rs +++ b/packages/eql/crates/eql-codegen/src/range.rs @@ -63,18 +63,26 @@ pub fn term_label(term: Term) -> &'static str { } /// The CHECK clauses one bound of a `term` range must satisfy: the shape the -/// single-bound query domain's term key has. An ORE bound is a non-empty -/// array of block terms, as `ob` is; an OPE bound is a hex string, as `op` is. +/// single-bound query domain's term key has in EQL v4, a JSON string that +/// starts with one of the term's prefixes ([`eql_domains::V4TermFormat`]). fn bound_checks(term: Term, bound: &str) -> Vec { + let format = term + .v4_format() + .unwrap_or_else(|| unreachable!("{term:?} has no range domain")); + let prefixes = format + .prefixes + .iter() + .map(|p| format!("'{}'", sql_str(p))) + .collect::>() + .join(", "); let bound = sql_str(bound); - match term { - Term::Ore => vec![ - format!("jsonb_typeof(VALUE -> '{bound}') = 'array'"), - format!("jsonb_array_length(VALUE -> '{bound}') > 0"), - ], - Term::Ope => vec![format!("jsonb_typeof(VALUE -> '{bound}') = 'string'")], - Term::Hm | Term::Bloom => unreachable!("{term:?} has no range domain"), - } + vec![ + format!("jsonb_typeof(VALUE -> '{bound}') = 'string'"), + format!( + "left(VALUE ->> '{bound}', {}) IN ({prefixes})", + format.prefixes[0].len() + ), + ] } /// The keys a range value must carry, in order: the envelope's `v` and `i`, @@ -389,10 +397,10 @@ mod tests { "VALUE ? 'lo'", "VALUE ? 'hi'", "NOT (VALUE ? 'c')", - "jsonb_typeof(VALUE -> 'lo') = 'array'", - "jsonb_array_length(VALUE -> 'lo') > 0", - "jsonb_typeof(VALUE -> 'hi') = 'array'", - "jsonb_array_length(VALUE -> 'hi') > 0", + "jsonb_typeof(VALUE -> 'lo') = 'string'", + "left(VALUE ->> 'lo', 4) IN ('0202', '0203')", + "jsonb_typeof(VALUE -> 'hi') = 'string'", + "left(VALUE ->> 'hi', 4) IN ('0202', '0203')", "COALESCE(VALUE->>'v' = '4', VALUE IS NULL)", ] { assert!(ore.contains(clause), "ORE range lacks {clause}:\n{ore}"); @@ -405,8 +413,11 @@ mod tests { .next() .unwrap(); assert!(ope.contains("jsonb_typeof(VALUE -> 'lo') = 'string'")); + assert!(ope.contains("left(VALUE ->> 'lo', 2) IN ('00', '01')")); assert!(ope.contains("jsonb_typeof(VALUE -> 'hi') = 'string'")); + assert!(ope.contains("left(VALUE ->> 'hi', 2) IN ('00', '01')")); assert!(!ope.contains("jsonb_array_length"), "{ope}"); + assert!(!ore.contains("jsonb_array_length"), "{ore}"); assert!(v4.contains( "COMMENT ON DOMAIN eql_v4.query_integer_range_ore IS 'EQL integer range query operand (block ORE)'" )); diff --git a/packages/eql/crates/eql-codegen/templates/query_types.sql.j2 b/packages/eql/crates/eql-codegen/templates/query_types.sql.j2 index db992f483..8187f10f7 100644 --- a/packages/eql/crates/eql-codegen/templates/query_types.sql.j2 +++ b/packages/eql/crates/eql-codegen/templates/query_types.sql.j2 @@ -39,19 +39,24 @@ BEGIN {%- for k in d.forbidden_keys %} AND NOT (VALUE ? '{{ k }}') {%- endfor %} - {%- if d.optional_term_keys and d.nonempty_array_keys %} + {%- if d.nonempty_array_keys %} {{ v3_only }} - {%- endif %} {%- for k in d.nonempty_array_keys %} AND jsonb_typeof(VALUE -> '{{ k }}') = 'array' AND jsonb_array_length(VALUE -> '{{ k }}') > 0 {%- endfor %} - {%- if d.optional_term_keys and d.nonempty_array_keys %} {{ v3_end }} + {%- endif %} + {%- if d.v4_hex_terms %} {{ v4_only }} - {%- for k in d.nonempty_array_keys %} - AND (NOT (VALUE ? '{{ k }}') OR (jsonb_typeof(VALUE -> '{{ k }}') = 'array' - AND jsonb_array_length(VALUE -> '{{ k }}') > 0)) + {%- for t in d.v4_hex_terms %} + {%- if d.optional_term_keys %} + AND (NOT (VALUE ? '{{ t.key }}') OR (jsonb_typeof(VALUE -> '{{ t.key }}') = 'string' + AND left(VALUE ->> '{{ t.key }}', {{ t.prefix_len }}) IN ({{ t.prefixes }}))) + {%- else %} + AND jsonb_typeof(VALUE -> '{{ t.key }}') = 'string' + AND left(VALUE ->> '{{ t.key }}', {{ t.prefix_len }}) IN ({{ t.prefixes }}) + {%- endif %} {%- endfor %} {{ v4_end }} {%- endif %} diff --git a/packages/eql/crates/eql-codegen/templates/types.sql.j2 b/packages/eql/crates/eql-codegen/templates/types.sql.j2 index 82a0d5085..490920fb0 100644 --- a/packages/eql/crates/eql-codegen/templates/types.sql.j2 +++ b/packages/eql/crates/eql-codegen/templates/types.sql.j2 @@ -18,10 +18,22 @@ BEGIN {%- for k in d.keys %} AND VALUE ? '{{ k }}' {%- endfor %} + {%- if d.nonempty_array_keys %} +{{ v3_only }} {%- for k in d.nonempty_array_keys %} AND jsonb_typeof(VALUE -> '{{ k }}') = 'array' AND jsonb_array_length(VALUE -> '{{ k }}') > 0 {%- endfor %} +{{ v3_end }} + {%- endif %} + {%- if d.v4_hex_terms %} +{{ v4_only }} + {%- for t in d.v4_hex_terms %} + AND jsonb_typeof(VALUE -> '{{ t.key }}') = 'string' + AND left(VALUE ->> '{{ t.key }}', {{ t.prefix_len }}) IN ({{ t.prefixes }}) + {%- endfor %} +{{ v4_end }} + {%- endif %} AND VALUE->>'v' = '{{ eql_version }}' ); END IF; diff --git a/packages/eql/crates/eql-codegen/tests/cli.rs b/packages/eql/crates/eql-codegen/tests/cli.rs index 86de21ef8..c8b41ea8e 100644 --- a/packages/eql/crates/eql-codegen/tests/cli.rs +++ b/packages/eql/crates/eql-codegen/tests/cli.rs @@ -52,11 +52,11 @@ fn bindings_subcommand_succeeds_and_reports_count() { let stdout = String::from_utf8_lossy(&out.stdout); // v3: scalar families + json_storage + payload + query_payload + // inventory. v4: every family with a scalar domain + targets + answers + - // inventory. + // inventory + term_formats. let expected = eql_domains::scalar_families().count() + 4 + eql_domains::families_with_scalar_domains().count() - + 3; + + 4; assert!( stdout.contains(&format!("bindings: ok ({expected} files)")), "expected 'bindings: ok ({expected} files)' in stdout, got:\n{stdout}" diff --git a/packages/eql/crates/eql-domains/src/lib.rs b/packages/eql/crates/eql-domains/src/lib.rs index 8f3f67ee1..9636b2222 100644 --- a/packages/eql/crates/eql-domains/src/lib.rs +++ b/packages/eql/crates/eql-domains/src/lib.rs @@ -26,6 +26,8 @@ mod fixtures; mod spec; mod term; +pub use term::V4TermFormat; + pub use fixtures::{ kind_for, BoundedIntKind, Fixture, ScalarKind, TypeFixtures, BIGINT_FIXTURES, BIGINT_VALUES, BOOLEAN_FIXTURES, DATE_FIXTURES, DOUBLE_FIXTURES, FIXTURES, INTEGER_FIXTURES, INTEGER_VALUES, diff --git a/packages/eql/crates/eql-domains/src/term.rs b/packages/eql/crates/eql-domains/src/term.rs index ea96331f8..a5aee9b49 100644 --- a/packages/eql/crates/eql-domains/src/term.rs +++ b/packages/eql/crates/eql-domains/src/term.rs @@ -140,6 +140,78 @@ impl Term { } } +/// How EQL v4 stores an order term: as the lowercase hex of a Stack Encrypt +/// term, one JSON string, where EQL v3 stores cipherstash-client's terms. +/// +/// One entry per term, read by both halves of the generated surface: the +/// bindings type the field with [`rust_type`](Self::rust_type) over the +/// family's plaintext and describe it to TypeScript and JSON Schema as +/// [`json_shape`](Self::json_shape), and the domain CHECKs refuse a value +/// that is not a string starting with one of [`prefixes`](Self::prefixes). +/// The same prefixes and [`unit_digits`](Self::unit_digits) make the JSON +/// Schema pattern, so the schema and the CHECK cannot disagree about which +/// strings are terms. +/// +/// The CHECK reads only the prefix and the type of the JSON value: cheap, +/// and enough to refuse an EQL v3 array or a term of another scheme. The +/// full structure (block count, length, padding) is checked where the term +/// is read: by `from_bytes` in Rust and by the comparator in SQL. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct V4TermFormat { + /// The Stack Encrypt type of the field, under `stack_encrypt::sem`, + /// generic over the plaintext it orders: `OreBlockTerm`, `OpeTerm`. + pub rust_type: &'static str, + /// The schema-only type naming the field's JSON in TypeScript and JSON + /// Schema: `OreBlockHex`, `OpeHex`. + pub json_shape: &'static str, + /// What the term is, for the schema's description. + pub description: &'static str, + /// The hex every term starts with, one of these: block ORE's wire + /// version and engine (`0202` fixed width, `0203` chained), OPE's carry + /// byte (`00`, `01`). + pub prefixes: &'static [&'static str], + /// The hex digits of the unit the rest of a term repeats: a byte for + /// block ORE, whose length the comparator checks, an 8-byte block for + /// OPE, one byte per bit of the value. + pub unit_digits: usize, +} + +impl V4TermFormat { + /// The JSON Schema pattern of a term: a prefix, then whole units of + /// lowercase hex, at least one. + pub fn pattern(&self) -> String { + format!( + "^({})([0-9a-f]{{{}}})+$", + self.prefixes.join("|"), + self.unit_digits + ) + } +} + +impl Term { + /// How EQL v4 stores this term, or `None` for a term v4 stores as v3 + /// does (`hm`, `bf`). See [`V4TermFormat`]. + pub const fn v4_format(self) -> Option { + match self { + Term::Ore => Some(V4TermFormat { + rust_type: "OreBlockTerm", + json_shape: "OreBlockHex", + description: "An ore-rs 6-bit block ORE ciphertext (wire version 2), as lowercase hex.", + prefixes: &["0202", "0203"], + unit_digits: 2, + }), + Term::Ope => Some(V4TermFormat { + rust_type: "OpeTerm", + json_shape: "OpeHex", + description: "A clww-ore CLWW OPE ciphertext, as lowercase hex: a carry byte, then 8 bytes per byte of the value.", + prefixes: &["00", "01"], + unit_digits: 16, + }), + Term::Hm | Term::Bloom => None, + } + } +} + impl Term { /// Stable dedupe — first occurrence wins. fn dedupe_preserving_order<'a>(items: impl IntoIterator) -> Vec<&'a str> { diff --git a/packages/eql/crates/eql-domains/src/tests.rs b/packages/eql/crates/eql-domains/src/tests.rs index c0d3da91b..d938d62ea 100644 --- a/packages/eql/crates/eql-domains/src/tests.rs +++ b/packages/eql/crates/eql-domains/src/tests.rs @@ -1592,4 +1592,23 @@ mod shape_tests { "JSON must be catalogued at the flip" ); } + + /// EQL v4 stores the two order terms as hex strings of Stack Encrypt's + /// types, and the others as EQL v3 does. A pattern accepts a prefix and + /// whole units of lowercase hex, and nothing else. + #[test] + fn v4_formats_name_the_order_terms_and_their_patterns() { + use crate::Term; + assert_eq!(Term::Hm.v4_format(), None); + assert_eq!(Term::Bloom.v4_format(), None); + let ore = Term::Ore.v4_format().expect("ob has a v4 format"); + assert_eq!( + (ore.rust_type, ore.json_shape), + ("OreBlockTerm", "OreBlockHex") + ); + assert_eq!(ore.pattern(), "^(0202|0203)([0-9a-f]{2})+$"); + let ope = Term::Ope.v4_format().expect("op has a v4 format"); + assert_eq!((ope.rust_type, ope.json_shape), ("OpeTerm", "OpeHex")); + assert_eq!(ope.pattern(), "^(00|01)([0-9a-f]{16})+$"); + } } diff --git a/packages/eql/src/v3/scalars/bigint/bigint_types.sql b/packages/eql/src/v3/scalars/bigint/bigint_types.sql index fd257d7f9..891f9970e 100644 --- a/packages/eql/src/v3/scalars/bigint/bigint_types.sql +++ b/packages/eql/src/v3/scalars/bigint/bigint_types.sql @@ -53,8 +53,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'ob' +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -73,6 +79,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -91,6 +101,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/bigint/query_bigint_range_types.sql b/packages/eql/src/v3/scalars/bigint/query_bigint_range_types.sql index 476d4f9c4..1a575330d 100644 --- a/packages/eql/src/v3/scalars/bigint/query_bigint_range_types.sql +++ b/packages/eql/src/v3/scalars/bigint/query_bigint_range_types.sql @@ -42,10 +42,10 @@ BEGIN AND VALUE ? 'lo' AND VALUE ? 'hi' AND NOT (VALUE ? 'c') - AND jsonb_typeof(VALUE -> 'lo') = 'array' - AND jsonb_array_length(VALUE -> 'lo') > 0 - AND jsonb_typeof(VALUE -> 'hi') = 'array' - AND jsonb_array_length(VALUE -> 'hi') > 0 + AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 4) IN ('0202', '0203') + AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 4) IN ('0202', '0203') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation @@ -70,7 +70,9 @@ BEGIN AND VALUE ? 'hi' AND NOT (VALUE ? 'c') AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 2) IN ('00', '01') AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 2) IN ('00', '01') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation diff --git a/packages/eql/src/v3/scalars/bigint/query_bigint_types.sql b/packages/eql/src/v3/scalars/bigint/query_bigint_types.sql index d1d4e14ed..283efd8a3 100644 --- a/packages/eql/src/v3/scalars/bigint/query_bigint_types.sql +++ b/packages/eql/src/v3/scalars/bigint/query_bigint_types.sql @@ -44,8 +44,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'ob' AND NOT (VALUE ? 'c') +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -64,6 +70,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -82,6 +92,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/date/date_types.sql b/packages/eql/src/v3/scalars/date/date_types.sql index 8ad3cb4ba..d103b52dc 100644 --- a/packages/eql/src/v3/scalars/date/date_types.sql +++ b/packages/eql/src/v3/scalars/date/date_types.sql @@ -53,8 +53,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'ob' +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -73,6 +79,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -91,6 +101,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/date/query_date_range_types.sql b/packages/eql/src/v3/scalars/date/query_date_range_types.sql index 0dcce4acc..a54bac820 100644 --- a/packages/eql/src/v3/scalars/date/query_date_range_types.sql +++ b/packages/eql/src/v3/scalars/date/query_date_range_types.sql @@ -42,10 +42,10 @@ BEGIN AND VALUE ? 'lo' AND VALUE ? 'hi' AND NOT (VALUE ? 'c') - AND jsonb_typeof(VALUE -> 'lo') = 'array' - AND jsonb_array_length(VALUE -> 'lo') > 0 - AND jsonb_typeof(VALUE -> 'hi') = 'array' - AND jsonb_array_length(VALUE -> 'hi') > 0 + AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 4) IN ('0202', '0203') + AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 4) IN ('0202', '0203') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation @@ -70,7 +70,9 @@ BEGIN AND VALUE ? 'hi' AND NOT (VALUE ? 'c') AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 2) IN ('00', '01') AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 2) IN ('00', '01') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation diff --git a/packages/eql/src/v3/scalars/date/query_date_types.sql b/packages/eql/src/v3/scalars/date/query_date_types.sql index 1f2f4edbf..df73b1936 100644 --- a/packages/eql/src/v3/scalars/date/query_date_types.sql +++ b/packages/eql/src/v3/scalars/date/query_date_types.sql @@ -44,8 +44,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'ob' AND NOT (VALUE ? 'c') +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -64,6 +70,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -82,6 +92,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/double/double_types.sql b/packages/eql/src/v3/scalars/double/double_types.sql index 95c4c3d85..9065091f0 100644 --- a/packages/eql/src/v3/scalars/double/double_types.sql +++ b/packages/eql/src/v3/scalars/double/double_types.sql @@ -53,8 +53,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'ob' +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -73,6 +79,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -91,6 +101,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/double/query_double_range_types.sql b/packages/eql/src/v3/scalars/double/query_double_range_types.sql index 5603183ee..5fe72c9ff 100644 --- a/packages/eql/src/v3/scalars/double/query_double_range_types.sql +++ b/packages/eql/src/v3/scalars/double/query_double_range_types.sql @@ -42,10 +42,10 @@ BEGIN AND VALUE ? 'lo' AND VALUE ? 'hi' AND NOT (VALUE ? 'c') - AND jsonb_typeof(VALUE -> 'lo') = 'array' - AND jsonb_array_length(VALUE -> 'lo') > 0 - AND jsonb_typeof(VALUE -> 'hi') = 'array' - AND jsonb_array_length(VALUE -> 'hi') > 0 + AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 4) IN ('0202', '0203') + AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 4) IN ('0202', '0203') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation @@ -70,7 +70,9 @@ BEGIN AND VALUE ? 'hi' AND NOT (VALUE ? 'c') AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 2) IN ('00', '01') AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 2) IN ('00', '01') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation diff --git a/packages/eql/src/v3/scalars/double/query_double_types.sql b/packages/eql/src/v3/scalars/double/query_double_types.sql index 9ab265bf5..a30e90d79 100644 --- a/packages/eql/src/v3/scalars/double/query_double_types.sql +++ b/packages/eql/src/v3/scalars/double/query_double_types.sql @@ -44,8 +44,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'ob' AND NOT (VALUE ? 'c') +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -64,6 +70,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -82,6 +92,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/integer/integer_types.sql b/packages/eql/src/v3/scalars/integer/integer_types.sql index b10c0ad7f..4f5c541b3 100644 --- a/packages/eql/src/v3/scalars/integer/integer_types.sql +++ b/packages/eql/src/v3/scalars/integer/integer_types.sql @@ -53,8 +53,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'ob' +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -73,6 +79,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -91,6 +101,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/integer/query_integer_range_types.sql b/packages/eql/src/v3/scalars/integer/query_integer_range_types.sql index 5c9fe2cee..9ab117d33 100644 --- a/packages/eql/src/v3/scalars/integer/query_integer_range_types.sql +++ b/packages/eql/src/v3/scalars/integer/query_integer_range_types.sql @@ -42,10 +42,10 @@ BEGIN AND VALUE ? 'lo' AND VALUE ? 'hi' AND NOT (VALUE ? 'c') - AND jsonb_typeof(VALUE -> 'lo') = 'array' - AND jsonb_array_length(VALUE -> 'lo') > 0 - AND jsonb_typeof(VALUE -> 'hi') = 'array' - AND jsonb_array_length(VALUE -> 'hi') > 0 + AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 4) IN ('0202', '0203') + AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 4) IN ('0202', '0203') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation @@ -70,7 +70,9 @@ BEGIN AND VALUE ? 'hi' AND NOT (VALUE ? 'c') AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 2) IN ('00', '01') AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 2) IN ('00', '01') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation diff --git a/packages/eql/src/v3/scalars/integer/query_integer_types.sql b/packages/eql/src/v3/scalars/integer/query_integer_types.sql index 62eb3a3f1..6f6638d53 100644 --- a/packages/eql/src/v3/scalars/integer/query_integer_types.sql +++ b/packages/eql/src/v3/scalars/integer/query_integer_types.sql @@ -44,8 +44,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'ob' AND NOT (VALUE ? 'c') +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -64,6 +70,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -82,6 +92,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/numeric/numeric_types.sql b/packages/eql/src/v3/scalars/numeric/numeric_types.sql index 37852ac11..f71b1a75c 100644 --- a/packages/eql/src/v3/scalars/numeric/numeric_types.sql +++ b/packages/eql/src/v3/scalars/numeric/numeric_types.sql @@ -53,8 +53,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'ob' +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -73,6 +79,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -91,6 +101,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/numeric/query_numeric_range_types.sql b/packages/eql/src/v3/scalars/numeric/query_numeric_range_types.sql index 2e47c91db..f1b35c592 100644 --- a/packages/eql/src/v3/scalars/numeric/query_numeric_range_types.sql +++ b/packages/eql/src/v3/scalars/numeric/query_numeric_range_types.sql @@ -42,10 +42,10 @@ BEGIN AND VALUE ? 'lo' AND VALUE ? 'hi' AND NOT (VALUE ? 'c') - AND jsonb_typeof(VALUE -> 'lo') = 'array' - AND jsonb_array_length(VALUE -> 'lo') > 0 - AND jsonb_typeof(VALUE -> 'hi') = 'array' - AND jsonb_array_length(VALUE -> 'hi') > 0 + AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 4) IN ('0202', '0203') + AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 4) IN ('0202', '0203') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation @@ -70,7 +70,9 @@ BEGIN AND VALUE ? 'hi' AND NOT (VALUE ? 'c') AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 2) IN ('00', '01') AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 2) IN ('00', '01') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation diff --git a/packages/eql/src/v3/scalars/numeric/query_numeric_types.sql b/packages/eql/src/v3/scalars/numeric/query_numeric_types.sql index b9646bcf8..8f4ebfde7 100644 --- a/packages/eql/src/v3/scalars/numeric/query_numeric_types.sql +++ b/packages/eql/src/v3/scalars/numeric/query_numeric_types.sql @@ -44,8 +44,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'ob' AND NOT (VALUE ? 'c') +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -64,6 +70,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -82,6 +92,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/real/query_real_range_types.sql b/packages/eql/src/v3/scalars/real/query_real_range_types.sql index e79669906..c71092354 100644 --- a/packages/eql/src/v3/scalars/real/query_real_range_types.sql +++ b/packages/eql/src/v3/scalars/real/query_real_range_types.sql @@ -42,10 +42,10 @@ BEGIN AND VALUE ? 'lo' AND VALUE ? 'hi' AND NOT (VALUE ? 'c') - AND jsonb_typeof(VALUE -> 'lo') = 'array' - AND jsonb_array_length(VALUE -> 'lo') > 0 - AND jsonb_typeof(VALUE -> 'hi') = 'array' - AND jsonb_array_length(VALUE -> 'hi') > 0 + AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 4) IN ('0202', '0203') + AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 4) IN ('0202', '0203') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation @@ -70,7 +70,9 @@ BEGIN AND VALUE ? 'hi' AND NOT (VALUE ? 'c') AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 2) IN ('00', '01') AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 2) IN ('00', '01') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation diff --git a/packages/eql/src/v3/scalars/real/query_real_types.sql b/packages/eql/src/v3/scalars/real/query_real_types.sql index a7b81b073..6198f2279 100644 --- a/packages/eql/src/v3/scalars/real/query_real_types.sql +++ b/packages/eql/src/v3/scalars/real/query_real_types.sql @@ -44,8 +44,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'ob' AND NOT (VALUE ? 'c') +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -64,6 +70,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -82,6 +92,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/real/real_types.sql b/packages/eql/src/v3/scalars/real/real_types.sql index 186fe4d4f..cf7804519 100644 --- a/packages/eql/src/v3/scalars/real/real_types.sql +++ b/packages/eql/src/v3/scalars/real/real_types.sql @@ -53,8 +53,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'ob' +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -73,6 +79,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -91,6 +101,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/smallint/query_smallint_range_types.sql b/packages/eql/src/v3/scalars/smallint/query_smallint_range_types.sql index bf467bef2..1d167b904 100644 --- a/packages/eql/src/v3/scalars/smallint/query_smallint_range_types.sql +++ b/packages/eql/src/v3/scalars/smallint/query_smallint_range_types.sql @@ -42,10 +42,10 @@ BEGIN AND VALUE ? 'lo' AND VALUE ? 'hi' AND NOT (VALUE ? 'c') - AND jsonb_typeof(VALUE -> 'lo') = 'array' - AND jsonb_array_length(VALUE -> 'lo') > 0 - AND jsonb_typeof(VALUE -> 'hi') = 'array' - AND jsonb_array_length(VALUE -> 'hi') > 0 + AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 4) IN ('0202', '0203') + AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 4) IN ('0202', '0203') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation @@ -70,7 +70,9 @@ BEGIN AND VALUE ? 'hi' AND NOT (VALUE ? 'c') AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 2) IN ('00', '01') AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 2) IN ('00', '01') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation diff --git a/packages/eql/src/v3/scalars/smallint/query_smallint_types.sql b/packages/eql/src/v3/scalars/smallint/query_smallint_types.sql index 382dc375f..e52403c68 100644 --- a/packages/eql/src/v3/scalars/smallint/query_smallint_types.sql +++ b/packages/eql/src/v3/scalars/smallint/query_smallint_types.sql @@ -44,8 +44,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'ob' AND NOT (VALUE ? 'c') +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -64,6 +70,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -82,6 +92,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/smallint/smallint_types.sql b/packages/eql/src/v3/scalars/smallint/smallint_types.sql index adc4887ca..103a397c4 100644 --- a/packages/eql/src/v3/scalars/smallint/smallint_types.sql +++ b/packages/eql/src/v3/scalars/smallint/smallint_types.sql @@ -53,8 +53,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'ob' +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -73,6 +79,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -91,6 +101,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/text/query_text_range_types.sql b/packages/eql/src/v3/scalars/text/query_text_range_types.sql index caaf084cf..0627fd499 100644 --- a/packages/eql/src/v3/scalars/text/query_text_range_types.sql +++ b/packages/eql/src/v3/scalars/text/query_text_range_types.sql @@ -42,10 +42,10 @@ BEGIN AND VALUE ? 'lo' AND VALUE ? 'hi' AND NOT (VALUE ? 'c') - AND jsonb_typeof(VALUE -> 'lo') = 'array' - AND jsonb_array_length(VALUE -> 'lo') > 0 - AND jsonb_typeof(VALUE -> 'hi') = 'array' - AND jsonb_array_length(VALUE -> 'hi') > 0 + AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 4) IN ('0202', '0203') + AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 4) IN ('0202', '0203') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation @@ -70,7 +70,9 @@ BEGIN AND VALUE ? 'hi' AND NOT (VALUE ? 'c') AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 2) IN ('00', '01') AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 2) IN ('00', '01') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation diff --git a/packages/eql/src/v3/scalars/text/query_text_types.sql b/packages/eql/src/v3/scalars/text/query_text_types.sql index f6f8a9b1d..2ae6b8aa8 100644 --- a/packages/eql/src/v3/scalars/text/query_text_types.sql +++ b/packages/eql/src/v3/scalars/text/query_text_types.sql @@ -63,8 +63,14 @@ BEGIN AND VALUE ? 'hm' AND VALUE ? 'ob' AND NOT (VALUE ? 'c') +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -84,6 +90,10 @@ BEGIN AND VALUE ? 'hm' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -103,6 +113,10 @@ BEGIN AND VALUE ? 'hm' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -133,8 +147,8 @@ BEGIN AND jsonb_array_length(VALUE -> 'ob') > 0 -- {{/v3}} -- {{#v4}} - AND (NOT (VALUE ? 'ob') OR (jsonb_typeof(VALUE -> 'ob') = 'array' - AND jsonb_array_length(VALUE -> 'ob') > 0)) + AND (NOT (VALUE ? 'ob') OR (jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203'))) -- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); @@ -161,6 +175,10 @@ BEGIN AND (VALUE ? 'hm' OR VALUE ? 'op' OR VALUE ? 'bf') -- {{/v4}} AND NOT (VALUE ? 'c') +-- {{#v4}} + AND (NOT (VALUE ? 'op') OR (jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01'))) +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/text/text_types.sql b/packages/eql/src/v3/scalars/text/text_types.sql index c4c93d202..65c5c216b 100644 --- a/packages/eql/src/v3/scalars/text/text_types.sql +++ b/packages/eql/src/v3/scalars/text/text_types.sql @@ -72,8 +72,14 @@ BEGIN AND VALUE ? 'c' AND VALUE ? 'hm' AND VALUE ? 'ob' +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -93,6 +99,10 @@ BEGIN AND VALUE ? 'c' AND VALUE ? 'hm' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -112,6 +122,10 @@ BEGIN AND VALUE ? 'c' AND VALUE ? 'hm' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -132,8 +146,14 @@ BEGIN AND VALUE ? 'hm' AND VALUE ? 'ob' AND VALUE ? 'bf' +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -154,6 +174,10 @@ BEGIN AND VALUE ? 'hm' AND VALUE ? 'op' AND VALUE ? 'bf' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_types.sql b/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_types.sql index 02a882baa..e58f726c2 100644 --- a/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_types.sql +++ b/packages/eql/src/v3/scalars/timestamp/query_timestamp_range_types.sql @@ -42,10 +42,10 @@ BEGIN AND VALUE ? 'lo' AND VALUE ? 'hi' AND NOT (VALUE ? 'c') - AND jsonb_typeof(VALUE -> 'lo') = 'array' - AND jsonb_array_length(VALUE -> 'lo') > 0 - AND jsonb_typeof(VALUE -> 'hi') = 'array' - AND jsonb_array_length(VALUE -> 'hi') > 0 + AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 4) IN ('0202', '0203') + AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 4) IN ('0202', '0203') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation @@ -70,7 +70,9 @@ BEGIN AND VALUE ? 'hi' AND NOT (VALUE ? 'c') AND jsonb_typeof(VALUE -> 'lo') = 'string' + AND left(VALUE ->> 'lo', 2) IN ('00', '01') AND jsonb_typeof(VALUE -> 'hi') = 'string' + AND left(VALUE ->> 'hi', 2) IN ('00', '01') -- `"v": null` makes the comparison NULL, and a CHECK that is NULL -- passes, so COALESCE refuses it. A NULL range still passes (no row -- matches it). The comparison keeps the one form the v4 isolation diff --git a/packages/eql/src/v3/scalars/timestamp/query_timestamp_types.sql b/packages/eql/src/v3/scalars/timestamp/query_timestamp_types.sql index 4a9978ee0..23a1bbc4d 100644 --- a/packages/eql/src/v3/scalars/timestamp/query_timestamp_types.sql +++ b/packages/eql/src/v3/scalars/timestamp/query_timestamp_types.sql @@ -44,8 +44,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'ob' AND NOT (VALUE ? 'c') +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -64,6 +70,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -82,6 +92,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'op' AND NOT (VALUE ? 'c') +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/scalars/timestamp/timestamp_types.sql b/packages/eql/src/v3/scalars/timestamp/timestamp_types.sql index c0d903b1b..d0bd59e57 100644 --- a/packages/eql/src/v3/scalars/timestamp/timestamp_types.sql +++ b/packages/eql/src/v3/scalars/timestamp/timestamp_types.sql @@ -53,8 +53,14 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'ob' +-- {{#v3}} AND jsonb_typeof(VALUE -> 'ob') = 'array' AND jsonb_array_length(VALUE -> 'ob') > 0 +-- {{/v3}} +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'ob') = 'string' + AND left(VALUE ->> 'ob', 4) IN ('0202', '0203') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -73,6 +79,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; @@ -91,6 +101,10 @@ BEGIN AND VALUE ? 'i' AND VALUE ? 'c' AND VALUE ? 'op' +-- {{#v4}} + AND jsonb_typeof(VALUE -> 'op') = 'string' + AND left(VALUE ->> 'op', 2) IN ('00', '01') +-- {{/v4}} AND VALUE->>'v' = '{{eql_version}}' ); END IF; diff --git a/packages/eql/src/v3/sem/ore_block_256/functions.sql b/packages/eql/src/v3/sem/ore_block_256/functions.sql index b77a4e48c..37445b4bc 100644 --- a/packages/eql/src/v3/sem/ore_block_256/functions.sql +++ b/packages/eql/src/v3/sem/ore_block_256/functions.sql @@ -13,6 +13,7 @@ --! deliberately avoid naming eql_v2 symbols so the self-containment grep stays --! clean.) +-- {{#v3}} --! @brief Convert JSONB array to ORE block composite type --! @internal --! @param val jsonb Array of hex-encoded ORE block terms @@ -64,6 +65,7 @@ $$ LANGUAGE plpgsql; --! this marker is the documented manual opt-in. COMMENT ON FUNCTION {{prefix}}_internal.jsonb_array_to_{{ore_block}}(jsonb) IS 'eql-inline-critical: per-encrypted-value ORE opclass-path helper; must stay unpinned (SET search_path adds per-call overhead)'; +-- {{/v3}} --! @brief Extract ORE block index term from JSONB payload @@ -79,7 +81,16 @@ AS $$ -- Declared STRICT: PostgreSQL returns NULL for a NULL argument without -- entering the body, so no explicit `val IS NULL` guard is needed. IF {{prefix}}_internal.has_{{ore_block}}(val) THEN +-- {{#v3}} RETURN {{prefix}}_internal.jsonb_array_to_{{ore_block}}(val->'ob'); +-- {{/v3}} +-- {{#v4}} + -- One term, the hex of the whole ore-rs ciphertext. The domain CHECK + -- has refused anything that is not a string with a block ORE prefix; + -- decode() raises on a character that is not hex, and the comparator + -- checks the rest of the structure. + RETURN ROW(ROW(decode(val->>'ob', 'hex'))::{{prefix}}_internal.{{ore_block}}_term)::{{prefix}}_internal.{{ore_block}}; +-- {{/v4}} END IF; RAISE 'Expected an ore index (ob) value in json: %', val; END; @@ -88,6 +99,7 @@ $$ LANGUAGE plpgsql; --! @brief Check if JSONB payload contains an ORE block index term --! @param val jsonb containing encrypted EQL payload +-- {{#v3}} --! @return boolean True only if the 'ob' field is present and is a JSON array --! @note A well-formed ORE index term is always a JSON array of block terms, so --! this guard treats a present-but-non-array `ob` (a scalar or object) as @@ -97,13 +109,27 @@ $$ LANGUAGE plpgsql; --! previous `val ->> 'ob' IS NOT NULL` form stringified scalars/objects and so --! reported them as present. `{}` (absent `ob`) and `{"ob": null}` (JSON null) --! both remain `false`. +-- {{/v3}} +-- {{#v4}} +--! @return boolean True only if the 'ob' field is present and is a JSON string +--! @note A well-formed ORE index term is one hex string, the whole ore-rs +--! ciphertext, so this guard treats a present `ob` of any other JSON type +--! (an EQL v3 array, a number, an object) as absent, and the extractor +--! `{{ore_block}}(val)` RAISEs on it. `{}` (absent `ob`) and +--! `{"ob": null}` (JSON null) are `false`. +-- {{/v4}} CREATE FUNCTION {{prefix}}_internal.has_{{ore_block}}(val jsonb) RETURNS boolean IMMUTABLE STRICT PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN +-- {{#v3}} RETURN COALESCE(jsonb_typeof(val -> 'ob') = 'array', false); +-- {{/v3}} +-- {{#v4}} + RETURN COALESCE(jsonb_typeof(val -> 'ob') = 'string', false); +-- {{/v4}} END; $$ LANGUAGE plpgsql; @@ -470,34 +496,6 @@ AS $$ + (latch > 0)::int * (2 * indicator - 1); END; $$ LANGUAGE plpgsql; - - ---! @brief Compare two EQL v4 ORE block values ---! @internal ---! @param a {{prefix}}_internal.{{ore_block}}_term[] First value's terms ---! @param b {{prefix}}_internal.{{ore_block}}_term[] Second value's terms ---! @return integer -1/0/1, or NULL if either array is NULL ---! @throws Exception unless each value holds exactly one term: Stack ---! Encrypt writes one 6-bit block ORE term per value, text included, where ---! EQL v3's text held up to six 8-bit terms -CREATE FUNCTION {{prefix}}_internal.compare_{{ore_block}}_terms(a {{prefix}}_internal.{{ore_block}}_term[], b {{prefix}}_internal.{{ore_block}}_term[]) -RETURNS integer - IMMUTABLE - SET search_path = pg_catalog, extensions, public -AS $$ - BEGIN - IF a IS NULL OR b IS NULL THEN - RETURN NULL; - END IF; - - IF cardinality(a) <> 1 OR cardinality(b) <> 1 THEN - RAISE EXCEPTION 'An ORE block value holds exactly one term, not % and %', - cardinality(a), cardinality(b); - END IF; - - RETURN {{prefix}}_internal.compare_{{ore_block}}_term(a[1], b[1]); - END -$$ LANGUAGE plpgsql; -- {{/v4}} @@ -512,6 +510,11 @@ RETURNS integer SET search_path = pg_catalog, extensions, public AS $$ BEGIN +-- {{#v3}} RETURN {{prefix}}_internal.compare_{{ore_block}}_terms(a.terms, b.terms); +-- {{/v3}} +-- {{#v4}} + RETURN {{prefix}}_internal.compare_{{ore_block}}_term(a.term, b.term); +-- {{/v4}} END $$ LANGUAGE plpgsql; diff --git a/packages/eql/src/v3/sem/ore_block_256/types.sql b/packages/eql/src/v3/sem/ore_block_256/types.sql index f9996c611..59954377c 100644 --- a/packages/eql/src/v3/sem/ore_block_256/types.sql +++ b/packages/eql/src/v3/sem/ore_block_256/types.sql @@ -15,6 +15,7 @@ CREATE TYPE {{prefix}}_internal.{{ore_block}}_term AS ( ); +-- {{#v3}} --! @brief ORE block index term type for range queries --! --! Composite type containing an array of ORE block terms. The array is stored @@ -24,3 +25,15 @@ CREATE TYPE {{prefix}}_internal.{{ore_block}}_term AS ( CREATE TYPE {{prefix}}_internal.{{ore_block}} AS ( terms {{prefix}}_internal.{{ore_block}}_term[] ); +-- {{/v3}} +-- {{#v4}} +--! @brief ORE block index term type for range queries +--! +--! Composite type holding one ORE block term: the ore-rs ciphertext that +--! the 'ob' field of an encrypted payload stores as one hex string. +--! +--! @note Transient type used only during query execution. +CREATE TYPE {{prefix}}_internal.{{ore_block}} AS ( + term {{prefix}}_internal.{{ore_block}}_term +); +-- {{/v4}} diff --git a/packages/eql/tests/encryption/tests/v4_ore_block.rs b/packages/eql/tests/encryption/tests/v4_ore_block.rs index d3321f46a..091dd7781 100644 --- a/packages/eql/tests/encryption/tests/v4_ore_block.rs +++ b/packages/eql/tests/encryption/tests/v4_ore_block.rs @@ -16,8 +16,13 @@ //! prefixes, and text in different padding buckets. They check the hash bit //! against ore.rs's vectors, that malformed and mismatched terms raise, that //! EQL admits the block counts ore-rs does up to 1366 blocks (the longest -//! term Stack Encrypt writes) and none past it, and that the domains answer -//! `<`, `=` and `ORDER BY` through a btree index. +//! term Stack Encrypt writes) and none past it, that the domains answer +//! `<`, `=` and `ORDER BY` through a btree index, and that a domain's CHECK +//! takes an order term only as one hex string with its scheme's prefix. +//! +//! An `ob` term is one hex string on the wire, the whole ore-rs ciphertext, +//! and the bindings hold it as Stack Encrypt's `OreBlockTerm`, which refuses +//! anything else when the payload is read. mod common; @@ -59,12 +64,11 @@ async fn stored(keyset: &Keyset<'_>, name: &str, value: ValueFrame) -> serde_jso serde_json::from_slice(&bytes).unwrap() } -/// The one block ORE term the named type stores for `value`, as bytes. +/// The block ORE term the named type stores for `value`, as bytes: one hex +/// string, the whole ore-rs ciphertext. async fn ob(keyset: &Keyset<'_>, name: &str, value: ValueFrame) -> Vec { let json = stored(keyset, name, value).await; - let terms = json["ob"].as_array().expect("ob is an array"); - assert_eq!(terms.len(), 1, "{name}: one block ORE term"); - unhex(terms[0].as_str().unwrap()) + unhex(json["ob"].as_str().expect("ob is one hex string")) } /// The terms of `values`, two writes each: the second write of a value is @@ -89,8 +93,8 @@ async fn text_terms(keyset: &Keyset<'_>, name: &str, values: &[&str]) -> Vec(a: &[u8], b: &[u8]) -> Ordering { - let a = OreBlockTerm::::from_bytes(a).unwrap(); - let b = OreBlockTerm::::from_bytes(b).unwrap(); + let a = OreBlockTerm::::try_from(a).unwrap(); + let b = OreBlockTerm::::try_from(b).unwrap(); a.compare(&b).expect("terms of one index compare") } @@ -159,11 +163,66 @@ async fn each_type_stores_one_block_ore_term_of_its_scheme() { .collect(); found.sort_unstable(); assert_eq!(found, keys, "{name}"); - let term = unhex(json["ob"][0].as_str().unwrap()); + let term = unhex(json["ob"].as_str().unwrap()); assert_eq!(term[..2], header, "{name}: the scheme's wire header"); } } +/// A v4 type reads an order term only as one hex string of its scheme: +/// the value Stack Encrypt writes reads back and writes out unchanged, and +/// an EQL v3 array, a term of the other scheme, a truncated term and text +/// that is not hex are refused when the payload is read. No error repeats +/// the term. +#[tokio::test] +async fn a_v4_type_reads_an_order_term_only_as_its_hex_string() { + use eql_bindings::v4::integer::{IntegerOrd, IntegerOrdOre}; + use serde_json::{json, Value}; + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let ore = stored(&keyset, "IntegerOrdOre", frame(7i32)).await; + let ope = stored(&keyset, "IntegerOrd", frame(7i32)).await; + let read: IntegerOrdOre = serde_json::from_value(ore.clone()).unwrap(); + assert_eq!(serde_json::to_value(&read).unwrap(), ore); + let read: IntegerOrd = serde_json::from_value(ope.clone()).unwrap(); + assert_eq!(serde_json::to_value(&read).unwrap(), ope); + + let ob = ore["ob"].as_str().unwrap().to_owned(); + let op = ope["op"].as_str().unwrap().to_owned(); + let with = |base: &Value, key: &str, term: Value| { + let mut value = base.clone(); + value[key] = term; + value + }; + for (what, value) in [ + ("an EQL v3 array", with(&ore, "ob", json!([ob]))), + ("an OPE term", with(&ore, "ob", json!(op))), + ( + "a truncated term", + with(&ore, "ob", json!(&ob[..ob.len() - 2])), + ), + ( + "text that is not hex", + with(&ore, "ob", json!(format!("zz{}", &ob[2..]))), + ), + ] { + let error = serde_json::from_value::(value) + .expect_err(what) + .to_string(); + assert!(!error.contains(&ob[8..]), "{what}: {error}"); + } + let carry = format!("02{}", &op[2..]); + for (what, value) in [ + ("an EQL v3 array", with(&ope, "op", json!([op]))), + ("a block ORE term", with(&ope, "op", json!(ob))), + ("a carry of 2", with(&ope, "op", json!(carry))), + ] { + let error = serde_json::from_value::(value) + .expect_err(what) + .to_string(); + assert!(!error.contains(&op[8..]), "{what}: {error}"); + } +} + /// Pairs whose order keys first differ in each block in turn: `x` with one /// bit of block `k` clear, and `x` with it set. An `i32`'s key is 32 bits, /// six 6-bit blocks (the last holds two bits); an `i64`'s is eleven. @@ -598,23 +657,24 @@ fn postgres_refuses_malformed_and_mismatched_terms() { let message = raises(&mut db, &text, &unpadded, "two padding policies"); assert!(message.contains("different policies"), "{message}"); - // A value holds exactly one term. + // A value holds one term, the hex string `ob` carries: the extractor + // reads it, and refuses an EQL v3 array as a value with no `ob`. + let extracted: Vec = db + .query_one( + "SELECT (eql_v4_internal.ore_block_v2(jsonb_build_object('ob', $1::text))).term.bytes", + &[&hex(&int)], + ) + .unwrap() + .get(0); + assert_eq!(extracted, int); let error = db .query_one( - "SELECT eql_v4_internal.compare_ore_block_v2_terms(\ - ARRAY[ROW($1::bytea), ROW($1::bytea)]::eql_v4_internal.ore_block_v2_term[], \ - ARRAY[ROW($1::bytea)]::eql_v4_internal.ore_block_v2_term[])", - &[&int], + "SELECT eql_v4_internal.ore_block_v2(jsonb_build_object('ob', jsonb_build_array($1::text)))", + &[&hex(&int)], ) .unwrap_err(); - assert!( - error - .as_db_error() - .unwrap() - .message() - .contains("exactly one term"), - "{error}" - ); + let message = error.as_db_error().unwrap().message().to_owned(); + assert!(message.contains("Expected an ore index (ob)"), "{message}"); } /// The longest chained term EQL v4 admits: 1366 blocks, a 1,024-byte string @@ -704,7 +764,7 @@ fn postgres_admits_the_block_counts_ore_rs_does_up_to_the_bound() { let mut both = 0; for (&(kind, policy, count), &sql) in cases.iter().zip(&admitted) { let term = zero_term(&chained_header(kind, policy, count), count); - let ore_rs = OreBlockTerm::::from_bytes(&term).is_ok(); + let ore_rs = OreBlockTerm::::try_from(term.as_slice()).is_ok(); let expected = ore_rs && (1..=MAX_CHAINED_BLOCKS).contains(&count); assert_eq!( sql, expected, @@ -719,11 +779,8 @@ fn postgres_admits_the_block_counts_ore_rs_does_up_to_the_bound() { .zip(&admitted) .any(|(&(_, _, count), &sql)| count > MAX_CHAINED_BLOCKS && !sql - && OreBlockTerm::::from_bytes(&zero_term( - &chained_header(0, 0, count), - count - )) - .is_ok())); + && OreBlockTerm::::from_bytes(zero_term(&chained_header(0, 0, count), count)) + .is_ok())); for count in 0..=20usize { let header = [2, 2, 0, u8::try_from(count).unwrap()]; @@ -732,7 +789,7 @@ fn postgres_admits_the_block_counts_ore_rs_does_up_to_the_bound() { .query_one("SELECT pg_temp.admits($1)", &[&term]) .unwrap() .get(0); - let ore_rs = OreBlockTerm::::from_bytes(&term).is_ok(); + let ore_rs = OreBlockTerm::::try_from(term.as_slice()).is_ok(); assert_eq!(sql, ore_rs, "fixed-width, {count} blocks"); } } @@ -772,7 +829,7 @@ fn postgres_compares_the_longest_term_and_refuses_one_past_it() { let count = MAX_CHAINED_BLOCKS + 1; let over = zero_term(&chained_header(0, 0, count), count); assert!( - OreBlockTerm::::from_bytes(&over).is_ok(), + OreBlockTerm::::try_from(over.as_slice()).is_ok(), "ore-rs admits it" ); let unpadded = zero_term( @@ -887,3 +944,92 @@ fn postgres_answers_the_orderings_through_a_btree_index() { assert_eq!(ordered, [1, 2, 3, 4, 5, 6], "{column}: ORDER BY"); } } + +/// SQLSTATE `check_violation`. +const CHECK_VIOLATION: &str = "23514"; + +/// Needs an EQL v4 installation. A domain takes an order term only as one +/// hex string with its scheme's prefix, so the CHECK refuses an EQL v3 +/// array, a term of another scheme or engine, an OPE carry other than 0 or +/// 1, and a value that is not a string, at the cast and before any +/// comparison. What Stack Encrypt writes passes, stored, queried and as a +/// range bound. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_domains_take_an_order_term_only_as_its_hex_string() { + use serde_json::{json, Value}; + let runtime = runtime(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + let ore = runtime.block_on(stored(&keyset, "IntegerOrdOre", frame(7i32))); + let ope = runtime.block_on(stored(&keyset, "IntegerOrd", frame(7i32))); + let ob = ore["ob"].as_str().unwrap().to_owned(); + let op = ope["op"].as_str().unwrap().to_owned(); + let query = |key: &str, term: Value| json!({"v": 4, "i": ore["i"], key: term}); + let range = |lo: Value, hi: Value| json!({"v": 4, "i": ore["i"], "lo": lo, "hi": hi}); + let with = |base: &Value, key: &str, term: Value| { + let mut value = base.clone(); + value[key] = term; + value + }; + let cast = |db: &mut postgres::Client, domain: &str, value: &Value| { + db.query_one( + &format!("SELECT $1::text::jsonb::{domain} IS NOT NULL"), + &[&value.to_string()], + ) + .map(drop) + }; + + let stored_ore = "public.eql_v4_integer_ord_ore"; + let stored_ope = "public.eql_v4_integer_ord"; + let query_ore = "eql_v4.query_integer_ord_ore"; + let query_ope = "eql_v4.query_integer_ord"; + let range_ore = "eql_v4.query_integer_range_ore"; + let range_ope = "eql_v4.query_integer_range_ope"; + for (domain, value) in [ + (stored_ore, ore.clone()), + (stored_ope, ope.clone()), + (query_ore, query("ob", json!(ob))), + (query_ope, query("op", json!(op))), + (range_ore, range(json!(ob), json!(ob))), + (range_ope, range(json!(op), json!(op))), + ] { + cast(&mut db, domain, &value).unwrap_or_else(|e| panic!("{domain}: {e}")); + } + + let engine = format!("0204{}", &ob[4..]); + let version = format!("01{}", &ob[2..]); + let carry = format!("02{}", &op[2..]); + for (what, domain, value) in [ + ("an EQL v3 array", stored_ore, with(&ore, "ob", json!([ob]))), + ( + "an unknown engine", + stored_ore, + with(&ore, "ob", json!(engine)), + ), + ( + "wire version 1", + stored_ore, + with(&ore, "ob", json!(version)), + ), + ("an OPE term", stored_ore, with(&ore, "ob", json!(op))), + ("a number", stored_ore, with(&ore, "ob", json!(7))), + ("an OPE array", stored_ope, with(&ope, "op", json!([op]))), + ("a carry of 2", stored_ope, with(&ope, "op", json!(carry))), + ("a block ORE term", stored_ope, with(&ope, "op", json!(ob))), + ("a query's v3 array", query_ore, query("ob", json!([ob]))), + ("a query's carry of 2", query_ope, query("op", json!(carry))), + ("a v3 array bound", range_ore, range(json!([ob]), json!(ob))), + ("an OPE bound", range_ore, range(json!(ob), json!(op))), + ( + "a carry of 2 bound", + range_ope, + range(json!(op), json!(carry)), + ), + ] { + let error = cast(&mut db, domain, &value).expect_err(what); + let state = error.code().map_or("", |code| code.code()); + assert_eq!(state, CHECK_VIOLATION, "{what}: {error}"); + } +} diff --git a/packages/eql/tests/encryption/tests/v4_types.rs b/packages/eql/tests/encryption/tests/v4_types.rs index 0eb777c1c..ed823cf1c 100644 --- a/packages/eql/tests/encryption/tests/v4_types.rs +++ b/packages/eql/tests/encryption/tests/v4_types.rs @@ -472,18 +472,13 @@ async fn every_producible_type_dispatches_by_name() { let term = term_of(key); assert!(json[term].is_string() || json[term].is_array(), "{name}"); if term == "ob" { - let one_hex = |value: &serde_json::Value| { - value - .as_array() - .is_some_and(|a| a.len() == 1 && a[0].is_string()) - }; assert!( - one_hex(&json[term]) && one_hex(&query[term]), - "{name} via {key}: one block ORE term each" + json[term].is_string() && query[term].is_string(), + "{name} via {key}: one block ORE term each, as hex" ); assert_eq!( - query[term][0].as_str().unwrap().len(), - json[term][0].as_str().unwrap().len(), + query[term].as_str().unwrap().len(), + json[term].as_str().unwrap().len(), "{name} via {key}: the query's term is the stored one's length" ); } else { From e7a2fad8be64716b475a9e2335fdebe9a5993440 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 19:39:44 +1100 Subject: [PATCH 3/4] docs(eql): an EQL v4 order term is one hex string, checked when written U-003 in the 3.1 upgrade guide says what changed for a hand-written eql_v4 payload: `ob` is one hex string, and every v4 domain carrying `ob` or `op` checks its prefix. A database that installed an earlier v4 bundle, such as the one attached to the Go module's alpha, keeps its stored domains across an install, so it must drop its stored order domains first; the guide gives the query that lists them. A changeset for @cipherstash/eql, CHANGELOG entries for eql-bindings and the Go module, and the bindings README and catalog reference describe the v4 term types. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- .changeset/eql-v4-order-term-shape.md | 5 + languages/golang/CHANGELOG.md | 7 ++ packages/eql/crates/eql-bindings/CHANGELOG.md | 15 ++- packages/eql/crates/eql-bindings/README.md | 12 ++- .../reference/catalog-driven-architecture.md | 7 ++ packages/eql/docs/upgrading/v3.1.md | 97 ++++++++++++++++++- 6 files changed, 135 insertions(+), 8 deletions(-) create mode 100644 .changeset/eql-v4-order-term-shape.md diff --git a/.changeset/eql-v4-order-term-shape.md b/.changeset/eql-v4-order-term-shape.md new file mode 100644 index 000000000..bfaa8afa3 --- /dev/null +++ b/.changeset/eql-v4-order-term-shape.md @@ -0,0 +1,5 @@ +--- +'@cipherstash/eql': minor +--- + +**An `eql_v4` order term is one hex string, and every `eql_v4` domain checks its shape when it is written.** A block ORE term (`ob`, and the `lo` and `hi` bounds of a block ORE range) is the lowercase hex of the whole ore-rs ciphertext, `"ob": "0202…"`, where it was a one-element array. Every `eql_v4` domain carrying `ob` must now hold a string starting `0202` or `0203`, and one carrying `op` a string starting `00` or `01`: an EQL v3 term or a term of the wrong scheme is refused with `check_violation` at the write or cast, not at the first comparison. The check reads only the prefix, so its cost does not grow with the term. In `eql-bindings`, a v4 `ob` field is Stack Encrypt's `OreBlockTerm` and an `op` field its `OpeTerm`, typed by the plaintext they order (`DateOrdOre::ob` is an `OreBlockTerm`), and reading a payload checks the term's structure. The `eql_v3` bundle is unchanged, byte for byte. A database that installed an earlier `eql_v4` bundle keeps its stored domains across an install, so it must drop its stored order domains first: see [U-003](../packages/eql/docs/upgrading/v3.1.md#u-003-an-order-term-is-one-hex-string-checked-when-it-is-written). diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 78dcb45a6..6a2a4f4d2 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -187,6 +187,13 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. ### Changed +- **An EQL v4 block ORE term is one hex string.** The `ob` of a value + encrypted into an `_ord_ore` or `text_search_ore` EQL type is + `"ob": "0202…"`, where it was a one-element array, and the EQL v4 bundle + checks the shape of every `ob` and `op` it stores. A database that + installed the EQL v4 bundle attached to `v0.1.0-alpha.1` must drop its + stored order domains before installing the new one (EQL's upgrade note + U-003). - **A call whose context ends fails on its own and leaves the client open.** A cancelled or timed-out call used to close the `encrypt.Client`, or the `auth` profile behind its strategy, for every later caller, and a diff --git a/packages/eql/crates/eql-bindings/CHANGELOG.md b/packages/eql/crates/eql-bindings/CHANGELOG.md index f73a220ac..97dfe1814 100644 --- a/packages/eql/crates/eql-bindings/CHANGELOG.md +++ b/packages/eql/crates/eql-bindings/CHANGELOG.md @@ -20,6 +20,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **A v4 order term is Stack Encrypt's own, typed by its plaintext** + (`stack-encrypt` feature). An `ob` field is + `stack_encrypt::sem::OreBlockTerm` and an `op` field + `stack_encrypt::sem::OpeTerm`, over the type's plaintext: + `DateOrdOre::ob` is an `OreBlockTerm`, `IntegerOrd::op` an + `OpeTerm`. Each is one lowercase hex string on the wire (`ob` was a + one-element array) and is decoded through `from_bytes` when a payload is + read, so an EQL v3 term, a term of the other scheme or a malformed one is + refused there. `v4::terms` no longer re-exports v3's `OreBlock256` and + `OpeCllw`; the generated `v4::terms::OreBlockHex` and `OpeHex` describe + the JSON to TypeScript and JSON Schema, with the pattern the domain CHECKs + require. + - **`v4::plaintext::Date`, `Timestamp`, `Decimal` and `RangeError` are Stack Encrypt's** (`stack_encrypt::value`), re-exported at the same path: the engine orders them, which it can only do for types of its own. @@ -33,7 +46,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 on for `smallint`, `bigint`, `date`, `timestamp`, `numeric`, `real` and `double`, listed by `v4::all_range_query()`. Each holds both bounds of one `BETWEEN`: each bound is the single-bound domain's `ob` (ORE, - `OreBlock256`) or `op` (OPE, `OpeCllw`) value, in a `{v, i, lo, hi}` + `OreBlockTerm`) or `op` (OPE, `OpeTerm`) value, in a `{v, i, lo, hi}` envelope, so a range binds as one parameter. The parameter must be cast to its exact range domain (`$1::eql_v4.query_integer_range_ore`); anything else fails with SQLSTATE 42725. They bind the diff --git a/packages/eql/crates/eql-bindings/README.md b/packages/eql/crates/eql-bindings/README.md index 1146823b4..abc4526d5 100644 --- a/packages/eql/crates/eql-bindings/README.md +++ b/packages/eql/crates/eql-bindings/README.md @@ -38,6 +38,14 @@ Shared wire fields are reusable newtypes in | `OreBlock256` | `ob` | `Vec` | `_ord_ore` domains, `text_search_ore` | | `BloomFilter` | `bf` | `Vec` (signed!) | `_match` domains | +EQL v4 shares these except its order terms. A v4 `ob` field is Stack +Encrypt's `stack_encrypt::sem::OreBlockTerm` and an `op` field its +`OpeTerm`, over the type's plaintext (`v4::date::DateOrdOre::ob` is an +`OreBlockTerm`). Each is one lowercase hex string on the wire, and +reading a payload checks the term's structure, so an EQL v3 term or a term +of the other scheme is refused. `v4::terms::OreBlockHex` and `OpeHex` +describe that JSON to TypeScript and JSON Schema. + Note "v3" names the SQL schema generation (`eql_v3.*`); the JSON envelope version is `v: 3` — the generated domain CHECKs assert `VALUE->>'v' = '3'`, and the wire field names are unchanged from v2 (the purpose-named rename in @@ -133,8 +141,8 @@ and `rust_decimal` features convert to and from `NaiveDate`, caller's, and is not wiped. Only a leap second, which a `DateTime` can hold and a `Timestamp` cannot, is refused in a conversion. On `NumericEq`, `1.5 = 1.50` (the scale is normalised); on `TimestampEq`, an instant is truncated to the microsecond (the nanoseconds below it are dropped, not rounded), so two instants in the same microsecond are equal. Each opens as -it was stored. The ordered date, timestamp and numeric types are refused -until the engine derives their order terms. +it was stored. Their ordered types order a date by its day, a timestamp by +the microsecond (truncated, as on `TimestampEq`) and a decimal by value. Each seals the same leaf the Go SDK seals for the same value, so values written here and by the Go SDK open each other. On the equality diff --git a/packages/eql/docs/reference/catalog-driven-architecture.md b/packages/eql/docs/reference/catalog-driven-architecture.md index 49c6fd5b2..2ffc430fa 100644 --- a/packages/eql/docs/reference/catalog-driven-architecture.md +++ b/packages/eql/docs/reference/catalog-driven-architecture.md @@ -143,10 +143,17 @@ A `Term` answers every question the generators need, via exhaustive `impl` metho | `extractor()` | `eq_term` | `ord_term_ore` | `match_term` | `ord_term` | | `ctor()` | `hmac_256` | `ore_block_256` | `bloom_filter` | `ope_cllw` | | `binding_newtype()` | `Hmac256` | `OreBlock256` | `BloomFilter` | `OpeCllw` | +| `v4_format()` | none | `OreBlockTerm`, `0202`/`0203` | none | `OpeTerm`, `00`/`01` | | `role()` | `Eq` | `Ord` | `Match` | `Ord` | | `operators()` | `= <>` | `= <> < <= > >=` | `@@` | `= <> < <= > >=` | | `provides_ordering()` | `false` | `true` | `false` | `true` | +`binding_newtype()` names the EQL v3 type. `v4_format()` is how EQL v4 stores an order +term instead: as one hex string of Stack Encrypt's own term (`OreBlockTerm`, +`OpeTerm`, over the type's plaintext), which must start with one of the listed +prefixes. Codegen writes the v4 Rust field, its JSON Schema pattern and the v4 domain +CHECKs from that one entry. + `Ope` is the CLLW-OPE term: a hex-encoded ciphertext that is natively `bytea`-sortable after hex-decode (no custom comparison protocol), so — like `Hm` — its extractor is the whole SEM surface. The extractor name tracks the *domain* it serves, not the cipher: `Ope` backs the diff --git a/packages/eql/docs/upgrading/v3.1.md b/packages/eql/docs/upgrading/v3.1.md index 9c8eb257a..c26b9bcb6 100644 --- a/packages/eql/docs/upgrading/v3.1.md +++ b/packages/eql/docs/upgrading/v3.1.md @@ -1,9 +1,9 @@ # Upgrading to EQL 3.1 -`3.1.0` is a minor release. It changes two things in the `eql_v4` surface that -a caller can meet without changing any SQL: which terms a text search query -must carry, and which block ORE ciphertext the `_ord_ore` and -`text_search_ore` domains hold. **The `eql_v3` bundle is unchanged, byte for +`3.1.0` is a minor release. It changes three things in the `eql_v4` surface +that a caller can meet without changing any SQL: which terms a text search +query must carry, which block ORE ciphertext the `_ord_ore` and +`text_search_ore` domains hold, and the shape every order term must have. **The `eql_v3` bundle is unchanged, byte for byte.** If you install only `eql_v3`, nothing here applies to you. Both changes land through Stack Encrypt as well as through the SQL: with @@ -24,6 +24,12 @@ before this release. previous `eql_v4` bundle compared; rows and queries holding them must be written again through Stack Encrypt — [U-002](#u-002-the-block-ore-term-is-ore-rss-6-bit-scheme). +3. An `eql_v4` order term is **one hex string**, `"ob": "0202…"` rather than a + one-element array, and every `eql_v4` domain with an order term **checks + its prefix** when it is written. A database that installed an earlier + `eql_v4` bundle must drop its stored order domains before installing this + one — + [U-003](#u-003-an-order-term-is-one-hex-string-checked-when-it-is-written). ## Compatibility @@ -35,7 +41,9 @@ before this release. | Stored `public.eql_v4_text_search` / `public.eql_v4_text_search_ore` values | **Unchanged.** A stored value still carries every term. Text too short for a match term stores the empty filter, `"bf": []`, which `=` and `<` find and no `@@` query does. | | `public.eql_v4_*_ord_ore`, `public.eql_v4_text_search_ore`, `eql_v4.query_*_ord_ore`, `eql_v4.query_*_range_ore` | **Term format changed.** The `ob` term is one ore-rs 6-bit block ORE ciphertext. The previous 8-bit layout is refused, not misread. Re-encryption of those columns is required where such rows exist. See U-002. | | `eql_v4_internal.ore_block_256`, `compare_ore_block_256_term` | **Renamed** in the v4 bundle to `eql_v4_internal.ore_block_v2` and `compare_ore_block_v2_term`, with a new body. Only the installer and the operator class name them. | -| `eql_v4` OPE (`op`) terms, equality (`hm`) terms, ciphertexts (`c`) | **Unchanged** by this release. | +| `ob` in every `eql_v4` block ORE domain and range bound | **Wire shape changed.** One hex string, `"ob": "0202…"`, where it was a one-element array. The CHECK requires a string starting `0202` or `0203`. See U-003. | +| `op` in every `eql_v4` OPE domain and range bound | **Tightened CHECK.** A string starting `00` or `01`, the term's carry byte. The term itself is unchanged. See U-003. | +| `eql_v4` equality (`hm`) terms, ciphertexts (`c`) | **Unchanged** by this release. | | Postgres 14, 15, 16, 17 | Supported. | | Supabase / managed Postgres (no superuser) | As before: the block ORE domains need the operator class the installer creates, so a non-superuser install disables them ([U-003 in the 3.0 guide](./v3.0.md#u-003-non-superuser-installs-disable-the-ore-backed-domains)). U-002 only concerns installs where they are enabled. | @@ -152,12 +160,91 @@ ORDER BY 1; -- ore_block_v2 names only; no ore_block_256 SELECT count(*) FROM users WHERE age <= age; -- equals count(*), no error ``` +### U-003: An order term is one hex string, checked when it is written + +**What changed.** An `eql_v4` block ORE term (`ob`, and the `lo` and `hi` +bounds of a `query_*_range_ore` operand) is one JSON string, the lowercase +hex of the whole ore-rs ciphertext: + +```json +{"v": 4, "i": {"t": "users", "c": "age"}, "c": "…", "ob": "02020006…"} +``` + +It was a one-element array (`"ob": ["02020006…"]`). Every `eql_v4` domain +that carries an order term now checks its shape in its CHECK, stored, query +and range alike: + +- `ob` must be a string starting `0202` (ore-rs's fixed-width engine) or + `0203` (its chained engine); +- `op` must be a string starting `00` or `01`, the OPE term's carry byte. + +A value of any other shape is refused when it is written or cast, +SQLSTATE `23514` (`check_violation`), before any comparison reads it. The +check reads the prefix only, so it costs the same whatever the term's +length. The comparator still checks the rest of a block ORE term's +structure, as U-002 describes. + +In Rust, eql-bindings holds these fields as Stack Encrypt's own terms, typed +by the plaintext they order: `DateOrdOre::ob` is a +`stack_encrypt::sem::OreBlockTerm` and `IntegerOrd::op` an +`OpeTerm`. Reading a payload checks the term's structure, so a +malformed term, or a term of the other scheme, is refused there too. + +**Why.** The array was cipherstash-client's shape for its 8-bit block ORE, +which held several terms for text. An ore-rs term is always one, and the +array added a level that said nothing and checked nothing. Checking the +prefix at write time catches an EQL v3 term or a term of the wrong scheme +where it enters the database, not at the first query that reads it. + +**Who is affected.** No Stack Encrypt release wrote an `eql_v4` order term +in the array shape: the ordered types became producible in this release. A +program that builds `eql_v4` payloads by hand must write `ob` as a string. + +A database that installed an earlier `eql_v4` bundle, such as the one +attached to the Go module's `v0.1.0-alpha.1` release, keeps its stored +`public.eql_v4_*` domains when this bundle is installed: they are created +only if they do not exist. Those domains still require `ob` to be an array +and do not check `op`, so they refuse every `ob` this release writes. + +**What to do.** In a database that installed an earlier `eql_v4` bundle, +drop the stored order domains before installing this one, so the install +creates them with the new CHECKs. A domain a column uses cannot be dropped; +move such a column to `jsonb` first and back afterwards, or write it again +into a new column: + +```sql +-- Lists the stored eql_v4 domains that carry an order term. +SELECT t.typname +FROM pg_type t +JOIN pg_constraint c ON c.contypid = t.oid +WHERE t.typnamespace = 'public'::regnamespace + AND t.typname LIKE 'eql\_v4\_%' + AND (pg_get_constraintdef(c.oid) LIKE '%''ob''%' + OR pg_get_constraintdef(c.oid) LIKE '%''op''%'); + +-- For each one no column uses: +DROP DOMAIN public.eql_v4_integer_ord_ore; +``` + +The query operand domains (`eql_v4.query_*`) are recreated by every +install, so they need nothing. + +**Verification.** + +```sql +-- The stored domain refuses the array shape: +SELECT '{"v": 4, "i": {"t": "t", "c": "c"}, "c": "x", "ob": ["0202"]}'::jsonb + ::public.eql_v4_integer_ord_ore; +-- ERROR: value for domain eql_v4_integer_ord_ore violates check constraint +``` + ## Verification checklist - [ ] The `eql_v4` install reports `3.1.0` (`SELECT obj_description('eql_v4'::regnamespace, 'pg_namespace')`). - [ ] A one-term `eql_v4.query_text_search` operand casts (U-001). - [ ] `eql_v4_internal` lists `ore_block_v2` objects and no `ore_block_256` (U-002). - [ ] Every `_ord_ore` / `text_search_ore` column passes the self-comparison scan (U-002). +- [ ] The stored `eql_v4` order domains require a hex string with a scheme prefix (U-003). - [ ] Grants and functional indexes have been re-applied after the install ([U-002 in the 3.0.5 guide](./v3.0.5.md#u-002-grants-and-dependent-objects-do-not-survive-an-install)). From eb415b4d70a1fda01b4e2788dc868a426bca2eda Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 22:35:33 +1100 Subject: [PATCH 4/4] docs(eql): link U-003 from the changelog's directory, and show string range bounds The changeset linked U-003 relative to the repository root, but Changesets copies it into packages/eql/packages/eql/CHANGELOG.md, where that path names nothing. It now uses ../../docs, as the U-001 and U-002 changesets do. The v4 range reference still said a block ORE range's bounds are ob arrays, which the range domains now refuse with check_violation. It now says both bounds are one hex string, names the prefixes, and links U-003. It also no longer says the envelope waits on the ordered types becoming producible: they are, and a caller builds the envelope from two single-bound queries. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- .changeset/eql-v4-order-term-shape.md | 2 +- packages/eql/docs/reference/eql-functions.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.changeset/eql-v4-order-term-shape.md b/.changeset/eql-v4-order-term-shape.md index bfaa8afa3..df869ac6a 100644 --- a/.changeset/eql-v4-order-term-shape.md +++ b/.changeset/eql-v4-order-term-shape.md @@ -2,4 +2,4 @@ '@cipherstash/eql': minor --- -**An `eql_v4` order term is one hex string, and every `eql_v4` domain checks its shape when it is written.** A block ORE term (`ob`, and the `lo` and `hi` bounds of a block ORE range) is the lowercase hex of the whole ore-rs ciphertext, `"ob": "0202…"`, where it was a one-element array. Every `eql_v4` domain carrying `ob` must now hold a string starting `0202` or `0203`, and one carrying `op` a string starting `00` or `01`: an EQL v3 term or a term of the wrong scheme is refused with `check_violation` at the write or cast, not at the first comparison. The check reads only the prefix, so its cost does not grow with the term. In `eql-bindings`, a v4 `ob` field is Stack Encrypt's `OreBlockTerm` and an `op` field its `OpeTerm`, typed by the plaintext they order (`DateOrdOre::ob` is an `OreBlockTerm`), and reading a payload checks the term's structure. The `eql_v3` bundle is unchanged, byte for byte. A database that installed an earlier `eql_v4` bundle keeps its stored domains across an install, so it must drop its stored order domains first: see [U-003](../packages/eql/docs/upgrading/v3.1.md#u-003-an-order-term-is-one-hex-string-checked-when-it-is-written). +**An `eql_v4` order term is one hex string, and every `eql_v4` domain checks its shape when it is written.** A block ORE term (`ob`, and the `lo` and `hi` bounds of a block ORE range) is the lowercase hex of the whole ore-rs ciphertext, `"ob": "0202…"`, where it was a one-element array. Every `eql_v4` domain carrying `ob` must now hold a string starting `0202` or `0203`, and one carrying `op` a string starting `00` or `01`: an EQL v3 term or a term of the wrong scheme is refused with `check_violation` at the write or cast, not at the first comparison. The check reads only the prefix, so its cost does not grow with the term. In `eql-bindings`, a v4 `ob` field is Stack Encrypt's `OreBlockTerm` and an `op` field its `OpeTerm`, typed by the plaintext they order (`DateOrdOre::ob` is an `OreBlockTerm`), and reading a payload checks the term's structure. The `eql_v3` bundle is unchanged, byte for byte. A database that installed an earlier `eql_v4` bundle keeps its stored domains across an install, so it must drop its stored order domains first: see [U-003](../../docs/upgrading/v3.1.md#u-003-an-order-term-is-one-hex-string-checked-when-it-is-written). diff --git a/packages/eql/docs/reference/eql-functions.md b/packages/eql/docs/reference/eql-functions.md index d65245783..27e57d28d 100644 --- a/packages/eql/docs/reference/eql-functions.md +++ b/packages/eql/docs/reference/eql-functions.md @@ -140,10 +140,10 @@ Every ordered family has a **range query domain** per ordering term, beside its | Range domain | Bounds | Column domains it applies to | |---|---|---| -| `eql_v4.query__range_ore` | two block-ORE terms (`ob` arrays) | `public.eql_v4__ord_ore`, `public.eql_v4_text_search_ore` | +| `eql_v4.query__range_ore` | two block-ORE terms (`ob` hex strings) | `public.eql_v4__ord_ore`, `public.eql_v4_text_search_ore` | | `eql_v4.query__range_ope` | two CLLW-OPE terms (`op` hex strings) | `public.eql_v4__ord`, `public.eql_v4__ord_ope`, `public.eql_v4_text_search` | -`` is any ordered family: `integer`, `smallint`, `bigint`, `date`, `timestamp`, `numeric`, `text`, `real`, `double`. A range value is `{"v": 4, "i": {...}, "lo": , "hi": }`: each bound is the single-bound query domain's `ob` (ORE) or `op` (OPE) value, in a `{v, i, lo, hi}` envelope, with no ciphertext `c`. Stack Encrypt does not build this envelope yet; it will when the ordered EQL types become producible. +`` is any ordered family: `integer`, `smallint`, `bigint`, `date`, `timestamp`, `numeric`, `text`, `real`, `double`. A range value is `{"v": 4, "i": {...}, "lo": , "hi": }`: each bound is the single-bound query domain's `ob` (ORE) or `op` (OPE) value, in a `{v, i, lo, hi}` envelope, with no ciphertext `c`. Both are one lowercase hex string: a block ORE bound starts `0202` or `0203` and an OPE bound `00` or `01`, and the range domain refuses any other bound with `check_violation` (an EQL v3 `ob` array among them; see [U-003](../upgrading/v3.1.md#u-003-an-order-term-is-one-hex-string-checked-when-it-is-written)). Stack Encrypt does not build this envelope yet: build it from the `ob` or `op` of two single-bound queries of the column's type. ```sql -- true when lo <= column <= hi, both bounds included