feat(stack-encrypt): answer a document field's JSON operators and open its reads from a data plan

.cargo/mutants.toml

@@ -116,6 +116,9 @@ exclude_re = [
   # A document always has a root, so its terms are never empty: `false` is
   # the value `is_empty` returns. The `true` replacement is still caught.
   'stack-encrypt/src/json/derive\.rs:\d+:\d+: replace JsonTerms::is_empty -> bool with false$',
+  # A JSON operator's input: the `JsonText` visitor's `expecting` writes a
+  # message `read_frame` discards for its fixed `Reason::JsonInput`.
+  'stack-encrypt/src/dynamic/json_query\.rs:\d+:\d+: replace <impl Deserialize<\x27de> for JsonText>::deserialize::<impl Visitor<\x27_> for V>::expecting -> std::fmt::Result with Ok\(Default::default\(\)\)$',
 ]
 
 # Headroom over the measured baseline before a slow-but-correct mutant is

.changeset/eql-encrypted-json-operations.md

@@ -0,0 +1,5 @@
+---
+"@cipherstash/eql": minor
+---
+
+**`JsonSearch` answers its JSON operators by name.** `eql-bindings`' target table row now tells Stack Encrypt the domain an ordering at a path binds as (`Target::leaf_query_sql_domain`, `eql_v4.query_json_leaf`), so a data plan's `lt_at`, `lte_at`, `gt_at`, `gte_at` and `between_at` bind their operand as that domain, beside `contains`, `contained_by` and `eq_at`, which bind the needle as `eql_v4.query_json`. No SQL changes.

languages/golang/CHANGELOG.md

@@ -12,6 +12,14 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`.
 
 ### Added
 
+- **Query and read an encrypted JSON document, at the record level**
+  (#1350, #1352). `Cipher.QueryTerm` derives a document field's operands:
+  `QueryTerm.Encoding` says what the term holds, and `QueryTerm.Text`
+  carries a selector, a node's token or a read's tokens.
+  `Cipher.JSONOpen` opens the rows a selector or subtree read returned
+  against the request that prepared it. `record.Operator.Input` names an
+  operator's input. Generated methods follow with `stashgen`.
+
 - **The EQL types for an encrypted JSON document** (#1348):
   `eql.JSONSearch` (`public.eql_v4_json_search`), its containment needle
   `eql.JSONSearchQuery`, its ordering operand

languages/golang/encrypt/checker.go

@@ -218,7 +218,7 @@ func parseTarget(entry vcvalue.Object) (record.Target, error) {
 
 // parseOperators reads a list of operators, an se_targets entry's or
 // se_operators's: a list of objects, each with exactly the keys operator,
-// fidelity, arity and index, every value one the engine writes.
+// fidelity, arity, index and input, every value one the engine writes.
 func parseOperators(value any) ([]record.Operator, error) {
 	list, ok := value.([]any)
 	if !ok {
@@ -256,14 +256,21 @@ func parseOperators(value any) ([]record.Operator, error) {
 				}
 			case "index":
 				op.Index, err = parseOperatorIndex(s)
+			case "input":
+				switch s {
+				case "value", "pair", "json", "path_json", "path_pair", "path", "path_bound":
+					op.Input = s
+				default:
+					return nil, fmt.Errorf("operator %d: unknown input %q", i, s)
+				}
 			default:
 				return nil, fmt.Errorf("operator %d: unknown key %q", i, f.Key)
 			}
 			if err != nil {
 				return nil, fmt.Errorf("operator %d: %v", i, err)
 			}
 		}
-		if len(seen) != 4 || op.Name == "" {
+		if len(seen) != 5 || op.Name == "" {
 			return nil, fmt.Errorf("operator %d: not every key, or no name", i)
 		}
 		operators = append(operators, op)
@@ -272,13 +279,17 @@ func parseOperators(value any) ([]record.Operator, error) {
 }
 
 // parseOperatorName reads an operator's name as the engine writes it: one
-// of eq, lt, lte, gt, gte, between and matches, the names the engine's
-// OpSpec::parse accepts. The catalog (se_targets) and se_query_term are
+// of the names the engine's OpSpec::parse accepts, the scalar operators
+// (eq, lt, lte, gt, gte, between, matches) and a document's (contains,
+// contained_by, eq_at, the orderings at a path, between_at, selector,
+// subtree, children, child_count). The catalog (se_targets) and se_query_term are
 // read through this, parseFidelity and parseOperatorIndex, so the two
 // readers refuse the same values.
 func parseOperatorName(s string) (string, error) {
 	switch s {
-	case "eq", "lt", "lte", "gt", "gte", "between", "matches":
+	case "eq", "lt", "lte", "gt", "gte", "between", "matches",
+		"contains", "contained_by", "eq_at", "lt_at", "lte_at", "gt_at", "gte_at", "between_at",
+		"selector", "subtree", "children", "child_count":
 		return s, nil
 	}
 	return "", fmt.Errorf("unknown operator %q", s)
@@ -294,10 +305,10 @@ func parseFidelity(s string) (record.Fidelity, error) {
 }
 
 // parseOperatorIndex reads the index that answers an operator: eq, match,
-// ore, ope or ore_block.
+// ore, ope, ore_block or json.
 func parseOperatorIndex(s string) (record.Output, error) {
 	switch o := record.Output(s); o {
-	case record.Equality, record.Match, record.Ore, record.Ope, record.OreBlock:
+	case record.Equality, record.Match, record.Ore, record.Ope, record.OreBlock, record.JSONIndex:
 		return o, nil
 	}
 	return "", fmt.Errorf("unknown index %q", s)

languages/golang/encrypt/guest.go

@@ -86,6 +86,7 @@ type instance struct {
 	cipherInit, shutdown, keyset  api.Function
 	term                          api.Function
 	queryTerm, verify, limit      api.Function
+	jsonOpen                      api.Function
 	encryptRecord, decryptRecord  api.Function
 	planCheck, targets, operators api.Function
 	// The step loop (step.go): take a paused export's step, resume it with
@@ -179,6 +180,7 @@ func newInstance(ctx context.Context, wasm []byte, t *transport, policy guest.Lo
 		"se_keyset":         &inst.keyset,
 		"se_term":           &inst.term,
 		"se_query_term":     &inst.queryTerm,
+		"se_json_open":      &inst.jsonOpen,
 		"se_verify":         &inst.verify,
 		"se_target_limit":   &inst.limit,
 		"se_encrypt_record": &inst.encryptRecord,

languages/golang/encrypt/guest/src/abi.rs

@@ -571,6 +571,49 @@ pub unsafe extern "C" fn se_query_term(
     })
 }
 
+/// Open the rows a read of a document field returned ([`ops::json_open`]):
+/// the codec-encoded rows (a list per database row of entries), the
+/// codec-encoded plan, the field's name (UTF-8 bytes) and the codec-encoded
+/// read request in, under the keyset `opts` selects (the keyset the read
+/// was prepared under); a codec-encoded list per row of JSON texts out. The
+/// output buffer contains **plaintext**: the host copies it out and
+/// immediately `se_dealloc`s it (which wipes it). The rows and the request
+/// hold no plaintext and are left untouched.
+///
+/// # Safety
+///
+/// As for [`se_term`].
+#[no_mangle]
+pub unsafe extern "C" fn se_json_open(
+    rows_ptr: *const u8,
+    rows_len: u32,
+    plan_ptr: *const u8,
+    plan_len: u32,
+    field_ptr: *const u8,
+    field_len: u32,
+    req_ptr: *const u8,
+    req_len: u32,
+    opt_ptr: *const u8,
+    opt_len: u32,
+) -> u64 {
+    export(|| {
+        // SAFETY: host-owned ranges the export was handed; the borrows end
+        // before it returns.
+        let rows = unsafe { input(rows_ptr, rows_len)? }.to_vec();
+        let plan = unsafe { input(plan_ptr, plan_len)? }.to_vec();
+        let field = unsafe { input(field_ptr, field_len)? }.to_vec();
+        let request = unsafe { input(req_ptr, req_len)? }.to_vec();
+        let opts = unsafe { input(opt_ptr, opt_len)? };
+        ops::validate::json_open(&rows, &plan, &field, &request)?;
+        let options = parse_options(opts, Side::Mint)?;
+        let cipher = cipher()?;
+        run(Box::pin(async move {
+            let keyset = options.keyset.resolve(&cipher).await?;
+            ops::json_open(&keyset, &rows, &plan, &field, &request).await
+        }))
+    })
+}
+
 /// Verify a page of decrypted values against a probabilistic query
 /// ([`ops::verify`]): the query's codec-encoded input and a codec-encoded
 /// list of values, the codec-encoded plan, and the field's and the

languages/golang/encrypt/guest/src/ops.rs

@@ -236,17 +236,21 @@ where
 ///
 /// ```text
 /// { "operator": "matches", "fidelity": "probabilistic", "index": "match",
-///   "domain": null, "term": <bytes> }
+///   "domain": null, "encoding": "bytes", "term": <bytes> }
 /// ```
 ///
 /// The input is one value for every operator but `between`, whose input is
-/// a list of two, low then high. It is read as the field's declared kind
-/// first, so a host's float `34.0` queries a `uint64` field. A field with
-/// indexes derives the term of the index that answers the operator, as
-/// bytes; `between` derives one per bound, as a list of two. A field that
-/// names an EQL type runs the type's query twin through this build's
+/// a list of two, low then high, and a document's operators, whose input
+/// is an object of named inputs (`{"json"}`, `{"path", "json"}`, ...; see
+/// [`dynamic::OpSpec::input`]). A value is read as the field's declared
+/// kind first, so a host's float `34.0` queries a `uint64` field. A field
+/// with indexes derives the term of the index that answers the operator,
+/// as bytes; `between` derives one per bound, as a list of two. A field
+/// that names an EQL type runs the type's query twin through this build's
 /// resolver, and `term` holds the twin's JSON bytes and `domain` its
-/// PostgreSQL domain. No data key is minted.
+/// PostgreSQL domain. A document's operator gives `encoding` `eql` (a
+/// needle or leaf operand), `text` (a selector or node token) or
+/// `text_array` (a subtree read's tokens). No data key is minted.
 pub async fn query_term<K>(
     cipher: &KeysetCipher<'_, K>,
     input: &[u8],
@@ -272,6 +276,42 @@ where
     encode_answer(&term)
 }
 
+/// Open the rows a read of a document field returned
+/// ([`dynamic::json_open`]): a codec-encoded list per database row of the
+/// entries the SQL returned (`public.eql_v4_json_entry` JSON, text or
+/// bytes), the plan, the field's name and the codec-encoded read request
+/// (`{"operation": "selector" | "subtree", "path", "wildcard_bound"?}`) in;
+/// a codec-encoded list per row of JSON texts (bytes) out, which is
+/// **plaintext**. The selector or the read key is derived again from the
+/// request under `keyset`, which the host names as it named it to prepare
+/// the read, and the rows open through that keyset only: a document sealed
+/// under another is refused before any key is retrieved. One batched key
+/// request for every row.
+pub async fn json_open<K>(
+    keyset: &KeysetCipher<'_, K>,
+    rows: &[u8],
+    plan: &[u8],
+    field: &[u8],
+    request: &[u8],
+) -> Result<Vec<u8>, u32>
+where
+    K: DataKeySource + Sync + 'static,
+{
+    let (plan, field) = parse_field(plan, field)?;
+    let opened = dynamic::json_open(
+        keyset,
+        &plan,
+        field,
+        parse_frame(request)?,
+        parse_frame(rows)?,
+        &resolver(),
+    )
+    .map_err(|e| fail_dynamic(&e))?
+    .await
+    .map_err(|e| fail_error(&e))?;
+    encode_frame(opened)
+}
+
 /// Verify a page of decrypted values against a probabilistic query: the
 /// query's codec-encoded input and a codec-encoded list of values, with the
 /// plan, the field's name and the operator's name, in; a codec-encoded list
@@ -332,18 +372,25 @@ pub fn target_limit(
     encode_answer(&limit.rows())
 }
 
-/// The plan, field name and operator a query names: the plan parsed against
-/// this build's resolver, the field's name UTF-8, and the operator one the
-/// engine knows ([`dynamic::OpSpec::parse`]); an unknown operator is
-/// refused naming the field, as an operator the field does not answer is.
+/// A plan and one of its fields' names: the plan parsed against this
+/// build's resolver, and the field's name UTF-8.
+fn parse_field<'f>(plan: &[u8], field: &'f [u8]) -> Result<(dynamic::record::Plan, &'f str), u32> {
+    let plan = parse_plan(plan)?;
+    let field = std::str::from_utf8(field)
+        .map_err(|_| last_error::malformed("the field name is not UTF-8"))?;
+    Ok((plan, field))
+}
+
+/// The plan, field name and operator a query names: [`parse_field`]'s two,
+/// and the operator one the engine knows ([`dynamic::OpSpec::parse`]); an
+/// unknown operator is refused naming the field, as an operator the field
+/// does not answer is.
 fn parse_query<'f>(
     plan: &[u8],
     field: &'f [u8],
     operator: &[u8],
 ) -> Result<(dynamic::record::Plan, &'f str, dynamic::OpSpec), u32> {
-    let plan = parse_plan(plan)?;
-    let field = std::str::from_utf8(field)
-        .map_err(|_| last_error::malformed("the field name is not UTF-8"))?;
+    let (plan, field) = parse_field(plan, field)?;
     let operator = std::str::from_utf8(operator)
         .ok()
         .and_then(dynamic::OpSpec::parse)
@@ -500,6 +547,21 @@ pub mod validate {
         .map_err(|e| fail_dynamic(&e))
     }
 
+    /// A read's inputs, as [`json_open`](super::json_open) takes them: the
+    /// plan parses, the field stores a document, and the request and rows
+    /// are the read's ([`dynamic::check_json_open`]).
+    pub fn json_open(rows: &[u8], plan: &[u8], field: &[u8], request: &[u8]) -> Result<(), u32> {
+        let (plan, field) = parse_field(plan, field)?;
+        dynamic::check_json_open(
+            &plan,
+            field,
+            parse_frame(request)?,
+            parse_frame(rows)?,
+            &resolver(),
+        )
+        .map_err(|e| fail_dynamic(&e))
+    }
+
     /// A record tree against its plan, as [`decrypt_record`] takes them:
     /// the plan parses, the tree decodes with well-formed leaves, every
     /// ciphertext-bearing field has a `"c"` node that is not a passthrough,
@@ -894,7 +956,8 @@ mod tests {
             panic!("a query term is an object")
         };
         assert!(matches!(&term[3].1, Tree::String(s) if s.as_bytes() == b"eql_v4.query_text_eq"));
-        let Tree::Bytes(query_bytes) = &term[4].1 else {
+        assert!(matches!(&term[4].1, Tree::String(s) if s.as_bytes() == b"eql"));
+        let Tree::Bytes(query_bytes) = &term[5].1 else {
             panic!("the term is the query value's bytes")
         };
         let probe_json: serde_json::Value =

languages/golang/encrypt/guest/src/status.rs

@@ -141,6 +141,13 @@ pub fn status_for_error(error: &stack_encrypt::Error) -> u32 {
         // one it was opened as: sealed as that type, not tampered with, so
         // the caller's input and not STATUS_AUTH.
         stack_encrypt::Error::OpenedType { .. } => STATUS_ENCODING,
+        // A JSON read's rows that are not the ones its request asked for,
+        // or a wildcard read of an array longer than its bound: statements
+        // about the rows the host handed over, decided before any key is
+        // retrieved, which the host acts on (asks again, raises the bound).
+        // The diagnostic's code tells the two apart.
+        stack_encrypt::Error::JsonWrongEntry { .. }
+        | stack_encrypt::Error::JsonWildcardTruncated { .. } => STATUS_ENCODING,
         _ => STATUS_INTERNAL,
     }
 }

languages/golang/encrypt/guest/src/targets.rs

@@ -110,9 +110,14 @@ fn describe(target: &eql_bindings::encryption::targets::Target) -> TargetDescrip
     );
     // A type with a match index answers `matches` through its twin only
     // when the engine knows the options its filter is built under.
-    match target.match_options() {
+    let descriptor = match target.match_options() {
         Some(options) => descriptor.with_match_options(options),
         None => descriptor,
+    };
+    // A document's orderings at a path bind as its leaf operand.
+    match target.leaf_query_sql_domain() {
+        Some(domain) => descriptor.with_leaf_query_sql_domain(domain),
+        None => descriptor,
     }
 }