From eb47bad0678028cf67d1f672ffd85609347a92f4 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 01:17:36 +1100 Subject: [PATCH 01/13] feat(go): generate encrypted JSON document fields A json.RawMessage field tagged encrypt_into=EncryptedJson, with optional ordered paths and sealed_root, is sealed by the engine from the field's data plan and gets generated methods for every json_tree query and read: Contains, ContainedBy, EqAt, Selector, Subtree, Children, ChildCount, and an At accessor per ordered path with Lt, Lte, Gt, Gte and Between. Selector and subtree reads return handles (encrypt.JSONSelection, encrypt.JSONRead) that keep the request they were prepared from and open the rows the database returns only as answers to it. TestJSONDocumentsQueryAndReadInPostgres runs every method against the EQL v4 bundle from the tree and joins go:test:postgres. Refs #1347, #1351, #1352 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 13 + languages/golang/cmd/stashgen/README.md | 31 +- languages/golang/cmd/stashgen/main_test.go | 66 ++++ languages/golang/encrypt/README.md | 30 +- languages/golang/encrypt/eql_postgres_test.go | 4 + languages/golang/encrypt/gensupport/codec.go | 2 +- .../golang/encrypt/gensupport/declaration.go | 45 +++ .../gensupport/gensupport_internal_test.go | 2 +- languages/golang/encrypt/gensupport/json.go | 118 +++++++ languages/golang/encrypt/gensupport/widen.go | 12 + .../internal/testusers/customer_stash.go | 283 +++++++++++++++++ .../encrypt/internal/testusers/customers.go | 15 + languages/golang/encrypt/json.go | 207 ++++++++++++ languages/golang/encrypt/json_fields_test.go | 133 ++++++++ .../golang/encrypt/json_postgres_test.go | 297 ++++++++++++++++++ languages/golang/encrypt/term.go | 18 +- languages/golang/stashgen/declaration.go | 11 +- languages/golang/stashgen/emit.go | 237 +++++++++++++- languages/golang/stashgen/engine.go | 7 +- languages/golang/stashgen/read.go | 11 +- languages/golang/stashgen/tag.go | 95 ++++++ .../testdata/stubsdk/encrypt/encrypt.go | 33 ++ .../stubsdk/encrypt/gensupport/gensupport.go | 42 +++ mise.toml | 6 +- 24 files changed, 1700 insertions(+), 18 deletions(-) create mode 100644 languages/golang/encrypt/gensupport/json.go create mode 100644 languages/golang/encrypt/internal/testusers/customer_stash.go create mode 100644 languages/golang/encrypt/internal/testusers/customers.go create mode 100644 languages/golang/encrypt/json_fields_test.go create mode 100644 languages/golang/encrypt/json_postgres_test.go diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index aec2688b0..273c7b01c 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -12,6 +12,19 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. ### Added +- **Encrypted JSON documents with `stashgen`** (#1351, #1352). A + `json.RawMessage` field tagged `encrypt_into=EncryptedJson` (options: + `ordered=$.a;$.b[*].c`, `sealed_root=false`) is stored as an + `eql.EncryptedJSON` in a `public.eql_v4_json_tree` column; a + `*json.RawMessage` field takes `null=sql`. Its `Fields` entry has + `Contains`, `ContainedBy`, `EqAt`, `Selector`, `Subtree`, `Children` and + `ChildCount`, and `At.` per ordered path with `Lt`, `Lte`, `Gt`, + `Gte` and `Between`. `encrypt.JSONSelection` and `encrypt.JSONRead` open + the rows `->` and `eql_v4.json_tree_subtree` return against the request + that prepared them; `encrypt.JSONSelector`, `JSONToken` and `JSONTokens` + bind as `text` and `text[]`. `gensupport.JSON`, `TargetOptions`, + `Declaration.Options` and the `JSON*` helpers are the generated code's. + - **Query and read an encrypted JSON document, at the record level** (#1350, #1352). `Cipher.QueryTerm` derives a document field's operands: `QueryTerm.Encoding` says what the term holds, and `QueryTerm.Text` diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 8a20c7b2e..8386a75ce 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -74,7 +74,8 @@ The first part of a tag is the field's name, which is the column name in a datab | `stash:"notes,encrypt"` | seal the field, with no index | | `stash:"email,encrypt,index=equality;match"` | seal the field, and derive each index beside it | | `stash:"score,index=ore"` | derive the index alone; no ciphertext is stored, so `Decrypt` leaves the field at its zero value | -| `stash:"attrs,index=json"` | refused in this build: the engine does not derive the `json` index yet | +| `stash:"profile,encrypt_into=EncryptedJson(ordered=$.age)"` | on a `json.RawMessage`: store the field as an encrypted JSON document; see "Encrypted JSON documents" | +| `stash:"attrs,index=json"` | refused: a JSON document is stored with `encrypt_into=EncryptedJson`, not as an index beside a ciphertext | | `stash:"id,passthrough"` | store the field as it is | | `stash:"tenant,context_field"` | the field's value is the context of every other field in the struct, in place of a `context=` tag; stored as it is | | `stash:"-"` | leave the field out | @@ -108,7 +109,33 @@ An index takes its options in parentheses after its name, separated by commas: ` This build refuses an index with options: a query term is derived with the default options only, so a stored term with other options would never match one. These words are the same as the Rust API's words for the same behaviour, with two that only Go has. `opaque` seals the whole struct as one `bytes` field holding a storage blob (see "What crosses the binding"). -`json` names the JSON index, which the Rust API does not declare yet; this build refuses it. +`json` names the JSON index, which a field reaches only through `encrypt_into=EncryptedJson`. + +### Encrypted JSON documents + +A `json.RawMessage` field with `encrypt_into=EncryptedJson` is stored as one encrypted JSON document, an `eql.EncryptedJSON` in a `public.eql_v4_json_tree` column. +The field holds the document's JSON text; the engine parses it, so an object that names a member twice, an integer outside the 64-bit range and text that is not JSON are refused with `encrypt.ErrEncoding`. +Values come back as JSON text holding the same values; key order and the spelling of numbers are not kept. +A `*json.RawMessage` field is optional and must say `null=sql`: a document has no fixed length to hide a missing one behind. A non-nil pointer to `null` is a stored JSON `null`. + +The type takes options in parentheses: `ordered=` names the paths whose numbers and strings carry an ordering term, separated by `;`, and `sealed_root=false` turns off the sealed root (on by default). +`encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at)` orders two paths. +A path in a tag is plain JSONPath: `$`, then `.key` steps of letters, digits and underscores, and `[n]` or `[*]` steps; an ordered path names no position. The engine checks every path when `stashgen` runs. + +The field's `Fields` entry has a method per operation: + +| Method | Returns | SQL | +|---|---|---| +| `Contains(ctx, c, needle)`, `ContainedBy(ctx, c, needle)` | `eql.EncryptedJSONQuery` | `col @> $1::eql_v4.query_json_tree`, `col <@ $1::eql_v4.query_json_tree` | +| `EqAt(ctx, c, path, v)` | `eql.EncryptedJSONQuery` | `col @> $1::eql_v4.query_json_tree` | +| `At.Age.Lt(ctx, c, v)`, `Lte`, `Gt`, `Gte`, `Between` | `eql.EncryptedJSONLeafQuery` | `(col -> $sel::text) < $1::eql_v4.query_json_tree_leaf` | +| `Selector(ctx, c, path)` | `encrypt.JSONSelection`: `Selector()` and `Open` | `col -> $1::text` | +| `Subtree(ctx, c, path, opts)` | `encrypt.JSONRead`: `Tokens()`, `Open` and `OpenOne` | `eql_v4.json_tree_subtree(col, $1::text[], 64)` | +| `Children(ctx, c, path)`, `ChildCount(ctx, c, path)` | `encrypt.JSONToken` | `eql_v4.json_tree_children(col, $1::text)`, `eql_v4.json_tree_child_count(col, $1::text)` | + +`At` has one accessor per ordered path, named for its keys (`$.visits[*].at` is `At.VisitsAt`); two paths with one name stop `stashgen`. +The needle and the values are `json.RawMessage`: write the JSON you mean, such as `json.RawMessage(`+"`"+`{"active":true}`+"`"+`)`. +A position in a path is refused for `EqAt` and `Selector`, whose selector reads every position as the wildcard; read a position with `Subtree`. ### Optional fields diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index 298a67a2b..5ed46be0f 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -276,3 +276,69 @@ func TestModelFlagsRepeat(t *testing.T) { t.Fatalf("String = %q", got) } } + +// A json.RawMessage field stored as an encrypted JSON document, against the +// embedded engine with the EQL types: the generated entry has the +// document's methods and an accessor per ordered path, and every refusal +// names the field before anything is written. +func TestRunGeneratesEncryptedJSONFields(t *testing.T) { + checker, err := encrypt.NewChecker(context.Background()) + if err != nil { + t.Skip(err) + } + _ = checker.Close() + source := func(fields string) string { + return "package users\n\nimport \"encoding/json\"\n\nvar _ json.RawMessage\n\ntype Customer struct {\n\t_ struct{} `stash:\"context=customers\"`\n\tID int64 `stash:\"id,passthrough\"`\n" + fields + "\n}\n" + } + generate := func(fields string) (string, string, int) { + dir := writeModule(t, map[string]string{"model.go": source(fields)}) + var stdout, stderr bytes.Buffer + code := run([]string{"-type", "Customer"}, dir, &stdout, &stderr, stashgen.GuestEngine) + written, _ := os.ReadFile(filepath.Join(dir, "customer_stash.go")) + return string(written), stderr.String(), code + } + written, stderr, code := generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at,sealed_root=false)\"`\n\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson,null=sql\"`") + if code != 0 { + t.Fatalf("exit %d\n%s", code, stderr) + } + for _, want := range []string{ + "Profile eql.EncryptedJSON", + "Prefs *eql.EncryptedJSON", + `EncryptInto("profile", gensupport.JSON, "EncryptedJson").`, + `Options("profile", gensupport.TargetOptions{SealedRoot: gensupport.Ptr(false), Ordered: []string{"$.age", "$.visits[*].at"}})`, + "func (f ProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error)", + "func (f ProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error)", + "Age ProfileFieldPath // $.age", + "VisitsAt ProfileFieldPath // $.visits[*].at", + "func (p ProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.EncryptedJSONLeafQuery], error)", + } { + if !strings.Contains(written, want) { + t.Errorf("the generated file lacks %q", want) + } + } + if strings.Contains(written, "PrefsFieldAt") { + t.Error("Prefs orders no path, and got an At") + } + for _, c := range []struct{ name, fields, want string }{ + {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson,null=sealed\"`", "Customer.Prefs"}, + {"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson\"`", "null=sql"}, + {"a position in an ordered path", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.scores[2])\"`", "the engine refuses the declaration"}, + {"a quoted key in a tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$[\\\"a b\\\"])\"`", "not a plain JSONPath"}, + {"an unknown option", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(bound=3)\"`", "no option \"bound\""}, + {"the Go name in the tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJSON\"`", "as the engine does, EncryptedJson"}, + {"two paths, one accessor", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.a_b;$.aB)\"`", "both name the accessor At.AB"}, + {"a document with no target", "\tProfile json.RawMessage `stash:\"profile,encrypt\"`", "the engine refuses the declaration"}, + {"a string into a document", "\tProfile string `stash:\"profile,encrypt_into=EncryptedJson\"`", "EncryptedJson seals a json"}, + {"options on a type that takes none", "\tEmail string `stash:\"email,encrypt_into=TextEq(ordered=$.a)\"`", "the engine refuses the declaration"}, + } { + t.Run(c.name, func(t *testing.T) { + written, stderr, code := generate(c.fields) + if code != 1 || written != "" { + t.Fatalf("exit %d, wrote %d bytes\n%s", code, len(written), stderr) + } + if !strings.Contains(stderr, c.want) { + t.Fatalf("stderr = %q, want %q", stderr, c.want) + } + }) + } +} diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index f68e8f2fb..5ba5516a5 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -576,7 +576,7 @@ the installer disables these domains, and the `Ord` types are the ones to use. Text whose fold is longer than 1023 bytes, or ends in NUL, has no block ORE term and is refused, stored or queried. -The rest of the catalog (ordered dates, timestamps and decimals, JSON) is listed in +The rest of the catalog (ordered dates, timestamps and decimals, the v3 JSON types) is listed in `eql.Types` with the reason it is not produced yet, and `stashgen` refuses it with that reason. The guest builds the value: the generated code imports `encrypt/eql`, which embeds the build of the engine that holds the @@ -591,6 +591,34 @@ with a tenant part refuses a struct with an `encrypt_into` field: an EQL value is stored under a table and a column, and the extended label has no column. +### Encrypted JSON documents + +`EncryptedJson` stores a whole JSON document in one `public.eql_v4_json_tree` +column, from a `json.RawMessage` field (`stash:"profile,encrypt_into=EncryptedJson(ordered=$.age)"`; +see the stashgen README). One entry per node is encrypted, so the database +can test containment, equality and ordering at a path, return one node with +`->`, and walk a subtree, without decrypting: + +```go +q, err := users.Fields.Profile.Contains(ctx, cipher, json.RawMessage(`{"plan":"pro"}`)) +rows, err := db.QueryContext(ctx, `SELECT id FROM users WHERE profile @> $1::eql_v4.query_json_tree`, q) + +read, err := users.Fields.Profile.Subtree(ctx, cipher, "$.visits[*].at", encrypt.JSONReadOptions{}) +// SELECT e FROM users, eql_v4.json_tree_subtree(profile, $1::text[], 64) AS e WHERE id = $2 +// with read.Tokens() as $1; collect each row's entries, then: +values, err := read.Open(ctx, entriesPerRow) +``` + +A selection or read opens rows only as answers to the request that prepared +it: the engine derives the selector or the read again from the field, the +path and the cipher's keyset, and refuses a row that is not one it asked +for. It does not prove the database returned every row, or rows of the +database row you meant. `encrypt.JSONTokens` binds as a `text[]` literal, +and a selector or token as `text`; cast each parameter, as above, so +PostgreSQL picks EQL's operator and not `jsonb`'s. Numbers order as `float64`, +so integers past 2^53 can tie at an ordered path; containment and `EqAt` +compare them exactly. + ## Building The package embeds `wasm/stack_encrypt_guest.wasm`, a build artefact of the diff --git a/languages/golang/encrypt/eql_postgres_test.go b/languages/golang/encrypt/eql_postgres_test.go index 9772237cc..af7136dc2 100644 --- a/languages/golang/encrypt/eql_postgres_test.go +++ b/languages/golang/encrypt/eql_postgres_test.go @@ -148,6 +148,10 @@ func TestEQLTypesRoundTripAndQueryInPostgres(t *testing.T) { ds[i] = d } runEQLType(h, gensupport.Decimal, ds, nil) + case "json": + // A document has operators of its own, not one query per + // index: TestJSONDocumentsQueryAndReadInPostgres runs them. + return default: t.Fatalf("no plaintexts for wire kind %q: add a case so %s is tested", typ.Plaintext, typ.Name) } diff --git a/languages/golang/encrypt/gensupport/codec.go b/languages/golang/encrypt/gensupport/codec.go index ab6aee541..4eba52782 100644 --- a/languages/golang/encrypt/gensupport/codec.go +++ b/languages/golang/encrypt/gensupport/codec.go @@ -26,7 +26,7 @@ type Output struct { Match encrypt.MatchTerm Ore encrypt.OreTerm Ope encrypt.OpeTerm - JSON encrypt.JSONTerm + JSON encrypt.JSONTerm //nolint:staticcheck // the deprecated output, kept so a declaration naming it fails into a Go type EQL []byte } diff --git a/languages/golang/encrypt/gensupport/declaration.go b/languages/golang/encrypt/gensupport/declaration.go index f110bb8ca..f74c054cf 100644 --- a/languages/golang/encrypt/gensupport/declaration.go +++ b/languages/golang/encrypt/gensupport/declaration.go @@ -48,6 +48,9 @@ const ( Timestamp Kind = "timestamp" String Kind = "string" Bytes Kind = "bytes" + // JSON is a json.RawMessage field stored as an EncryptedJSON document: + // it crosses as its JSON text, which the engine parses. + JSON Kind = "json" ) // OpaqueField is the one field of an opaque declaration: the whole struct, @@ -78,6 +81,19 @@ type field struct { indexes []encrypt.Index eqlType string null NullMode + options *TargetOptions +} + +// TargetOptions are the options an EncryptInto field gives its EQL type: +// for EncryptedJson, the JSON index's. The engine checks them when the +// declaration is first used (stashgen asked it when the file was written). +type TargetOptions struct { + // SealedRoot says whether the document's root is sealed; nil is the + // default, sealed. + SealedRoot *bool + // Ordered are the JSONPath paths whose number and string leaves carry + // an ordering term. + Ordered []string } // NullMode is how a sealed field declared over a pointer (*T) stores nil: @@ -170,6 +186,28 @@ func (d Declaration) EncryptInto(name string, kind Kind, eqlType string) Declara return d.add(field{name: name, kind: kind, verb: verbEncryptInto, eqlType: eqlType}) } +// Options gives the named EncryptInto field's EQL type its options: the +// `encrypt_into=EncryptedJson(ordered=...)` tag. +func (d Declaration) Options(name string, options TargetOptions) Declaration { + if d.err != nil { + return d + } + for i := range d.fields { + if d.fields[i].name != name { + continue + } + if d.fields[i].verb != verbEncryptInto { + d.err = fmt.Errorf("gensupport: Options(%q): only an EncryptInto field's EQL type takes options", name) + return d + } + d.fields = slices.Clone(d.fields) + d.fields[i].options = &options + return d + } + d.err = fmt.Errorf("gensupport: Options(%q): no such field", name) + return d +} + // Null says how the named field, declared over a pointer, stores nil. The // engine decides whether the field can, when the declaration is first used: // a sealed null needs a field whose only index is equality, of a kind of @@ -293,6 +331,9 @@ func (d Declaration) plan() (*record.Plan, error) { // is the engine's to say, at the first call (stashgen asked it // when the file was written). rf.Target = f.eqlType + if f.options != nil { + rf.TargetOptions = &record.TargetOptions{SealedRoot: f.options.SealedRoot, Ordered: slices.Clone(f.options.Ordered)} + } case verbEncrypt: rf.Outputs = []record.Output{record.Ciphertext} case verbEncryptIndex: @@ -320,3 +361,7 @@ func (d Declaration) plan() (*record.Plan, error) { var termOutputs = map[record.Output]struct{}{ record.Equality: {}, record.Match: {}, record.Ore: {}, record.Ope: {}, } + +// Ptr is a pointer to v: what a generated TargetOptions literal spells an +// option set to false with. +func Ptr[T any](v T) *T { return &v } diff --git a/languages/golang/encrypt/gensupport/gensupport_internal_test.go b/languages/golang/encrypt/gensupport/gensupport_internal_test.go index 0c2655799..a109e5c17 100644 --- a/languages/golang/encrypt/gensupport/gensupport_internal_test.go +++ b/languages/golang/encrypt/gensupport/gensupport_internal_test.go @@ -73,7 +73,7 @@ func TestDeclarationRefusals(t *testing.T) { "identity of no field": Declare("u").Encrypt("a", String).Identity("b", "x"), "seals nothing": Declare("u").Passthrough("id"), "EQL type unnamed": Declare("u").EncryptInto("email", String, ""), - "json index": Declare("u").EncryptIndex("a", String, encrypt.JSON()), + "json index": Declare("u").EncryptIndex("a", String, encrypt.JSON()), //nolint:staticcheck // the deprecated index is still refused "name not a label": Declare("u").Encrypt("1a", String), } for name, d := range cases { diff --git a/languages/golang/encrypt/gensupport/json.go b/languages/golang/encrypt/gensupport/json.go new file mode 100644 index 000000000..f04391529 --- /dev/null +++ b/languages/golang/encrypt/gensupport/json.go @@ -0,0 +1,118 @@ +package gensupport + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/vitaminc/bindings/go/vcvalue" +) + +// The operators of a field stored as an EncryptedJSON document, for the +// generated methods of its Fields entry. Each sends the operator's named +// inputs, JSON values as their JSON text, and checks the engine answered +// in the encoding the method returns. + +// jsonTerm derives one JSON operator's term on the field. +func (f Field[T]) jsonTerm(ctx context.Context, c *encrypt.Cipher, operator, encoding string, input vcvalue.Object) (encrypt.QueryTerm, error) { + if f.err != nil { + return encrypt.QueryTerm{}, f.err + } + if c == nil { + return encrypt.QueryTerm{}, fmt.Errorf("%w: gensupport: %s: a query needs a cipher", encrypt.ErrEncoding, f.name) + } + term, err := c.QueryTerm(ctx, f.plan, f.name, operator, input) + if err != nil { + return encrypt.QueryTerm{}, err + } + if term.Encoding != encoding || term.Fidelity != "exact" { + return encrypt.QueryTerm{}, fmt.Errorf("gensupport: %s: %s came back as a %s %s term, not an exact %s one", f.name, operator, term.Fidelity, term.Encoding, encoding) + } + return term, nil +} + +// jsonText is a JSON value as an input carries it: its text, as bytes. +func jsonText(v json.RawMessage) []byte { return []byte(v) } + +// JSONQuery derives a containment operator's query on a JSON field +// (contains, contained_by): the needle's projection, Q. +func JSONQuery[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator string, needle json.RawMessage) (Q, error) { + term, err := f.jsonTerm(ctx, c, operator, "eql", vcvalue.Object{{Key: "json", Value: jsonText(needle)}}) + if err != nil || len(term.Terms) != 1 { + return nil, oneTermErr(f.name, operator, err) + } + return Q(term.Terms[0]), nil +} + +// JSONQueryAt derives an operator at a path on a JSON field (eq_at, lt_at, +// lte_at, gt_at, gte_at): Q is the needle for eq_at and the leaf operand +// for the orderings. +func JSONQueryAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator, path string, v json.RawMessage) (Q, error) { + term, err := f.jsonTerm(ctx, c, operator, "eql", vcvalue.Object{{Key: "path", Value: path}, {Key: "json", Value: jsonText(v)}}) + if err != nil || len(term.Terms) != 1 { + return nil, oneTermErr(f.name, operator, err) + } + return Q(term.Terms[0]), nil +} + +// JSONBetweenAt derives between_at on a JSON field: the two leaf operands, +// low then high, inclusive. +func JSONBetweenAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string, lo, hi json.RawMessage) (encrypt.JSONRange[Q], error) { + term, err := f.jsonTerm(ctx, c, "between_at", "eql", vcvalue.Object{{Key: "path", Value: path}, {Key: "low", Value: jsonText(lo)}, {Key: "high", Value: jsonText(hi)}}) + if err != nil { + return encrypt.JSONRange[Q]{}, err + } + if len(term.Terms) != 2 { + return encrypt.JSONRange[Q]{}, fmt.Errorf("gensupport: %s: between_at came back as %d terms, not 2", f.name, len(term.Terms)) + } + return encrypt.JSONRange[Q]{Low: Q(term.Terms[0]), High: Q(term.Terms[1])}, nil +} + +// JSONSelect prepares `->` at a path on a JSON field: the selector to +// bind, and what opens the entries it returns. E is the field's entry type. +func JSONSelect[E ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string) (encrypt.JSONSelection[E], error) { + term, err := f.jsonTerm(ctx, c, "selector", "text", vcvalue.Object{{Key: "path", Value: path}}) + if err == nil && len(term.Text) != 1 { + err = fmt.Errorf("gensupport: %s: selector came back as %d texts, not 1", f.name, len(term.Text)) + } + if err != nil { + return encrypt.JSONSelection[E]{}, err + } + return encrypt.NewJSONSelection[E](c, f.plan, f.name, path, term.Text[0]), nil +} + +// JSONSubtree prepares a subtree read at a path on a JSON field: the +// tokens to bind, and what opens the rows they return. +func JSONSubtree[E ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[E], error) { + input := vcvalue.Object{{Key: "path", Value: path}} + if opts.WildcardBound != 0 { + input = append(input, vcvalue.Field{Key: "wildcard_bound", Value: opts.WildcardBound}) + } + term, err := f.jsonTerm(ctx, c, "subtree", "text_array", input) + if err != nil { + return encrypt.JSONRead[E]{}, err + } + return encrypt.NewJSONRead[E](c, f.plan, f.name, path, opts.WildcardBound, term.Text), nil +} + +// JSONNodeToken derives the token of the node at a path with no wildcard +// (children, child_count), which json_tree_children and +// json_tree_child_count take. +func JSONNodeToken[T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator, path string) (encrypt.JSONToken, error) { + term, err := f.jsonTerm(ctx, c, operator, "text", vcvalue.Object{{Key: "path", Value: path}}) + if err == nil && len(term.Text) != 1 { + err = fmt.Errorf("gensupport: %s: %s came back as %d texts, not 1", f.name, operator, len(term.Text)) + } + if err != nil { + return "", err + } + return encrypt.JSONToken(term.Text[0]), nil +} + +func oneTermErr(field, operator string, err error) error { + if err != nil { + return err + } + return fmt.Errorf("gensupport: %s: %s came back as other than one term", field, operator) +} diff --git a/languages/golang/encrypt/gensupport/widen.go b/languages/golang/encrypt/gensupport/widen.go index 02e13c735..ca9f0564e 100644 --- a/languages/golang/encrypt/gensupport/widen.go +++ b/languages/golang/encrypt/gensupport/widen.go @@ -1,6 +1,7 @@ package gensupport import ( + "encoding/json" "fmt" "math" "reflect" @@ -31,6 +32,17 @@ func asKind(kind Kind, v any) (any, error) { if err := checkValue(v); err != nil { return nil, err } + if kind == JSON { + // A document crosses as its JSON text, in a bytes value; the + // engine parses it. + switch raw := v.(type) { + case json.RawMessage: + return []byte(raw), nil + case []byte: + return raw, nil + } + return nil, fmt.Errorf("%w: a JSON field takes a json.RawMessage, not a %T", encrypt.ErrEncoding, v) + } if !isInteger(kind) { return v, nil } diff --git a/languages/golang/encrypt/internal/testusers/customer_stash.go b/languages/golang/encrypt/internal/testusers/customer_stash.go new file mode 100644 index 000000000..b8ebbf437 --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/customer_stash.go @@ -0,0 +1,283 @@ +// Code generated by stashgen. DO NOT EDIT. + +package testusers + +import ( + "context" + "encoding/json" + "log/slog" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" +) + +// Stops compiling when the library does not accept this version of generated file. +const _ = gensupport.GeneratedVersion1 + +// Customer prints its sealed fields in the clear: it has no String or LogValue +// method. Write them, or run stashgen with -redact. + +type EncryptedCustomer struct { + ID int64 + Profile eql.EncryptedJSON + Prefs *eql.EncryptedJSON +} + +func (e EncryptedCustomer) String() string { + return gensupport.Redacted("EncryptedCustomer", map[string]any{"ID": e.ID}, "Profile", "Prefs") +} + +func (e EncryptedCustomer) LogValue() slog.Value { + return gensupport.RedactedLog(map[string]any{"ID": e.ID}, "Profile", "Prefs") +} + +// Stops compiling when Customer gains, loses, reorders or retypes a field. +var _ = customerShape(Customer{}) + +type customerShape struct { + _ struct{} + ID int64 + Profile json.RawMessage + Prefs *json.RawMessage +} + +var customerDeclaration = gensupport.Declare("customers"). + Passthrough("id"). + EncryptInto("profile", gensupport.JSON, "EncryptedJson"). + Options("profile", gensupport.TargetOptions{Ordered: []string{"$.age", "$.visits[*].at"}}). + EncryptInto("prefs", gensupport.JSON, "EncryptedJson"). + Null("prefs", gensupport.NullSQL) + +var customerCodec = gensupport.New(gensupport.Generated[Customer, EncryptedCustomer]{ + TypeName: "Customer", + Declaration: customerDeclaration, + PrintsPlaintext: true, + Source: func(v Customer) gensupport.Values { + return gensupport.Values{ + "id": v.ID, + "profile": v.Profile, + "prefs": gensupport.Deref(v.Prefs), + } + }, + Seal: func(rec gensupport.Record) (EncryptedCustomer, error) { + var e EncryptedCustomer + var err error + if e.ID, err = gensupport.Passthrough[int64](rec, "id"); err != nil { + return EncryptedCustomer{}, err + } + e.Profile = eql.EncryptedJSON(rec["profile"].EQL) + if o, ok := rec["prefs"]; ok { + v := eql.EncryptedJSON(o.EQL) + e.Prefs = &v + } + return e, nil + }, + Open: func(e EncryptedCustomer) gensupport.Record { + rec := gensupport.Record{ + "id": {Value: e.ID}, + "profile": {EQL: e.Profile}, + } + if e.Prefs != nil { + rec["prefs"] = gensupport.Output{EQL: *e.Prefs} + } + return rec + }, + Value: func(e EncryptedCustomer, vals gensupport.Values) (Customer, error) { + var v Customer + var err error + if v.ID, err = gensupport.Get[int64](vals, "id"); err != nil { + return Customer{}, err + } + if v.Profile, err = gensupport.Get[json.RawMessage](vals, "profile"); err != nil { + return Customer{}, err + } + if v.Prefs, err = gensupport.GetPtr[json.RawMessage](vals, "prefs"); err != nil { + return Customer{}, err + } + return v, nil + }, +}) + +// EncryptCustomer seals each Customer, with one ZeroKMS request for each 500 +// sealed values, plus one the first time a keyset is used. The result has one +// element for each input, in the same order. +func EncryptCustomer(ctx context.Context, cipher *encrypt.Cipher, values []Customer) ([]EncryptedCustomer, error) { + return customerCodec.Encrypt(ctx, cipher, values) +} + +// DecryptCustomer opens each EncryptedCustomer, with one ZeroKMS request for +// each 500 sealed values, plus one the first time a keyset is used. +func DecryptCustomer(ctx context.Context, d encrypt.Decrypter, encrypted []EncryptedCustomer) ([]Customer, error) { + return customerCodec.Decrypt(ctx, d, encrypted) +} + +var CustomerFields = struct { + Profile CustomerProfileField + Prefs CustomerPrefsField +}{ + Profile: newCustomerProfileField(gensupport.NewField[json.RawMessage](customerDeclaration, "profile")), + Prefs: newCustomerPrefsField(gensupport.NewField[json.RawMessage](customerDeclaration, "prefs")), +} + +type CustomerProfileField struct { + field gensupport.Field[json.RawMessage] + // At has one accessor per path the field orders, whose methods + // are the orderings there. + At CustomerProfileFieldAt +} + +func newCustomerProfileField(f gensupport.Field[json.RawMessage]) CustomerProfileField { + return CustomerProfileField{field: f, At: CustomerProfileFieldAt{ + Age: CustomerProfileFieldPath{field: f, path: "$.age"}, + VisitsAt: CustomerProfileFieldPath{field: f, path: "$.visits[*].at"}, + }} +} + +func (f CustomerProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSON, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.EncryptedJSON(out.EQL), err +} + +// Contains derives the query for documents that contain needle, a JSON value: +// col @> query. Each array element of the needle must sit inside one element of +// the document's array. +func (f CustomerProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contains", needle) +} + +// ContainedBy derives the query for documents contained by needle, a JSON +// value: col <@ query. +func (f CustomerProfileField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contained_by", needle) +} + +// EqAt derives the query for documents holding the scalar v at path: col @> +// query. A wildcard step matches when any element holds it; a position ([2]) is +// refused, since the stored selector reads every position as the wildcard. +func (f CustomerProfileField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONQuery](ctx, f.field, c, "eq_at", path, v) +} + +// Selector prepares `->` at path: bind Selector() as col -> $1::text, and open +// the entries it returns with Open. A position ([2]) is refused; read one with +// Subtree. +func (f CustomerProfileField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, f.field, c, path) +} + +// Subtree prepares a read of the value at path: bind Tokens() as +// eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it +// returns with Open or OpenOne. +func (f CustomerProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSubtree[eql.EncryptedJSONEntry](ctx, f.field, c, path, opts) +} + +// Children derives the token of the node at path, a path with no wildcard, +// which eql_v4.json_tree_children(col, $1::text) takes. Its rows are encrypted +// entries one level down. +func (f CustomerProfileField) Children(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "children", path) +} + +// ChildCount derives the token of the node at path, which +// eql_v4.json_tree_child_count(col, $1::text) takes: zero for a scalar or an +// empty container, NULL where the row has no node there or for the sealed root. +func (f CustomerProfileField) ChildCount(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "child_count", path) +} + +type CustomerProfileFieldAt struct { + Age CustomerProfileFieldPath // $.age + VisitsAt CustomerProfileFieldPath // $.visits[*].at +} + +// CustomerProfileFieldPath is one ordered path of Profile: its orderings +// compare the first value `->` extracts there, by the leaf's ordering term. +// Numbers order as float64, so integers past 2^53 can tie; text orders case- +// and accent-folded. +type CustomerProfileFieldPath struct { + field gensupport.Field[json.RawMessage] + path string +} + +func (p CustomerProfileFieldPath) Lt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "lt_at", p.path, v) +} + +func (p CustomerProfileFieldPath) Lte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "lte_at", p.path, v) +} + +func (p CustomerProfileFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "gt_at", p.path, v) +} + +func (p CustomerProfileFieldPath) Gte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "gte_at", p.path, v) +} + +func (p CustomerProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.EncryptedJSONLeafQuery], error) { + return gensupport.JSONBetweenAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, p.path, lo, hi) +} + +type CustomerPrefsField struct { + field gensupport.Field[json.RawMessage] +} + +func newCustomerPrefsField(f gensupport.Field[json.RawMessage]) CustomerPrefsField { + return CustomerPrefsField{field: f} +} + +func (f CustomerPrefsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSON, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.EncryptedJSON(out.EQL), err +} + +// Contains derives the query for documents that contain needle, a JSON value: +// col @> query. Each array element of the needle must sit inside one element of +// the document's array. +func (f CustomerPrefsField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contains", needle) +} + +// ContainedBy derives the query for documents contained by needle, a JSON +// value: col <@ query. +func (f CustomerPrefsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contained_by", needle) +} + +// EqAt derives the query for documents holding the scalar v at path: col @> +// query. A wildcard step matches when any element holds it; a position ([2]) is +// refused, since the stored selector reads every position as the wildcard. +func (f CustomerPrefsField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONQuery](ctx, f.field, c, "eq_at", path, v) +} + +// Selector prepares `->` at path: bind Selector() as col -> $1::text, and open +// the entries it returns with Open. A position ([2]) is refused; read one with +// Subtree. +func (f CustomerPrefsField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, f.field, c, path) +} + +// Subtree prepares a read of the value at path: bind Tokens() as +// eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it +// returns with Open or OpenOne. +func (f CustomerPrefsField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSubtree[eql.EncryptedJSONEntry](ctx, f.field, c, path, opts) +} + +// Children derives the token of the node at path, a path with no wildcard, +// which eql_v4.json_tree_children(col, $1::text) takes. Its rows are encrypted +// entries one level down. +func (f CustomerPrefsField) Children(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "children", path) +} + +// ChildCount derives the token of the node at path, which +// eql_v4.json_tree_child_count(col, $1::text) takes: zero for a scalar or an +// empty container, NULL where the row has no node there or for the sealed root. +func (f CustomerPrefsField) ChildCount(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "child_count", path) +} diff --git a/languages/golang/encrypt/internal/testusers/customers.go b/languages/golang/encrypt/internal/testusers/customers.go new file mode 100644 index 000000000..f6a330798 --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/customers.go @@ -0,0 +1,15 @@ +package testusers + +import "encoding/json" + +//go:generate go run github.com/cipherstash/stack/languages/golang/cmd/stashgen -type Customer -name Customer + +// Customer holds two encrypted JSON documents (the EQL type EncryptedJson, +// stored in public.eql_v4_json_tree): a profile whose $.age and every +// visit's $.at order, and optional preferences, stored as SQL NULL when nil. +type Customer struct { + _ struct{} `stash:"context=customers"` + ID int64 `stash:"id,passthrough"` + Profile json.RawMessage `stash:"profile,encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at)"` + Prefs *json.RawMessage `stash:"prefs,encrypt_into=EncryptedJson,null=sql"` +} diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 988a50547..ca17273ac 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -2,7 +2,10 @@ package encrypt import ( "context" + "database/sql/driver" + "encoding/json" "fmt" + "strings" "github.com/cipherstash/stack/languages/golang/internal/record" "github.com/cipherstash/vitaminc/bindings/go/vcffi" @@ -91,3 +94,207 @@ func (cph *Cipher) JSONOpen(ctx context.Context, plan *record.Plan, field string } return opened, nil } + +// JSONSelector is a JSON path's selector: what `->`, `->>` and the path +// functions of an EncryptedJSON column take. It binds as SQL text; cast it, +// `col -> $1::text`, so PostgreSQL picks EQL's operator and not jsonb's. +// It is column-wide: the same path has the same selector in every row. +type JSONSelector string + +// Value binds the selector as text. +func (s JSONSelector) Value() (driver.Value, error) { return string(s), nil } + +// JSONToken is one node's token: what eql_v4.json_tree_children and +// eql_v4.json_tree_child_count take. It binds as SQL text. A token is a +// capability over the node's subtree in every row of the column: send it +// to the database for a query, and do not log it. +type JSONToken string + +// Value binds the token as text. +func (t JSONToken) Value() (driver.Value, error) { return string(t), nil } + +// JSONTokens are a subtree read's tokens: what eql_v4.json_tree_subtree +// takes. They bind as a PostgreSQL text[] literal; cast the parameter, +// `$2::text[]`. Like a [JSONToken], each is a capability: do not log them. +type JSONTokens []string + +// Value binds the tokens as a text[] literal. A token is base64, so it +// needs no escape inside the quotes. +func (t JSONTokens) Value() (driver.Value, error) { + var b strings.Builder + b.WriteByte('{') + for i, token := range t { + if i > 0 { + b.WriteByte(',') + } + b.WriteByte('"') + b.WriteString(token) + b.WriteByte('"') + } + b.WriteByte('}') + return b.String(), nil +} + +// JSONValue is one value a read reached, or none. Found is false for no +// value at the path (the row has no node there, or SQL NULL); a JSON null +// that is there is Found, with JSON "null". JSON is the value's JSON text: +// plaintext, which the caller owns. +type JSONValue struct { + Found bool + JSON json.RawMessage +} + +// JSONReadOptions are a subtree read's options. +type JSONReadOptions struct { + // WildcardBound is how many elements each [*] in the path is expanded + // to; 0 is the engine's default, 64. A read of an array longer than the + // bound fails rather than open to part of it ([ErrEncoding], with the + // diagnostic code stack_encrypt::json_wildcard_truncated). Two wildcard + // levels send bound² + bound + 1 tokens, and the engine refuses a read + // past 8,192. + WildcardBound uint32 +} + +// JSONRange is both bounds of a between at a path, each an ordering +// operand Q, low then high, inclusive. +type JSONRange[Q ~[]byte] struct { + Low Q + High Q +} + +// JSONSelection is a prepared `->` on an EncryptedJSON field: the selector +// to bind, and what opens the entries the database returns for it. It keeps +// the request it was prepared from (the field's declaration, the path and +// the cipher's keyset), so the entries open only as answers to it: the +// engine derives the selector again and refuses an entry that does not +// carry it. E is the column's entry type, eql.EncryptedJSONEntry. +type JSONSelection[E ~[]byte] struct { + cipher *Cipher + plan *record.Plan + field string + path string + selector JSONSelector +} + +// NewJSONSelection is a selection prepared by [Cipher.QueryTerm]'s +// selector operator. For generated code. +func NewJSONSelection[E ~[]byte](c *Cipher, plan *record.Plan, field, path, selector string) JSONSelection[E] { + return JSONSelection[E]{cipher: c, plan: plan, field: field, path: path, selector: JSONSelector(selector)} +} + +// Selector is the selector to bind: `col -> $1::text`. +func (s JSONSelection[E]) Selector() JSONSelector { return s.selector } + +// Open opens the entries `col -> selector` returned, one per database row, +// nil for a row where it returned SQL NULL, with one ZeroKMS request. Each +// opens to the scalar it holds (Found), or for the sealed root at "$" the +// whole document; a nil opens to no value. A container's entry is refused: +// read a container with a subtree read. An entry that is not the selector's +// is [ErrEncoding]. +func (s JSONSelection[E]) Open(ctx context.Context, rows []*E) ([]JSONValue, error) { + if s.cipher == nil { + return nil, fmt.Errorf("%w: a zero JSONSelection has no request to open against", ErrEncoding) + } + groups := make([][][]byte, len(rows)) + for i, row := range rows { + if row != nil { + groups[i] = [][]byte{[]byte(*row)} + } + } + opened, err := s.cipher.JSONOpen(ctx, s.plan, s.field, vcvalue.Object{{Key: "operation", Value: "selector"}, {Key: "path", Value: s.path}}, groups) + if err != nil { + return nil, err + } + out := make([]JSONValue, len(opened)) + for i, values := range opened { + if len(values) == 1 { + out[i] = JSONValue{Found: true, JSON: values[0]} + } + } + return out, nil +} + +// JSONRead is a prepared subtree read of an EncryptedJSON field: the tokens +// to bind, and what opens the rows eql_v4.json_tree_subtree returns for +// them. It keeps the request it was prepared from (the field's declaration, +// the path, the bound and the cipher's keyset); the engine derives the read +// again from it, and checks every row against it. E is the column's entry +// type, eql.EncryptedJSONEntry. +type JSONRead[E ~[]byte] struct { + cipher *Cipher + plan *record.Plan + field string + path string + bound uint32 + tokens JSONTokens +} + +// NewJSONRead is a read prepared by [Cipher.QueryTerm]'s subtree operator. +// For generated code. +func NewJSONRead[E ~[]byte](c *Cipher, plan *record.Plan, field, path string, bound uint32, tokens []string) JSONRead[E] { + return JSONRead[E]{cipher: c, plan: plan, field: field, path: path, bound: bound, tokens: JSONTokens(tokens)} +} + +// Tokens are the tokens to bind: +// `eql_v4.json_tree_subtree(col, $1::text[], 64)`. +func (r JSONRead[E]) Tokens() JSONTokens { return r.tokens } + +// Open opens the rows the read returned, one group per database row, with +// one ZeroKMS request: per row, the value at each concrete path the read +// named that the row has, in document order. A row with no node there +// opens to no value. Past its walk budget the database returns the sealed +// root instead, and the read's paths are evaluated over the whole document +// to the same values; only those values come back. A row that is not one +// the read asked for, or a wildcard read of an array longer than the bound, +// is [ErrEncoding]. +// +// Opening checks each row against the read, not that the database +// returned every row it should have, nor that the rows came from the row +// the caller meant. +func (r JSONRead[E]) Open(ctx context.Context, rows [][]E) ([][]json.RawMessage, error) { + if r.cipher == nil { + return nil, fmt.Errorf("%w: a zero JSONRead has no request to open against", ErrEncoding) + } + groups := make([][][]byte, len(rows)) + for i, row := range rows { + groups[i] = make([][]byte, len(row)) + for j, entry := range row { + groups[i][j] = []byte(entry) + } + } + request := vcvalue.Object{{Key: "operation", Value: "subtree"}, {Key: "path", Value: r.path}} + if r.bound != 0 { + request = append(request, vcvalue.Field{Key: "wildcard_bound", Value: r.bound}) + } + opened, err := r.cipher.JSONOpen(ctx, r.plan, r.field, request, groups) + if err != nil { + return nil, err + } + out := make([][]json.RawMessage, len(opened)) + for i, values := range opened { + out[i] = make([]json.RawMessage, len(values)) + for j, v := range values { + out[i][j] = v + } + } + return out, nil +} + +// OpenOne opens one row's rows to the one value at the read's path: Found +// with its JSON, or not Found where the row has no node there. A read that +// reaches more than one value (a wildcard) is [ErrEncoding]; open it with +// [JSONRead.Open]. +func (r JSONRead[E]) OpenOne(ctx context.Context, rows []E) (JSONValue, error) { + opened, err := r.Open(ctx, [][]E{rows}) + if err != nil { + return JSONValue{}, err + } + switch values := opened[0]; len(values) { + case 0: + return JSONValue{}, nil + case 1: + return JSONValue{Found: true, JSON: values[0]}, nil + default: + return JSONValue{}, fmt.Errorf("%w: the read reached %d values, not one; open it with Open", ErrEncoding, len(values)) + } +} diff --git a/languages/golang/encrypt/json_fields_test.go b/languages/golang/encrypt/json_fields_test.go new file mode 100644 index 000000000..12fab79b0 --- /dev/null +++ b/languages/golang/encrypt/json_fields_test.go @@ -0,0 +1,133 @@ +package encrypt_test + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +// Generated code for json.RawMessage fields stored as EncryptedJson: +// testusers.Customer's Profile, ordered at $.age and $.visits[*].at, and its +// optional Prefs. + +func rawPtr(s string) *json.RawMessage { + r := json.RawMessage(s) + return &r +} + +// sameJSON reports whether two JSON texts hold the same value: the +// document is sealed as values, so its text is not promised back byte for +// byte. +func sameJSON(t *testing.T, a, b []byte) bool { + t.Helper() + var x, y any + if err := json.Unmarshal(a, &x); err != nil { + t.Fatalf("%s: %v", a, err) + } + if err := json.Unmarshal(b, &y); err != nil { + t.Fatalf("%s: %v", b, err) + } + return reflect.DeepEqual(x, y) +} + +func TestACustomersDocumentsRoundTrip(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + in := []testusers.Customer{ + {ID: 1, Profile: json.RawMessage(`{"name":"Alice","age":34,"visits":[{"at":3},{"at":1}]}`), Prefs: rawPtr(`{"theme":"dark"}`)}, + {ID: 2, Profile: json.RawMessage(`[1,2,3]`)}, + {ID: 3, Profile: json.RawMessage(`null`), Prefs: rawPtr(`null`)}, + } + encrypted, err := testusers.EncryptCustomer(ctx, c.DefaultKeyset(), in) + if err != nil { + t.Fatal(err) + } + if encrypted[1].Prefs != nil || encrypted[2].Prefs == nil { + t.Fatalf("a nil document is SQL NULL, a JSON null is a document: %v %v", encrypted[1].Prefs, encrypted[2].Prefs) + } + if s := encrypted[0].String(); strings.Contains(s, "Alice") { + t.Fatalf("String shows the document: %s", s) + } + back, err := testusers.DecryptCustomer(ctx, c, encrypted) + if err != nil { + t.Fatal(err) + } + for i := range in { + if back[i].ID != in[i].ID || !sameJSON(t, back[i].Profile, in[i].Profile) { + t.Fatalf("row %d: %s", i, back[i].Profile) + } + if (back[i].Prefs == nil) != (in[i].Prefs == nil) || (in[i].Prefs != nil && !sameJSON(t, *back[i].Prefs, *in[i].Prefs)) { + t.Fatalf("row %d prefs: %v", i, back[i].Prefs) + } + } + // Empty bytes are not a document: a required field takes JSON. + if _, err := testusers.EncryptCustomer(ctx, c.DefaultKeyset(), []testusers.Customer{{ID: 4}}); !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("an empty document: %v", err) + } +} + +func TestACustomersFieldsDeriveEachQuery(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + f := testusers.CustomerFields.Profile + contains, err := f.Contains(ctx, cipher, json.RawMessage(`{"name":"Alice"}`)) + if err != nil || len(contains) == 0 { + t.Fatalf("Contains: %v", err) + } + if v, err := contains.Value(); err != nil || !strings.Contains(v.(string), `"sv"`) { + t.Fatalf("Contains binds as JSON: %v %v", v, err) + } + if _, err := f.ContainedBy(ctx, cipher, json.RawMessage(`{"name":"Alice","age":34}`)); err != nil { + t.Fatal(err) + } + if _, err := f.EqAt(ctx, cipher, "$.name", json.RawMessage(`"Alice"`)); err != nil { + t.Fatal(err) + } + if _, err := f.At.Age.Lt(ctx, cipher, json.RawMessage(`40`)); err != nil { + t.Fatal(err) + } + rng, err := f.At.VisitsAt.Between(ctx, cipher, json.RawMessage(`1`), json.RawMessage(`2`)) + if err != nil || len(rng.Low) == 0 || len(rng.High) == 0 { + t.Fatalf("Between: %+v %v", rng, err) + } + sel, err := f.Selector(ctx, cipher, "$.name") + if err != nil || sel.Selector() == "" { + t.Fatalf("Selector: %v", err) + } + read, err := f.Subtree(ctx, cipher, "$.visits[*]", encrypt.JSONReadOptions{WildcardBound: 2}) + if err != nil || len(read.Tokens()) != 3 { + t.Fatalf("Subtree: %v %v", read.Tokens(), err) + } + if v, err := read.Tokens().Value(); err != nil || !strings.HasPrefix(v.(string), `{"`) { + t.Fatalf("Tokens bind as a text[] literal: %v %v", v, err) + } + if _, err := f.Children(ctx, cipher, "$.visits"); err != nil { + t.Fatal(err) + } + // What the engine refuses: an ordering on a value no ordering takes, a + // container at a path, a position in a selector, a wildcard where one + // node is named. Each is a returned error naming no value. + for name, err := range map[string]error{ + "bool ordering": func() error { _, err := f.At.Age.Gt(ctx, cipher, json.RawMessage(`true`)); return err }(), + "container eq": func() error { _, err := f.EqAt(ctx, cipher, "$.name", json.RawMessage(`{"secret":1}`)); return err }(), + "position": func() error { _, err := f.Selector(ctx, cipher, "$.visits[1].at"); return err }(), + "wildcard node": func() error { _, err := f.ChildCount(ctx, cipher, "$.visits[*]"); return err }(), + "not JSON": func() error { _, err := f.Contains(ctx, cipher, json.RawMessage(`{"secret"`)); return err }(), + } { + if err == nil || strings.Contains(err.Error(), "secret") { + t.Errorf("%s: %v", name, err) + } + } + // Prefs orders no path: its entry has no At, and the engine answers no + // ordering on it. + if _, ok := reflect.TypeOf(testusers.CustomerFields.Prefs).FieldByName("At"); ok { + t.Fatal("Prefs has an At") + } +} diff --git a/languages/golang/encrypt/json_postgres_test.go b/languages/golang/encrypt/json_postgres_test.go new file mode 100644 index 000000000..946f937af --- /dev/null +++ b/languages/golang/encrypt/json_postgres_test.go @@ -0,0 +1,297 @@ +package encrypt_test + +import ( + "context" + "database/sql" + "encoding/json" + "os" + "slices" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +// Encrypted JSON documents (EncryptedJson, public.eql_v4_json_tree) the Go +// SDK stores, found by the queries its generated methods derive and read by +// its selections and reads, inside PostgreSQL with EQL v4 installed. Each +// query's rows are the ones the plaintexts say. Skipped unless DATABASE_URL +// is set, as eql_postgres_test.go is. + +var jsonCustomers = []testusers.Customer{ + {ID: 1, Profile: json.RawMessage(`{"name":"Alice","age":34,"visits":[{"at":3},{"at":1}],"tags":["a"]}`), Prefs: rawPtr(`{"theme":"dark"}`)}, + {ID: 2, Profile: json.RawMessage(`{"name":"Bob","age":7,"visits":[{"at":5}]}`)}, + {ID: 3, Profile: json.RawMessage(`{"name":"Carol","age":61,"visits":[]}`), Prefs: rawPtr(`null`)}, + {ID: 4, Profile: json.RawMessage(`{"name":"Dan"}`)}, +} + +func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { + url := os.Getenv("DATABASE_URL") + if url == "" { + t.Skip("DATABASE_URL is not set") + } + ctx := context.Background() + c := deterministicEQLClient(t) + cipher := c.DefaultKeyset() + conn := eqlPostgresConn(t, ctx, url) + exec := func(q string, args ...any) { + t.Helper() + if _, err := conn.ExecContext(ctx, q, args...); err != nil { + t.Fatalf("%s: %v", q, err) + } + } + exec(`CREATE TEMP TABLE customers (id bigint PRIMARY KEY, profile public.eql_v4_json_tree NOT NULL, prefs public.eql_v4_json_tree)`) + t.Cleanup(func() { _, _ = conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS customers`) }) + encrypted, err := testusers.EncryptCustomer(ctx, cipher, jsonCustomers) + if err != nil { + t.Fatal(err) + } + for _, e := range encrypted { + // Each value binds through its driver.Valuer; a nil Prefs is NULL. + exec(`INSERT INTO customers VALUES ($1, $2, $3)`, e.ID, e.Profile, e.Prefs) + } + ids := func(q string, args ...any) []int64 { + t.Helper() + rows, err := conn.QueryContext(ctx, q, args...) + if err != nil { + t.Fatalf("%s: %v", q, err) + } + defer rows.Close() + var out []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + t.Fatal(err) + } + out = append(out, id) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return out + } + want := func(name string, got []int64, want ...int64) { + t.Helper() + if !slices.Equal(got, want) { + t.Errorf("%s: rows %v, want %v", name, got, want) + } + } + f := testusers.CustomerFields.Profile + must := func(v []byte, err error) []byte { + t.Helper() + if err != nil { + t.Fatal(err) + } + return v + } + + // Stored values read back, re-serialised by PostgreSQL, open as they were. + rows, err := conn.QueryContext(ctx, `SELECT id, profile, prefs FROM customers ORDER BY id`) + if err != nil { + t.Fatal(err) + } + var stored []testusers.EncryptedCustomer + for rows.Next() { + var e testusers.EncryptedCustomer + if err := rows.Scan(&e.ID, &e.Profile, &e.Prefs); err != nil { + t.Fatal(err) + } + stored = append(stored, e) + } + _ = rows.Close() + back, err := testusers.DecryptCustomer(ctx, c, stored) + if err != nil { + t.Fatal(err) + } + for i := range jsonCustomers { + if !sameJSON(t, back[i].Profile, jsonCustomers[i].Profile) || (back[i].Prefs == nil) != (jsonCustomers[i].Prefs == nil) { + t.Fatalf("row %d read back as %s %v", i, back[i].Profile, back[i].Prefs) + } + } + + // Containment, both ways. + q := eql.EncryptedJSONQuery(must(f.Contains(ctx, cipher, json.RawMessage(`{"name":"Alice"}`)))) + want("contains name", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json_tree ORDER BY id`, q), 1) + q = eql.EncryptedJSONQuery(must(f.Contains(ctx, cipher, json.RawMessage(`{"visits":[{"at":5}]}`)))) + want("contains an element", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json_tree ORDER BY id`, q), 2) + q = eql.EncryptedJSONQuery(must(f.ContainedBy(ctx, cipher, json.RawMessage(`{"name":"Dan","x":1}`)))) + want("contained by", ids(`SELECT id FROM customers WHERE profile <@ $1::eql_v4.query_json_tree ORDER BY id`, q), 4) + // Equality at a wildcard path matches any element. + q = eql.EncryptedJSONQuery(must(f.EqAt(ctx, cipher, "$.visits[*].at", json.RawMessage(`1`)))) + want("eq_at", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json_tree ORDER BY id`, q), 1) + // Containment on the optional column: SQL NULL rows match nothing. + pq, err := testusers.CustomerFields.Prefs.Contains(ctx, cipher, json.RawMessage(`{"theme":"dark"}`)) + if err != nil { + t.Fatal(err) + } + want("prefs contains", ids(`SELECT id FROM customers WHERE prefs @> $1::eql_v4.query_json_tree ORDER BY id`, pq), 1) + + // Orderings at an ordered path compare the leaf `->` extracts there. + age, err := f.Selector(ctx, cipher, "$.age") + if err != nil { + t.Fatal(err) + } + lt, err := f.At.Age.Lt(ctx, cipher, json.RawMessage(`40`)) + if err != nil { + t.Fatal(err) + } + want("age < 40", ids(`SELECT id FROM customers WHERE (profile -> $1::text) < $2::eql_v4.query_json_tree_leaf ORDER BY id`, age.Selector(), lt), 1, 2) + rng, err := f.At.Age.Between(ctx, cipher, json.RawMessage(`30`), json.RawMessage(`70`)) + if err != nil { + t.Fatal(err) + } + want("age between", ids(`SELECT id FROM customers WHERE (profile -> $1::text) >= $2::eql_v4.query_json_tree_leaf AND (profile -> $1::text) <= $3::eql_v4.query_json_tree_leaf ORDER BY id`, age.Selector(), rng.Low, rng.High), 1, 3) + at, err := f.Selector(ctx, cipher, "$.visits[*].at") + if err != nil { + t.Fatal(err) + } + gt, err := f.At.VisitsAt.Gt(ctx, cipher, json.RawMessage(`2`)) + if err != nil { + t.Fatal(err) + } + want("first visit > 2", ids(`SELECT id FROM customers WHERE (profile -> $1::text) > $2::eql_v4.query_json_tree_leaf ORDER BY id`, at.Selector(), gt), 1, 2) + + // `->` opened against the selection that asked for it: each row's + // value, and no value where the row has none. + selected := func(sel encrypt.JSONSelection[eql.EncryptedJSONEntry]) []encrypt.JSONValue { + t.Helper() + rows, err := conn.QueryContext(ctx, `SELECT profile -> $1::text FROM customers ORDER BY id`, sel.Selector()) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var entries []*eql.EncryptedJSONEntry + for rows.Next() { + var e *eql.EncryptedJSONEntry + if err := rows.Scan(&e); err != nil { + t.Fatal(err) + } + entries = append(entries, e) + } + values, err := sel.Open(ctx, entries) + if err != nil { + t.Fatal(err) + } + return values + } + name, err := f.Selector(ctx, cipher, "$.name") + if err != nil { + t.Fatal(err) + } + var names []string + for _, v := range selected(name) { + names = append(names, string(v.JSON)) + } + if !slices.Equal(names, []string{`"Alice"`, `"Bob"`, `"Carol"`, `"Dan"`}) { + t.Errorf("->: %v", names) + } + ages := selected(age) + if !ages[0].Found || string(ages[0].JSON) != "34" || ages[3].Found { + t.Errorf("-> $.age: %+v", ages) + } + + // A subtree read: per row, every visit's `at`, in document order, with + // the walk and with the sealed-root fallback a budget of 1 forces. + read, err := f.Subtree(ctx, cipher, "$.visits[*].at", encrypt.JSONReadOptions{}) + if err != nil { + t.Fatal(err) + } + walk := func(budget int) [][]json.RawMessage { + t.Helper() + groups := make([][]eql.EncryptedJSONEntry, len(jsonCustomers)) + for i, cust := range jsonCustomers { + rows, err := conn.QueryContext(ctx, `SELECT e FROM customers, eql_v4.json_tree_subtree(profile, $2::text[], $3) AS e WHERE id = $1`, cust.ID, read.Tokens(), budget) + if err != nil { + t.Fatal(err) + } + for rows.Next() { + var e eql.EncryptedJSONEntry + if err := rows.Scan(&e); err != nil { + t.Fatal(err) + } + groups[i] = append(groups[i], e) + } + _ = rows.Close() + } + opened, err := read.Open(ctx, groups) + if err != nil { + t.Fatal(err) + } + return opened + } + text := func(groups [][]json.RawMessage) [][]string { + out := make([][]string, len(groups)) + for i, g := range groups { + out[i] = []string{} + for _, v := range g { + out[i] = append(out[i], string(v)) + } + } + return out + } + wantAts := [][]string{{"3", "1"}, {"5"}, {}, {}} + for _, budget := range []int{64, 1} { + if got := text(walk(budget)); !slicesEqual2(got, wantAts) { + t.Errorf("subtree with budget %d: %v, want %v", budget, got, wantAts) + } + } + + // A concrete read of a container, opened to one value or none. + visits, err := f.Subtree(ctx, cipher, "$.visits", encrypt.JSONReadOptions{}) + if err != nil { + t.Fatal(err) + } + var entries []eql.EncryptedJSONEntry + rs, err := conn.QueryContext(ctx, `SELECT e FROM customers, eql_v4.json_tree_subtree(profile, $1::text[]) AS e WHERE id = 1`, visits.Tokens()) + if err != nil { + t.Fatal(err) + } + for rs.Next() { + var e eql.EncryptedJSONEntry + if err := rs.Scan(&e); err != nil { + t.Fatal(err) + } + entries = append(entries, e) + } + _ = rs.Close() + one, err := visits.OpenOne(ctx, entries) + if err != nil || !one.Found || !sameJSON(t, one.JSON, []byte(`[{"at":3},{"at":1}]`)) { + t.Errorf("OpenOne $.visits: %+v %v", one, err) + } + + // Child counts: two visits, one, an empty array, and none. + token, err := f.ChildCount(ctx, cipher, "$.visits") + if err != nil { + t.Fatal(err) + } + cr, err := conn.QueryContext(ctx, `SELECT eql_v4.json_tree_child_count(profile, $1::text) FROM customers ORDER BY id`, token) + if err != nil { + t.Fatal(err) + } + var counts []string + for cr.Next() { + var n sql.NullInt64 + if err := cr.Scan(&n); err != nil { + t.Fatal(err) + } + if n.Valid { + counts = append(counts, string(rune('0'+n.Int64))) + } else { + counts = append(counts, "NULL") + } + } + _ = cr.Close() + if !slices.Equal(counts, []string{"2", "1", "0", "NULL"}) { + t.Errorf("child counts: %v", counts) + } + children, err := f.Children(ctx, cipher, "$.visits") + if err != nil { + t.Fatal(err) + } + want("children rows", ids(`SELECT id FROM customers, eql_v4.json_tree_children(profile, $1::text) AS e ORDER BY id`, children), 1, 1, 2) +} + +func slicesEqual2(a, b [][]string) bool { + return slices.EqualFunc(a, b, func(x, y []string) bool { return slices.Equal(x, y) }) +} diff --git a/languages/golang/encrypt/term.go b/languages/golang/encrypt/term.go index 8e3bf55a0..e30d1c97d 100644 --- a/languages/golang/encrypt/term.go +++ b/languages/golang/encrypt/term.go @@ -70,9 +70,12 @@ type JSONOption interface { jsonOption() } -// JSON is the index over a JSON document. The engine does not derive it -// yet: a declaration that names it is refused by stashgen, and at run time -// by the engine. +// JSON is the index over a JSON document as an output beside a ciphertext, +// which the engine does not derive: a declaration that names it is refused +// by stashgen, and at run time by the engine. +// +// Deprecated: store a JSON document as a json.RawMessage field with +// `encrypt_into=EncryptedJson`, whose EQL type carries the JSON index. func JSON(options ...JSONOption) Index { _ = options return index("json") @@ -151,9 +154,12 @@ func (t OpeTerm) Compare(other OpeTerm) int { return bytes.Compare(t, other) } // Less reports whether t's plaintext orders before other's. func (t OpeTerm) Less(other OpeTerm) bool { return t.Compare(other) < 0 } -// JSONTerm is the term of the json index. The engine does not derive it -// yet; the type exists so a generated declaration that names [JSON] has a -// Go type to fail into. +// JSONTerm is the term of the json index as an output. The engine does not +// derive it; the type exists so a generated declaration that names [JSON] +// has a Go type to fail into. +// +// Deprecated: an EncryptedJson field's queries are eql.EncryptedJSONQuery +// and eql.EncryptedJSONLeafQuery. type JSONTerm []byte // compareCLLW is cllw-ore's compare_lex: compare_slice over the common diff --git a/languages/golang/stashgen/declaration.go b/languages/golang/stashgen/declaration.go index d19d612e7..de10e50eb 100644 --- a/languages/golang/stashgen/declaration.go +++ b/languages/golang/stashgen/declaration.go @@ -3,6 +3,8 @@ package stashgen import ( "fmt" "strings" + + "github.com/cipherstash/stack/languages/golang/internal/record" ) // Declaration is how each field of one struct is encrypted: the context of @@ -43,6 +45,9 @@ type Field struct { Indexes []Index // EQLType is the EQL type an EncryptInto field seals into. EQLType string + // TargetOptions are the EQL type's options, for a type that takes them + // (EncryptedJson); nil for its defaults. + TargetOptions *record.TargetOptions // Identity is the field's part of its context when it differs from Name: // a column that was renamed keeps the identity it was first written // under. "" means Name. Only a policy sets it. @@ -270,6 +275,9 @@ const ( KindTimestamp // KindDecimal is encrypt.Decimal. KindDecimal + // KindJSON is json.RawMessage: a JSON document, stored as an + // EncryptedJSON value. + KindJSON ) var kindNames = [...]string{ @@ -289,6 +297,7 @@ var kindNames = [...]string{ KindDate: "date", KindTimestamp: "timestamp", KindDecimal: "decimal", + KindJSON: "json", } // String returns the kind's name. @@ -303,7 +312,7 @@ func (k Kind) String() string { func (k Kind) Scalar() bool { switch k { case KindString, KindBool, KindInt, KindUint, KindFloat, KindBytes, - KindInt128, KindUint128, KindDate, KindTimestamp, KindDecimal: + KindInt128, KindUint128, KindDate, KindTimestamp, KindDecimal, KindJSON: return true } return false diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index e91a6d6d3..04849b653 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -263,6 +263,8 @@ func kindConst(k record.Kind) string { return "String" case record.Bytes: return "Bytes" + case record.JSON: + return "JSON" } return "Untyped" } @@ -295,8 +297,9 @@ func (w *writer) declaration(f *genFile) { } } for i, fld := range fields { + last := i == len(fields)-1 end := "." - if i == len(fields)-1 && fld.Identity == "" && fld.Null == NotNull { + if last && fld.Identity == "" && fld.Null == NotNull && fld.TargetOptions == nil { end = "" } switch fld.Verb { @@ -312,6 +315,13 @@ func (w *writer) declaration(f *genFile) { kind = "gensupport." + kindConst(wire) } w.p("\tEncryptInto(%q, %s, %q)%s", fld.Name, kind, fld.EQLType, end) + if fld.TargetOptions != nil { + end = "." + if last && fld.Identity == "" && fld.Null == NotNull { + end = "" + } + w.p("\tOptions(%q, %s)%s", fld.Name, targetOptionsExpr(fld.TargetOptions), end) + } case VerbEncryptIndex, VerbIndex: call := "EncryptIndex" if fld.Verb == VerbIndex { @@ -340,6 +350,23 @@ func (w *writer) declaration(f *genFile) { } } +// targetOptionsExpr is an EQL type's options as a gensupport.TargetOptions +// literal. +func targetOptionsExpr(o *record.TargetOptions) string { + var parts []string + if o.SealedRoot != nil { + parts = append(parts, fmt.Sprintf("SealedRoot: gensupport.Ptr(%t)", *o.SealedRoot)) + } + if o.Ordered != nil { + quoted := make([]string, len(o.Ordered)) + for i, p := range o.Ordered { + quoted[i] = fmt.Sprintf("%q", p) + } + parts = append(parts, "Ordered: []string{"+strings.Join(quoted, ", ")+"}") + } + return "gensupport.TargetOptions{" + strings.Join(parts, ", ") + "}" +} + // nullExpr is a null mode's constant in gensupport. func nullExpr(n Null) string { if n == NullSQL { @@ -638,11 +665,19 @@ func (w *writer) fieldEntries(f *genFile) { } w.p("}{") for _, g := range sealed { + if g.isJSON() { + w.p("\t%s: new%s(gensupport.NewField[%s](%s, %q)),", g.GoName, g.fieldType, g.valueExpr, f.declVar, g.Name) + continue + } w.p("\t%s: %s{gensupport.NewField[%s](%s, %q)},", g.GoName, g.fieldType, g.valueExpr, f.declVar, g.Name) } w.p("}") for _, g := range sealed { w.nl() + if g.isJSON() { + w.jsonEntry(g) + continue + } w.p("type %s struct {", g.fieldType) w.p("\tfield gensupport.Field[%s]", g.valueExpr) w.p("}") @@ -716,6 +751,14 @@ func planQueries(f *genFile) error { for i := range f.fields { g := &f.fields[i] g.queries = nil + if g.isJSON() { + // A document's operators have methods of their own: see + // jsonEntry. + if err := planJSON(f, g); err != nil { + return err + } + continue + } for _, op := range g.operators { q, err := queryFor(g, op) if err != nil { @@ -927,3 +970,195 @@ func joinNames(names []string) string { } return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1] } + +// isJSON reports whether the field is stored as an EncryptedJSON document: +// a json.RawMessage sealed into an EQL type. +func (g genField) isJSON() bool { + return g.Verb == VerbEncryptInto && g.GoType.Kind == KindJSON +} + +// jsonOperators are the operators a document field answers, and what each +// is a method of. +var jsonOperators = map[string]bool{ + "contains": true, "contained_by": true, "eq_at": true, "selector": true, + "subtree": true, "children": true, "child_count": true, + "lt_at": true, "lte_at": true, "gt_at": true, "gte_at": true, "between_at": true, +} + +// planJSON checks a document field's operators against the methods the +// generator writes for them, and names each ordered path's accessor. +func planJSON(f *genFile, g *genField) error { + for _, op := range g.operators { + if !jsonOperators[op.Name] { + return fieldErr(f.typeName, g.GoName, "the engine answers %s on %s, which the generator has no method for", op.Name, g.EQLType) + } + } + g.jsonPaths = nil + seen := map[string]string{} + if g.TargetOptions != nil { + for _, path := range g.TargetOptions.Ordered { + name, err := jsonPathName(path) + if err != nil { + return fieldErr(f.typeName, g.GoName, "%v", err) + } + if prior, ok := seen[name]; ok { + return fieldErr(f.typeName, g.GoName, "the ordered paths %s and %s both name the accessor At.%s; order one of them only", prior, path, name) + } + seen[name] = path + g.jsonPaths = append(g.jsonPaths, jsonPath{name: name, path: path}) + } + } + f.imports.add("encoding/json", "json") + return nil +} + +// jsonPathName is an ordered path's accessor name: its keys in Go case, +// array steps dropped. $.age is Age, $.visits[*].at is VisitsAt, $ is Root. +func jsonPathName(path string) (string, error) { + var b strings.Builder + for _, step := range strings.FieldsFunc(strings.TrimPrefix(path, "$"), func(r rune) bool { return r == '.' || r == '[' || r == ']' }) { + if step == "*" || strings.Trim(step, "0123456789") == "" { + continue + } + for _, word := range strings.Split(step, "_") { + if word != "" { + b.WriteString(strings.ToUpper(word[:1]) + word[1:]) + } + } + } + name := b.String() + if name == "" { + name = "Root" + } + if !isIdent(name) || name[0] < 'A' || name[0] > 'Z' { + return "", fmt.Errorf("the ordered path %s has no Go accessor name: a key starts with a digit", path) + } + return name, nil +} + +// jsonPath is one ordered path's accessor on a document field's entry. +type jsonPath struct { + name string // "Age" + path string // "$.age" +} + +// jsonEntry writes a document field's Fields entry: its query methods, a +// method for each read, and an accessor under At for each ordered path, +// whose methods are the orderings there. +func (w *writer) jsonEntry(g genField) { + answered := map[string]bool{} + for _, op := range g.operators { + answered[op.Name] = true + } + eqlName := strings.TrimPrefix(g.outputType, "eql.") + needle := g.queryType + leaf := "eql." + eqlName + "LeafQuery" + entry := "eql." + eqlName + "Entry" + ordered := len(g.jsonPaths) > 0 && answered["lt_at"] + field := "gensupport.Field[" + g.valueExpr + "]" + + w.p("type %s struct {", g.fieldType) + w.p("\tfield %s", field) + if ordered { + w.p("\t// At has one accessor per path the field orders, whose methods") + w.p("\t// are the orderings there.") + w.p("\tAt %sAt", g.fieldType) + } + w.p("}") + w.nl() + w.p("func new%s(f %s) %s {", g.fieldType, field, g.fieldType) + if ordered { + w.p("\treturn %s{field: f, At: %sAt{", g.fieldType, g.fieldType) + for _, p := range g.jsonPaths { + w.p("\t\t%s: %sPath{field: f, path: %q},", p.name, g.fieldType, p.path) + } + w.p("\t}}") + } else { + w.p("\treturn %s{field: f}", g.fieldType) + } + w.p("}") + w.nl() + w.p("func (f %s) Encrypt(ctx context.Context, c *encrypt.Cipher, v %s) (%s, error) {", g.fieldType, g.valueExpr, g.outputType) + w.sealedOutput(g, "f.field.Encrypt(ctx, c, v)") + w.p("}") + recv := "func (f " + g.fieldType + ") " + if answered["contains"] { + w.nl() + w.b.WriteString(wrapComment("Contains derives the query for documents that contain needle, a JSON value: col @> query. Each array element of the needle must sit inside one element of the document's array.", 80)) + w.p("%sContains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (%s, error) {", recv, needle) + w.p("\treturn gensupport.JSONQuery[%s](ctx, f.field, c, \"contains\", needle)", needle) + w.p("}") + } + if answered["contained_by"] { + w.nl() + w.b.WriteString(wrapComment("ContainedBy derives the query for documents contained by needle, a JSON value: col <@ query.", 80)) + w.p("%sContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (%s, error) {", recv, needle) + w.p("\treturn gensupport.JSONQuery[%s](ctx, f.field, c, \"contained_by\", needle)", needle) + w.p("}") + } + if answered["eq_at"] { + w.nl() + w.b.WriteString(wrapComment("EqAt derives the query for documents holding the scalar v at path: col @> query. A wildcard step matches when any element holds it; a position ([2]) is refused, since the stored selector reads every position as the wildcard.", 80)) + w.p("%sEqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (%s, error) {", recv, needle) + w.p("\treturn gensupport.JSONQueryAt[%s](ctx, f.field, c, \"eq_at\", path, v)", needle) + w.p("}") + } + if answered["selector"] { + w.nl() + w.b.WriteString(wrapComment("Selector prepares `->` at path: bind Selector() as col -> $1::text, and open the entries it returns with Open. A position ([2]) is refused; read one with Subtree.", 80)) + w.p("%sSelector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[%s], error) {", recv, entry) + w.p("\treturn gensupport.JSONSelect[%s](ctx, f.field, c, path)", entry) + w.p("}") + } + if answered["subtree"] { + w.nl() + w.b.WriteString(wrapComment("Subtree prepares a read of the value at path: bind Tokens() as eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it returns with Open or OpenOne.", 80)) + w.p("%sSubtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[%s], error) {", recv, entry) + w.p("\treturn gensupport.JSONSubtree[%s](ctx, f.field, c, path, opts)", entry) + w.p("}") + } + if answered["children"] { + w.nl() + w.b.WriteString(wrapComment("Children derives the token of the node at path, a path with no wildcard, which eql_v4.json_tree_children(col, $1::text) takes. Its rows are encrypted entries one level down.", 80)) + w.p("%sChildren(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) {", recv) + w.p("\treturn gensupport.JSONNodeToken(ctx, f.field, c, \"children\", path)") + w.p("}") + } + if answered["child_count"] { + w.nl() + w.b.WriteString(wrapComment("ChildCount derives the token of the node at path, which eql_v4.json_tree_child_count(col, $1::text) takes: zero for a scalar or an empty container, NULL where the row has no node there or for the sealed root.", 80)) + w.p("%sChildCount(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) {", recv) + w.p("\treturn gensupport.JSONNodeToken(ctx, f.field, c, \"child_count\", path)") + w.p("}") + } + if !ordered { + return + } + w.nl() + w.p("type %sAt struct {", g.fieldType) + for _, p := range g.jsonPaths { + w.p("\t%s %sPath // %s", p.name, g.fieldType, p.path) + } + w.p("}") + w.nl() + w.b.WriteString(wrapComment(fmt.Sprintf("%sPath is one ordered path of %s: its orderings compare the first value `->` extracts there, by the leaf's ordering term. Numbers order as float64, so integers past 2^53 can tie; text orders case- and accent-folded.", g.fieldType, g.GoName), 80)) + w.p("type %sPath struct {", g.fieldType) + w.p("\tfield %s", field) + w.p("\tpath string") + w.p("}") + for _, op := range []struct{ name, method string }{{"lt_at", "Lt"}, {"lte_at", "Lte"}, {"gt_at", "Gt"}, {"gte_at", "Gte"}} { + if !answered[op.name] { + continue + } + w.nl() + w.p("func (p %sPath) %s(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (%s, error) {", g.fieldType, op.method, leaf) + w.p("\treturn gensupport.JSONQueryAt[%s](ctx, p.field, c, %q, p.path, v)", leaf, op.name) + w.p("}") + } + if answered["between_at"] { + w.nl() + w.p("func (p %sPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[%s], error) {", g.fieldType, leaf) + w.p("\treturn gensupport.JSONBetweenAt[%s](ctx, p.field, c, p.path, lo, hi)", leaf) + w.p("}") + } +} diff --git a/languages/golang/stashgen/engine.go b/languages/golang/stashgen/engine.go index b99a197ae..3609e4dbf 100644 --- a/languages/golang/stashgen/engine.go +++ b/languages/golang/stashgen/engine.go @@ -250,6 +250,7 @@ func lowerDeclaration(d Declaration, eqlTypes []EQLType) (*record.Plan, error) { return nil, &FieldError{Type: d.Type, Field: f.GoName, Reason: fmt.Sprintf("%s is stored under a table and a column, and the context %q has %d segments, not one", f.EQLType, d.Context, len(plan.Context))} } rf.Target = f.EQLType + rf.TargetOptions = f.TargetOptions case VerbEncrypt, VerbEncryptIndex: rf.Outputs = append(rf.Outputs, record.Ciphertext) } @@ -281,7 +282,7 @@ var outputOfIndex = map[IndexName]record.Output{ // no entry in outputOfIndex, so no tag asks for it. var indexOfOutput = map[record.Output]IndexName{ record.Equality: IndexEquality, record.Match: IndexMatch, record.Ore: IndexOre, record.Ope: IndexOpe, - record.OreBlock: IndexOreBlock, + record.OreBlock: IndexOreBlock, record.JSONIndex: IndexJSON, } // wireKind is the record kind of a Go type: the same mapping the emitter @@ -330,6 +331,8 @@ func wireKind(t GoType) record.Kind { return record.Timestamp case KindDecimal: return record.Decimal + case KindJSON: + return record.JSON } return record.Untyped } @@ -360,6 +363,8 @@ func kindOfWire(k record.Kind) Kind { return KindTimestamp case record.Decimal: return KindDecimal + case record.JSON: + return KindJSON } return KindOther } diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index 663b36191..841554f9c 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -105,6 +105,7 @@ type genField struct { queries []queryMethod // the field's query methods, one for each operator fieldType string // "EmailField" pathType string // the type with its package path, for model checks + jsonPaths []jsonPath // a document field's ordered paths, each with its accessor } // definedScalar reports whether the field's type is a named type over a @@ -708,7 +709,7 @@ func (r *reader) buildFields(c *collected) error { return fieldErr(typeName, cf.goName, "two fields write the name %q: %s and %s", name, prev, cf.goName) } seen[name] = cf.goName - field := Field{Name: name, GoName: cf.goName, GoType: r.goType(cf.typ), Verb: t.Verb, Indexes: t.Indexes, EQLType: t.EQLType, Identity: t.Identity} + field := Field{Name: name, GoName: cf.goName, GoType: r.goType(cf.typ), Verb: t.Verb, Indexes: t.Indexes, EQLType: t.EQLType, TargetOptions: t.EQLOptions, Identity: t.Identity} if t.Omit { field.Verb = VerbOmit } @@ -760,6 +761,11 @@ func (r *reader) buildFields(c *collected) error { if len(r.eql) == 0 { return fieldErr(typeName, cf.goName, "EQL types are not available yet; this build of the engine produces none") } + for _, known := range r.eql { + if EQLGoName(known.Name) == field.EQLType && known.Name != field.EQLType { + return fieldErr(typeName, cf.goName, "encrypt_into names the EQL type as the engine does, %s; eql.%s is its Go type", known.Name, field.EQLType) + } + } return fieldErr(typeName, cf.goName, "the engine has no EQL type %s", field.EQLType) } if !eqlType.Producible { @@ -954,6 +960,9 @@ func valueKind(t types.Type) (Kind, bool) { if path == "time" && name == "Time" { return KindTimestamp, true } + if path == "encoding/json" && name == "RawMessage" { + return KindJSON, true + } if path != encryptPath && path != vcvaluePath { continue } diff --git a/languages/golang/stashgen/tag.go b/languages/golang/stashgen/tag.go index 655419ba4..925b5a093 100644 --- a/languages/golang/stashgen/tag.go +++ b/languages/golang/stashgen/tag.go @@ -4,7 +4,10 @@ import ( "errors" "fmt" "reflect" + "strconv" "strings" + + "github.com/cipherstash/stack/languages/golang/internal/record" ) // The key of the struct tag the generator reads. @@ -23,6 +26,8 @@ const tagKey = "stash" // stash:"tenant,context_field" Name, VerbContextField // stash:",passthrough" VerbPassthrough with no Name, on an embedded struct // stash:"dob,encrypt_into=Date,null=sql" Null, on a sealed field declared over a pointer +// stash:"profile,encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at,sealed_root=false)" +// EQLType and EQLOptions type tag struct { Omit bool Context string @@ -31,6 +36,9 @@ type tag struct { Verb Verb Indexes []Index EQLType string + // EQLOptions are the EQL type's options, from the parentheses after its + // name; nil when the tag gives none. + EQLOptions *record.TargetOptions // Null is how a sealed field declared over a pointer stores nil, when // the tag says; the generator picks NullSealed where it can. Null Null @@ -96,6 +104,16 @@ func parseTagValue(value string) (tag, error) { } sawInto = true t.EQLType = strings.TrimPrefix(item, "encrypt_into=") + if open := strings.IndexByte(t.EQLType, '('); open >= 0 { + if !strings.HasSuffix(t.EQLType, ")") { + return tag{}, fmt.Errorf("tag %q: encrypt_into's options open with ( and do not close", value) + } + t.EQLOptions, err = parseEQLOptions(t.EQLType[open+1 : len(t.EQLType)-1]) + if err != nil { + return tag{}, fmt.Errorf("tag %q: %w", value, err) + } + t.EQLType = t.EQLType[:open] + } if !isIdent(t.EQLType) { return tag{}, fmt.Errorf("tag %q: encrypt_into names an EQL type, and %q is not a type name", value, t.EQLType) } @@ -184,6 +202,83 @@ func parseContextTag(items []string) (tag, error) { return t, nil } +// parseEQLOptions reads an EQL type's options: `ordered=$.age;$.visits[*].at` +// and `sealed_root=true|false`, separated by commas. The engine checks the +// paths when the declaration is checked. A path in a tag is plain JSONPath: +// keys of letters, digits and underscores, [n] and [*]; a quoted key, or a +// key holding a comma, semicolon or parenthesis, has no tag spelling. +func parseEQLOptions(list string) (*record.TargetOptions, error) { + if list == "" { + return nil, errors.New("encrypt_into has empty options; drop the parentheses") + } + options := &record.TargetOptions{} + seen := map[string]bool{} + for _, opt := range strings.Split(list, ",") { + key, value, ok := strings.Cut(opt, "=") + if !ok || value == "" { + return nil, fmt.Errorf("encrypt_into option %q is not key=value", opt) + } + if seen[key] { + return nil, fmt.Errorf("encrypt_into option %s is given twice", key) + } + seen[key] = true + switch key { + case "ordered": + for _, path := range strings.Split(value, ";") { + if !plainPath(path) { + return nil, fmt.Errorf("encrypt_into ordered path %q is not a plain JSONPath ($ then .key, [n] or [*] steps, keys of letters, digits and underscores)", path) + } + options.Ordered = append(options.Ordered, path) + } + case "sealed_root": + b, err := strconv.ParseBool(value) + if err != nil || (value != "true" && value != "false") { + return nil, fmt.Errorf("encrypt_into sealed_root is true or false, not %q", value) + } + options.SealedRoot = &b + default: + return nil, fmt.Errorf("encrypt_into has no option %q; the options are ordered and sealed_root", key) + } + } + return options, nil +} + +// plainPath reports whether s is a JSONPath a tag can spell: $, then .key +// steps (letters, digits, underscores) and [n] or [*] steps. +func plainPath(s string) bool { + if !strings.HasPrefix(s, "$") { + return false + } + for i := 1; i < len(s); { + switch s[i] { + case '.': + j := i + 1 + for j < len(s) && (s[j] == '_' || s[j] >= 'a' && s[j] <= 'z' || s[j] >= 'A' && s[j] <= 'Z' || s[j] >= '0' && s[j] <= '9') { + j++ + } + if j == i+1 { + return false + } + i = j + case '[': + end := strings.IndexByte(s[i:], ']') + if end < 0 { + return false + } + inner := s[i+1 : i+end] + if inner != "*" { + if _, err := strconv.ParseUint(inner, 10, 32); err != nil { + return false + } + } + i += end + 1 + default: + return false + } + } + return true +} + // parseIndexes reads `equality;match(k=3,m=2048)`. func parseIndexes(list string) ([]Index, error) { parts, err := splitOutsideParens(list, ';') diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go index 97273a058..6ddfbe7bc 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go @@ -119,3 +119,36 @@ type ( Negative bool } ) + +// JSONSelector is the selector `col -> $1::text` binds. +type JSONSelector string + +// JSONToken is one node's token. +type JSONToken string + +// JSONTokens are a subtree read's tokens. +type JSONTokens []string + +// JSONValue is one value a read reached, or none. +type JSONValue struct { + Found bool + JSON []byte +} + +// JSONReadOptions are a subtree read's options. +type JSONReadOptions struct{ WildcardBound uint32 } + +// JSONRange is both bounds of a between at a path. +type JSONRange[Q ~[]byte] struct{ Low, High Q } + +// JSONSelection is a prepared `->` on an EncryptedJSON field. +type JSONSelection[E ~[]byte] struct{ _ E } + +func (JSONSelection[E]) Selector() JSONSelector { return "" } +func (JSONSelection[E]) Open(context.Context, []*E) ([]JSONValue, error) { return nil, nil } + +// JSONRead is a prepared subtree read on an EncryptedJSON field. +type JSONRead[E ~[]byte] struct{ _ E } + +func (JSONRead[E]) Tokens() JSONTokens { return nil } +func (JSONRead[E]) OpenOne(context.Context, []E) (JSONValue, error) { return JSONValue{}, nil } diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go index 83abf8f0f..76ae60221 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go @@ -7,6 +7,7 @@ package gensupport import ( "context" + "encoding/json" "log/slog" "github.com/cipherstash/stack/languages/golang/encrypt" @@ -44,6 +45,7 @@ const ( Timestamp Kind = "timestamp" String Kind = "string" Bytes Kind = "bytes" + JSON Kind = "json" ) // Redacted formats a value with its sealed fields hidden. @@ -87,6 +89,7 @@ func (d Declaration) Encrypt(name string, kind Kind) Declaration func (d Declaration) EncryptIndex(name string, kind Kind, idx ...encrypt.Index) Declaration { return d } func (d Declaration) Index(name string, kind Kind, idx ...encrypt.Index) Declaration { return d } func (d Declaration) EncryptInto(name string, kind Kind, eqlType string) Declaration { return d } +func (d Declaration) Options(name string, options TargetOptions) Declaration { return d } func (d Declaration) Omit(name string) Declaration { return d } func (d Declaration) Err() error { return nil } @@ -196,3 +199,42 @@ func (f Field[T]) NullTerm(context.Context, *encrypt.Cipher, string) ([]byte, er // GetPtr reads one opened field declared over a pointer. func GetPtr[T any](Values, string) (*T, error) { return nil, nil } + +// TargetOptions are the options an EncryptInto field gives its EQL type. +type TargetOptions struct { + SealedRoot *bool + Ordered []string +} + +// Ptr is a pointer to v. +func Ptr[T any](v T) *T { return &v } + +// JSONQuery derives a containment operator's query on a JSON field. +func JSONQuery[Q ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string, json.RawMessage) (Q, error) { + return nil, nil +} + +// JSONQueryAt derives an operator at a path on a JSON field. +func JSONQueryAt[Q ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string, string, json.RawMessage) (Q, error) { + return nil, nil +} + +// JSONBetweenAt derives between_at on a JSON field. +func JSONBetweenAt[Q ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string, json.RawMessage, json.RawMessage) (encrypt.JSONRange[Q], error) { + return encrypt.JSONRange[Q]{}, nil +} + +// JSONSelect prepares `->` at a path on a JSON field. +func JSONSelect[E ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string) (encrypt.JSONSelection[E], error) { + return encrypt.JSONSelection[E]{}, nil +} + +// JSONSubtree prepares a subtree read at a path on a JSON field. +func JSONSubtree[E ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string, encrypt.JSONReadOptions) (encrypt.JSONRead[E], error) { + return encrypt.JSONRead[E]{}, nil +} + +// JSONNodeToken derives the token of the node at a path. +func JSONNodeToken[T any](context.Context, Field[T], *encrypt.Cipher, string, string) (encrypt.JSONToken, error) { + return "", nil +} diff --git a/mise.toml b/mise.toml index da66a306a..930072ba3 100644 --- a/mise.toml +++ b/mise.toml @@ -368,7 +368,7 @@ description = "Format check, vet and test the Go module (languages/golang: encry run = "scripts/go-binding-test.sh languages/golang" [tasks."go:test:postgres"] -description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names and run the Go module's PostgreSQL tests against it: rows the Go SDK stores in every EQL type it produces, found by the queries it derives, and random values in every ordered type, ordered as their plaintexts (EQL_ORDER_SEED replays a seed); needs `wasm:guest:build:deterministic` and `wasm:guest:build:eql:deterministic` first" +description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names and run the Go module's PostgreSQL tests against it: rows the Go SDK stores in every EQL type it produces, found by the queries it derives, encrypted JSON documents queried and read through their generated methods, and random values in every ordered type, ordered as their plaintexts (EQL_ORDER_SEED replays a seed); needs `wasm:guest:build:deterministic` and `wasm:guest:build:eql:deterministic` first" # The bundle is built from packages/eql, as test:encryption:postgres builds # it: the copy under crates/eql-bindings/sql is the RELEASED bundle and moves # only at the lockstep bump, so it cannot show a SQL change in the same pull @@ -384,7 +384,7 @@ cd languages/golang # The tests skip without DATABASE_URL and without their guest builds; -v and # the grep turn either skip into a failure here, where both are provided. DATABASE_URL="$EQL_TEST_DATABASE_URL" CGO_ENABLED=0 go test -count=1 -v \ - -run '^(TestEQLTypesRoundTripAndQueryInPostgres|TestEQLOrderMatchesPlaintextInPostgres|TestQueriesRunAgainstPostgres)$' ./encrypt \ + -run '^(TestEQLTypesRoundTripAndQueryInPostgres|TestEQLOrderMatchesPlaintextInPostgres|TestQueriesRunAgainstPostgres|TestJSONDocumentsQueryAndReadInPostgres)$' ./encrypt \ | tee "${TMPDIR:-/tmp}/go-test-postgres.log" if grep -q -- '--- SKIP' "${TMPDIR:-/tmp}/go-test-postgres.log"; then echo "error: a PostgreSQL test skipped; build the deterministic guests first" >&2 @@ -392,7 +392,7 @@ if grep -q -- '--- SKIP' "${TMPDIR:-/tmp}/go-test-postgres.log"; then fi # A -run pattern that matches no test (one renamed, say) prints "no tests # to run" and PASS, so each test has to be seen to pass by name. -for test in TestEQLTypesRoundTripAndQueryInPostgres TestEQLOrderMatchesPlaintextInPostgres TestQueriesRunAgainstPostgres; do +for test in TestEQLTypesRoundTripAndQueryInPostgres TestEQLOrderMatchesPlaintextInPostgres TestQueriesRunAgainstPostgres TestJSONDocumentsQueryAndReadInPostgres; do if ! grep -q -- "^--- PASS: $test " "${TMPDIR:-/tmp}/go-test-postgres.log"; then echo "error: $test did not run and pass; was it renamed?" >&2 exit 1 From 3ae71660c53b2913596c9d4db575d3dfcc519719 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 06:06:03 +1100 Subject: [PATCH 02/13] fix(go): close the encrypted JSON fields review findings - A json.RawMessage is a document only under encrypt_into; sealed or indexed any other way it stays bytes, so existing models generate as before. - An ordered path's accessor has its own Selector, so an ordering and the value it compares come from one path; the README says why. - JSONRead keeps the subtree input it was prepared from and sends it back when it opens; OpenOne refuses a wildcard path before opening anything. - JSONSelection.Open makes no request for a page with no entries and refuses more than one value for a row; a zero JSONSelector or JSONToken is refused rather than bound as ''. - gensupport's JSON helpers check the term's encoding, fidelity (record.Exact) and part count in one place. - stashgen: imports are re-checked after planning adds encoding/json; a document field needs the engine to answer selector on it; an ordered path given twice, or with no name, is refused by name; a key of digits names its accessor; index=json points at encrypt_into=EncryptedJson. - Tests check rows.Err() and no longer assert a driver value's type. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 11 ++-- languages/golang/cmd/stashgen/README.md | 7 +- languages/golang/cmd/stashgen/main_test.go | 17 ++++- languages/golang/encrypt/gensupport/json.go | 65 ++++++++----------- .../internal/testusers/customer_stash.go | 7 ++ languages/golang/encrypt/json.go | 54 +++++++++++---- languages/golang/encrypt/json_fields_test.go | 31 ++++++++- .../golang/encrypt/json_postgres_test.go | 22 ++++++- languages/golang/stashgen/emit.go | 40 ++++++++++-- languages/golang/stashgen/engine.go | 6 +- .../golang/stashgen/enginetest/enginetest.go | 2 +- languages/golang/stashgen/read.go | 11 ++++ languages/golang/stashgen/refusal_test.go | 2 +- 13 files changed, 202 insertions(+), 73 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 4da8a5e57..5fad839ff 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -19,10 +19,13 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. `*json.RawMessage` field takes `null=sql`. Its `Fields` entry has `Contains`, `ContainedBy`, `EqAt`, `Selector`, `Subtree`, `Children` and `ChildCount`, and `At.` per ordered path with `Lt`, `Lte`, `Gt`, - `Gte` and `Between`. `encrypt.JSONSelection` and `encrypt.JSONRead` open - the rows `->` and `eql_v4.json_tree_subtree` return against the request - that prepared them; `encrypt.JSONSelector`, `JSONToken` and `JSONTokens` - bind as `text` and `text[]`. `gensupport.JSON`, `TargetOptions`, + `Gte`, `Between` and the path's own `Selector`. `encrypt.JSONSelection` + and `encrypt.JSONRead` open the rows `->` and `eql_v4.json_tree_subtree` + return against the request that prepared them (`OpenOne` refuses a + wildcard read up front); `encrypt.JSONSelector`, `JSONToken` and + `JSONTokens` bind as `text` and `text[]`, and a zero selector or token is + refused rather than bound. A `json.RawMessage` sealed any other way stays + bytes. `gensupport.JSON`, `TargetOptions`, `Declaration.Options` and the `JSON*` helpers are the generated code's. - **Query and read an encrypted JSON document, at the record level** diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index fe859e01c..17ca31938 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -117,6 +117,7 @@ A `json.RawMessage` field with `encrypt_into=EncryptedJson` is stored as one enc The field holds the document's JSON text; the engine parses it, so an object that names a member twice, an integer outside the 64-bit range and text that is not JSON are refused with `encrypt.ErrEncoding`. Values come back as JSON text holding the same values; key order and the spelling of numbers are not kept. A `*json.RawMessage` field is optional and must say `null=sql`: a document has no fixed length to hide a missing one behind. A non-nil pointer to `null` is a stored JSON `null`. +A `json.RawMessage` sealed any other way (`encrypt`, `index=equality`) is bytes, as any type defined over `[]byte` is. The type takes options in parentheses: `ordered=` names the paths whose numbers and strings carry an ordering term, separated by `;`, and `sealed_root=false` turns off the sealed root (on by default). `encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at)` orders two paths. @@ -128,12 +129,14 @@ The field's `Fields` entry has a method per operation: |---|---|---| | `Contains(ctx, c, needle)`, `ContainedBy(ctx, c, needle)` | `eql.EncryptedJSONQuery` | `col @> $1::eql_v4.query_json_tree`, `col <@ $1::eql_v4.query_json_tree` | | `EqAt(ctx, c, path, v)` | `eql.EncryptedJSONQuery` | `col @> $1::eql_v4.query_json_tree` | -| `At.Age.Lt(ctx, c, v)`, `Lte`, `Gt`, `Gte`, `Between` | `eql.EncryptedJSONLeafQuery` | `(col -> $sel::text) < $1::eql_v4.query_json_tree_leaf` | +| `At.Age.Lt(ctx, c, v)`, `Lte`, `Gt`, `Gte`, `Between` | `eql.EncryptedJSONLeafQuery` | `(col -> $sel::text) < $1::eql_v4.query_json_tree_leaf`, `$sel` from `At.Age.Selector(ctx, c)` | | `Selector(ctx, c, path)` | `encrypt.JSONSelection`: `Selector()` and `Open` | `col -> $1::text` | | `Subtree(ctx, c, path, opts)` | `encrypt.JSONRead`: `Tokens()`, `Open` and `OpenOne` | `eql_v4.json_tree_subtree(col, $1::text[], 64)` | | `Children(ctx, c, path)`, `ChildCount(ctx, c, path)` | `encrypt.JSONToken` | `eql_v4.json_tree_children(col, $1::text)`, `eql_v4.json_tree_child_count(col, $1::text)` | -`At` has one accessor per ordered path, named for its keys (`$.visits[*].at` is `At.VisitsAt`); two paths with one name stop `stashgen`. +`At` has one accessor per ordered path, named for its keys (`$.visits[*].at` is `At.VisitsAt`, `$.a.1` is `At.A1`); two paths with one name, or one path given twice, stop `stashgen`. +An ordering compares the value `->` extracts at its path, and EQL's `<` compares ordering terms alone, so take the selector from the same accessor (`At.Age.Selector`) rather than spelling the path again. +`OpenOne` reads one value and refuses a read whose path has a wildcard before anything is opened; open a wildcard read with `Open`. The needle and the values are `json.RawMessage`: write the JSON you mean, such as `json.RawMessage(`+"`"+`{"active":true}`+"`"+`)`. A position in a path is refused for `EqAt` and `Selector`, whose selector reads every position as the wildcard; read a position with `Subtree`. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index ac3cb928d..f2f27d73c 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -333,6 +333,20 @@ func TestRunGeneratesEncryptedJSONFields(t *testing.T) { if strings.Contains(written, "PrefsFieldAt") { t.Error("Prefs orders no path, and got an At") } + if !strings.Contains(written, "func (p ProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error)") { + t.Error("an ordered path has no Selector of its own") + } + // A json.RawMessage sealed any other way is bytes, as it was before + // documents; an object key of digits names its accessor. + written, stderr, code = generate("\tData json.RawMessage `stash:\"data,encrypt,index=equality\"`\n\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.a.1)\"`") + if code != 0 { + t.Fatalf("exit %d\n%s", code, stderr) + } + for _, want := range []string{`EncryptIndex("data", gensupport.Bytes`, "A1 ProfileFieldPath // $.a.1"} { + if !strings.Contains(written, want) { + t.Errorf("the generated file lacks %q", want) + } + } for _, c := range []struct{ name, fields, want string }{ {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson,null=sealed\"`", "Customer.Prefs"}, {"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson\"`", "null=sql"}, @@ -341,7 +355,8 @@ func TestRunGeneratesEncryptedJSONFields(t *testing.T) { {"an unknown option", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(bound=3)\"`", "no option \"bound\""}, {"the Go name in the tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJSON\"`", "as the engine does, EncryptedJson"}, {"two paths, one accessor", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.a_b;$.aB)\"`", "both name the accessor At.AB"}, - {"a document with no target", "\tProfile json.RawMessage `stash:\"profile,encrypt\"`", "the engine refuses the declaration"}, + {"the same ordered path twice", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.a;$.a)\"`", "the ordered path $.a is given twice"}, + {"a path with no accessor name", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.__)\"`", "no Go accessor name"}, {"a string into a document", "\tProfile string `stash:\"profile,encrypt_into=EncryptedJson\"`", "EncryptedJson seals a json"}, {"options on a type that takes none", "\tEmail string `stash:\"email,encrypt_into=TextEq(ordered=$.a)\"`", "the engine refuses the declaration"}, } { diff --git a/languages/golang/encrypt/gensupport/json.go b/languages/golang/encrypt/gensupport/json.go index f04391529..717e1821b 100644 --- a/languages/golang/encrypt/gensupport/json.go +++ b/languages/golang/encrypt/gensupport/json.go @@ -6,16 +6,20 @@ import ( "fmt" "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/internal/record" "github.com/cipherstash/vitaminc/bindings/go/vcvalue" ) // The operators of a field stored as an EncryptedJSON document, for the // generated methods of its Fields entry. Each sends the operator's named -// inputs, JSON values as their JSON text, and checks the engine answered -// in the encoding the method returns. +// inputs, JSON values as their JSON text ([]byte of the json.RawMessage), +// and checks the engine answered in the encoding and with the number of +// parts the method returns. -// jsonTerm derives one JSON operator's term on the field. -func (f Field[T]) jsonTerm(ctx context.Context, c *encrypt.Cipher, operator, encoding string, input vcvalue.Object) (encrypt.QueryTerm, error) { +// jsonTerm derives one JSON operator's term on the field: an exact term in +// encoding, of parts byte strings ("eql") or texts ("text"); parts 0 takes +// any number of texts ("text_array"). +func (f Field[T]) jsonTerm(ctx context.Context, c *encrypt.Cipher, operator, encoding string, parts int, input vcvalue.Object) (encrypt.QueryTerm, error) { if f.err != nil { return encrypt.QueryTerm{}, f.err } @@ -26,21 +30,22 @@ func (f Field[T]) jsonTerm(ctx context.Context, c *encrypt.Cipher, operator, enc if err != nil { return encrypt.QueryTerm{}, err } - if term.Encoding != encoding || term.Fidelity != "exact" { - return encrypt.QueryTerm{}, fmt.Errorf("gensupport: %s: %s came back as a %s %s term, not an exact %s one", f.name, operator, term.Fidelity, term.Encoding, encoding) + got := len(term.Terms) + if encoding == "text" || encoding == "text_array" { + got = len(term.Text) + } + if term.Encoding != encoding || term.Fidelity != record.Exact || (parts != 0 && got != parts) { + return encrypt.QueryTerm{}, fmt.Errorf("gensupport: %s: %s came back as a %s %s term of %d parts, not an exact %s one of %d", f.name, operator, term.Fidelity, term.Encoding, got, encoding, parts) } return term, nil } -// jsonText is a JSON value as an input carries it: its text, as bytes. -func jsonText(v json.RawMessage) []byte { return []byte(v) } - // JSONQuery derives a containment operator's query on a JSON field // (contains, contained_by): the needle's projection, Q. func JSONQuery[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator string, needle json.RawMessage) (Q, error) { - term, err := f.jsonTerm(ctx, c, operator, "eql", vcvalue.Object{{Key: "json", Value: jsonText(needle)}}) - if err != nil || len(term.Terms) != 1 { - return nil, oneTermErr(f.name, operator, err) + term, err := f.jsonTerm(ctx, c, operator, "eql", 1, vcvalue.Object{{Key: "json", Value: []byte(needle)}}) + if err != nil { + return nil, err } return Q(term.Terms[0]), nil } @@ -49,9 +54,9 @@ func JSONQuery[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cip // lte_at, gt_at, gte_at): Q is the needle for eq_at and the leaf operand // for the orderings. func JSONQueryAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator, path string, v json.RawMessage) (Q, error) { - term, err := f.jsonTerm(ctx, c, operator, "eql", vcvalue.Object{{Key: "path", Value: path}, {Key: "json", Value: jsonText(v)}}) - if err != nil || len(term.Terms) != 1 { - return nil, oneTermErr(f.name, operator, err) + term, err := f.jsonTerm(ctx, c, operator, "eql", 1, vcvalue.Object{{Key: "path", Value: path}, {Key: "json", Value: []byte(v)}}) + if err != nil { + return nil, err } return Q(term.Terms[0]), nil } @@ -59,23 +64,17 @@ func JSONQueryAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.C // JSONBetweenAt derives between_at on a JSON field: the two leaf operands, // low then high, inclusive. func JSONBetweenAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string, lo, hi json.RawMessage) (encrypt.JSONRange[Q], error) { - term, err := f.jsonTerm(ctx, c, "between_at", "eql", vcvalue.Object{{Key: "path", Value: path}, {Key: "low", Value: jsonText(lo)}, {Key: "high", Value: jsonText(hi)}}) + term, err := f.jsonTerm(ctx, c, "between_at", "eql", 2, vcvalue.Object{{Key: "path", Value: path}, {Key: "low", Value: []byte(lo)}, {Key: "high", Value: []byte(hi)}}) if err != nil { return encrypt.JSONRange[Q]{}, err } - if len(term.Terms) != 2 { - return encrypt.JSONRange[Q]{}, fmt.Errorf("gensupport: %s: between_at came back as %d terms, not 2", f.name, len(term.Terms)) - } return encrypt.JSONRange[Q]{Low: Q(term.Terms[0]), High: Q(term.Terms[1])}, nil } // JSONSelect prepares `->` at a path on a JSON field: the selector to // bind, and what opens the entries it returns. E is the field's entry type. func JSONSelect[E ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string) (encrypt.JSONSelection[E], error) { - term, err := f.jsonTerm(ctx, c, "selector", "text", vcvalue.Object{{Key: "path", Value: path}}) - if err == nil && len(term.Text) != 1 { - err = fmt.Errorf("gensupport: %s: selector came back as %d texts, not 1", f.name, len(term.Text)) - } + term, err := f.jsonTerm(ctx, c, "selector", "text", 1, vcvalue.Object{{Key: "path", Value: path}}) if err != nil { return encrypt.JSONSelection[E]{}, err } @@ -83,36 +82,28 @@ func JSONSelect[E ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Ci } // JSONSubtree prepares a subtree read at a path on a JSON field: the -// tokens to bind, and what opens the rows they return. +// tokens to bind, and what opens the rows they return. The request is +// built once, here, and the read keeps it: opening sends the engine the +// same request the tokens were derived from. func JSONSubtree[E ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[E], error) { input := vcvalue.Object{{Key: "path", Value: path}} if opts.WildcardBound != 0 { input = append(input, vcvalue.Field{Key: "wildcard_bound", Value: opts.WildcardBound}) } - term, err := f.jsonTerm(ctx, c, "subtree", "text_array", input) + term, err := f.jsonTerm(ctx, c, "subtree", "text_array", 0, input) if err != nil { return encrypt.JSONRead[E]{}, err } - return encrypt.NewJSONRead[E](c, f.plan, f.name, path, opts.WildcardBound, term.Text), nil + return encrypt.NewJSONRead[E](c, f.plan, f.name, path, input, term.Text), nil } // JSONNodeToken derives the token of the node at a path with no wildcard // (children, child_count), which json_tree_children and // json_tree_child_count take. func JSONNodeToken[T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator, path string) (encrypt.JSONToken, error) { - term, err := f.jsonTerm(ctx, c, operator, "text", vcvalue.Object{{Key: "path", Value: path}}) - if err == nil && len(term.Text) != 1 { - err = fmt.Errorf("gensupport: %s: %s came back as %d texts, not 1", f.name, operator, len(term.Text)) - } + term, err := f.jsonTerm(ctx, c, operator, "text", 1, vcvalue.Object{{Key: "path", Value: path}}) if err != nil { return "", err } return encrypt.JSONToken(term.Text[0]), nil } - -func oneTermErr(field, operator string, err error) error { - if err != nil { - return err - } - return fmt.Errorf("gensupport: %s: %s came back as other than one term", field, operator) -} diff --git a/languages/golang/encrypt/internal/testusers/customer_stash.go b/languages/golang/encrypt/internal/testusers/customer_stash.go index a6afbe086..50b266aa1 100644 --- a/languages/golang/encrypt/internal/testusers/customer_stash.go +++ b/languages/golang/encrypt/internal/testusers/customer_stash.go @@ -221,6 +221,13 @@ func (p CustomerProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher return gensupport.JSONBetweenAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, p.path, lo, hi) } +// Selector prepares `->` at this path: the value an ordering here compares is +// col -> Selector(), so bind it beside the ordering's operand, from the same +// path. +func (p CustomerProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, p.field, c, p.path) +} + type CustomerPrefsField struct { field gensupport.Field[json.RawMessage] } diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index ca17273ac..37ed84527 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -101,8 +101,14 @@ func (cph *Cipher) JSONOpen(ctx context.Context, plan *record.Plan, field string // It is column-wide: the same path has the same selector in every row. type JSONSelector string -// Value binds the selector as text. -func (s JSONSelector) Value() (driver.Value, error) { return string(s), nil } +// Value binds the selector as text. A zero JSONSelector (one never +// prepared) is refused rather than bound as ”, which matches no row. +func (s JSONSelector) Value() (driver.Value, error) { + if s == "" { + return nil, fmt.Errorf("%w: an empty JSONSelector: prepare one with the field's Selector", ErrEncoding) + } + return string(s), nil +} // JSONToken is one node's token: what eql_v4.json_tree_children and // eql_v4.json_tree_child_count take. It binds as SQL text. A token is a @@ -110,8 +116,14 @@ func (s JSONSelector) Value() (driver.Value, error) { return string(s), nil } // to the database for a query, and do not log it. type JSONToken string -// Value binds the token as text. -func (t JSONToken) Value() (driver.Value, error) { return string(t), nil } +// Value binds the token as text. A zero JSONToken is refused, as a zero +// JSONSelector is. +func (t JSONToken) Value() (driver.Value, error) { + if t == "" { + return nil, fmt.Errorf("%w: an empty JSONToken: derive one with the field's Children or ChildCount", ErrEncoding) + } + return string(t), nil +} // JSONTokens are a subtree read's tokens: what eql_v4.json_tree_subtree // takes. They bind as a PostgreSQL text[] literal; cast the parameter, @@ -196,19 +208,29 @@ func (s JSONSelection[E]) Open(ctx context.Context, rows []*E) ([]JSONValue, err return nil, fmt.Errorf("%w: a zero JSONSelection has no request to open against", ErrEncoding) } groups := make([][][]byte, len(rows)) + entries := false for i, row := range rows { if row != nil { groups[i] = [][]byte{[]byte(*row)} + entries = true } } + // No row has an entry: nothing to open, and no key to ask for. + if !entries { + return make([]JSONValue, len(rows)), nil + } opened, err := s.cipher.JSONOpen(ctx, s.plan, s.field, vcvalue.Object{{Key: "operation", Value: "selector"}, {Key: "path", Value: s.path}}, groups) if err != nil { return nil, err } out := make([]JSONValue, len(opened)) for i, values := range opened { - if len(values) == 1 { + switch len(values) { + case 0: + case 1: out[i] = JSONValue{Found: true, JSON: values[0]} + default: + return nil, fmt.Errorf("%w: se_json_open opened %d values for row %d of a selector read", ErrInternal, len(values), i) } } return out, nil @@ -225,14 +247,16 @@ type JSONRead[E ~[]byte] struct { plan *record.Plan field string path string - bound uint32 + // input is the subtree operator's input the tokens were derived from; + // opening sends it back as the read's request, unchanged. + input vcvalue.Object tokens JSONTokens } -// NewJSONRead is a read prepared by [Cipher.QueryTerm]'s subtree operator. -// For generated code. -func NewJSONRead[E ~[]byte](c *Cipher, plan *record.Plan, field, path string, bound uint32, tokens []string) JSONRead[E] { - return JSONRead[E]{cipher: c, plan: plan, field: field, path: path, bound: bound, tokens: JSONTokens(tokens)} +// NewJSONRead is a read prepared by [Cipher.QueryTerm]'s subtree operator +// from input, its {"path", "wildcard_bound"?} object. For generated code. +func NewJSONRead[E ~[]byte](c *Cipher, plan *record.Plan, field, path string, input vcvalue.Object, tokens []string) JSONRead[E] { + return JSONRead[E]{cipher: c, plan: plan, field: field, path: path, input: input, tokens: JSONTokens(tokens)} } // Tokens are the tokens to bind: @@ -262,10 +286,7 @@ func (r JSONRead[E]) Open(ctx context.Context, rows [][]E) ([][]json.RawMessage, groups[i][j] = []byte(entry) } } - request := vcvalue.Object{{Key: "operation", Value: "subtree"}, {Key: "path", Value: r.path}} - if r.bound != 0 { - request = append(request, vcvalue.Field{Key: "wildcard_bound", Value: r.bound}) - } + request := append(vcvalue.Object{{Key: "operation", Value: "subtree"}}, r.input...) opened, err := r.cipher.JSONOpen(ctx, r.plan, r.field, request, groups) if err != nil { return nil, err @@ -285,6 +306,11 @@ func (r JSONRead[E]) Open(ctx context.Context, rows [][]E) ([][]json.RawMessage, // reaches more than one value (a wildcard) is [ErrEncoding]; open it with // [JSONRead.Open]. func (r JSONRead[E]) OpenOne(ctx context.Context, rows []E) (JSONValue, error) { + // A wildcard reaches a value per element: refused before anything is + // opened, not only for the rows that happen to hold two. + if strings.Contains(r.path, "[*]") { + return JSONValue{}, fmt.Errorf("%w: OpenOne reads one value, and %s has a wildcard; open it with Open", ErrEncoding, r.path) + } opened, err := r.Open(ctx, [][]E{rows}) if err != nil { return JSONValue{}, err diff --git a/languages/golang/encrypt/json_fields_test.go b/languages/golang/encrypt/json_fields_test.go index 12fab79b0..976ad14c5 100644 --- a/languages/golang/encrypt/json_fields_test.go +++ b/languages/golang/encrypt/json_fields_test.go @@ -4,11 +4,13 @@ import ( "context" "encoding/json" "errors" + "fmt" "reflect" "strings" "testing" "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" ) @@ -81,7 +83,7 @@ func TestACustomersFieldsDeriveEachQuery(t *testing.T) { if err != nil || len(contains) == 0 { t.Fatalf("Contains: %v", err) } - if v, err := contains.Value(); err != nil || !strings.Contains(v.(string), `"sv"`) { + if v, err := contains.Value(); err != nil || !strings.Contains(fmt.Sprint(v), `"sv"`) { t.Fatalf("Contains binds as JSON: %v %v", v, err) } if _, err := f.ContainedBy(ctx, cipher, json.RawMessage(`{"name":"Alice","age":34}`)); err != nil { @@ -105,12 +107,37 @@ func TestACustomersFieldsDeriveEachQuery(t *testing.T) { if err != nil || len(read.Tokens()) != 3 { t.Fatalf("Subtree: %v %v", read.Tokens(), err) } - if v, err := read.Tokens().Value(); err != nil || !strings.HasPrefix(v.(string), `{"`) { + if v, err := read.Tokens().Value(); err != nil || !strings.HasPrefix(fmt.Sprint(v), `{"`) { t.Fatalf("Tokens bind as a text[] literal: %v %v", v, err) } if _, err := f.Children(ctx, cipher, "$.visits"); err != nil { t.Fatal(err) } + // An ordered path's own selector is the field's at that path. + pathSel, err := f.At.VisitsAt.Selector(ctx, cipher) + if err != nil { + t.Fatal(err) + } + if fieldSel, _ := f.Selector(ctx, cipher, "$.visits[*].at"); pathSel.Selector() != fieldSel.Selector() { + t.Fatal("the accessor's selector is another path's") + } + // OpenOne refuses a wildcard read before anything is opened, whatever + // the rows hold; a zero selector or token binds nothing. + if _, err := read.OpenOne(ctx, nil); !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("OpenOne on a wildcard read: %v", err) + } + if _, err := (encrypt.JSONSelector("")).Value(); !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("a zero selector binds: %v", err) + } + if _, err := (encrypt.JSONToken("")).Value(); !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("a zero token binds: %v", err) + } + // A selection over rows with no entry opens to no value without a + // request. + none, err := sel.Open(ctx, make([]*eql.EncryptedJSONEntry, 3)) + if err != nil || len(none) != 3 || none[0].Found || none[2].Found { + t.Fatalf("an empty page: %+v %v", none, err) + } // What the engine refuses: an ordering on a value no ordering takes, a // container at a path, a position in a selector, a wildcard where one // node is named. Each is a returned error naming no value. diff --git a/languages/golang/encrypt/json_postgres_test.go b/languages/golang/encrypt/json_postgres_test.go index 946f937af..ba9864ee2 100644 --- a/languages/golang/encrypt/json_postgres_test.go +++ b/languages/golang/encrypt/json_postgres_test.go @@ -99,6 +99,9 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } stored = append(stored, e) } + if err := rows.Err(); err != nil { + t.Fatal(err) + } _ = rows.Close() back, err := testusers.DecryptCustomer(ctx, c, stored) if err != nil { @@ -127,8 +130,9 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } want("prefs contains", ids(`SELECT id FROM customers WHERE prefs @> $1::eql_v4.query_json_tree ORDER BY id`, pq), 1) - // Orderings at an ordered path compare the leaf `->` extracts there. - age, err := f.Selector(ctx, cipher, "$.age") + // Orderings at an ordered path compare the leaf `->` extracts there, + // selected by the accessor's own Selector, so the path is named once. + age, err := f.At.Age.Selector(ctx, cipher) if err != nil { t.Fatal(err) } @@ -142,7 +146,7 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { t.Fatal(err) } want("age between", ids(`SELECT id FROM customers WHERE (profile -> $1::text) >= $2::eql_v4.query_json_tree_leaf AND (profile -> $1::text) <= $3::eql_v4.query_json_tree_leaf ORDER BY id`, age.Selector(), rng.Low, rng.High), 1, 3) - at, err := f.Selector(ctx, cipher, "$.visits[*].at") + at, err := f.At.VisitsAt.Selector(ctx, cipher) if err != nil { t.Fatal(err) } @@ -169,6 +173,9 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } entries = append(entries, e) } + if err := rows.Err(); err != nil { + t.Fatal(err) + } values, err := sel.Open(ctx, entries) if err != nil { t.Fatal(err) @@ -212,6 +219,9 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } groups[i] = append(groups[i], e) } + if err := rows.Err(); err != nil { + t.Fatal(err) + } _ = rows.Close() } opened, err := read.Open(ctx, groups) @@ -254,6 +264,9 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } entries = append(entries, e) } + if err := rs.Err(); err != nil { + t.Fatal(err) + } _ = rs.Close() one, err := visits.OpenOne(ctx, entries) if err != nil || !one.Found || !sameJSON(t, one.JSON, []byte(`[{"at":3},{"at":1}]`)) { @@ -281,6 +294,9 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { counts = append(counts, "NULL") } } + if err := cr.Err(); err != nil { + t.Fatal(err) + } _ = cr.Close() if !slices.Equal(counts, []string{"2", "1", "0", "NULL"}) { t.Errorf("child counts: %v", counts) diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index 04253b0c1..314781003 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -25,6 +25,11 @@ func emit(f *genFile) ([]byte, error) { if err := planQueries(f); err != nil { return nil, err } + // Planning adds imports (encoding/json for a document field): a clash + // it makes is refused here, before anything is written. + if f.imports.err != nil { + return nil, f.imports.err + } w := &writer{} w.header(f) w.encryptedType(f) @@ -989,15 +994,27 @@ var jsonOperators = map[string]bool{ // planJSON checks a document field's operators against the methods the // generator writes for them, and names each ordered path's accessor. func planJSON(f *genFile, g *genField) error { + document := false for _, op := range g.operators { if !jsonOperators[op.Name] { return fieldErr(f.typeName, g.GoName, "the engine answers %s on %s, which the generator has no method for", op.Name, g.EQLType) } + document = document || op.Name == "selector" + } + // The engine answers selector for a field whose type stores a json_tree + // document; the leaf and entry types below are that family's. + if !document { + return fieldErr(f.typeName, g.GoName, "the engine answers no document operator on %s, and the generator writes json.RawMessage fields of EncryptedJson only", g.EQLType) } g.jsonPaths = nil seen := map[string]string{} + given := map[string]bool{} if g.TargetOptions != nil { for _, path := range g.TargetOptions.Ordered { + if given[path] { + return fieldErr(f.typeName, g.GoName, "the ordered path %s is given twice", path) + } + given[path] = true name, err := jsonPathName(path) if err != nil { return fieldErr(f.typeName, g.GoName, "%v", err) @@ -1014,11 +1031,16 @@ func planJSON(f *genFile, g *genField) error { } // jsonPathName is an ordered path's accessor name: its keys in Go case, -// array steps dropped. $.age is Age, $.visits[*].at is VisitsAt, $ is Root. +// wildcard steps dropped. $.age is Age, $.visits[*].at is VisitsAt, $.a.1 +// is A1, and $ is Root. An ordered path names no position (the engine +// refuses one), so a step of digits is an object key. func jsonPathName(path string) (string, error) { + if path == "$" { + return "Root", nil + } var b strings.Builder for _, step := range strings.FieldsFunc(strings.TrimPrefix(path, "$"), func(r rune) bool { return r == '.' || r == '[' || r == ']' }) { - if step == "*" || strings.Trim(step, "0123456789") == "" { + if step == "*" { continue } for _, word := range strings.Split(step, "_") { @@ -1028,11 +1050,8 @@ func jsonPathName(path string) (string, error) { } } name := b.String() - if name == "" { - name = "Root" - } - if !isIdent(name) || name[0] < 'A' || name[0] > 'Z' { - return "", fmt.Errorf("the ordered path %s has no Go accessor name: a key starts with a digit", path) + if name == "" || !isIdent(name) || name[0] < 'A' || name[0] > 'Z' { + return "", fmt.Errorf("the ordered path %s has no Go accessor name: its keys give no name that starts with a letter", path) } return name, nil } @@ -1162,4 +1181,11 @@ func (w *writer) jsonEntry(g genField) { w.p("\treturn gensupport.JSONBetweenAt[%s](ctx, p.field, c, p.path, lo, hi)", leaf) w.p("}") } + if answered["selector"] { + w.nl() + w.b.WriteString(wrapComment("Selector prepares `->` at this path: the value an ordering here compares is col -> Selector(), so bind it beside the ordering's operand, from the same path.", 80)) + w.p("func (p %sPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[%s], error) {", g.fieldType, entry) + w.p("\treturn gensupport.JSONSelect[%s](ctx, p.field, c, p.path)", entry) + w.p("}") + } } diff --git a/languages/golang/stashgen/engine.go b/languages/golang/stashgen/engine.go index db598931c..08275aaa1 100644 --- a/languages/golang/stashgen/engine.go +++ b/languages/golang/stashgen/engine.go @@ -269,7 +269,11 @@ func lowerDeclaration(d Declaration, eqlTypes []EQLType) (*record.Plan, error) { } out, ok := outputOfIndex[idx.Name] if !ok { - return nil, &FieldError{Type: d.Type, Field: f.GoName, Reason: fmt.Sprintf("the engine does not derive the %s index yet", idx.Name)} + reason := fmt.Sprintf("the engine does not derive the %s index yet", idx.Name) + if idx.Name == IndexJSON { + reason = "the json index is not an output beside a ciphertext: store a JSON document as a json.RawMessage field with encrypt_into=EncryptedJson, whose EQL type carries the index" + } + return nil, &FieldError{Type: d.Type, Field: f.GoName, Reason: reason} } rf.Outputs = append(rf.Outputs, out) } diff --git a/languages/golang/stashgen/enginetest/enginetest.go b/languages/golang/stashgen/enginetest/enginetest.go index 2e5905a74..2f91c56d3 100644 --- a/languages/golang/stashgen/enginetest/enginetest.go +++ b/languages/golang/stashgen/enginetest/enginetest.go @@ -278,7 +278,7 @@ func indexApplies(name stashgen.IndexName, t stashgen.GoType) error { return nil } case stashgen.IndexJSON: - return fmt.Errorf("the engine cannot derive the json index yet") + return fmt.Errorf("the json index is not an output beside a ciphertext: store a JSON document with encrypt_into=EncryptedJson") } return fmt.Errorf("%s does not apply to %s", name, t) } diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index 810bdd8bf..2a98fe518 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -714,6 +714,17 @@ func (r *reader) buildFields(c *collected) error { if t.Omit { field.Verb = VerbOmit } + // A json.RawMessage is a document only where it names an EQL type + // that stores one; sealed or indexed any other way it is bytes, as + // it was before documents were. + for g := &field.GoType; g != nil; g = g.Elem { + if g.Kind == KindJSON && field.Verb != VerbEncryptInto { + g.Kind, g.Basic = KindBytes, "[]byte" + } + if g.Kind != KindPointer { + break + } + } // A sealed field declared over a pointer to one scalar may be nil: // it is that scalar's field, and says how it stores nil (#1298). pointer := field.Sealed() && field.GoType.Kind == KindPointer && field.GoType.Elem != nil && field.GoType.Elem.Kind.Scalar() diff --git a/languages/golang/stashgen/refusal_test.go b/languages/golang/stashgen/refusal_test.go index 67f161491..3297fdc9c 100644 --- a/languages/golang/stashgen/refusal_test.go +++ b/languages/golang/stashgen/refusal_test.go @@ -113,7 +113,7 @@ func TestRefusals(t *testing.T) { {"an opaque struct with a map keyed by a struct", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tM map[hidden]int") + "\ntype hidden struct{ A int }\n", stashgen.Request{Type: "User"}, "M", "map key that is not a string or an integer"}, {"an EQL type the engine cannot produce yet", user(ctx + "\tAge int64 `stash:\"age,encrypt_into=BigintOrdOre\"`"), stashgen.Request{Type: "User"}, "Age", "has no EQL type BigintOrdOre"}, {"an index on a composite", user(ctx + "\tAttrs map[string]string `stash:\"attrs,encrypt,index=equality\"`"), stashgen.Request{Type: "User"}, "Attrs", "seals only as part of an opaque struct"}, - {"the json index, not in the engine yet", user(ctx + "\tAttrs string `stash:\"attrs,index=json\"`"), stashgen.Request{Type: "User"}, "Attrs", "cannot derive the json index yet"}, + {"the json index as an output", user(ctx + "\tAttrs string `stash:\"attrs,index=json\"`"), stashgen.Request{Type: "User"}, "Attrs", "encrypt_into=EncryptedJson"}, {"an index option a query term cannot use", user(ctx + "\tEmail string `stash:\"email,encrypt,index=match(k=3)\"`"), stashgen.Request{Type: "User"}, "Email", "a query term uses only the default index options"}, {"a passthrough field that has an index", user(ctx + "\tID int64 `stash:\"id,passthrough,index=equality\"`"), stashgen.Request{Type: "User"}, "ID", "passthrough field has no index"}, {"an embedded struct from another package with no tag", user(ctx + "\tgorm.Model\n\tEmail string `stash:\"email,encrypt\"`"), stashgen.Request{Type: "User"}, "Model", "cannot carry tags; tag the field"}, From c502848a7a3f2c619eaba8754809080fd8a13dcf Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 11:29:15 +1100 Subject: [PATCH 03/13] test(go): sort rows by an ordered JSON path in PostgreSQL The PostgreSQL test compared the ordered paths $.age and $.visits[*].at only through <, <=, >= and >, so nothing showed that rows sort by them. Ordering by the entry `->` returns sorts by its ciphertext, which the blockers cannot stop, so the form EQL documents is ORDER BY eql_v4.ord_term(col -> sel), and that form had no Go test. The test now inserts two more rows, giving each path a tie broken by id and a document whose first visit (2) is neither its least nor its greatest element, then sorts by both paths ascending and descending and compares the ids with the order computed from the plaintexts. A row with no value at the path (no age, or no visits) has a NULL term and sorts last ascending and first descending, as PostgreSQL sorts NULL by default. At the wildcard path the term is the first element's, which is what `->` returns. Sorting by the entry itself instead fails all four orderings. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .../golang/encrypt/json_postgres_test.go | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/languages/golang/encrypt/json_postgres_test.go b/languages/golang/encrypt/json_postgres_test.go index ba9864ee2..60a607efa 100644 --- a/languages/golang/encrypt/json_postgres_test.go +++ b/languages/golang/encrypt/json_postgres_test.go @@ -1,6 +1,7 @@ package encrypt_test import ( + "cmp" "context" "database/sql" "encoding/json" @@ -306,6 +307,97 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { t.Fatal(err) } want("children rows", ids(`SELECT id FROM customers, eql_v4.json_tree_children(profile, $1::text) AS e ORDER BY id`, children), 1, 1, 2) + + // Sorting by an ordered path, in the form EQL documents: + // ORDER BY eql_v4.ord_term(col -> sel). Ordering by the entry itself + // would sort by its ciphertext. Two more rows give each path a tie, + // broken by id, and Eve's visits (first 2, last 9) tell the first match + // `->` takes at the wildcard path from the least or greatest element. + more := []testusers.Customer{ + {ID: 5, Profile: json.RawMessage(`{"name":"Eve","age":34,"visits":[{"at":2},{"at":9}]}`)}, + {ID: 6, Profile: json.RawMessage(`{"name":"Fay","age":7,"visits":[{"at":3}]}`)}, + } + extra, err := testusers.EncryptCustomer(ctx, cipher, more) + if err != nil { + t.Fatal(err) + } + for _, e := range extra { + exec(`INSERT INTO customers VALUES ($1, $2, $3)`, e.ID, e.Profile, e.Prefs) + } + type plain struct { + Age *float64 `json:"age"` + Visits []struct { + At *float64 `json:"at"` + } `json:"visits"` + } + ageOf := func(p plain) *float64 { return p.Age } + firstAtOf := func(p plain) *float64 { + for _, v := range p.Visits { + if v.At != nil { + return v.At + } + } + return nil + } + // The ids in the order PostgreSQL's defaults put the plaintexts in: a + // row with no value at the path has a NULL term, which sorts as the + // greatest value, so last ascending and first descending; ties by id. + plaintextOrder := func(key func(plain) *float64, desc bool) []int64 { + type row struct { + id int64 + v *float64 + } + var rows []row + for _, cust := range append(slices.Clone(jsonCustomers), more...) { + var p plain + if err := json.Unmarshal(cust.Profile, &p); err != nil { + t.Fatal(err) + } + rows = append(rows, row{cust.ID, key(p)}) + } + slices.SortFunc(rows, func(a, b row) int { + var c int + switch { + case a.v == nil && b.v == nil: + case a.v == nil: + c = 1 + case b.v == nil: + c = -1 + default: + c = cmp.Compare(*a.v, *b.v) + } + if desc { + c = -c + } + if c == 0 { + c = cmp.Compare(a.id, b.id) + } + return c + }) + out := make([]int64, len(rows)) + for i, r := range rows { + out[i] = r.id + } + return out + } + for _, tc := range []struct { + name string + selector encrypt.JSONSelector + key func(plain) *float64 + asc, desc []int64 // what plaintextOrder gives, written out + }{ + // Bob and Fay tie at 7, Alice and Eve at 34; Dan has no age. + {"$.age", age.Selector(), ageOf, []int64{2, 6, 1, 5, 3, 4}, []int64{4, 3, 1, 5, 2, 6}}, + // First visits: Eve 2, Alice and Fay 3, Bob 5; Carol's visits are + // empty and Dan has none. + {"$.visits[*].at", at.Selector(), firstAtOf, []int64{5, 1, 6, 2, 3, 4}, []int64{3, 4, 2, 1, 6, 5}}, + } { + asc, desc := plaintextOrder(tc.key, false), plaintextOrder(tc.key, true) + want("plaintext order by "+tc.name, asc, tc.asc...) + want("plaintext order by "+tc.name+" DESC", desc, tc.desc...) + want("ORDER BY "+tc.name, ids(`SELECT id FROM customers ORDER BY eql_v4.ord_term(profile -> $1::text), id`, tc.selector), asc...) + want("ORDER BY "+tc.name+" DESC", ids(`SELECT id FROM customers ORDER BY eql_v4.ord_term(profile -> $1::text) DESC, id`, tc.selector), desc...) + } } func slicesEqual2(a, b [][]string) bool { From e7310dac965658a0f20f1ead0b5bcf3aafbe7357 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 00:33:50 +0000 Subject: [PATCH 04/13] feat(go): tag a JSON document field encrypt_into=JsonSearch With the base's rename the EQL type a document field names is `JsonSearch` (Go `eql.JSONSearch`, over `public.eql_v4_json_search`). The tag is now `encrypt_into=JsonSearch(ordered=$.a;$.b[*].c,sealed_root=false)`; the Go spelling `JSONSearch` in a tag is refused with the same hint as before, naming the engine's spelling. The generator named a document's companion types by appending to the stored type's Go name, `LeafQuery` and `Entry`. Under the v3 names the entry is not `JSONSearchEntry` but `JSONEntry`, named for the plaintext as `public.eql_v4_json_entry` is, so the leaf operand is now derived from the needle (`JSONSearchQuery` to `JSONSearchLeafQuery`) and the entry is `eql.JSONEntry`. The test model, its regenerated customer_stash.go, the READMEs, the CHANGELOG and the PostgreSQL test's SQL (`eql_v4.query_json`, `eql_v4.query_json_leaf`) follow. Refs #1351 Claude-Session: https://claude.ai/code/session_01HLuZxys1NDRmAkv3uhnuoJ --- languages/golang/CHANGELOG.md | 4 +- languages/golang/cmd/stashgen/README.md | 16 ++-- languages/golang/cmd/stashgen/main_test.go | 46 +++++----- languages/golang/encrypt/README.md | 6 +- .../golang/encrypt/gensupport/declaration.go | 6 +- languages/golang/encrypt/gensupport/json.go | 2 +- .../internal/testusers/customer_stash.go | 84 +++++++++---------- .../encrypt/internal/testusers/customers.go | 8 +- languages/golang/encrypt/json.go | 10 +-- languages/golang/encrypt/json_fields_test.go | 4 +- .../golang/encrypt/json_postgres_test.go | 42 +++++----- languages/golang/encrypt/term.go | 6 +- languages/golang/stashgen/declaration.go | 4 +- languages/golang/stashgen/emit.go | 14 ++-- languages/golang/stashgen/engine.go | 2 +- .../golang/stashgen/enginetest/enginetest.go | 2 +- languages/golang/stashgen/refusal_test.go | 2 +- languages/golang/stashgen/tag.go | 2 +- .../testdata/stubsdk/encrypt/encrypt.go | 4 +- 19 files changed, 134 insertions(+), 130 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 2db63af13..38d1c0782 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -13,9 +13,9 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. ### Added - **Encrypted JSON documents with `stashgen`** (#1351, #1352). A - `json.RawMessage` field tagged `encrypt_into=EncryptedJson` (options: + `json.RawMessage` field tagged `encrypt_into=JsonSearch` (options: `ordered=$.a;$.b[*].c`, `sealed_root=false`) is stored as an - `eql.EncryptedJSON` in a `public.eql_v4_json_tree` column; a + `eql.JSONSearch` in a `public.eql_v4_json_search` column; a `*json.RawMessage` field takes `null=sql`. Its `Fields` entry has `Contains`, `ContainedBy`, `EqAt`, `Selector`, `Subtree`, `Children` and `ChildCount`, and `At.` per ordered path with `Lt`, `Lte`, `Gt`, diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 17ca31938..079a37722 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -74,8 +74,8 @@ The first part of a tag is the field's name, which is the column name in a datab | `stash:"notes,encrypt"` | seal the field, with no index | | `stash:"email,encrypt,index=equality;match"` | seal the field, and derive each index beside it | | `stash:"score,index=ore"` | derive the index alone; no ciphertext is stored, so `Decrypt` leaves the field at its zero value | -| `stash:"profile,encrypt_into=EncryptedJson(ordered=$.age)"` | on a `json.RawMessage`: store the field as an encrypted JSON document; see "Encrypted JSON documents" | -| `stash:"attrs,index=json"` | refused: a JSON document is stored with `encrypt_into=EncryptedJson`, not as an index beside a ciphertext | +| `stash:"profile,encrypt_into=JsonSearch(ordered=$.age)"` | on a `json.RawMessage`: store the field as an encrypted JSON document; see "Encrypted JSON documents" | +| `stash:"attrs,index=json"` | refused: a JSON document is stored with `encrypt_into=JsonSearch`, not as an index beside a ciphertext | | `stash:"id,passthrough"` | store the field as it is | | `stash:"tenant,context_field"` | the field's value is the context of every other field in the struct, in place of a `context=` tag; stored as it is | | `stash:"-"` | leave the field out | @@ -109,27 +109,27 @@ An index takes its options in parentheses after its name, separated by commas: ` This build refuses an index with options: a query term is derived with the default options only, so a stored term with other options would never match one. These words are the same as the Rust API's words for the same behaviour, with two that only Go has. `opaque` seals the whole struct as one `bytes` field holding a storage blob (see "What crosses the binding"). -`json` names the JSON index, which a field reaches only through `encrypt_into=EncryptedJson`. +`json` names the JSON index, which a field reaches only through `encrypt_into=JsonSearch`. ### Encrypted JSON documents -A `json.RawMessage` field with `encrypt_into=EncryptedJson` is stored as one encrypted JSON document, an `eql.EncryptedJSON` in a `public.eql_v4_json_tree` column. +A `json.RawMessage` field with `encrypt_into=JsonSearch` is stored as one encrypted JSON document, an `eql.JSONSearch` in a `public.eql_v4_json_search` column. The field holds the document's JSON text; the engine parses it, so an object that names a member twice, an integer outside the 64-bit range and text that is not JSON are refused with `encrypt.ErrEncoding`. Values come back as JSON text holding the same values; key order and the spelling of numbers are not kept. A `*json.RawMessage` field is optional and must say `null=sql`: a document has no fixed length to hide a missing one behind. A non-nil pointer to `null` is a stored JSON `null`. A `json.RawMessage` sealed any other way (`encrypt`, `index=equality`) is bytes, as any type defined over `[]byte` is. The type takes options in parentheses: `ordered=` names the paths whose numbers and strings carry an ordering term, separated by `;`, and `sealed_root=false` turns off the sealed root (on by default). -`encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at)` orders two paths. +`encrypt_into=JsonSearch(ordered=$.age;$.visits[*].at)` orders two paths. A path in a tag is plain JSONPath: `$`, then `.key` steps of letters, digits and underscores, and `[n]` or `[*]` steps; an ordered path names no position. The engine checks every path when `stashgen` runs. The field's `Fields` entry has a method per operation: | Method | Returns | SQL | |---|---|---| -| `Contains(ctx, c, needle)`, `ContainedBy(ctx, c, needle)` | `eql.EncryptedJSONQuery` | `col @> $1::eql_v4.query_json_tree`, `col <@ $1::eql_v4.query_json_tree` | -| `EqAt(ctx, c, path, v)` | `eql.EncryptedJSONQuery` | `col @> $1::eql_v4.query_json_tree` | -| `At.Age.Lt(ctx, c, v)`, `Lte`, `Gt`, `Gte`, `Between` | `eql.EncryptedJSONLeafQuery` | `(col -> $sel::text) < $1::eql_v4.query_json_tree_leaf`, `$sel` from `At.Age.Selector(ctx, c)` | +| `Contains(ctx, c, needle)`, `ContainedBy(ctx, c, needle)` | `eql.JSONSearchQuery` | `col @> $1::eql_v4.query_json`, `col <@ $1::eql_v4.query_json` | +| `EqAt(ctx, c, path, v)` | `eql.JSONSearchQuery` | `col @> $1::eql_v4.query_json` | +| `At.Age.Lt(ctx, c, v)`, `Lte`, `Gt`, `Gte`, `Between` | `eql.JSONSearchLeafQuery` | `(col -> $sel::text) < $1::eql_v4.query_json_leaf`, `$sel` from `At.Age.Selector(ctx, c)` | | `Selector(ctx, c, path)` | `encrypt.JSONSelection`: `Selector()` and `Open` | `col -> $1::text` | | `Subtree(ctx, c, path, opts)` | `encrypt.JSONRead`: `Tokens()`, `Open` and `OpenOne` | `eql_v4.json_tree_subtree(col, $1::text[], 64)` | | `Children(ctx, c, path)`, `ChildCount(ctx, c, path)` | `encrypt.JSONToken` | `eql_v4.json_tree_children(col, $1::text)`, `eql_v4.json_tree_child_count(col, $1::text)` | diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index f2f27d73c..c83f34fbd 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -216,13 +216,13 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { } // A document type over another Go type is refused by naming the Go // types that carry one, not by a kind mismatch. - document := strings.Replace(userSource, "Email string `stash:\"email,encrypt_into=TextEq\"`", "Profile []byte `stash:\"profile,encrypt_into=EncryptedJson\"`", 1) + document := strings.Replace(userSource, "Email string `stash:\"email,encrypt_into=TextEq\"`", "Profile []byte `stash:\"profile,encrypt_into=JsonSearch\"`", 1) dir = writeModule(t, map[string]string{"model.go": document}) stderr.Reset() if code := run([]string{"-type", "User"}, dir, &stdout, &stderr, stashgen.GuestEngine); code != 1 { t.Fatalf("exit %d, want 1\n%s", code, stderr.String()) } - if !strings.Contains(stderr.String(), "User.Profile") || !strings.Contains(stderr.String(), "EncryptedJson seals a json document: declare the field json.RawMessage") { + if !strings.Contains(stderr.String(), "User.Profile") || !strings.Contains(stderr.String(), "JsonSearch seals a json document: declare the field json.RawMessage") { t.Fatalf("stderr = %q", stderr.String()) } // A context of two segments leaves an EQL field no column: refused by @@ -271,7 +271,7 @@ func TestEQLGoNameAgreesWithTheGeneratedPackage(t *testing.T) { t.Fatal("the rule renamed nothing: the Json family is spelled JSON") } if stashgen.EQLGoName("Json") != "JSON" || stashgen.EQLGoName("TextEq") != "TextEq" || stashgen.EQLGoName("SteVecQuery") != "SteVecQuery" || - stashgen.EQLGoName("EncryptedJsonLeafQuery") != "EncryptedJSONLeafQuery" || stashgen.EQLGoName("Jsonify") != "Jsonify" || + stashgen.EQLGoName("JsonSearchLeafQuery") != "JSONSearchLeafQuery" || stashgen.EQLGoName("Jsonify") != "Jsonify" || // A letter after the word is read as a character, as Rust reads it, // not as its first byte. stashgen.EQLGoName("Jsonécrit") != "Jsonécrit" || stashgen.EQLGoName("JsonÉcrit") != "JSONÉcrit" { @@ -295,7 +295,7 @@ func TestModelFlagsRepeat(t *testing.T) { // embedded engine with the EQL types: the generated entry has the // document's methods and an accessor per ordered path, and every refusal // names the field before anything is written. -func TestRunGeneratesEncryptedJSONFields(t *testing.T) { +func TestRunGeneratesJSONSearchFields(t *testing.T) { checker, err := encrypt.NewChecker(context.Background()) if err != nil { t.Skip(err) @@ -311,20 +311,20 @@ func TestRunGeneratesEncryptedJSONFields(t *testing.T) { written, _ := os.ReadFile(filepath.Join(dir, "customer_stash.go")) return string(written), stderr.String(), code } - written, stderr, code := generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at,sealed_root=false)\"`\n\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson,null=sql\"`") + written, stderr, code := generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.age;$.visits[*].at,sealed_root=false)\"`\n\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch,null=sql\"`") if code != 0 { t.Fatalf("exit %d\n%s", code, stderr) } for _, want := range []string{ - "Profile eql.EncryptedJSON", - "Prefs *eql.EncryptedJSON", - `EncryptInto("profile", gensupport.JSON, "EncryptedJson").`, + "Profile eql.JSONSearch", + "Prefs *eql.JSONSearch", + `EncryptInto("profile", gensupport.JSON, "JsonSearch").`, `Options("profile", gensupport.TargetOptions{SealedRoot: gensupport.Ptr(false), Ordered: []string{"$.age", "$.visits[*].at"}})`, - "func (f ProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error)", - "func (f ProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error)", + "func (f ProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.JSONSearchQuery, error)", + "func (f ProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.JSONEntry], error)", "Age ProfileFieldPath // $.age", "VisitsAt ProfileFieldPath // $.visits[*].at", - "func (p ProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.EncryptedJSONLeafQuery], error)", + "func (p ProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.JSONSearchLeafQuery], error)", } { if !strings.Contains(written, want) { t.Errorf("the generated file lacks %q", want) @@ -333,12 +333,12 @@ func TestRunGeneratesEncryptedJSONFields(t *testing.T) { if strings.Contains(written, "PrefsFieldAt") { t.Error("Prefs orders no path, and got an At") } - if !strings.Contains(written, "func (p ProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error)") { + if !strings.Contains(written, "func (p ProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.JSONEntry], error)") { t.Error("an ordered path has no Selector of its own") } // A json.RawMessage sealed any other way is bytes, as it was before // documents; an object key of digits names its accessor. - written, stderr, code = generate("\tData json.RawMessage `stash:\"data,encrypt,index=equality\"`\n\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.a.1)\"`") + written, stderr, code = generate("\tData json.RawMessage `stash:\"data,encrypt,index=equality\"`\n\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a.1)\"`") if code != 0 { t.Fatalf("exit %d\n%s", code, stderr) } @@ -348,16 +348,16 @@ func TestRunGeneratesEncryptedJSONFields(t *testing.T) { } } for _, c := range []struct{ name, fields, want string }{ - {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson,null=sealed\"`", "Customer.Prefs"}, - {"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson\"`", "null=sql"}, - {"a position in an ordered path", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.scores[2])\"`", "the engine refuses the declaration"}, - {"a quoted key in a tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$[\\\"a b\\\"])\"`", "not a plain JSONPath"}, - {"an unknown option", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(bound=3)\"`", "no option \"bound\""}, - {"the Go name in the tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJSON\"`", "as the engine does, EncryptedJson"}, - {"two paths, one accessor", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.a_b;$.aB)\"`", "both name the accessor At.AB"}, - {"the same ordered path twice", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.a;$.a)\"`", "the ordered path $.a is given twice"}, - {"a path with no accessor name", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.__)\"`", "no Go accessor name"}, - {"a string into a document", "\tProfile string `stash:\"profile,encrypt_into=EncryptedJson\"`", "EncryptedJson seals a json"}, + {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch,null=sealed\"`", "Customer.Prefs"}, + {"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch\"`", "null=sql"}, + {"a position in an ordered path", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.scores[2])\"`", "the engine refuses the declaration"}, + {"a quoted key in a tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$[\\\"a b\\\"])\"`", "not a plain JSONPath"}, + {"an unknown option", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(bound=3)\"`", "no option \"bound\""}, + {"the Go name in the tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JSONSearch\"`", "as the engine does, JsonSearch"}, + {"two paths, one accessor", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a_b;$.aB)\"`", "both name the accessor At.AB"}, + {"the same ordered path twice", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a;$.a)\"`", "the ordered path $.a is given twice"}, + {"a path with no accessor name", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.__)\"`", "no Go accessor name"}, + {"a string into a document", "\tProfile string `stash:\"profile,encrypt_into=JsonSearch\"`", "JsonSearch seals a json"}, {"options on a type that takes none", "\tEmail string `stash:\"email,encrypt_into=TextEq(ordered=$.a)\"`", "the engine refuses the declaration"}, } { t.Run(c.name, func(t *testing.T) { diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 5ba5516a5..72ec0be90 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -593,15 +593,15 @@ column. ### Encrypted JSON documents -`EncryptedJson` stores a whole JSON document in one `public.eql_v4_json_tree` -column, from a `json.RawMessage` field (`stash:"profile,encrypt_into=EncryptedJson(ordered=$.age)"`; +`JsonSearch` stores a whole JSON document in one `public.eql_v4_json_search` +column, from a `json.RawMessage` field (`stash:"profile,encrypt_into=JsonSearch(ordered=$.age)"`; see the stashgen README). One entry per node is encrypted, so the database can test containment, equality and ordering at a path, return one node with `->`, and walk a subtree, without decrypting: ```go q, err := users.Fields.Profile.Contains(ctx, cipher, json.RawMessage(`{"plan":"pro"}`)) -rows, err := db.QueryContext(ctx, `SELECT id FROM users WHERE profile @> $1::eql_v4.query_json_tree`, q) +rows, err := db.QueryContext(ctx, `SELECT id FROM users WHERE profile @> $1::eql_v4.query_json`, q) read, err := users.Fields.Profile.Subtree(ctx, cipher, "$.visits[*].at", encrypt.JSONReadOptions{}) // SELECT e FROM users, eql_v4.json_tree_subtree(profile, $1::text[], 64) AS e WHERE id = $2 diff --git a/languages/golang/encrypt/gensupport/declaration.go b/languages/golang/encrypt/gensupport/declaration.go index f74c054cf..9ac0b6a1d 100644 --- a/languages/golang/encrypt/gensupport/declaration.go +++ b/languages/golang/encrypt/gensupport/declaration.go @@ -48,7 +48,7 @@ const ( Timestamp Kind = "timestamp" String Kind = "string" Bytes Kind = "bytes" - // JSON is a json.RawMessage field stored as an EncryptedJSON document: + // JSON is a json.RawMessage field stored as a JSONSearch document: // it crosses as its JSON text, which the engine parses. JSON Kind = "json" ) @@ -85,7 +85,7 @@ type field struct { } // TargetOptions are the options an EncryptInto field gives its EQL type: -// for EncryptedJson, the JSON index's. The engine checks them when the +// for JsonSearch, the JSON index's. The engine checks them when the // declaration is first used (stashgen asked it when the file was written). type TargetOptions struct { // SealedRoot says whether the document's root is sealed; nil is the @@ -187,7 +187,7 @@ func (d Declaration) EncryptInto(name string, kind Kind, eqlType string) Declara } // Options gives the named EncryptInto field's EQL type its options: the -// `encrypt_into=EncryptedJson(ordered=...)` tag. +// `encrypt_into=JsonSearch(ordered=...)` tag. func (d Declaration) Options(name string, options TargetOptions) Declaration { if d.err != nil { return d diff --git a/languages/golang/encrypt/gensupport/json.go b/languages/golang/encrypt/gensupport/json.go index 717e1821b..3212138ba 100644 --- a/languages/golang/encrypt/gensupport/json.go +++ b/languages/golang/encrypt/gensupport/json.go @@ -10,7 +10,7 @@ import ( "github.com/cipherstash/vitaminc/bindings/go/vcvalue" ) -// The operators of a field stored as an EncryptedJSON document, for the +// The operators of a field stored as a JSONSearch document, for the // generated methods of its Fields entry. Each sends the operator's named // inputs, JSON values as their JSON text ([]byte of the json.RawMessage), // and checks the engine answered in the encoding and with the number of diff --git a/languages/golang/encrypt/internal/testusers/customer_stash.go b/languages/golang/encrypt/internal/testusers/customer_stash.go index 50b266aa1..86bb5579d 100644 --- a/languages/golang/encrypt/internal/testusers/customer_stash.go +++ b/languages/golang/encrypt/internal/testusers/customer_stash.go @@ -20,8 +20,8 @@ const _ = gensupport.GeneratedVersion2 type EncryptedCustomer struct { ID int64 - Profile eql.EncryptedJSON - Prefs *eql.EncryptedJSON + Profile eql.JSONSearch + Prefs *eql.JSONSearch } func (e EncryptedCustomer) String() string { @@ -44,9 +44,9 @@ type customerShape struct { var customerDeclaration = gensupport.Declare("customers"). Passthrough("id"). - EncryptInto("profile", gensupport.JSON, "EncryptedJson"). + EncryptInto("profile", gensupport.JSON, "JsonSearch"). Options("profile", gensupport.TargetOptions{Ordered: []string{"$.age", "$.visits[*].at"}}). - EncryptInto("prefs", gensupport.JSON, "EncryptedJson"). + EncryptInto("prefs", gensupport.JSON, "JsonSearch"). Null("prefs", gensupport.NullSQL) var customerCodec = gensupport.New(gensupport.Generated[Customer, EncryptedCustomer]{ @@ -66,9 +66,9 @@ var customerCodec = gensupport.New(gensupport.Generated[Customer, EncryptedCusto if e.ID, err = gensupport.Passthrough[int64](rec, "id"); err != nil { return EncryptedCustomer{}, err } - e.Profile = eql.EncryptedJSON(rec["profile"].EQL) + e.Profile = eql.JSONSearch(rec["profile"].EQL) if o, ok := rec["prefs"]; ok { - v := eql.EncryptedJSON(o.EQL) + v := eql.JSONSearch(o.EQL) e.Prefs = &v } return e, nil @@ -134,43 +134,43 @@ func newCustomerProfileField(f gensupport.Field[json.RawMessage]) CustomerProfil }} } -func (f CustomerProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSON, error) { +func (f CustomerProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearch, error) { out, err := f.field.Encrypt(ctx, c, v) - return eql.EncryptedJSON(out.EQL), err + return eql.JSONSearch(out.EQL), err } // Contains derives the query for documents that contain needle, a JSON value: // col @> query. Each array element of the needle must sit inside one element of // the document's array. -func (f CustomerProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { - return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contains", needle) +func (f CustomerProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.JSONSearchQuery, error) { + return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) } // ContainedBy derives the query for documents contained by needle, a JSON // value: col <@ query. -func (f CustomerProfileField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { - return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contained_by", needle) +func (f CustomerProfileField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.JSONSearchQuery, error) { + return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) } // EqAt derives the query for documents holding the scalar v at path: col @> // query. A wildcard step matches when any element holds it; a position ([2]) is // refused, since the stored selector reads every position as the wildcard. -func (f CustomerProfileField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (eql.EncryptedJSONQuery, error) { - return gensupport.JSONQueryAt[eql.EncryptedJSONQuery](ctx, f.field, c, "eq_at", path, v) +func (f CustomerProfileField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (eql.JSONSearchQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchQuery](ctx, f.field, c, "eq_at", path, v) } // Selector prepares `->` at path: bind Selector() as col -> $1::text, and open // the entries it returns with Open. A position ([2]) is refused; read one with // Subtree. -func (f CustomerProfileField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { - return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, f.field, c, path) +func (f CustomerProfileField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.JSONEntry], error) { + return gensupport.JSONSelect[eql.JSONEntry](ctx, f.field, c, path) } // Subtree prepares a read of the value at path: bind Tokens() as // eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it // returns with Open or OpenOne. -func (f CustomerProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error) { - return gensupport.JSONSubtree[eql.EncryptedJSONEntry](ctx, f.field, c, path, opts) +func (f CustomerProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.JSONEntry], error) { + return gensupport.JSONSubtree[eql.JSONEntry](ctx, f.field, c, path, opts) } // Children derives the token of the node at path, a path with no wildcard, @@ -201,31 +201,31 @@ type CustomerProfileFieldPath struct { path string } -func (p CustomerProfileFieldPath) Lt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { - return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "lt_at", p.path, v) +func (p CustomerProfileFieldPath) Lt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "lt_at", p.path, v) } -func (p CustomerProfileFieldPath) Lte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { - return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "lte_at", p.path, v) +func (p CustomerProfileFieldPath) Lte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "lte_at", p.path, v) } -func (p CustomerProfileFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { - return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "gt_at", p.path, v) +func (p CustomerProfileFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "gt_at", p.path, v) } -func (p CustomerProfileFieldPath) Gte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { - return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "gte_at", p.path, v) +func (p CustomerProfileFieldPath) Gte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "gte_at", p.path, v) } -func (p CustomerProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.EncryptedJSONLeafQuery], error) { - return gensupport.JSONBetweenAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, p.path, lo, hi) +func (p CustomerProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.JSONSearchLeafQuery], error) { + return gensupport.JSONBetweenAt[eql.JSONSearchLeafQuery](ctx, p.field, c, p.path, lo, hi) } // Selector prepares `->` at this path: the value an ordering here compares is // col -> Selector(), so bind it beside the ordering's operand, from the same // path. -func (p CustomerProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { - return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, p.field, c, p.path) +func (p CustomerProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.JSONEntry], error) { + return gensupport.JSONSelect[eql.JSONEntry](ctx, p.field, c, p.path) } type CustomerPrefsField struct { @@ -236,43 +236,43 @@ func newCustomerPrefsField(f gensupport.Field[json.RawMessage]) CustomerPrefsFie return CustomerPrefsField{field: f} } -func (f CustomerPrefsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSON, error) { +func (f CustomerPrefsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearch, error) { out, err := f.field.Encrypt(ctx, c, v) - return eql.EncryptedJSON(out.EQL), err + return eql.JSONSearch(out.EQL), err } // Contains derives the query for documents that contain needle, a JSON value: // col @> query. Each array element of the needle must sit inside one element of // the document's array. -func (f CustomerPrefsField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { - return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contains", needle) +func (f CustomerPrefsField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.JSONSearchQuery, error) { + return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) } // ContainedBy derives the query for documents contained by needle, a JSON // value: col <@ query. -func (f CustomerPrefsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { - return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contained_by", needle) +func (f CustomerPrefsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.JSONSearchQuery, error) { + return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) } // EqAt derives the query for documents holding the scalar v at path: col @> // query. A wildcard step matches when any element holds it; a position ([2]) is // refused, since the stored selector reads every position as the wildcard. -func (f CustomerPrefsField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (eql.EncryptedJSONQuery, error) { - return gensupport.JSONQueryAt[eql.EncryptedJSONQuery](ctx, f.field, c, "eq_at", path, v) +func (f CustomerPrefsField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (eql.JSONSearchQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchQuery](ctx, f.field, c, "eq_at", path, v) } // Selector prepares `->` at path: bind Selector() as col -> $1::text, and open // the entries it returns with Open. A position ([2]) is refused; read one with // Subtree. -func (f CustomerPrefsField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { - return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, f.field, c, path) +func (f CustomerPrefsField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.JSONEntry], error) { + return gensupport.JSONSelect[eql.JSONEntry](ctx, f.field, c, path) } // Subtree prepares a read of the value at path: bind Tokens() as // eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it // returns with Open or OpenOne. -func (f CustomerPrefsField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error) { - return gensupport.JSONSubtree[eql.EncryptedJSONEntry](ctx, f.field, c, path, opts) +func (f CustomerPrefsField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.JSONEntry], error) { + return gensupport.JSONSubtree[eql.JSONEntry](ctx, f.field, c, path, opts) } // Children derives the token of the node at path, a path with no wildcard, diff --git a/languages/golang/encrypt/internal/testusers/customers.go b/languages/golang/encrypt/internal/testusers/customers.go index f6a330798..e32245de8 100644 --- a/languages/golang/encrypt/internal/testusers/customers.go +++ b/languages/golang/encrypt/internal/testusers/customers.go @@ -4,12 +4,12 @@ import "encoding/json" //go:generate go run github.com/cipherstash/stack/languages/golang/cmd/stashgen -type Customer -name Customer -// Customer holds two encrypted JSON documents (the EQL type EncryptedJson, -// stored in public.eql_v4_json_tree): a profile whose $.age and every +// Customer holds two encrypted JSON documents (the EQL type JsonSearch, +// stored in public.eql_v4_json_search): a profile whose $.age and every // visit's $.at order, and optional preferences, stored as SQL NULL when nil. type Customer struct { _ struct{} `stash:"context=customers"` ID int64 `stash:"id,passthrough"` - Profile json.RawMessage `stash:"profile,encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at)"` - Prefs *json.RawMessage `stash:"prefs,encrypt_into=EncryptedJson,null=sql"` + Profile json.RawMessage `stash:"profile,encrypt_into=JsonSearch(ordered=$.age;$.visits[*].at)"` + Prefs *json.RawMessage `stash:"prefs,encrypt_into=JsonSearch,null=sql"` } diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 3e13bee40..50b9de025 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -96,7 +96,7 @@ func (cph *Cipher) JSONOpen(ctx context.Context, plan *record.Plan, field string } // JSONSelector is a JSON path's selector: what `->`, `->>` and the path -// functions of an EncryptedJSON column take. It binds as SQL text; cast it, +// functions of a JSONSearch column take. It binds as SQL text; cast it, // `col -> $1::text`, so PostgreSQL picks EQL's operator and not jsonb's. // It is column-wide: the same path has the same selector in every row. type JSONSelector string @@ -174,12 +174,12 @@ type JSONRange[Q ~[]byte] struct { High Q } -// JSONSelection is a prepared `->` on an EncryptedJSON field: the selector +// JSONSelection is a prepared `->` on a JSONSearch field: the selector // to bind, and what opens the entries the database returns for it. It keeps // the request it was prepared from (the field's declaration, the path and // the cipher's keyset), so the entries open only as answers to it: the // engine derives the selector again and refuses an entry that does not -// carry it. E is the column's entry type, eql.EncryptedJSONEntry. +// carry it. E is the column's entry type, eql.JSONEntry. type JSONSelection[E ~[]byte] struct { cipher *Cipher plan *record.Plan @@ -236,12 +236,12 @@ func (s JSONSelection[E]) Open(ctx context.Context, rows []*E) ([]JSONValue, err return out, nil } -// JSONRead is a prepared subtree read of an EncryptedJSON field: the tokens +// JSONRead is a prepared subtree read of a JSONSearch field: the tokens // to bind, and what opens the rows eql_v4.json_tree_subtree returns for // them. It keeps the request it was prepared from (the field's declaration, // the path, the bound and the cipher's keyset); the engine derives the read // again from it, and checks every row against it. E is the column's entry -// type, eql.EncryptedJSONEntry. +// type, eql.JSONEntry. type JSONRead[E ~[]byte] struct { cipher *Cipher plan *record.Plan diff --git a/languages/golang/encrypt/json_fields_test.go b/languages/golang/encrypt/json_fields_test.go index 976ad14c5..3dde05150 100644 --- a/languages/golang/encrypt/json_fields_test.go +++ b/languages/golang/encrypt/json_fields_test.go @@ -14,7 +14,7 @@ import ( "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" ) -// Generated code for json.RawMessage fields stored as EncryptedJson: +// Generated code for json.RawMessage fields stored as JsonSearch: // testusers.Customer's Profile, ordered at $.age and $.visits[*].at, and its // optional Prefs. @@ -134,7 +134,7 @@ func TestACustomersFieldsDeriveEachQuery(t *testing.T) { } // A selection over rows with no entry opens to no value without a // request. - none, err := sel.Open(ctx, make([]*eql.EncryptedJSONEntry, 3)) + none, err := sel.Open(ctx, make([]*eql.JSONEntry, 3)) if err != nil || len(none) != 3 || none[0].Found || none[2].Found { t.Fatalf("an empty page: %+v %v", none, err) } diff --git a/languages/golang/encrypt/json_postgres_test.go b/languages/golang/encrypt/json_postgres_test.go index ba9864ee2..ad3704c63 100644 --- a/languages/golang/encrypt/json_postgres_test.go +++ b/languages/golang/encrypt/json_postgres_test.go @@ -13,7 +13,7 @@ import ( "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" ) -// Encrypted JSON documents (EncryptedJson, public.eql_v4_json_tree) the Go +// Encrypted JSON documents (JsonSearch, public.eql_v4_json_search) the Go // SDK stores, found by the queries its generated methods derive and read by // its selections and reads, inside PostgreSQL with EQL v4 installed. Each // query's rows are the ones the plaintexts say. Skipped unless DATABASE_URL @@ -41,7 +41,7 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { t.Fatalf("%s: %v", q, err) } } - exec(`CREATE TEMP TABLE customers (id bigint PRIMARY KEY, profile public.eql_v4_json_tree NOT NULL, prefs public.eql_v4_json_tree)`) + exec(`CREATE TEMP TABLE customers (id bigint PRIMARY KEY, profile public.eql_v4_json_search NOT NULL, prefs public.eql_v4_json_search)`) t.Cleanup(func() { _, _ = conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS customers`) }) encrypted, err := testusers.EncryptCustomer(ctx, cipher, jsonCustomers) if err != nil { @@ -114,21 +114,21 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } // Containment, both ways. - q := eql.EncryptedJSONQuery(must(f.Contains(ctx, cipher, json.RawMessage(`{"name":"Alice"}`)))) - want("contains name", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json_tree ORDER BY id`, q), 1) - q = eql.EncryptedJSONQuery(must(f.Contains(ctx, cipher, json.RawMessage(`{"visits":[{"at":5}]}`)))) - want("contains an element", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json_tree ORDER BY id`, q), 2) - q = eql.EncryptedJSONQuery(must(f.ContainedBy(ctx, cipher, json.RawMessage(`{"name":"Dan","x":1}`)))) - want("contained by", ids(`SELECT id FROM customers WHERE profile <@ $1::eql_v4.query_json_tree ORDER BY id`, q), 4) + q := eql.JSONSearchQuery(must(f.Contains(ctx, cipher, json.RawMessage(`{"name":"Alice"}`)))) + want("contains name", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json ORDER BY id`, q), 1) + q = eql.JSONSearchQuery(must(f.Contains(ctx, cipher, json.RawMessage(`{"visits":[{"at":5}]}`)))) + want("contains an element", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json ORDER BY id`, q), 2) + q = eql.JSONSearchQuery(must(f.ContainedBy(ctx, cipher, json.RawMessage(`{"name":"Dan","x":1}`)))) + want("contained by", ids(`SELECT id FROM customers WHERE profile <@ $1::eql_v4.query_json ORDER BY id`, q), 4) // Equality at a wildcard path matches any element. - q = eql.EncryptedJSONQuery(must(f.EqAt(ctx, cipher, "$.visits[*].at", json.RawMessage(`1`)))) - want("eq_at", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json_tree ORDER BY id`, q), 1) + q = eql.JSONSearchQuery(must(f.EqAt(ctx, cipher, "$.visits[*].at", json.RawMessage(`1`)))) + want("eq_at", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json ORDER BY id`, q), 1) // Containment on the optional column: SQL NULL rows match nothing. pq, err := testusers.CustomerFields.Prefs.Contains(ctx, cipher, json.RawMessage(`{"theme":"dark"}`)) if err != nil { t.Fatal(err) } - want("prefs contains", ids(`SELECT id FROM customers WHERE prefs @> $1::eql_v4.query_json_tree ORDER BY id`, pq), 1) + want("prefs contains", ids(`SELECT id FROM customers WHERE prefs @> $1::eql_v4.query_json ORDER BY id`, pq), 1) // Orderings at an ordered path compare the leaf `->` extracts there, // selected by the accessor's own Selector, so the path is named once. @@ -140,12 +140,12 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { if err != nil { t.Fatal(err) } - want("age < 40", ids(`SELECT id FROM customers WHERE (profile -> $1::text) < $2::eql_v4.query_json_tree_leaf ORDER BY id`, age.Selector(), lt), 1, 2) + want("age < 40", ids(`SELECT id FROM customers WHERE (profile -> $1::text) < $2::eql_v4.query_json_leaf ORDER BY id`, age.Selector(), lt), 1, 2) rng, err := f.At.Age.Between(ctx, cipher, json.RawMessage(`30`), json.RawMessage(`70`)) if err != nil { t.Fatal(err) } - want("age between", ids(`SELECT id FROM customers WHERE (profile -> $1::text) >= $2::eql_v4.query_json_tree_leaf AND (profile -> $1::text) <= $3::eql_v4.query_json_tree_leaf ORDER BY id`, age.Selector(), rng.Low, rng.High), 1, 3) + want("age between", ids(`SELECT id FROM customers WHERE (profile -> $1::text) >= $2::eql_v4.query_json_leaf AND (profile -> $1::text) <= $3::eql_v4.query_json_leaf ORDER BY id`, age.Selector(), rng.Low, rng.High), 1, 3) at, err := f.At.VisitsAt.Selector(ctx, cipher) if err != nil { t.Fatal(err) @@ -154,20 +154,20 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { if err != nil { t.Fatal(err) } - want("first visit > 2", ids(`SELECT id FROM customers WHERE (profile -> $1::text) > $2::eql_v4.query_json_tree_leaf ORDER BY id`, at.Selector(), gt), 1, 2) + want("first visit > 2", ids(`SELECT id FROM customers WHERE (profile -> $1::text) > $2::eql_v4.query_json_leaf ORDER BY id`, at.Selector(), gt), 1, 2) // `->` opened against the selection that asked for it: each row's // value, and no value where the row has none. - selected := func(sel encrypt.JSONSelection[eql.EncryptedJSONEntry]) []encrypt.JSONValue { + selected := func(sel encrypt.JSONSelection[eql.JSONEntry]) []encrypt.JSONValue { t.Helper() rows, err := conn.QueryContext(ctx, `SELECT profile -> $1::text FROM customers ORDER BY id`, sel.Selector()) if err != nil { t.Fatal(err) } defer rows.Close() - var entries []*eql.EncryptedJSONEntry + var entries []*eql.JSONEntry for rows.Next() { - var e *eql.EncryptedJSONEntry + var e *eql.JSONEntry if err := rows.Scan(&e); err != nil { t.Fatal(err) } @@ -206,14 +206,14 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } walk := func(budget int) [][]json.RawMessage { t.Helper() - groups := make([][]eql.EncryptedJSONEntry, len(jsonCustomers)) + groups := make([][]eql.JSONEntry, len(jsonCustomers)) for i, cust := range jsonCustomers { rows, err := conn.QueryContext(ctx, `SELECT e FROM customers, eql_v4.json_tree_subtree(profile, $2::text[], $3) AS e WHERE id = $1`, cust.ID, read.Tokens(), budget) if err != nil { t.Fatal(err) } for rows.Next() { - var e eql.EncryptedJSONEntry + var e eql.JSONEntry if err := rows.Scan(&e); err != nil { t.Fatal(err) } @@ -252,13 +252,13 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { if err != nil { t.Fatal(err) } - var entries []eql.EncryptedJSONEntry + var entries []eql.JSONEntry rs, err := conn.QueryContext(ctx, `SELECT e FROM customers, eql_v4.json_tree_subtree(profile, $1::text[]) AS e WHERE id = 1`, visits.Tokens()) if err != nil { t.Fatal(err) } for rs.Next() { - var e eql.EncryptedJSONEntry + var e eql.JSONEntry if err := rs.Scan(&e); err != nil { t.Fatal(err) } diff --git a/languages/golang/encrypt/term.go b/languages/golang/encrypt/term.go index e30d1c97d..608f7004b 100644 --- a/languages/golang/encrypt/term.go +++ b/languages/golang/encrypt/term.go @@ -75,7 +75,7 @@ type JSONOption interface { // by stashgen, and at run time by the engine. // // Deprecated: store a JSON document as a json.RawMessage field with -// `encrypt_into=EncryptedJson`, whose EQL type carries the JSON index. +// `encrypt_into=JsonSearch`, whose EQL type carries the JSON index. func JSON(options ...JSONOption) Index { _ = options return index("json") @@ -158,8 +158,8 @@ func (t OpeTerm) Less(other OpeTerm) bool { return t.Compare(other) < 0 } // derive it; the type exists so a generated declaration that names [JSON] // has a Go type to fail into. // -// Deprecated: an EncryptedJson field's queries are eql.EncryptedJSONQuery -// and eql.EncryptedJSONLeafQuery. +// Deprecated: a JsonSearch field's queries are eql.JSONSearchQuery +// and eql.JSONSearchLeafQuery. type JSONTerm []byte // compareCLLW is cllw-ore's compare_lex: compare_slice over the common diff --git a/languages/golang/stashgen/declaration.go b/languages/golang/stashgen/declaration.go index de10e50eb..9b8070767 100644 --- a/languages/golang/stashgen/declaration.go +++ b/languages/golang/stashgen/declaration.go @@ -46,7 +46,7 @@ type Field struct { // EQLType is the EQL type an EncryptInto field seals into. EQLType string // TargetOptions are the EQL type's options, for a type that takes them - // (EncryptedJson); nil for its defaults. + // (JsonSearch); nil for its defaults. TargetOptions *record.TargetOptions // Identity is the field's part of its context when it differs from Name: // a column that was renamed keeps the identity it was first written @@ -276,7 +276,7 @@ const ( // KindDecimal is encrypt.Decimal. KindDecimal // KindJSON is json.RawMessage: a JSON document, stored as an - // EncryptedJSON value. + // JSONSearch value. KindJSON ) diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index 314781003..080966b31 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -977,7 +977,7 @@ func joinNames(names []string) string { return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1] } -// isJSON reports whether the field is stored as an EncryptedJSON document: +// isJSON reports whether the field is stored as a JSONSearch document: // a json.RawMessage sealed into an EQL type. func (g genField) isJSON() bool { return g.Verb == VerbEncryptInto && g.GoType.Kind == KindJSON @@ -1004,7 +1004,7 @@ func planJSON(f *genFile, g *genField) error { // The engine answers selector for a field whose type stores a json_tree // document; the leaf and entry types below are that family's. if !document { - return fieldErr(f.typeName, g.GoName, "the engine answers no document operator on %s, and the generator writes json.RawMessage fields of EncryptedJson only", g.EQLType) + return fieldErr(f.typeName, g.GoName, "the engine answers no document operator on %s, and the generator writes json.RawMessage fields of JsonSearch only", g.EQLType) } g.jsonPaths = nil seen := map[string]string{} @@ -1070,10 +1070,14 @@ func (w *writer) jsonEntry(g genField) { for _, op := range g.operators { answered[op.Name] = true } - eqlName := strings.TrimPrefix(g.outputType, "eql.") needle := g.queryType - leaf := "eql." + eqlName + "LeafQuery" - entry := "eql." + eqlName + "Entry" + // The document's companion types carry the names EQL v3 gives its own + // (eql_domains::JSON_TREE): the leaf operand puts Leaf before the + // needle's Query (JSONSearchQuery, JSONSearchLeafQuery), and the entry + // is named for the plaintext it holds one node of (JSONEntry, over + // public.eql_v4_json_entry), not for the stored type. + leaf := strings.TrimSuffix(needle, "Query") + "LeafQuery" + entry := "eql." + EQLGoName("JsonEntry") ordered := len(g.jsonPaths) > 0 && answered["lt_at"] field := "gensupport.Field[" + g.valueExpr + "]" diff --git a/languages/golang/stashgen/engine.go b/languages/golang/stashgen/engine.go index 08275aaa1..0368d876f 100644 --- a/languages/golang/stashgen/engine.go +++ b/languages/golang/stashgen/engine.go @@ -271,7 +271,7 @@ func lowerDeclaration(d Declaration, eqlTypes []EQLType) (*record.Plan, error) { if !ok { reason := fmt.Sprintf("the engine does not derive the %s index yet", idx.Name) if idx.Name == IndexJSON { - reason = "the json index is not an output beside a ciphertext: store a JSON document as a json.RawMessage field with encrypt_into=EncryptedJson, whose EQL type carries the index" + reason = "the json index is not an output beside a ciphertext: store a JSON document as a json.RawMessage field with encrypt_into=JsonSearch, whose EQL type carries the index" } return nil, &FieldError{Type: d.Type, Field: f.GoName, Reason: reason} } diff --git a/languages/golang/stashgen/enginetest/enginetest.go b/languages/golang/stashgen/enginetest/enginetest.go index 2f91c56d3..e3a582f49 100644 --- a/languages/golang/stashgen/enginetest/enginetest.go +++ b/languages/golang/stashgen/enginetest/enginetest.go @@ -278,7 +278,7 @@ func indexApplies(name stashgen.IndexName, t stashgen.GoType) error { return nil } case stashgen.IndexJSON: - return fmt.Errorf("the json index is not an output beside a ciphertext: store a JSON document with encrypt_into=EncryptedJson") + return fmt.Errorf("the json index is not an output beside a ciphertext: store a JSON document with encrypt_into=JsonSearch") } return fmt.Errorf("%s does not apply to %s", name, t) } diff --git a/languages/golang/stashgen/refusal_test.go b/languages/golang/stashgen/refusal_test.go index 3297fdc9c..3d4855ff5 100644 --- a/languages/golang/stashgen/refusal_test.go +++ b/languages/golang/stashgen/refusal_test.go @@ -113,7 +113,7 @@ func TestRefusals(t *testing.T) { {"an opaque struct with a map keyed by a struct", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tM map[hidden]int") + "\ntype hidden struct{ A int }\n", stashgen.Request{Type: "User"}, "M", "map key that is not a string or an integer"}, {"an EQL type the engine cannot produce yet", user(ctx + "\tAge int64 `stash:\"age,encrypt_into=BigintOrdOre\"`"), stashgen.Request{Type: "User"}, "Age", "has no EQL type BigintOrdOre"}, {"an index on a composite", user(ctx + "\tAttrs map[string]string `stash:\"attrs,encrypt,index=equality\"`"), stashgen.Request{Type: "User"}, "Attrs", "seals only as part of an opaque struct"}, - {"the json index as an output", user(ctx + "\tAttrs string `stash:\"attrs,index=json\"`"), stashgen.Request{Type: "User"}, "Attrs", "encrypt_into=EncryptedJson"}, + {"the json index as an output", user(ctx + "\tAttrs string `stash:\"attrs,index=json\"`"), stashgen.Request{Type: "User"}, "Attrs", "encrypt_into=JsonSearch"}, {"an index option a query term cannot use", user(ctx + "\tEmail string `stash:\"email,encrypt,index=match(k=3)\"`"), stashgen.Request{Type: "User"}, "Email", "a query term uses only the default index options"}, {"a passthrough field that has an index", user(ctx + "\tID int64 `stash:\"id,passthrough,index=equality\"`"), stashgen.Request{Type: "User"}, "ID", "passthrough field has no index"}, {"an embedded struct from another package with no tag", user(ctx + "\tgorm.Model\n\tEmail string `stash:\"email,encrypt\"`"), stashgen.Request{Type: "User"}, "Model", "cannot carry tags; tag the field"}, diff --git a/languages/golang/stashgen/tag.go b/languages/golang/stashgen/tag.go index 925b5a093..0e7584c94 100644 --- a/languages/golang/stashgen/tag.go +++ b/languages/golang/stashgen/tag.go @@ -26,7 +26,7 @@ const tagKey = "stash" // stash:"tenant,context_field" Name, VerbContextField // stash:",passthrough" VerbPassthrough with no Name, on an embedded struct // stash:"dob,encrypt_into=Date,null=sql" Null, on a sealed field declared over a pointer -// stash:"profile,encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at,sealed_root=false)" +// stash:"profile,encrypt_into=JsonSearch(ordered=$.age;$.visits[*].at,sealed_root=false)" // EQLType and EQLOptions type tag struct { Omit bool diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go index 6ddfbe7bc..aa1117201 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go @@ -141,13 +141,13 @@ type JSONReadOptions struct{ WildcardBound uint32 } // JSONRange is both bounds of a between at a path. type JSONRange[Q ~[]byte] struct{ Low, High Q } -// JSONSelection is a prepared `->` on an EncryptedJSON field. +// JSONSelection is a prepared `->` on a JSONSearch field. type JSONSelection[E ~[]byte] struct{ _ E } func (JSONSelection[E]) Selector() JSONSelector { return "" } func (JSONSelection[E]) Open(context.Context, []*E) ([]JSONValue, error) { return nil, nil } -// JSONRead is a prepared subtree read on an EncryptedJSON field. +// JSONRead is a prepared subtree read on a JSONSearch field. type JSONRead[E ~[]byte] struct{ _ E } func (JSONRead[E]) Tokens() JSONTokens { return nil } From ed858fbbc171cb51e9b30533ed4d9bd46ad03ae1 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:10:15 +1100 Subject: [PATCH 05/13] fix(go): decide OpenOne's wildcard from the engine's token count JSONRead.OpenOne refused a wildcard read by looking for "[*]" in the path text. That copies a rule of the engine's path parser into Go, and gets it wrong for a quoted key: $["a[*]"] is one member key, and OpenOne refused it. The engine already answers the question. It derives exactly one token for a path that names one node, and for a wildcard one token per element below the bound plus a probe at the bound, which is two or more at any bound of 1 or more. Generated code never sends a bound of 0. OpenOne now refuses a read with more or fewer than one token. A read prepared by hand with a bound of 0 passes the check, but it reaches no value: a row with an element there is refused as truncated, and any other opens to none. The refusal no longer prints the path, which can hold a customer's key. The parameter is named entries, for what it holds: one database row's json_tree_subtree entries. JSONRead no longer keeps the path, which nothing reads now, so NewJSONRead stops taking it. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/encrypt/gensupport/json.go | 2 +- languages/golang/encrypt/json.go | 34 ++++++++++++-------- languages/golang/encrypt/json_fields_test.go | 14 ++++++-- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/languages/golang/encrypt/gensupport/json.go b/languages/golang/encrypt/gensupport/json.go index 3212138ba..04c034f60 100644 --- a/languages/golang/encrypt/gensupport/json.go +++ b/languages/golang/encrypt/gensupport/json.go @@ -94,7 +94,7 @@ func JSONSubtree[E ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.C if err != nil { return encrypt.JSONRead[E]{}, err } - return encrypt.NewJSONRead[E](c, f.plan, f.name, path, input, term.Text), nil + return encrypt.NewJSONRead[E](c, f.plan, f.name, input, term.Text), nil } // JSONNodeToken derives the token of the node at a path with no wildcard diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 50b9de025..800f700ec 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -246,7 +246,6 @@ type JSONRead[E ~[]byte] struct { cipher *Cipher plan *record.Plan field string - path string // input is the subtree operator's input the tokens were derived from; // opening sends it back as the read's request, unchanged. input vcvalue.Object @@ -255,8 +254,8 @@ type JSONRead[E ~[]byte] struct { // NewJSONRead is a read prepared by [Cipher.QueryTerm]'s subtree operator // from input, its {"path", "wildcard_bound"?} object. For generated code. -func NewJSONRead[E ~[]byte](c *Cipher, plan *record.Plan, field, path string, input vcvalue.Object, tokens []string) JSONRead[E] { - return JSONRead[E]{cipher: c, plan: plan, field: field, path: path, input: input, tokens: JSONTokens(tokens)} +func NewJSONRead[E ~[]byte](c *Cipher, plan *record.Plan, field string, input vcvalue.Object, tokens []string) JSONRead[E] { + return JSONRead[E]{cipher: c, plan: plan, field: field, input: input, tokens: JSONTokens(tokens)} } // Tokens are the tokens to bind: @@ -301,17 +300,26 @@ func (r JSONRead[E]) Open(ctx context.Context, rows [][]E) ([][]json.RawMessage, return out, nil } -// OpenOne opens one row's rows to the one value at the read's path: Found -// with its JSON, or not Found where the row has no node there. A read that -// reaches more than one value (a wildcard) is [ErrEncoding]; open it with -// [JSONRead.Open]. -func (r JSONRead[E]) OpenOne(ctx context.Context, rows []E) (JSONValue, error) { - // A wildcard reaches a value per element: refused before anything is - // opened, not only for the rows that happen to hold two. - if strings.Contains(r.path, "[*]") { - return JSONValue{}, fmt.Errorf("%w: OpenOne reads one value, and %s has a wildcard; open it with Open", ErrEncoding, r.path) +// OpenOne opens one database row's entries to the one value at the read's +// path: Found with its JSON, or not Found where the row has no node there. +// A read that reaches more than one value (a wildcard) is [ErrEncoding]; +// open it with [JSONRead.Open]. +func (r JSONRead[E]) OpenOne(ctx context.Context, entries []E) (JSONValue, error) { + // The engine derives one token for a path that names one node, and for + // a wildcard one per element below the bound plus a probe at it: two + // or more at any bound of 1 or more. The generated Subtree never sends + // a bound of 0 (WildcardBound 0 sends none, and the engine's default + // is 64). So the count is the engine's own answer to whether the path + // names one node: a quoted key that spells [*] is one member. A + // wildcard is refused here, before anything is opened, not only for + // the rows that happen to hold two values. A read prepared with a + // bound of 0 expands a wildcard to the probe alone and passes this + // check, but it reaches no value: a row with an element there is + // refused as truncated, and any other opens to none. + if r.cipher != nil && len(r.tokens) != 1 { + return JSONValue{}, fmt.Errorf("%w: OpenOne reads one value, and this read's path has a wildcard; open it with Open", ErrEncoding) } - opened, err := r.Open(ctx, [][]E{rows}) + opened, err := r.Open(ctx, [][]E{entries}) if err != nil { return JSONValue{}, err } diff --git a/languages/golang/encrypt/json_fields_test.go b/languages/golang/encrypt/json_fields_test.go index 3dde05150..e0609e0c5 100644 --- a/languages/golang/encrypt/json_fields_test.go +++ b/languages/golang/encrypt/json_fields_test.go @@ -122,10 +122,20 @@ func TestACustomersFieldsDeriveEachQuery(t *testing.T) { t.Fatal("the accessor's selector is another path's") } // OpenOne refuses a wildcard read before anything is opened, whatever - // the rows hold; a zero selector or token binds nothing. - if _, err := read.OpenOne(ctx, nil); !errors.Is(err, encrypt.ErrEncoding) { + // the rows hold, and its error does not name the path, which can hold + // a customer's key. A quoted key that spells [*] names one member: the + // engine derives one token for it, and OpenOne takes it. A zero + // selector or token binds nothing. + if _, err := read.OpenOne(ctx, nil); !errors.Is(err, encrypt.ErrEncoding) || strings.Contains(err.Error(), "visits") { t.Fatalf("OpenOne on a wildcard read: %v", err) } + quoted, err := f.Subtree(ctx, cipher, `$["a[*]"]`, encrypt.JSONReadOptions{}) + if err != nil || len(quoted.Tokens()) != 1 { + t.Fatalf("Subtree at a quoted key: %v %v", quoted.Tokens(), err) + } + if v, err := quoted.OpenOne(ctx, nil); err != nil || v.Found { + t.Fatalf("OpenOne at a quoted key that spells [*]: %+v %v", v, err) + } if _, err := (encrypt.JSONSelector("")).Value(); !errors.Is(err, encrypt.ErrEncoding) { t.Fatalf("a zero selector binds: %v", err) } From 247bd3239dc1e90f4c52b10330bb21c7d4e80422 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:10:46 +1100 Subject: [PATCH 06/13] fix(go): refuse empty JSONTokens rather than bind them as {} A zero JSONSelector or JSONToken is refused with ErrEncoding when bound, because the empty value matches no row without an error. JSONTokens did not follow the rule: a nil or empty list bound as the text[] literal {}, so a zero JSONRead's tokens read nothing from every row and said nothing. Every read the engine prepares has at least one token, so an empty list is always a read that was never prepared. It is now refused the same way, and the changelog's sentence covers the token list too. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 4 ++-- languages/golang/encrypt/json.go | 7 ++++++- languages/golang/encrypt/json_fields_test.go | 8 +++++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 55a41957f..a2e9ffb1c 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -23,8 +23,8 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. and `encrypt.JSONRead` open the rows `->` and `eql_v4.json_tree_subtree` return against the request that prepared them (`OpenOne` refuses a wildcard read up front); `encrypt.JSONSelector`, `JSONToken` and - `JSONTokens` bind as `text` and `text[]`, and a zero selector or token is - refused rather than bound. A `json.RawMessage` sealed any other way stays + `JSONTokens` bind as `text` and `text[]`, and a zero selector, token or + token list is refused rather than bound. A `json.RawMessage` sealed any other way stays bytes. `gensupport.JSON`, `TargetOptions`, `Declaration.Options` and the `JSON*` helpers are the generated code's. diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 800f700ec..20f072aac 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -131,8 +131,13 @@ func (t JSONToken) Value() (driver.Value, error) { type JSONTokens []string // Value binds the tokens as a text[] literal. A token is base64, so it -// needs no escape inside the quotes. +// needs no escape inside the quotes. Empty JSONTokens (a zero JSONRead's) +// are refused, as a zero [JSONToken] is: bound as {}, they would read +// nothing from every row. func (t JSONTokens) Value() (driver.Value, error) { + if len(t) == 0 { + return nil, fmt.Errorf("%w: empty JSONTokens: prepare a read with the field's Subtree", ErrEncoding) + } var b strings.Builder b.WriteByte('{') for i, token := range t { diff --git a/languages/golang/encrypt/json_fields_test.go b/languages/golang/encrypt/json_fields_test.go index e0609e0c5..eb38d31c8 100644 --- a/languages/golang/encrypt/json_fields_test.go +++ b/languages/golang/encrypt/json_fields_test.go @@ -125,7 +125,7 @@ func TestACustomersFieldsDeriveEachQuery(t *testing.T) { // the rows hold, and its error does not name the path, which can hold // a customer's key. A quoted key that spells [*] names one member: the // engine derives one token for it, and OpenOne takes it. A zero - // selector or token binds nothing. + // selector, token or tokens binds nothing. if _, err := read.OpenOne(ctx, nil); !errors.Is(err, encrypt.ErrEncoding) || strings.Contains(err.Error(), "visits") { t.Fatalf("OpenOne on a wildcard read: %v", err) } @@ -142,6 +142,12 @@ func TestACustomersFieldsDeriveEachQuery(t *testing.T) { if _, err := (encrypt.JSONToken("")).Value(); !errors.Is(err, encrypt.ErrEncoding) { t.Fatalf("a zero token binds: %v", err) } + if _, err := (encrypt.JSONTokens(nil)).Value(); !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("no tokens bind: %v", err) + } + if _, err := (encrypt.JSONRead[eql.JSONEntry]{}).Tokens().Value(); !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("a zero read's tokens bind: %v", err) + } // A selection over rows with no entry opens to no value without a // request. none, err := sel.Open(ctx, make([]*eql.JSONEntry, 3)) From 2424891309cef3d030ba6ccb788019fc3b67c8c6 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:12:21 +1100 Subject: [PATCH 07/13] fix(go): print and log the JSON read types redacted JSONToken and JSONTokens are documented as capabilities: a token opens the structure of its node's subtree in every row of the column. They still printed in the clear with %v and through slog. JSONSelection and JSONRead printed their selector, tokens and path with %+v, and a path's keys can be a customer's data (Go principle 12). Each now has String, GoString, Format and LogValue, as the query types in query.go do, so every fmt verb and slog record shows the type, the field and a token count, with [redacted] for the tokens, the selector and the path. The values still bind through driver.Valuer, which pgx and database/sql consult before a Stringer. JSONValue's doc now says it holds plaintext JSON that prints in the clear, as the generated plaintext types' docs do. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 5 +- languages/golang/encrypt/json.go | 97 +++++++++++++++++++- languages/golang/encrypt/json_fields_test.go | 48 ++++++++++ 3 files changed, 146 insertions(+), 4 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index a2e9ffb1c..22e73c11d 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -24,7 +24,10 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. return against the request that prepared them (`OpenOne` refuses a wildcard read up front); `encrypt.JSONSelector`, `JSONToken` and `JSONTokens` bind as `text` and `text[]`, and a zero selector, token or - token list is refused rather than bound. A `json.RawMessage` sealed any other way stays + token list is refused rather than bound. `JSONToken`, `JSONTokens`, + `JSONSelection` and `JSONRead` print and log with the tokens, the + selector and the path redacted; `encrypt.JSONValue` holds plaintext and + prints it. A `json.RawMessage` sealed any other way stays bytes. `gensupport.JSON`, `TargetOptions`, `Declaration.Options` and the `JSON*` helpers are the generated code's. diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 20f072aac..648ec3a37 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -5,6 +5,8 @@ import ( "database/sql/driver" "encoding/json" "fmt" + "io" + "log/slog" "strings" "github.com/cipherstash/stack/languages/golang/internal/record" @@ -113,7 +115,8 @@ func (s JSONSelector) Value() (driver.Value, error) { // JSONToken is one node's token: what eql_v4.json_tree_children and // eql_v4.json_tree_child_count take. It binds as SQL text. A token is a // capability over the node's subtree in every row of the column: send it -// to the database for a query, and do not log it. +// to the database for a query, and nowhere else. It prints and logs +// redacted. type JSONToken string // Value binds the token as text. A zero JSONToken is refused, as a zero @@ -125,9 +128,31 @@ func (t JSONToken) Value() (driver.Value, error) { return string(t), nil } +// redacted stands in for a token, a selector or a path in what the JSON +// read types print and log. +const redacted = "[redacted]" + +// String hides the token: it is a capability, so it does not print. +func (t JSONToken) String() string { + if t == "" { + return "encrypt.JSONToken{}" + } + return "encrypt.JSONToken{" + redacted + "}" +} + +// GoString is what %#v prints: [JSONToken.String]. +func (t JSONToken) GoString() string { return t.String() } + +// Format prints [JSONToken.String] for every verb fmt hands it. +func (t JSONToken) Format(f fmt.State, _ rune) { _, _ = io.WriteString(f, t.String()) } + +// LogValue is what slog records: [redacted], never the token. +func (t JSONToken) LogValue() slog.Value { return slog.StringValue(redacted) } + // JSONTokens are a subtree read's tokens: what eql_v4.json_tree_subtree // takes. They bind as a PostgreSQL text[] literal; cast the parameter, -// `$2::text[]`. Like a [JSONToken], each is a capability: do not log them. +// `$2::text[]`. Like a [JSONToken], each is a capability, and they print +// and log redacted. type JSONTokens []string // Value binds the tokens as a text[] literal. A token is base64, so it @@ -152,10 +177,27 @@ func (t JSONTokens) Value() (driver.Value, error) { return b.String(), nil } +// String counts the tokens and hides them: each is a capability. +func (t JSONTokens) String() string { + return fmt.Sprintf("encrypt.JSONTokens{%d tokens: %s}", len(t), redacted) +} + +// GoString is what %#v prints: [JSONTokens.String]. +func (t JSONTokens) GoString() string { return t.String() } + +// Format prints [JSONTokens.String] for every verb fmt hands it. +func (t JSONTokens) Format(f fmt.State, _ rune) { _, _ = io.WriteString(f, t.String()) } + +// LogValue is what slog records: how many tokens, never the tokens. +func (t JSONTokens) LogValue() slog.Value { + return slog.GroupValue(slog.Int("tokens", len(t))) +} + // JSONValue is one value a read reached, or none. Found is false for no // value at the path (the row has no node there, or SQL NULL); a JSON null // that is there is Found, with JSON "null". JSON is the value's JSON text: -// plaintext, which the caller owns. +// plaintext, which the caller owns. JSONValue prints its JSON in the +// clear: it has no String or LogValue method, so do not log one. type JSONValue struct { Found bool JSON json.RawMessage @@ -202,6 +244,30 @@ func NewJSONSelection[E ~[]byte](c *Cipher, plan *record.Plan, field, path, sele // Selector is the selector to bind: `col -> $1::text`. func (s JSONSelection[E]) Selector() JSONSelector { return s.selector } +// String names the field and hides the path and the selector: a path's +// keys can be a customer's data. +func (s JSONSelection[E]) String() string { + if s.cipher == nil { + return "encrypt.JSONSelection{}" + } + return fmt.Sprintf("encrypt.JSONSelection{field: %q, path: %s, selector: %s}", s.field, redacted, redacted) +} + +// GoString is what %#v prints: [JSONSelection.String]. +func (s JSONSelection[E]) GoString() string { return s.String() } + +// Format prints [JSONSelection.String] for every verb fmt hands it. +func (s JSONSelection[E]) Format(f fmt.State, _ rune) { _, _ = io.WriteString(f, s.String()) } + +// LogValue is what slog records: the field, and [redacted] for the path +// and the selector. +func (s JSONSelection[E]) LogValue() slog.Value { + if s.cipher == nil { + return slog.GroupValue() + } + return slog.GroupValue(slog.String("field", s.field), slog.String("path", redacted), slog.String("selector", redacted)) +} + // Open opens the entries `col -> selector` returned, one per database row, // nil for a row where it returned SQL NULL, with one ZeroKMS request. Each // opens to the scalar it holds (Found), or for the sealed root at "$" the @@ -267,6 +333,31 @@ func NewJSONRead[E ~[]byte](c *Cipher, plan *record.Plan, field string, input vc // `eql_v4.json_tree_subtree(col, $1::text[], 64)`. func (r JSONRead[E]) Tokens() JSONTokens { return r.tokens } +// String names the field and counts the tokens, and hides the path and +// the tokens: a path's keys can be a customer's data, and a token is a +// capability. +func (r JSONRead[E]) String() string { + if r.cipher == nil { + return "encrypt.JSONRead{}" + } + return fmt.Sprintf("encrypt.JSONRead{field: %q, path: %s, tokens: %d}", r.field, redacted, len(r.tokens)) +} + +// GoString is what %#v prints: [JSONRead.String]. +func (r JSONRead[E]) GoString() string { return r.String() } + +// Format prints [JSONRead.String] for every verb fmt hands it. +func (r JSONRead[E]) Format(f fmt.State, _ rune) { _, _ = io.WriteString(f, r.String()) } + +// LogValue is what slog records: the field, [redacted] for the path, and +// how many tokens. +func (r JSONRead[E]) LogValue() slog.Value { + if r.cipher == nil { + return slog.GroupValue() + } + return slog.GroupValue(slog.String("field", r.field), slog.String("path", redacted), slog.Int("tokens", len(r.tokens))) +} + // Open opens the rows the read returned, one group per database row, with // one ZeroKMS request: per row, the value at each concrete path the read // named that the row has, in document order. A row with no node there diff --git a/languages/golang/encrypt/json_fields_test.go b/languages/golang/encrypt/json_fields_test.go index eb38d31c8..6384c92d7 100644 --- a/languages/golang/encrypt/json_fields_test.go +++ b/languages/golang/encrypt/json_fields_test.go @@ -1,10 +1,12 @@ package encrypt_test import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "log/slog" "reflect" "strings" "testing" @@ -174,3 +176,49 @@ func TestACustomersFieldsDeriveEachQuery(t *testing.T) { t.Fatal("Prefs has an At") } } + +// A token is a capability and a path's keys can be a customer's data: the +// read types print and log neither, under any verb or through slog. +func TestJSONReadTypesPrintAndLogRedacted(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + f := testusers.CustomerFields.Profile + sel, err := f.Selector(ctx, cipher, "$.secretkey") + if err != nil { + t.Fatal(err) + } + read, err := f.Subtree(ctx, cipher, "$.secretkey[*]", encrypt.JSONReadOptions{WildcardBound: 2}) + if err != nil { + t.Fatal(err) + } + token, err := f.Children(ctx, cipher, "$.secretkey") + if err != nil { + t.Fatal(err) + } + hidden := append([]string{"secretkey", string(sel.Selector()), string(token)}, read.Tokens()...) + var logged bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&logged, nil)) + for name, v := range map[string]any{"selection": sel, "read": read, "token": token, "tokens": read.Tokens()} { + logger.Info(name, "v", v) + for _, verb := range []string{"%v", "%+v", "%#v", "%s", "%q", "%x", "%d"} { + out := fmt.Sprintf(verb, v) + for _, h := range hidden { + if strings.Contains(out, h) { + t.Errorf("%s printed with %s shows %q: %s", name, verb, h, out) + } + } + if !strings.Contains(out, "encrypt.JSON") { + t.Errorf("%s printed with %s does not name its type: %s", name, verb, out) + } + } + } + for _, h := range hidden { + if strings.Contains(logged.String(), h) { + t.Errorf("slog shows %q: %s", h, logged.String()) + } + } + if !strings.Contains(logged.String(), `"field":"profile"`) || !strings.Contains(logged.String(), `"tokens":3`) { + t.Errorf("slog records no field or token count: %s", logged.String()) + } +} From e33cf3e9f786f58d618408a0c30d318495cebe64 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:12:44 +1100 Subject: [PATCH 08/13] docs(go): name the JSON read constructors' declaration parameter NewJSONSelection and NewJSONRead are exported, so their parameter names are part of what godoc shows. The parameter was named plan, a word for the engine's internal form that a user never sees (Go principle 7). It is now declaration, for what it is: the declaration of the record type the field belongs to, and the docs say so. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/encrypt/json.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 648ec3a37..77c3de098 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -236,9 +236,10 @@ type JSONSelection[E ~[]byte] struct { } // NewJSONSelection is a selection prepared by [Cipher.QueryTerm]'s -// selector operator. For generated code. -func NewJSONSelection[E ~[]byte](c *Cipher, plan *record.Plan, field, path, selector string) JSONSelection[E] { - return JSONSelection[E]{cipher: c, plan: plan, field: field, path: path, selector: JSONSelector(selector)} +// selector operator on field of the record type that declaration +// declares, at path. For generated code. +func NewJSONSelection[E ~[]byte](c *Cipher, declaration *record.Plan, field, path, selector string) JSONSelection[E] { + return JSONSelection[E]{cipher: c, plan: declaration, field: field, path: path, selector: JSONSelector(selector)} } // Selector is the selector to bind: `col -> $1::text`. @@ -324,9 +325,10 @@ type JSONRead[E ~[]byte] struct { } // NewJSONRead is a read prepared by [Cipher.QueryTerm]'s subtree operator -// from input, its {"path", "wildcard_bound"?} object. For generated code. -func NewJSONRead[E ~[]byte](c *Cipher, plan *record.Plan, field string, input vcvalue.Object, tokens []string) JSONRead[E] { - return JSONRead[E]{cipher: c, plan: plan, field: field, input: input, tokens: JSONTokens(tokens)} +// on field of the record type that declaration declares, from input, its +// {"path", "wildcard_bound"?} object. For generated code. +func NewJSONRead[E ~[]byte](c *Cipher, declaration *record.Plan, field string, input vcvalue.Object, tokens []string) JSONRead[E] { + return JSONRead[E]{cipher: c, plan: declaration, field: field, input: input, tokens: JSONTokens(tokens)} } // Tokens are the tokens to bind: From 8736697e2da1abc0536beac006a85376059df448 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:14:10 +1100 Subject: [PATCH 09/13] fix(go): refuse ordered paths the engine answers no ordering on stashgen wrote a document field's At accessors only when the engine answered lt_at on it. A tag that names ordered paths on a field the engine answers no ordering on means the generator and the engine it embeds disagree, and the generator then wrote the field with no At and said nothing: the paths the user asked to order had no methods, and the build still passed. planJSON now refuses that field by name, so the drift fails generation instead of producing a quieter API. jsonEntry writes At whenever the field has ordered paths, since planJSON has already checked the engine answers on them. The test runs the embedded engine with lt_at removed from its answer, which is the one way to reach the refusal: the real engine always answers the orderings on an ordered field. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/cmd/stashgen/main_test.go | 42 ++++++++++++++++++++++ languages/golang/stashgen/emit.go | 19 +++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index c83f34fbd..df224efc5 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -347,6 +347,20 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { t.Errorf("the generated file lacks %q", want) } } + // An engine that answers no ordering on a field whose tag orders paths + // disagrees with the generator: refused by name, not written as a field + // with no At. + dir := writeModule(t, map[string]string{"model.go": source("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.age)\"`")}) + stderrBuf := bytes.Buffer{} + if code := run([]string{"-type", "Customer"}, dir, &bytes.Buffer{}, &stderrBuf, withoutOperator("lt_at")); code != 1 { + t.Fatalf("exit %d, want 1\n%s", code, stderrBuf.String()) + } + if got := stderrBuf.String(); !strings.Contains(got, "Customer.Profile") || !strings.Contains(got, "the engine answers no ordering (lt_at)") { + t.Fatalf("stderr = %q", got) + } + if _, err := os.Stat(filepath.Join(dir, "customer_stash.go")); !os.IsNotExist(err) { + t.Fatal("a file was written after the refusal") + } for _, c := range []struct{ name, fields, want string }{ {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch,null=sealed\"`", "Customer.Prefs"}, {"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch\"`", "null=sql"}, @@ -371,3 +385,31 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { }) } } + +// withoutOperator is the embedded engine with op removed from every field's +// answer: an engine that disagrees with the generator. +func withoutOperator(op string) func(context.Context) (stashgen.Engine, error) { + return func(ctx context.Context) (stashgen.Engine, error) { + engine, err := stashgen.GuestEngine(ctx) + return droppingEngine{Engine: engine, op: op}, err + } +} + +type droppingEngine struct { + stashgen.Engine + op string +} + +func (e droppingEngine) Check(ctx context.Context, d stashgen.Declaration) (map[string][]stashgen.Operator, error) { + answered, err := e.Engine.Check(ctx, d) + for field, ops := range answered { + kept := ops[:0] + for _, o := range ops { + if o.Name != e.op { + kept = append(kept, o) + } + } + answered[field] = kept + } + return answered, err +} diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index 080966b31..f31fcdb42 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -1026,10 +1026,26 @@ func planJSON(f *genFile, g *genField) error { g.jsonPaths = append(g.jsonPaths, jsonPath{name: name, path: path}) } } + // The engine answers the orderings on a field that orders a path. A + // tag that orders one and gets none is the engine and the generator + // disagreeing: refused, rather than written as a field with no At. + if len(g.jsonPaths) > 0 && !answers(g, "lt_at") { + return fieldErr(f.typeName, g.GoName, "the tag orders %d paths, and the engine answers no ordering (lt_at) on %s: the generator and its engine disagree, which is a bug in this stashgen", len(g.jsonPaths), g.EQLType) + } f.imports.add("encoding/json", "json") return nil } +// answers reports whether the engine answers op on the field. +func answers(g *genField, op string) bool { + for _, o := range g.operators { + if o.Name == op { + return true + } + } + return false +} + // jsonPathName is an ordered path's accessor name: its keys in Go case, // wildcard steps dropped. $.age is Age, $.visits[*].at is VisitsAt, $.a.1 // is A1, and $ is Root. An ordered path names no position (the engine @@ -1078,7 +1094,8 @@ func (w *writer) jsonEntry(g genField) { // public.eql_v4_json_entry), not for the stored type. leaf := strings.TrimSuffix(needle, "Query") + "LeafQuery" entry := "eql." + EQLGoName("JsonEntry") - ordered := len(g.jsonPaths) > 0 && answered["lt_at"] + // planJSON refuses ordered paths the engine answers no ordering on. + ordered := len(g.jsonPaths) > 0 field := "gensupport.Field[" + g.valueExpr + "]" w.p("type %s struct {", g.fieldType) From ef1bbd41685f5a0a5b29305aaaad9304a1b3543a Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:14:51 +1100 Subject: [PATCH 10/13] test(go): assert the reason stashgen gives for a sealed null document The "a sealed null" case asserted only that stderr named Customer.Prefs, so any refusal of that field passed it, including one for the wrong reason. It now asserts the field and the reason together: a document field carries an index other than equality, a nil has only an equality term, so a sealed nil would show which documents are absent, and the field must say null=sql. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/cmd/stashgen/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index df224efc5..df5410e40 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -362,7 +362,7 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { t.Fatal("a file was written after the refusal") } for _, c := range []struct{ name, fields, want string }{ - {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch,null=sealed\"`", "Customer.Prefs"}, + {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch,null=sealed\"`", "Customer.Prefs: a nil has an equality term and no other, so another index's column would show which values are absent; write null=sql"}, {"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch\"`", "null=sql"}, {"a position in an ordered path", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.scores[2])\"`", "the engine refuses the declaration"}, {"a quoted key in a tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$[\\\"a b\\\"])\"`", "not a plain JSONPath"}, From 66a81a1bf8b2d8b5bc890f9853993d9ed44a549b Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:15:50 +1100 Subject: [PATCH 11/13] docs(go): say JsonSearch refuses a type defined over json.RawMessage stashgen accepts a scalar field declared with any type defined over the scalar (type Email string), so a reader would expect type Doc json.RawMessage to work for JsonSearch too. It does not: stashgen knows a document by the json.RawMessage type itself, so a defined type is bytes, and JsonSearch refuses it with "declare the field json.RawMessage". An alias (type Doc = json.RawMessage) is the type itself and is accepted. Both READMEs now say so, and the stashgen test pins both: the alias generates an eql.JSONSearch field, and the defined type is refused by name with that reason. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/cmd/stashgen/README.md | 1 + languages/golang/cmd/stashgen/main_test.go | 9 ++++++++- languages/golang/encrypt/README.md | 4 +++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index a72f31923..d5585c9b6 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -118,6 +118,7 @@ The field holds the document's JSON text; the engine parses it, so an object tha Values come back as JSON text holding the same values; key order and the spelling of numbers are not kept. A `*json.RawMessage` field is optional and must say `null=sql`: a document has no fixed length to hide a missing one behind. A non-nil pointer to `null` is a stored JSON `null`. A `json.RawMessage` sealed any other way (`encrypt`, `index=equality`) is bytes, as any type defined over `[]byte` is. +`JsonSearch` takes `json.RawMessage` itself, or an alias of it (`type Doc = json.RawMessage`), and refuses a type defined over it (`type Doc json.RawMessage`): `stashgen` knows a document by the `json.RawMessage` type, where a scalar type is accepted with any type defined over it. The type takes options in parentheses: `ordered=` names the paths whose numbers and strings carry an ordering term, separated by `;`, and `sealed_root=false` turns off the sealed root (on by default). `encrypt_into=JsonSearch(ordered=$.age;$.visits[*].at)` orders two paths. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index df5410e40..860e8d309 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -302,7 +302,7 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { } _ = checker.Close() source := func(fields string) string { - return "package users\n\nimport \"encoding/json\"\n\nvar _ json.RawMessage\n\ntype Customer struct {\n\t_ struct{} `stash:\"context=customers\"`\n\tID int64 `stash:\"id,passthrough\"`\n" + fields + "\n}\n" + return "package users\n\nimport \"encoding/json\"\n\ntype Doc json.RawMessage\n\ntype DocAlias = json.RawMessage\n\ntype Customer struct {\n\t_ struct{} `stash:\"context=customers\"`\n\tID int64 `stash:\"id,passthrough\"`\n" + fields + "\n}\n" } generate := func(fields string) (string, string, int) { dir := writeModule(t, map[string]string{"model.go": source(fields)}) @@ -347,6 +347,12 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { t.Errorf("the generated file lacks %q", want) } } + // An alias of json.RawMessage is the type itself, and is a document; a + // type defined over it is refused below. + written, stderr, code = generate("\tProfile DocAlias `stash:\"profile,encrypt_into=JsonSearch\"`") + if code != 0 || !strings.Contains(written, "Profile eql.JSONSearch") { + t.Fatalf("an alias of json.RawMessage: exit %d\n%s", code, stderr) + } // An engine that answers no ordering on a field whose tag orders paths // disagrees with the generator: refused by name, not written as a field // with no At. @@ -371,6 +377,7 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { {"two paths, one accessor", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a_b;$.aB)\"`", "both name the accessor At.AB"}, {"the same ordered path twice", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a;$.a)\"`", "the ordered path $.a is given twice"}, {"a path with no accessor name", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.__)\"`", "no Go accessor name"}, + {"a type defined over json.RawMessage", "\tProfile Doc `stash:\"profile,encrypt_into=JsonSearch\"`", "Customer.Profile: JsonSearch seals a json document: declare the field json.RawMessage"}, {"a string into a document", "\tProfile string `stash:\"profile,encrypt_into=JsonSearch\"`", "JsonSearch seals a json"}, {"options on a type that takes none", "\tEmail string `stash:\"email,encrypt_into=TextEq(ordered=$.a)\"`", "the engine refuses the declaration"}, } { diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 72ec0be90..96343c0ca 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -595,7 +595,9 @@ column. `JsonSearch` stores a whole JSON document in one `public.eql_v4_json_search` column, from a `json.RawMessage` field (`stash:"profile,encrypt_into=JsonSearch(ordered=$.age)"`; -see the stashgen README). One entry per node is encrypted, so the database +see the stashgen README). Unlike the types above, a type defined over +`json.RawMessage` (`type Doc json.RawMessage`) is refused; an alias of it +(`type Doc = json.RawMessage`) is accepted. One entry per node is encrypted, so the database can test containment, equality and ordering at a path, return one node with `->`, and walk a subtree, without decrypting: From 726e3a3b6eda4aa07036c88bccf5fe93d916c8c7 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:16:11 +1100 Subject: [PATCH 12/13] docs(go): say what the JSON read types print and refuse to bind The encrypt README's JSON section said how the selector, token and token list bind, but not that a zero one is refused, nor that the read types print and log redacted while JSONValue prints the plaintext it holds. A reader deciding what is safe to log needs both, and the README is where they look before the godoc. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/encrypt/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 96343c0ca..2376a2db5 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -617,7 +617,12 @@ path and the cipher's keyset, and refuses a row that is not one it asked for. It does not prove the database returned every row, or rows of the database row you meant. `encrypt.JSONTokens` binds as a `text[]` literal, and a selector or token as `text`; cast each parameter, as above, so -PostgreSQL picks EQL's operator and not `jsonb`'s. Numbers order as `float64`, +PostgreSQL picks EQL's operator and not `jsonb`'s; a zero one is refused +rather than bound. A token is a capability over its node's subtree in every +row of the column, and a path's keys can be your customer's data, so +`JSONToken`, `JSONTokens`, `JSONSelection` and `JSONRead` print and log with +the tokens, the selector and the path redacted. A `JSONValue` holds the +plaintext a read opened, and prints it. Numbers order as `float64`, so integers past 2^53 can tie at an ordered path; containment and `EqAt` compare them exactly. From 955b9a1f5f0f953d6e86020a0c003e1daac105a7 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:51:59 +1100 Subject: [PATCH 13/13] fix(go): write orderings at a wildcard path over every match A JSON document's entries are now stored in a random order (#1379), so `col -> sel` at a wildcard path, and an ordering through it, sees an unspecified element of each row rather than the first. The At accessors stay: their operands are what a comparison over every match takes, and the engine stores ordering terms at a wildcard path when asked to, so refusing ordered= there would leave those terms unreachable from Go. What changes is the SQL they are documented and tested with. - The generated XPath comment gives both forms: col -> Selector() at a path with no array step, and at a wildcard path EXISTS over eql_v4.json_tree_path_query for a comparison and the least or greatest match's ordering term for ORDER BY. Lt, Lte, Gt, Gte and Between get doc comments of their own, which they lacked, and Selector says which form it binds into. The READMEs and the CHANGELOG say the same. - TestJSONDocumentsQueryAndReadInPostgres compares at $.visits[*].at with EXISTS (some visit > 2, and some visit < 2, which only Alice's second visit meets), orders rows by their least visit ascending and greatest descending against the plaintexts' order, and reads a wildcard past its bound from PostgreSQL's own rows, which Open refuses as ErrEncoding. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 4 +- languages/golang/cmd/stashgen/README.md | 12 +++ languages/golang/encrypt/README.md | 5 +- .../internal/testusers/customer_stash.go | 38 +++++++-- .../golang/encrypt/json_postgres_test.go | 80 ++++++++++++++----- languages/golang/stashgen/emit.go | 8 +- 6 files changed, 117 insertions(+), 30 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 22e73c11d..cdcbb0763 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -19,7 +19,9 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. `*json.RawMessage` field takes `null=sql`. Its `Fields` entry has `Contains`, `ContainedBy`, `EqAt`, `Selector`, `Subtree`, `Children` and `ChildCount`, and `At.` per ordered path with `Lt`, `Lte`, `Gt`, - `Gte`, `Between` and the path's own `Selector`. `encrypt.JSONSelection` + `Gte`, `Between` and the path's own `Selector`; at a wildcard path their + operands are compared over every match (`eql_v4.json_tree_path_query`), + since `->` there returns an unspecified element. `encrypt.JSONSelection` and `encrypt.JSONRead` open the rows `->` and `eql_v4.json_tree_subtree` return against the request that prepared them (`OpenOne` refuses a wildcard read up front); `encrypt.JSONSelector`, `JSONToken` and diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index d5585c9b6..45d473b53 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -137,6 +137,18 @@ The field's `Fields` entry has a method per operation: `At` has one accessor per ordered path, named for its keys (`$.visits[*].at` is `At.VisitsAt`, `$.a.1` is `At.A1`); two paths with one name, or one path given twice, stop `stashgen`. An ordering compares the value `->` extracts at its path, and EQL's `<` compares ordering terms alone, so take the selector from the same accessor (`At.Age.Selector`) rather than spelling the path again. +At a wildcard path (`$.visits[*].at`), `->` returns an unspecified element of each row (the entries are stored in a random order), so write the comparison over every match and order by the least or greatest one: + +```sql +-- rows with some visit after $2 +SELECT id FROM customers WHERE EXISTS ( + SELECT 1 FROM eql_v4.json_tree_path_query(profile, $1::text) AS e WHERE e > $2::eql_v4.query_json_leaf) +-- rows by their earliest visit +SELECT id FROM customers ORDER BY ( + SELECT eql_v4.ord_term(e) FROM eql_v4.json_tree_path_query(profile, $1::text) AS e ORDER BY 1 LIMIT 1) +``` + +with `$1` from `At.VisitsAt.Selector(ctx, c)` and `$2` from `At.VisitsAt.Gt(ctx, c, v)`; for the latest visit, order by `ORDER BY 1 DESC NULLS LAST` inside and `DESC` outside. `OpenOne` reads one value and refuses a read whose path has a wildcard before anything is opened; open a wildcard read with `Open`. The needle and the values are `json.RawMessage`: write the JSON you mean, such as `json.RawMessage(`+"`"+`{"active":true}`+"`"+`)`. A position in a path is refused for `EqAt` and `Selector`, whose selector reads every position as the wildcard; read a position with `Subtree`. diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 2376a2db5..80261ab83 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -624,7 +624,10 @@ row of the column, and a path's keys can be your customer's data, so the tokens, the selector and the path redacted. A `JSONValue` holds the plaintext a read opened, and prints it. Numbers order as `float64`, so integers past 2^53 can tie at an ordered path; containment and `EqAt` -compare them exactly. +compare them exactly. At a wildcard path, `->` returns an unspecified +element of each row, so an ordering over every element is written over +`eql_v4.json_tree_path_query(col, $sel)`, its matches (the stashgen README +shows the SQL). ## Building diff --git a/languages/golang/encrypt/internal/testusers/customer_stash.go b/languages/golang/encrypt/internal/testusers/customer_stash.go index 86bb5579d..20acbf6f9 100644 --- a/languages/golang/encrypt/internal/testusers/customer_stash.go +++ b/languages/golang/encrypt/internal/testusers/customer_stash.go @@ -192,38 +192,62 @@ type CustomerProfileFieldAt struct { VisitsAt CustomerProfileFieldPath // $.visits[*].at } -// CustomerProfileFieldPath is one ordered path of Profile: its orderings -// compare the first value `->` extracts there, by the leaf's ordering term. -// Numbers order as float64, so integers past 2^53 can tie; text orders case- -// and accent-folded. +// CustomerProfileFieldPath is one ordered path of Profile. Its orderings derive +// the leaf operand (eql_v4.query_json_leaf) a stored leaf's ordering term is +// compared with, and Selector the path's selector. At a path with no array +// step, compare col -> Selector() with the operand, and order by +// eql_v4.ord_term(col -> Selector()). At a wildcard path ([*]) col -> +// Selector() is an unspecified element of each row, so write the comparison +// over every match, EXISTS (SELECT 1 FROM eql_v4.json_tree_path_query(col, +// $sel) e WHERE e < $1::eql_v4.query_json_leaf), and order by the least or +// greatest match's term: (SELECT eql_v4.ord_term(e) FROM +// eql_v4.json_tree_path_query(col, $sel) e ORDER BY 1 LIMIT 1). Numbers order +// as float64, so integers past 2^53 can tie; text orders case- and +// accent-folded. type CustomerProfileFieldPath struct { field gensupport.Field[json.RawMessage] path string } +// Lt derives the operand that finds a leaf at this path less than v: (col -> +// $sel) < $1::eql_v4.query_json_leaf, or at a wildcard path the same comparison +// over every match (see CustomerProfileFieldPath). func (p CustomerProfileFieldPath) Lt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "lt_at", p.path, v) } +// Lte derives the operand that finds a leaf at this path at most v: (col -> +// $sel) <= $1::eql_v4.query_json_leaf, or at a wildcard path the same +// comparison over every match (see CustomerProfileFieldPath). func (p CustomerProfileFieldPath) Lte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "lte_at", p.path, v) } +// Gt derives the operand that finds a leaf at this path greater than v: (col -> +// $sel) > $1::eql_v4.query_json_leaf, or at a wildcard path the same comparison +// over every match (see CustomerProfileFieldPath). func (p CustomerProfileFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "gt_at", p.path, v) } +// Gte derives the operand that finds a leaf at this path at least v: (col -> +// $sel) >= $1::eql_v4.query_json_leaf, or at a wildcard path the same +// comparison over every match (see CustomerProfileFieldPath). func (p CustomerProfileFieldPath) Gte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "gte_at", p.path, v) } +// Between derives the two operands that find a leaf at this path from lo to hi, +// inclusive: (col -> $sel) >= Low AND (col -> $sel) <= High, or at a wildcard +// path the same over every match, with both bounds on one element (see +// CustomerProfileFieldPath). func (p CustomerProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.JSONSearchLeafQuery], error) { return gensupport.JSONBetweenAt[eql.JSONSearchLeafQuery](ctx, p.field, c, p.path, lo, hi) } -// Selector prepares `->` at this path: the value an ordering here compares is -// col -> Selector(), so bind it beside the ordering's operand, from the same -// path. +// Selector prepares `->` at this path: bind Selector() beside the ordering's +// operand, from the same path, as col -> $sel, or at a wildcard path as +// eql_v4.json_tree_path_query(col, $sel), whose matches are every element. func (p CustomerProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.JSONEntry], error) { return gensupport.JSONSelect[eql.JSONEntry](ctx, p.field, c, p.path) } diff --git a/languages/golang/encrypt/json_postgres_test.go b/languages/golang/encrypt/json_postgres_test.go index 25cdb0370..f9e2a130a 100644 --- a/languages/golang/encrypt/json_postgres_test.go +++ b/languages/golang/encrypt/json_postgres_test.go @@ -5,6 +5,8 @@ import ( "context" "database/sql" "encoding/json" + "errors" + "fmt" "os" "slices" "testing" @@ -155,7 +157,16 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { if err != nil { t.Fatal(err) } - want("first visit > 2", ids(`SELECT id FROM customers WHERE (profile -> $1::text) > $2::eql_v4.query_json_leaf ORDER BY id`, at.Selector(), gt), 1, 2) + // At a wildcard path `->` sees an unspecified element of each row, so a + // comparison over the elements is written over every match. + some := `SELECT id FROM customers WHERE EXISTS (SELECT 1 FROM eql_v4.json_tree_path_query(profile, $1::text) AS e WHERE e %s $2::eql_v4.query_json_leaf) ORDER BY id` + want("some visit > 2", ids(fmt.Sprintf(some, ">"), at.Selector(), gt), 1, 2) + lt2, err := f.At.VisitsAt.Lt(ctx, cipher, json.RawMessage(`2`)) + if err != nil { + t.Fatal(err) + } + // Alice's second visit, at 1, is the only one before 2. + want("some visit < 2", ids(fmt.Sprintf(some, "<"), at.Selector(), lt2), 1) // `->` opened against the selection that asked for it: each row's // value, and no value where the row has none. @@ -248,6 +259,29 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } } + // A wildcard read of an array longer than its bound is refused, not + // opened to the elements below the bound: Alice has two visits. + short, err := f.Subtree(ctx, cipher, "$.visits[*].at", encrypt.JSONReadOptions{WildcardBound: 1}) + if err != nil { + t.Fatal(err) + } + var shortRows []eql.JSONEntry + sr, err := conn.QueryContext(ctx, `SELECT e FROM customers, eql_v4.json_tree_subtree(profile, $1::text[]) AS e WHERE id = 1`, short.Tokens()) + if err != nil { + t.Fatal(err) + } + for sr.Next() { + var e eql.JSONEntry + if err := sr.Scan(&e); err != nil { + t.Fatal(err) + } + shortRows = append(shortRows, e) + } + _ = sr.Close() + if _, err := short.Open(ctx, [][]eql.JSONEntry{shortRows}); !errors.Is(err, encrypt.ErrEncoding) { + t.Errorf("a wildcard read past its bound: %v, want ErrEncoding", err) + } + // A concrete read of a container, opened to one value or none. visits, err := f.Subtree(ctx, cipher, "$.visits", encrypt.JSONReadOptions{}) if err != nil { @@ -309,10 +343,12 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { want("children rows", ids(`SELECT id FROM customers, eql_v4.json_tree_children(profile, $1::text) AS e ORDER BY id`, children), 1, 1, 2) // Sorting by an ordered path, in the form EQL documents: - // ORDER BY eql_v4.ord_term(col -> sel). Ordering by the entry itself - // would sort by its ciphertext. Two more rows give each path a tie, - // broken by id, and Eve's visits (first 2, last 9) tell the first match - // `->` takes at the wildcard path from the least or greatest element. + // ORDER BY eql_v4.ord_term(col -> sel) at a path with no array step. + // Ordering by the entry itself would sort by its ciphertext. At the + // wildcard path `->` sees an unspecified element, so a row is ordered by + // its least match's term ascending and its greatest descending, over + // json_tree_path_query. Two more rows give each path a tie, broken by + // id, and Eve's visits (2 and 9) tell the least from the greatest. more := []testusers.Customer{ {ID: 5, Profile: json.RawMessage(`{"name":"Eve","age":34,"visits":[{"at":2},{"at":9}]}`)}, {ID: 6, Profile: json.RawMessage(`{"name":"Fay","age":7,"visits":[{"at":3}]}`)}, @@ -331,13 +367,17 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } `json:"visits"` } ageOf := func(p plain) *float64 { return p.Age } - firstAtOf := func(p plain) *float64 { - for _, v := range p.Visits { - if v.At != nil { - return v.At + // The least or greatest `at` of a row's visits, nil for none. + atOf := func(greatest bool) func(plain) *float64 { + return func(p plain) *float64 { + var out *float64 + for _, v := range p.Visits { + if v.At != nil && (out == nil || (*v.At > *out) == greatest) { + out = v.At + } } + return out } - return nil } // The ids in the order PostgreSQL's defaults put the plaintexts in: a // row with no value at the path has a NULL term, which sorts as the @@ -380,23 +420,27 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } return out } + byTerm := `SELECT id FROM customers ORDER BY eql_v4.ord_term(profile -> $1::text) %[1]s, id` + byMatch := `SELECT id FROM customers ORDER BY (SELECT eql_v4.ord_term(e) FROM eql_v4.json_tree_path_query(profile, $1::text) AS e ORDER BY 1 %[1]s NULLS LAST LIMIT 1) %[1]s, id` for _, tc := range []struct { name string + sql string selector encrypt.JSONSelector - key func(plain) *float64 + ascKey func(plain) *float64 + descKey func(plain) *float64 asc, desc []int64 // what plaintextOrder gives, written out }{ // Bob and Fay tie at 7, Alice and Eve at 34; Dan has no age. - {"$.age", age.Selector(), ageOf, []int64{2, 6, 1, 5, 3, 4}, []int64{4, 3, 1, 5, 2, 6}}, - // First visits: Eve 2, Alice and Fay 3, Bob 5; Carol's visits are - // empty and Dan has none. - {"$.visits[*].at", at.Selector(), firstAtOf, []int64{5, 1, 6, 2, 3, 4}, []int64{3, 4, 2, 1, 6, 5}}, + {"$.age", byTerm, age.Selector(), ageOf, ageOf, []int64{2, 6, 1, 5, 3, 4}, []int64{4, 3, 1, 5, 2, 6}}, + // Least visits: Alice 1, Eve 2, Fay 3, Bob 5; greatest: Eve 9, + // Bob 5, Alice and Fay 3. Carol's visits are empty and Dan has none. + {"$.visits[*].at", byMatch, at.Selector(), atOf(false), atOf(true), []int64{1, 5, 6, 2, 3, 4}, []int64{3, 4, 5, 2, 1, 6}}, } { - asc, desc := plaintextOrder(tc.key, false), plaintextOrder(tc.key, true) + asc, desc := plaintextOrder(tc.ascKey, false), plaintextOrder(tc.descKey, true) want("plaintext order by "+tc.name, asc, tc.asc...) want("plaintext order by "+tc.name+" DESC", desc, tc.desc...) - want("ORDER BY "+tc.name, ids(`SELECT id FROM customers ORDER BY eql_v4.ord_term(profile -> $1::text), id`, tc.selector), asc...) - want("ORDER BY "+tc.name+" DESC", ids(`SELECT id FROM customers ORDER BY eql_v4.ord_term(profile -> $1::text) DESC, id`, tc.selector), desc...) + want("ORDER BY "+tc.name, ids(fmt.Sprintf(tc.sql, "ASC"), tc.selector), asc...) + want("ORDER BY "+tc.name+" DESC", ids(fmt.Sprintf(tc.sql, "DESC"), tc.selector), desc...) } } diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index f31fcdb42..ce34e5ca6 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -1182,29 +1182,31 @@ func (w *writer) jsonEntry(g genField) { } w.p("}") w.nl() - w.b.WriteString(wrapComment(fmt.Sprintf("%sPath is one ordered path of %s: its orderings compare the first value `->` extracts there, by the leaf's ordering term. Numbers order as float64, so integers past 2^53 can tie; text orders case- and accent-folded.", g.fieldType, g.GoName), 80)) + w.b.WriteString(wrapComment(fmt.Sprintf("%sPath is one ordered path of %s. Its orderings derive the leaf operand (eql_v4.query_json_leaf) a stored leaf's ordering term is compared with, and Selector the path's selector. At a path with no array step, compare col -> Selector() with the operand, and order by eql_v4.ord_term(col -> Selector()). At a wildcard path ([*]) col -> Selector() is an unspecified element of each row, so write the comparison over every match, EXISTS (SELECT 1 FROM eql_v4.json_tree_path_query(col, $sel) e WHERE e < $1::eql_v4.query_json_leaf), and order by the least or greatest match's term: (SELECT eql_v4.ord_term(e) FROM eql_v4.json_tree_path_query(col, $sel) e ORDER BY 1 LIMIT 1). Numbers order as float64, so integers past 2^53 can tie; text orders case- and accent-folded.", g.fieldType, g.GoName), 80)) w.p("type %sPath struct {", g.fieldType) w.p("\tfield %s", field) w.p("\tpath string") w.p("}") - for _, op := range []struct{ name, method string }{{"lt_at", "Lt"}, {"lte_at", "Lte"}, {"gt_at", "Gt"}, {"gte_at", "Gte"}} { + for _, op := range []struct{ name, method, sign, words string }{{"lt_at", "Lt", "<", "less than"}, {"lte_at", "Lte", "<=", "at most"}, {"gt_at", "Gt", ">", "greater than"}, {"gte_at", "Gte", ">=", "at least"}} { if !answered[op.name] { continue } w.nl() + w.b.WriteString(wrapComment(fmt.Sprintf("%s derives the operand that finds a leaf at this path %s v: (col -> $sel) %s $1::eql_v4.query_json_leaf, or at a wildcard path the same comparison over every match (see %sPath).", op.method, op.words, op.sign, g.fieldType), 80)) w.p("func (p %sPath) %s(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (%s, error) {", g.fieldType, op.method, leaf) w.p("\treturn gensupport.JSONQueryAt[%s](ctx, p.field, c, %q, p.path, v)", leaf, op.name) w.p("}") } if answered["between_at"] { w.nl() + w.b.WriteString(wrapComment(fmt.Sprintf("Between derives the two operands that find a leaf at this path from lo to hi, inclusive: (col -> $sel) >= Low AND (col -> $sel) <= High, or at a wildcard path the same over every match, with both bounds on one element (see %sPath).", g.fieldType), 80)) w.p("func (p %sPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[%s], error) {", g.fieldType, leaf) w.p("\treturn gensupport.JSONBetweenAt[%s](ctx, p.field, c, p.path, lo, hi)", leaf) w.p("}") } if answered["selector"] { w.nl() - w.b.WriteString(wrapComment("Selector prepares `->` at this path: the value an ordering here compares is col -> Selector(), so bind it beside the ordering's operand, from the same path.", 80)) + w.b.WriteString(wrapComment("Selector prepares `->` at this path: bind Selector() beside the ordering's operand, from the same path, as col -> $sel, or at a wildcard path as eql_v4.json_tree_path_query(col, $sel), whose matches are every element.", 80)) w.p("func (p %sPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[%s], error) {", g.fieldType, entry) w.p("\treturn gensupport.JSONSelect[%s](ctx, p.field, c, p.path)", entry) w.p("}")