From 4a35f6f5ea4b74ffe8d046f6dfa77ce9de4b455b Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 00:00:09 +1100 Subject: [PATCH 1/4] feat(go): seal an opaque struct as a vitaminc storage blob An opaque struct is one column holding the whole struct. It crossed the binding as the JSON document encoding/json wrote: bytes that are kept, in a format nothing versions. vitaminc#520/#526 add the storage profile, a versioned blob format for kept values that Go and Rust read alike, so gensupport now writes the shape struct with vcffi.MarshalStorage. Rows sealed by earlier releases still open: Opaque reads a value that begins with the storage header ("VCS") as a blob and anything else as the JSON it used to be. A blob of a later storage version is refused as such (vcffi.ErrUnsupportedVersion), never misread as JSON. The generated shape struct carries `vc` tags beside its `json` ones, and stashgen checks a field against both formats: notStorable refuses a type that writes itself only through MarshalJSON over a struct with unexported fields (a blob would drop them) and a struct with no exported field. The blob writes a float by its bits, so NaN and the infinities now seal; a nil slice or map comes back empty. The CHANGELOG lists both under Breaking. This is kept apart from the JSON-tree work: the storage profile is for opaque values, and json_tree's encrypted nodes and EQL envelope are a separate, already frozen format. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 13 ++++ languages/golang/cmd/stashgen/README.md | 12 ++-- languages/golang/encrypt/ciphertext.go | 2 +- languages/golang/encrypt/doc.go | 5 +- languages/golang/encrypt/gensupport/codec.go | 43 ++++++++---- .../golang/encrypt/gensupport/declaration.go | 9 +-- .../gensupport/gensupport_internal_test.go | 66 ++++++++++++++++--- .../internal/testusers/document_stash.go | 11 ++-- .../internal/testusers/everything_stash.go | 61 ++++++++--------- .../encrypt/internal/testusers/kinds.go | 5 +- languages/golang/encrypt/kinds_test.go | 35 ++++++---- languages/golang/stashgen/emit.go | 10 +-- languages/golang/stashgen/engine.go | 2 +- languages/golang/stashgen/read.go | 57 ++++++++++++++-- languages/golang/stashgen/refusal_test.go | 2 + .../cases/documents/document_stash.go.golden | 11 ++-- 16 files changed, 250 insertions(+), 94 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 6a2a4f4d2..ea21c5ed1 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -42,6 +42,19 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. ### Breaking +- **An opaque struct seals as a storage blob, and reads the JSON it used + to (#1347).** `stash:"context=...,opaque"` now seals the struct as one + blob in vitaminc's storage profile, version 1 (`vcffi.MarshalStorage`), + the format vitaminc keeps readable across releases and languages, where + it sealed the JSON document `encoding/json` wrote. Rows sealed before + still decrypt: `Decrypt` reads a value that is not a blob as JSON. Run + stashgen again: the generated shape struct gains `vc` tags. What + changes for a program: a NaN or infinite float now seals (JSON refused + it with `ErrEncoding`); a nil slice or map comes back empty, not nil, + from a blob; and `stashgen` refuses a field type that only + `MarshalJSON` carries over a struct with unexported fields, which a + blob would write without them. This takes vcffi from the unreleased + cipherstash/vitaminc#526 by pseudo-version until vitaminc releases it. - **Every order term changes: re-derive stored `ore`, `ope` and `ore_block` index terms.** The engine moved to vitaminc 0.8, clww-ore 0.6 (formerly cllw-ore) and ore-rs 0.10, and both order schemes now derive diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 91a563489..8a20c7b2e 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -107,7 +107,7 @@ An EQL `TextMatch` field is stricter: see below. An index takes its options in parentheses after its name, separated by commas: `index=equality;match(k=3)`. This build refuses an index with options: a query term is derived with the default options only, so a stored term with other options would never match one. These words are the same as the Rust API's words for the same behaviour, with two that only Go has. -`opaque` seals the whole struct as one `bytes` field holding a JSON document (see "What crosses the binding"). +`opaque` seals the whole struct as one `bytes` field holding a storage blob (see "What crosses the binding"). `json` names the JSON index, which the Rust API does not declare yet; this build refuses it. ### Optional fields @@ -227,7 +227,7 @@ The error names the type and the field, and never a value. - an embedded struct from another package with no tag; - a struct whose every field is left out; - a struct, slice or map field with `encrypt` or `index=` outside an opaque struct; -- a field of an opaque struct whose type JSON cannot carry both ways. +- a field of an opaque struct whose type the storage profile or JSON cannot carry both ways. ## Declarations from a policy @@ -273,8 +273,12 @@ No warning, error or log line holds a plaintext value. Generated code sends the engine every sealed field with its value, under the declaration lowered to data: each field's label (`/`), its outputs and its wire type (`int64`, `string`, `bytes`, ...), which `stashgen` chose from the field's Go type. A passthrough field stays on the host: the engine does nothing to it a program could observe, and the FFI codec cannot carry every Go type a program stores beside a ciphertext (a `driver.Valuer`, a `sql.NullTime`). -An `opaque` struct crosses as one JSON document and is one column, decoded back into the exact Go types the struct declares; a field may be any type `encoding/json` carries both ways (a scalar or a type defined over one, `[]byte`, slices, maps with string or integer keys, pointers, structs whose fields are all exported, `time.Time`). -A float in an opaque struct must be finite: `encoding/json` refuses NaN and the infinities, so `Encrypt` fails for the batch with `encrypt.ErrEncoding`. A sealed float field outside an opaque struct carries them. +An `opaque` struct crosses as one storage blob and is one column, read back into the exact Go types the struct declares. +The blob is vitaminc's storage profile, version 1 (`vcffi.MarshalStorage`), the format vitaminc keeps readable across releases and languages. +A field may be any type that both the storage profile and `encoding/json` carry both ways: a scalar or a type defined over one, `[]byte`, slices, maps with string or integer keys, pointers, structs whose fields are all exported, and types with `MarshalText` and `UnmarshalText`, such as `time.Time`. +`encoding/json` still matters because a row an earlier release sealed holds a JSON document, and `Decrypt` reads it as JSON. +A float in an opaque struct may be NaN or an infinity: the blob writes a float by its bits. +A nil slice or map in an opaque struct comes back empty, not nil, from a blob; from an earlier release's JSON document it comes back nil. A sealed field outside an opaque struct is one scalar, a type defined over one, or a pointer to either (see "Optional fields"); a struct, slice or map field seals only inside an opaque struct. The scalars are `string`, `[]byte`, `bool`, every integer type (each sealed at its own width, `int` and `uint` as 64-bit), `encrypt.Int128` and `encrypt.Uint128`, `float32`, `float64`, `encrypt.Decimal`, `encrypt.Date` and `time.Time` (a timestamp). The wire type `stashgen` writes for each is the field's own: `int8` is `gensupport.Int8`, `time.Time` is `gensupport.Timestamp`, and so on. A decimal orders by value, so `1.5` ties `1.50`; a `time.Time` orders by the microsecond, so instants within one microsecond tie. diff --git a/languages/golang/encrypt/ciphertext.go b/languages/golang/encrypt/ciphertext.go index 77816f727..0ddca58aa 100644 --- a/languages/golang/encrypt/ciphertext.go +++ b/languages/golang/encrypt/ciphertext.go @@ -19,7 +19,7 @@ import ( // null as long as a value or stored as SQL NULL — and seals as vitaminc's // tagged leaf, which a // Rust record opens when its field is a Value (not a bare u32 or String). A struct, slice or map seals only as part of an opaque struct, -// which crosses as one JSON document and is one leaf; stashgen refuses it +// which crosses as one storage blob and is one leaf; stashgen refuses it // anywhere else, because the engine would seal it as a tree of leaves and a // column holds one. type Ciphertext []byte diff --git a/languages/golang/encrypt/doc.go b/languages/golang/encrypt/doc.go index 04a9a4349..2f9c1e509 100644 --- a/languages/golang/encrypt/doc.go +++ b/languages/golang/encrypt/doc.go @@ -131,8 +131,9 @@ // Passthrough fields never cross the binding: the engine does nothing to a // passthrough value that a program could observe, and the FFI codec cannot // carry every Go type a program stores beside a ciphertext (a -// driver.Valuer, a sql.NullTime). An opaque struct crosses as one JSON -// document and is one column. +// driver.Valuer, a sql.NullTime). An opaque struct crosses as one storage +// blob (vitaminc's storage profile, vcffi.MarshalStorage) and is one column; +// a row an earlier release sealed as a JSON document still opens. // // # Terms // diff --git a/languages/golang/encrypt/gensupport/codec.go b/languages/golang/encrypt/gensupport/codec.go index 657458264..ab6aee541 100644 --- a/languages/golang/encrypt/gensupport/codec.go +++ b/languages/golang/encrypt/gensupport/codec.go @@ -1,6 +1,7 @@ package gensupport import ( + "bytes" "context" "encoding/json" "fmt" @@ -8,6 +9,7 @@ import ( "github.com/cipherstash/stack/languages/golang/encrypt" "github.com/cipherstash/stack/languages/golang/internal/record" + "github.com/cipherstash/vitaminc/bindings/go/vcffi" ) // Values is a struct's field values by declared name: what Source gives and @@ -377,23 +379,32 @@ func (c *RecordsCodec[P, R]) Decrypt(ctx context.Context, d encrypt.Decrypter, r } // opaqueBytes is an opaque struct's fields as the one value the engine -// seals: a JSON document of the generated shape struct, so the struct is one -// column and every field type encoding/json round-trips comes back as it -// was. +// seals: a storage blob (vitaminc's storage profile, version 1, written by +// vcffi.MarshalStorage) of the generated shape struct, so the struct is one +// column and every field comes back at the Go type the struct declares. The +// storage profile is the format for bytes that are kept: unlike the +// transport, its version 1 is a compatibility commitment shared with Rust, +// and it carries every float, NaN and the infinities included. func opaqueBytes(fields any) ([]byte, error) { - encoded, err := json.Marshal(fields) + encoded, err := vcffi.MarshalStorage(fields) if err != nil { - // encoding/json refuses NaN and the infinities, which a sealed float - // field outside an opaque struct accepts. - // Its error is not wrapped: encoding/json's text quotes the value. - return nil, fmt.Errorf("%w: the opaque value does not encode as JSON (NaN, an infinity, or a type encoding/json refuses)", encrypt.ErrEncoding) + // Not wrapped: the reason is the kind of value and where, and a + // value is never quoted, but the opaque struct is plaintext and + // nothing here needs more than the class. + return nil, fmt.Errorf("%w: the opaque value cannot be written as a storage blob (a type the storage profile has no form for, a repeated map key, or text that is not UTF-8)", encrypt.ErrEncoding) } return encoded, nil } -// Opaque reads an opened opaque value into the generated shape struct: the -// JSON document the engine returned, decoded into the exact Go types the -// struct declares. Generated code calls it from Value. +// Opaque reads an opened opaque value into the generated shape struct, at +// the exact Go types the struct declares. Generated code calls it from +// Value. +// +// The value is a storage blob, which begins "VCS". A value an earlier +// release sealed is the JSON document encoding/json wrote, which begins +// "{": it is read as JSON, so stored rows open without being encrypted +// again. Each is read by its own rules: a storage blob gives back an empty +// slice or map where the struct held nil, and JSON gives back nil. func Opaque[T any](vals Values, out *T) error { v, ok := vals[OpaqueField] if !ok { @@ -403,6 +414,16 @@ func Opaque[T any](vals Values, out *T) error { if !ok { return fmt.Errorf("gensupport: %w: the opaque value opened as %T, not bytes", encrypt.ErrEncoding, v) } + if bytes.HasPrefix(encoded, vcffi.StorageHeader[:3]) { + if err := vcffi.UnmarshalStorage(encoded, out); err != nil { + // The vcffi error names the kind found and where, never the + // value; it is wrapped so errors.Is reads its class + // (vcffi.ErrUnsupportedVersion for a blob a later release + // wrote, say). + return fmt.Errorf("gensupport: %w: the opaque value does not read into a %T: %w", encrypt.ErrEncoding, *out, err) + } + return nil + } if err := json.Unmarshal(encoded, out); err != nil { // Its error is not wrapped: encoding/json's text quotes the value. return fmt.Errorf("gensupport: %w: the opaque value does not decode into a %T", encrypt.ErrEncoding, *out) diff --git a/languages/golang/encrypt/gensupport/declaration.go b/languages/golang/encrypt/gensupport/declaration.go index 679927df9..f110bb8ca 100644 --- a/languages/golang/encrypt/gensupport/declaration.go +++ b/languages/golang/encrypt/gensupport/declaration.go @@ -14,7 +14,7 @@ import ( // derives; every field seals as vitaminc's tagged leaf whatever its kind, // which a Rust record opens when its field is a Value. Every sealed field has a // scalar kind: stashgen refuses a struct, slice or map outside an opaque -// struct, and an opaque struct seals as Bytes (one JSON document). Untyped +// struct, and an opaque struct seals as Bytes (one storage blob). Untyped // names a field with no declared type; generated code never writes it, and // the engine refuses a plan whose indexed field has no type. type Kind string @@ -52,9 +52,10 @@ const ( // OpaqueField is the one field of an opaque declaration: the whole struct, // sealed as one value under /value. The struct crosses the binding -// as one JSON document — the engine seals a composite value as a tree of -// leaves, and an opaque struct is one column — so its field types are what -// JSON carries: scalars, []byte (base64), slices and maps of them. +// as one storage blob (vitaminc's storage profile, version 1) — the engine +// seals a composite value as a tree of leaves, and an opaque struct is one +// column. A value an earlier release sealed is a JSON document, which +// [Opaque] still reads. const OpaqueField = "value" type verb uint8 diff --git a/languages/golang/encrypt/gensupport/gensupport_internal_test.go b/languages/golang/encrypt/gensupport/gensupport_internal_test.go index 09cc9bd1d..0c2655799 100644 --- a/languages/golang/encrypt/gensupport/gensupport_internal_test.go +++ b/languages/golang/encrypt/gensupport/gensupport_internal_test.go @@ -1,6 +1,7 @@ package gensupport import ( + "bytes" "context" "encoding/json" "errors" @@ -148,21 +149,38 @@ func TestConvertStaysWithinAFamily(t *testing.T) { if err := convert(obj, &x); err == nil { t.Fatal("a struct target was accepted") } - // An opaque value decodes straight into the generated shape type. + // An opaque value is a storage blob of the generated shape type, and + // decodes straight into it. type shape struct { - Title string `json:"title"` - N int32 `json:"n"` - Tags []string `json:"tags"` - Inner map[string]string `json:"inner"` + Title string `vc:"title" json:"title"` + N int32 `vc:"n" json:"n"` + Tags []string `vc:"tags" json:"tags"` + Inner map[string]string `vc:"inner" json:"inner"` } encoded, err := opaqueBytes(shape{Title: "x", N: 4, Tags: []string{"a"}, Inner: map[string]string{"k": "v"}}) if err != nil { t.Fatal(err) } + if !bytes.HasPrefix(encoded, vcffi.StorageHeader[:]) { + t.Fatalf("opaqueBytes wrote % x, not a version 1 storage blob", encoded[:4]) + } var back shape if err := Opaque(Values{OpaqueField: encoded}, &back); err != nil || back.Title != "x" || back.N != 4 || back.Inner["k"] != "v" { t.Fatalf("Opaque: %v %+v", err, back) } + // A value an earlier release sealed is the JSON document it wrote, and + // still opens. + var legacy shape + if err := Opaque(Values{OpaqueField: []byte(`{"title":"x","n":4,"tags":["a"],"inner":{"k":"v"}}`)}, &legacy); err != nil || !reflect.DeepEqual(legacy, back) { + t.Fatalf("Opaque of a JSON document: %v %+v", err, legacy) + } + // A blob of a storage version this release does not read is refused as + // such, not misread as JSON. + future := append([]byte(nil), encoded...) + future[3] = vcffi.StorageVersion + 1 + if err := Opaque(Values{OpaqueField: future}, &back); !errors.Is(err, encrypt.ErrEncoding) || !errors.Is(err, vcffi.ErrUnsupportedVersion) { + t.Fatalf("Opaque of a version %d blob: %v", vcffi.StorageVersion+1, err) + } if err := Opaque(Values{OpaqueField: "not bytes"}, &back); err == nil { t.Fatal("Opaque accepted a string") } @@ -259,10 +277,18 @@ func TestDecryptErrorsHoldNoPlaintext(t *testing.T) { {"json uint", json.Number("-4242"), func(v Values) error { _, err := Get[uint64](v, "f"); return err }, "4242"}, {"json float", json.Number("4242e999"), func(v Values) error { _, err := Get[float64](v, "f"); return err }, "4242"}, {"map key", Values{"secret-key-4242": "x"}, func(v Values) error { _, err := Get[map[string]int](v, "f"); return err }, "4242"}, - {"opaque bytes", []byte(`{"n":4242}`), func(v Values) error { + {"opaque JSON bytes", []byte(`{"n":4242}`), func(v Values) error { var out struct{ N uint8 } return Opaque(Values{OpaqueField: v["f"]}, &out) }, "4242"}, + {"opaque storage bytes", mustStorage(t, struct { + N uint32 `vc:"n"` + }{4242}), func(v Values) error { + var out struct { + N uint8 `vc:"n"` + } + return Opaque(Values{OpaqueField: v["f"]}, &out) + }, "4242"}, } for _, tc := range cases { err := tc.read(Values{"f": tc.value}) @@ -276,11 +302,31 @@ func TestDecryptErrorsHoldNoPlaintext(t *testing.T) { t.Errorf("%s: the error holds the value: %v", tc.name, err) } } - // encoding/json's own error ("json: unsupported value: NaN") quotes the - // value, so it is not wrapped. - if _, err := opaqueBytes(struct{ F float64 }{math.NaN()}); !errors.Is(err, encrypt.ErrEncoding) || strings.Contains(err.Error(), "json:") { - t.Errorf("opaqueBytes NaN: %v", err) + // A value the storage profile has no form for is ErrEncoding, and the + // error names the class, not the value. + if _, err := opaqueBytes(struct{ C chan int }{make(chan int)}); !errors.Is(err, encrypt.ErrEncoding) { + t.Errorf("opaqueBytes chan: %v", err) + } + // The storage profile writes a float by its bits: NaN and the + // infinities, which encoding/json refused, are values like any other. + blob, err := opaqueBytes(struct{ F float64 }{math.NaN()}) + if err != nil { + t.Fatalf("opaqueBytes NaN: %v", err) + } + var nan struct{ F float64 } + if err := Opaque(Values{OpaqueField: blob}, &nan); err != nil || !math.IsNaN(nan.F) { + t.Fatalf("NaN came back as %v, %v", nan.F, err) + } +} + +// mustStorage is v as a storage blob. +func mustStorage(t *testing.T, v any) []byte { + t.Helper() + b, err := vcffi.MarshalStorage(v) + if err != nil { + t.Fatal(err) } + return b } func TestSplitFailsClosedInBothDirections(t *testing.T) { diff --git a/languages/golang/encrypt/internal/testusers/document_stash.go b/languages/golang/encrypt/internal/testusers/document_stash.go index 086fa0941..3c37e969b 100644 --- a/languages/golang/encrypt/internal/testusers/document_stash.go +++ b/languages/golang/encrypt/internal/testusers/document_stash.go @@ -38,12 +38,13 @@ type documentShape struct { Tags []string } -// The opaque value as it crosses the binding: one JSON document of the -// struct's fields, by their declared names. +// The opaque value as it crosses the binding: one storage blob of the +// struct's fields, by their declared names. The json names read a value +// an earlier release sealed as a JSON document. type documentOpaque struct { - Title string `json:"title"` - Body string `json:"body"` - Tags []string `json:"tags"` + Title string `vc:"title" json:"title"` + Body string `vc:"body" json:"body"` + Tags []string `vc:"tags" json:"tags"` } var documentDeclaration = gensupport.DeclareOpaque("documents/v2/body") diff --git a/languages/golang/encrypt/internal/testusers/everything_stash.go b/languages/golang/encrypt/internal/testusers/everything_stash.go index 8be911965..d832eda24 100644 --- a/languages/golang/encrypt/internal/testusers/everything_stash.go +++ b/languages/golang/encrypt/internal/testusers/everything_stash.go @@ -65,37 +65,38 @@ type everythingShape struct { Arr [2]uint8 } -// The opaque value as it crosses the binding: one JSON document of the -// struct's fields, by their declared names. +// The opaque value as it crosses the binding: one storage blob of the +// struct's fields, by their declared names. The json names read a value +// an earlier release sealed as a JSON document. type everythingOpaque struct { - S string `json:"s"` - E Email `json:"e"` - Bo bool `json:"bo"` - I8 int8 `json:"i8"` - I int `json:"i"` - U64 uint64 `json:"u64"` - F32 float32 `json:"f32"` - F64 float64 `json:"f64"` - By []byte `json:"by"` - Bl Blob `json:"bl"` - Sc Score `json:"sc"` - Tags []string `json:"tags"` - Ints []int `json:"ints"` - F32s []float32 `json:"f32s"` - St Status `json:"st"` - D time.Duration `json:"d"` - Emails []Email `json:"emails"` - Counts map[string]int `json:"counts"` - Labels map[string]string `json:"labels"` - ByKey map[int]string `json:"by_key"` - In Inner `json:"in"` - Ins []Inner `json:"ins"` - P *string `json:"p"` - PI *Inner `json:"pi"` - T time.Time `json:"t"` - N sql.NullTime `json:"n"` - Nested [][]byte `json:"nested"` - Arr [2]uint8 `json:"arr"` + S string `vc:"s" json:"s"` + E Email `vc:"e" json:"e"` + Bo bool `vc:"bo" json:"bo"` + I8 int8 `vc:"i8" json:"i8"` + I int `vc:"i" json:"i"` + U64 uint64 `vc:"u64" json:"u64"` + F32 float32 `vc:"f32" json:"f32"` + F64 float64 `vc:"f64" json:"f64"` + By []byte `vc:"by" json:"by"` + Bl Blob `vc:"bl" json:"bl"` + Sc Score `vc:"sc" json:"sc"` + Tags []string `vc:"tags" json:"tags"` + Ints []int `vc:"ints" json:"ints"` + F32s []float32 `vc:"f32s" json:"f32s"` + St Status `vc:"st" json:"st"` + D time.Duration `vc:"d" json:"d"` + Emails []Email `vc:"emails" json:"emails"` + Counts map[string]int `vc:"counts" json:"counts"` + Labels map[string]string `vc:"labels" json:"labels"` + ByKey map[int]string `vc:"by_key" json:"by_key"` + In Inner `vc:"in" json:"in"` + Ins []Inner `vc:"ins" json:"ins"` + P *string `vc:"p" json:"p"` + PI *Inner `vc:"pi" json:"pi"` + T time.Time `vc:"t" json:"t"` + N sql.NullTime `vc:"n" json:"n"` + Nested [][]byte `vc:"nested" json:"nested"` + Arr [2]uint8 `vc:"arr" json:"arr"` } var everythingDeclaration = gensupport.DeclareOpaque("everything") diff --git a/languages/golang/encrypt/internal/testusers/kinds.go b/languages/golang/encrypt/internal/testusers/kinds.go index eab99226b..9a8c2f352 100644 --- a/languages/golang/encrypt/internal/testusers/kinds.go +++ b/languages/golang/encrypt/internal/testusers/kinds.go @@ -108,8 +108,9 @@ type Lookup struct { //go:generate go run github.com/cipherstash/stack/languages/golang/cmd/stashgen -type Everything -name Everything -// Everything is an opaque struct with a field of every type JSON carries -// both ways. It crosses as one document and comes back as it was. +// Everything is an opaque struct with a field of every type the storage +// profile and JSON carry both ways. It crosses as one storage blob and comes +// back as it was, save that a nil slice or map comes back empty. type Everything struct { _ struct{} `stash:"context=everything,opaque"` S string diff --git a/languages/golang/encrypt/kinds_test.go b/languages/golang/encrypt/kinds_test.go index 065f878af..9370d83ee 100644 --- a/languages/golang/encrypt/kinds_test.go +++ b/languages/golang/encrypt/kinds_test.go @@ -148,7 +148,7 @@ func TestEveryOpaqueFieldTypeRoundTrips(t *testing.T) { In: testusers.Inner{Name: "in", N: 5}, Ins: []testusers.Inner{{Name: "i", N: 6}}, P: &hello, PI: &testusers.Inner{Name: "pi", N: 7}, T: when, N: sql.NullTime{Time: when, Valid: true}, Nested: [][]byte{{1}, {2, 3}}, Arr: [2]uint8{9, 8}, }, { - // Zero values, with the slices and maps nil: JSON null comes back as nil. + // Zero values, with the slices and maps nil. }} encrypted, err := testusers.EncryptEverything(ctx, cipher, in) if err != nil { @@ -158,8 +158,14 @@ func TestEveryOpaqueFieldTypeRoundTrips(t *testing.T) { if err != nil { t.Fatal(err) } - if !reflect.DeepEqual(back, in) { - t.Fatalf("DecryptEverything =\n%+v\nwant\n%+v", back, in) + // A storage blob writes a nil slice or map as an empty one, so the zero + // value's come back empty; its nil pointers stay nil. + want := []testusers.Everything{in[0], { + By: []byte{}, Bl: testusers.Blob{}, Tags: []string{}, Ints: []int{}, F32s: []float32{}, Emails: []testusers.Email{}, + Counts: map[string]int{}, Labels: map[string]string{}, ByKey: map[int]string{}, Ins: []testusers.Inner{}, Nested: [][]byte{}, + }} + if !reflect.DeepEqual(back, want) { + t.Fatalf("DecryptEverything =\n%+v\nwant\n%+v", back, want) } } @@ -227,16 +233,23 @@ func TestAnEmptyBatchRoundTrips(t *testing.T) { } } -// encoding/json refuses NaN and the infinities, so an opaque struct holding -// one is an ErrEncoding that names the value, not an unclassified error. -func TestANonFiniteFloatInAnOpaqueStructIsErrEncoding(t *testing.T) { +// A storage blob writes a float by its bits, so an opaque struct holding NaN +// or an infinity, which the JSON document of earlier releases refused, seals +// and comes back as it was. +func TestANonFiniteFloatInAnOpaqueStructRoundTrips(t *testing.T) { c := deterministicClient(t) ctx := context.Background() - for _, v := range []testusers.Everything{{F64: math.NaN()}, {F32: float32(math.Inf(1))}} { - _, err := testusers.EncryptEverything(ctx, c.DefaultKeyset(), []testusers.Everything{v}) - if !errors.Is(err, encrypt.ErrEncoding) || !strings.Contains(err.Error(), "value 0") { - t.Fatalf("err = %v, want ErrEncoding naming value 0", err) - } + in := []testusers.Everything{{F64: math.NaN()}, {F32: float32(math.Inf(1)), F64: math.Inf(-1)}} + encrypted, err := testusers.EncryptEverything(ctx, c.DefaultKeyset(), in) + if err != nil { + t.Fatal(err) + } + back, err := testusers.DecryptEverything(ctx, c, encrypted) + if err != nil { + t.Fatal(err) + } + if !math.IsNaN(back[0].F64) || !math.IsInf(float64(back[1].F32), 1) || !math.IsInf(back[1].F64, -1) { + t.Fatalf("came back as %v, %v, %v", back[0].F64, back[1].F32, back[1].F64) } } diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index 1b080c122..e91a6d6d3 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -167,17 +167,19 @@ func (w *writer) printMethods(f *genFile) { } // opaqueShape writes the struct an opaque value crosses the binding as: the -// struct's fields by their declared names, as one JSON document. +// struct's fields by their declared names, written as one storage blob and +// read from a blob or from the JSON document an earlier release wrote. func (w *writer) opaqueShape(f *genFile) { if !f.decl.Opaque { return } w.nl() - w.p("// The opaque value as it crosses the binding: one JSON document of the") - w.p("// struct's fields, by their declared names.") + w.p("// The opaque value as it crosses the binding: one storage blob of the") + w.p("// struct's fields, by their declared names. The json names read a value") + w.p("// an earlier release sealed as a JSON document.") w.p("type %s struct {", f.opaqueShape) for _, g := range f.opaque { - w.p("\t%s %s `json:%q`", g.GoName, g.typeExpr, g.Name) + w.p("\t%s %s `vc:%q json:%q`", g.GoName, g.typeExpr, g.Name, g.Name) } w.p("}") } diff --git a/languages/golang/stashgen/engine.go b/languages/golang/stashgen/engine.go index 22294ffea..b99a197ae 100644 --- a/languages/golang/stashgen/engine.go +++ b/languages/golang/stashgen/engine.go @@ -184,7 +184,7 @@ func lowerDeclaration(d Declaration, eqlTypes []EQLType) (*record.Plan, error) { plan.Context = segments } if d.Opaque { - // gensupport seals an opaque struct as one JSON document under + // gensupport seals an opaque struct as one storage blob under // /value; see gensupport.OpaqueField. plan.Fields = []record.Field{{Name: "value", Kind: record.Bytes, Outputs: []record.Output{record.Ciphertext}}} return plan, nil diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index ebbfe769b..43e3aa2ce 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -664,7 +664,10 @@ func (r *reader) buildFields(c *collected) error { } g := genField{Field: Field{Name: cf.tag.Name, GoName: cf.goName, GoType: r.goType(cf.typ), Verb: VerbEncrypt}, typeExpr: r.typeExpr(cf.typ)} if reason := notJSONEncodable(cf.typ, map[types.Type]bool{}); reason != "" { - return fieldErr(typeName, cf.goName, "an opaque struct crosses as one JSON document, and %s %s", g.typeExpr, reason) + return fieldErr(typeName, cf.goName, "an opaque struct is read back from the JSON document earlier releases sealed, and %s %s", g.typeExpr, reason) + } + if reason := notStorable(cf.typ, map[types.Type]bool{}); reason != "" { + return fieldErr(typeName, cf.goName, "an opaque struct is sealed as one storage blob, and %s %s", g.typeExpr, reason) } if prev, dup := seen[g.Name]; dup { return fieldErr(typeName, cf.goName, "two fields write the name %q: %s and %s", g.Name, prev, cf.goName) @@ -1051,9 +1054,10 @@ func itOrEach(fields []string) string { } // notJSONEncodable says why encoding/json cannot carry a type both ways, or -// "" when it can: every field of an opaque struct must, because the struct -// crosses the binding as one JSON document and comes back by decoding it -// into the generated shape struct. A type with its own MarshalJSON and +// "" when it can: every field of an opaque struct must, because a row an +// earlier release sealed holds the struct as one JSON document, which +// comes back by decoding it into the generated shape struct. The struct +// is sealed as a storage blob now, which notStorable checks. A type with its own MarshalJSON and // UnmarshalJSON (time.Time) carries itself; otherwise a scalar, a slice or // array of an encodable type, a map with string or integer keys, a pointer, // or a struct whose every field is exported and encodable. A struct with an @@ -1100,6 +1104,51 @@ func notJSONEncodable(t types.Type, seen map[types.Type]bool) string { return "is not a type JSON carries" } +// notStorable says why the storage profile (vcffi.MarshalStorage) cannot +// carry a type both ways when encoding/json can, or "" when it can. The +// storage profile writes a struct field by field, by name, and a value +// with MarshalText as its text; it knows nothing of MarshalJSON. So a type +// that carries itself through JSON alone, over a struct with unexported +// fields, would be written without them, and a struct with no exported +// field at all has no storage form. notJSONEncodable has refused every +// other type the two disagree on. +func notStorable(t types.Type, seen map[types.Type]bool) string { + if seen[t] { + return "" + } + seen[t] = true + defer delete(seen, t) + if hasMethod(types.NewPointer(t), "UnmarshalText") && hasMethod(t, "MarshalText") { + return "" + } + switch u := t.Underlying().(type) { + case *types.Slice: + return notStorable(u.Elem(), seen) + case *types.Array: + return notStorable(u.Elem(), seen) + case *types.Pointer: + return notStorable(u.Elem(), seen) + case *types.Map: + return notStorable(u.Elem(), seen) + case *types.Struct: + exported := 0 + for i := range u.NumFields() { + fld := u.Field(i) + if !fld.Exported() { + return fmt.Sprintf("has an unexported field %s that a storage blob would drop (give the type MarshalText and UnmarshalText)", fld.Name()) + } + exported++ + if reason := notStorable(fld.Type(), seen); reason != "" { + return "has a field " + fld.Name() + " that " + reason + } + } + if exported == 0 { + return "has no exported field, and a storage blob writes a struct by its exported fields" + } + } + return "" +} + // valueBaseName is the plaintext type's bare name: User for User, *pb.Individual, crm.Contact. func valueBaseName(f *genFile) string { name := strings.TrimPrefix(f.typeExpr, "*") diff --git a/languages/golang/stashgen/refusal_test.go b/languages/golang/stashgen/refusal_test.go index 287f3b8f2..043ca3852 100644 --- a/languages/golang/stashgen/refusal_test.go +++ b/languages/golang/stashgen/refusal_test.go @@ -104,6 +104,8 @@ func TestRefusals(t *testing.T) { {"an opaque struct with a channel", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tDone chan int"), stashgen.Request{Type: "User"}, "Done", "is not a type JSON carries"}, {"an opaque struct with an interface", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tAny any"), stashgen.Request{Type: "User"}, "Any", "is not a type JSON carries"}, {"an opaque struct with a struct JSON would truncate", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tTx hidden") + "\ntype hidden struct {\n\tA int\n\tb int\n}\n", stashgen.Request{Type: "User"}, "Tx", "JSON would drop"}, + {"an opaque struct with a type that writes itself only as JSON", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tTx jsonOnly") + "\ntype jsonOnly struct {\n\tA int\n\tb int\n}\n\nfunc (j jsonOnly) MarshalJSON() ([]byte, error) { return nil, nil }\n\nfunc (j *jsonOnly) UnmarshalJSON([]byte) error { return nil }\n", stashgen.Request{Type: "User"}, "Tx", "a storage blob would drop"}, + {"an opaque struct with a struct of no exported field", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tE empty") + "\ntype empty struct{}\n", stashgen.Request{Type: "User"}, "E", "has no exported field"}, {"an opaque struct with a map keyed by a struct", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tM map[hidden]int") + "\ntype hidden struct{ A int }\n", stashgen.Request{Type: "User"}, "M", "map key that is not a string or an integer"}, {"an EQL type the engine cannot produce yet", user(ctx + "\tAge int64 `stash:\"age,encrypt_into=BigintOrdOre\"`"), stashgen.Request{Type: "User"}, "Age", "has no EQL type BigintOrdOre"}, {"an index on a composite", user(ctx + "\tAttrs map[string]string `stash:\"attrs,encrypt,index=equality\"`"), stashgen.Request{Type: "User"}, "Attrs", "seals only as part of an opaque struct"}, diff --git a/languages/golang/stashgen/testdata/cases/documents/document_stash.go.golden b/languages/golang/stashgen/testdata/cases/documents/document_stash.go.golden index d2ff0276a..97445ba49 100644 --- a/languages/golang/stashgen/testdata/cases/documents/document_stash.go.golden +++ b/languages/golang/stashgen/testdata/cases/documents/document_stash.go.golden @@ -38,12 +38,13 @@ type documentShape struct { Tags []string } -// The opaque value as it crosses the binding: one JSON document of the -// struct's fields, by their declared names. +// The opaque value as it crosses the binding: one storage blob of the +// struct's fields, by their declared names. The json names read a value +// an earlier release sealed as a JSON document. type documentOpaque struct { - Title string `json:"title"` - Body string `json:"body"` - Tags []string `json:"tags"` + Title string `vc:"title" json:"title"` + Body string `vc:"body" json:"body"` + Tags []string `vc:"tags" json:"tags"` } var declaration = gensupport.DeclareOpaque("documents/v2/body") From 2cd83ea3290efb12d5e34a105ef533d81929fadc Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 02:08:40 +1100 Subject: [PATCH 2/4] fix(go): close the opaque storage blob's review findings - Generated files name gensupport.GeneratedVersion2: a file generated before has no vc tags and would seal blobs keyed by Go field names that a regenerated file cannot read, so it no longer compiles. - The shape struct tags each field vc:"name,default", so a field added to an opaque struct reads as its zero value from an older blob. - stashgen's storage check follows vcffi's rules: encrypt.Decimal and encrypt.Date (no storage form), a type with only one of MarshalText and UnmarshalText, and a map key that is not an integer, string, bytes or text are refused; an empty struct (map[string]struct{}) is accepted. - opaqueBytes wraps vcffi's error, which names a type or kind and never a value; encrypt.ErrStorageVersion names vcffi.ErrUnsupportedVersion. - The CHANGELOG says to upgrade readers before writers, lists the new Encrypt-time refusals, and says no release while vcffi is a pseudo-version. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 35 ++++-- languages/golang/cmd/stashgen/README.md | 7 +- languages/golang/encrypt/errors.go | 11 +- .../golang/encrypt/example/user_stash.go | 2 +- languages/golang/encrypt/gensupport/codec.go | 10 +- .../golang/encrypt/gensupport/gensupport.go | 13 ++- .../gensupport/gensupport_internal_test.go | 21 ++++ .../encrypt/gensupport/gensupport_test.go | 8 +- .../internal/testpolicy/member_stash.go | 2 +- .../internal/testusers/account_stash.go | 2 +- .../internal/testusers/birthday_stash.go | 2 +- .../internal/testusers/contact_stash.go | 2 +- .../internal/testusers/document_stash.go | 13 ++- .../encrypt/internal/testusers/entry_stash.go | 2 +- .../internal/testusers/everything_stash.go | 63 +++++------ .../encrypt/internal/testusers/kinds_stash.go | 2 +- .../internal/testusers/ledger_stash.go | 2 +- .../internal/testusers/lookup_stash.go | 2 +- .../internal/testusers/memberstash_stash.go | 2 +- .../encrypt/internal/testusers/note_stash.go | 2 +- .../internal/testusers/person_stash.go | 2 +- .../encrypt/internal/testusers/probe_stash.go | 2 +- .../internal/testusers/profile_stash.go | 2 +- .../internal/testusers/queryprofile_stash.go | 2 +- .../internal/testusers/ranking_stash.go | 2 +- .../internal/testusers/secret_stash.go | 2 +- .../encrypt/internal/testusers/tally_stash.go | 2 +- .../encrypt/internal/testusers/user_stash.go | 2 +- .../internal/testusers/valuekinds_stash.go | 2 +- .../internal/testusers/widened_stash.go | 2 +- languages/golang/encrypt/kinds_test.go | 17 +++ languages/golang/stashgen/doc.go | 2 +- languages/golang/stashgen/emit.go | 9 +- languages/golang/stashgen/read.go | 104 +++++++++++++++--- languages/golang/stashgen/refusal_test.go | 18 ++- .../cases/accounts/account_stash.go.golden | 2 +- .../contacts/contactstash_stash.go.golden | 2 +- .../cases/directory/entry_stash.go.golden | 2 +- .../cases/documents/document_stash.go.golden | 13 ++- .../cases/embedded/patient_stash.go.golden | 2 +- .../foreign/individualstash_stash.go.golden | 2 +- .../cases/orders/order_stash.go.golden | 2 +- .../cases/orders/refund_stash.go.golden | 2 +- .../cases/tenants/note_stash.go.golden | 2 +- .../testdata/cases/users/user_stash.go.golden | 2 +- .../policy_individual_stash.go.golden | 2 +- .../stubsdk/encrypt/gensupport/gensupport.go | 4 +- 47 files changed, 288 insertions(+), 122 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 9166162cd..ce6cd16c9 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -47,14 +47,33 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. blob in vitaminc's storage profile, version 1 (`vcffi.MarshalStorage`), the format vitaminc keeps readable across releases and languages, where it sealed the JSON document `encoding/json` wrote. Rows sealed before - still decrypt: `Decrypt` reads a value that is not a blob as JSON. Run - stashgen again: the generated shape struct gains `vc` tags. What - changes for a program: a NaN or infinite float now seals (JSON refused - it with `ErrEncoding`); a nil slice or map comes back empty, not nil, - from a blob; and `stashgen` refuses a field type that only - `MarshalJSON` carries over a struct with unexported fields, which a - blob would write without them. This takes vcffi from the unreleased - cipherstash/vitaminc#526 by pseudo-version until vitaminc releases it. + still decrypt: `Decrypt` reads a value that is not a blob as JSON. + **Run stashgen again:** generated files name + `gensupport.GeneratedVersion2`, so a file generated before does not + compile against this release (it would seal blobs keyed by Go field + names that the regenerated file could not read). The generated shape + struct names each field `vc:"name,default"`, so a field added to the + struct later reads as its zero value from a blob sealed before it, as + it did from JSON. A field added to a nested struct type has no such + tag: give it a pointer type, or rows sealed before it do not decrypt. + **Upgrade every reader before any writer:** an earlier release cannot + read a blob, so a rolling deploy or a rollback that runs an earlier + release after this one has sealed a row fails to decrypt that row. + What else changes for a program: a NaN or infinite float now seals + (JSON refused it with `ErrEncoding`); a nil slice or map comes back + empty, not nil, from a blob; `Encrypt` refuses, as `ErrEncoding` + wrapping `vcffi.ErrUnwritable`, a string that is not UTF-8 (JSON wrote + U+FFFD), a value nested more than 128 levels deep, and two map keys + whose `MarshalText` spells alike. `stashgen` refuses in an opaque + struct `encrypt.Decimal` and `encrypt.Date` (the storage profile has no + form for them), a type with only one of `MarshalText` and + `UnmarshalText`, a map key that is not an integer, a string, bytes or + text, and a type that only `MarshalJSON` carries over a struct with + unexported fields. A blob a later release writes is + `encrypt.ErrStorageVersion`. This takes vcffi from the unreleased + cipherstash/vitaminc#526 by pseudo-version; the storage format is a + compatibility commitment only once vitaminc releases it, so no Go + release should be cut while the pin is a pseudo-version. - **Every order term changes: re-derive stored `ore`, `ope` and `ore_block` index terms.** The engine moved to vitaminc 0.8, clww-ore 0.6 (formerly cllw-ore) and ore-rs 0.10, and both order schemes now derive diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 8a20c7b2e..5967fda76 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -275,7 +275,10 @@ Generated code sends the engine every sealed field with its value, under the dec A passthrough field stays on the host: the engine does nothing to it a program could observe, and the FFI codec cannot carry every Go type a program stores beside a ciphertext (a `driver.Valuer`, a `sql.NullTime`). An `opaque` struct crosses as one storage blob and is one column, read back into the exact Go types the struct declares. The blob is vitaminc's storage profile, version 1 (`vcffi.MarshalStorage`), the format vitaminc keeps readable across releases and languages. -A field may be any type that both the storage profile and `encoding/json` carry both ways: a scalar or a type defined over one, `[]byte`, slices, maps with string or integer keys, pointers, structs whose fields are all exported, and types with `MarshalText` and `UnmarshalText`, such as `time.Time`. +A field may be any type that both the storage profile and `encoding/json` carry both ways: a scalar or a type defined over one, `[]byte`, slices, maps with string or integer keys, pointers, structs whose fields are all exported, and types with both `MarshalText` and `UnmarshalText`, such as `time.Time`. +`encrypt.Decimal` and `encrypt.Date` have no storage form and are refused in an opaque struct: store them as text, or by their parts. +The generated shape struct tags each field `vc:"name,default"`, so a field added to the opaque struct reads as its zero value from a row sealed before it; a field added to a nested struct type needs a pointer type for the same. +`Encrypt` refuses a string that is not UTF-8 and a value nested more than 128 levels deep, as `encrypt.ErrEncoding`. `encoding/json` still matters because a row an earlier release sealed holds a JSON document, and `Decrypt` reads it as JSON. A float in an opaque struct may be NaN or an infinity: the blob writes a float by its bits. A nil slice or map in an opaque struct comes back empty, not nil, from a blob; from an earlier release's JSON document it comes back nil. @@ -283,7 +286,7 @@ A sealed field outside an opaque struct is one scalar, a type defined over one, The scalars are `string`, `[]byte`, `bool`, every integer type (each sealed at its own width, `int` and `uint` as 64-bit), `encrypt.Int128` and `encrypt.Uint128`, `float32`, `float64`, `encrypt.Decimal`, `encrypt.Date` and `time.Time` (a timestamp). The wire type `stashgen` writes for each is the field's own: `int8` is `gensupport.Int8`, `time.Time` is `gensupport.Timestamp`, and so on. A decimal orders by value, so `1.5` ties `1.50`; a `time.Time` orders by the microsecond, so instants within one microsecond tie. The zero `encrypt.Date` is not a date: a date field must be set, and `Encrypt` refuses a zero one with `encrypt.ErrEncoding`, naming the field. An optional date is a `*encrypt.Date` field (see "Optional fields"). -A nil `[]byte` in a sealed field, or a nil slice of a type defined over `[]byte`, comes back from `Decrypt` as an empty, non-nil slice; inside an opaque struct, nil comes back as nil. +A nil `[]byte` in a sealed field, or a nil slice of a type defined over `[]byte`, comes back from `Decrypt` as an empty, non-nil slice; inside an opaque struct it comes back empty from a blob too, and nil only from an earlier release's JSON document. A protobuf message with a `oneof` cannot be generated from a policy: protoc-gen-go puts its members in wrapper types, not in the message struct, and `Generate` refuses it. ## Status diff --git a/languages/golang/encrypt/errors.go b/languages/golang/encrypt/errors.go index 0a21d54af..68960418a 100644 --- a/languages/golang/encrypt/errors.go +++ b/languages/golang/encrypt/errors.go @@ -1,6 +1,9 @@ package encrypt -import "github.com/cipherstash/stack/languages/golang/internal/guest" +import ( + "github.com/cipherstash/stack/languages/golang/internal/guest" + "github.com/cipherstash/vitaminc/bindings/go/vcffi" +) // Failure kinds surfaced across the boundary: the guest reports a status // code, and these are what the codes decode to. They separate a tampered @@ -66,6 +69,12 @@ var ( ErrMemoryLock = guest.ErrMemoryLock ) +// ErrStorageVersion is an opaque struct sealed as a storage blob of a +// version this release does not read: a later release wrote it. Its error +// wraps [ErrEncoding] too. It is vcffi.ErrUnsupportedVersion, named here so +// a program can check for it without importing vitaminc's bindings. +var ErrStorageVersion = vcffi.ErrUnsupportedVersion + // Diagnostic is the full error behind a failure the guest reports, beside // the kind: every such failure is a *Diagnostic wrapping one of the // sentinels above, so errors.Is matches the kind and errors.As reads the diff --git a/languages/golang/encrypt/example/user_stash.go b/languages/golang/encrypt/example/user_stash.go index 4dc7ec03a..28820c2c2 100644 --- a/languages/golang/encrypt/example/user_stash.go +++ b/languages/golang/encrypt/example/user_stash.go @@ -11,7 +11,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // User prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/gensupport/codec.go b/languages/golang/encrypt/gensupport/codec.go index ab6aee541..e7f9acdec 100644 --- a/languages/golang/encrypt/gensupport/codec.go +++ b/languages/golang/encrypt/gensupport/codec.go @@ -388,10 +388,10 @@ func (c *RecordsCodec[P, R]) Decrypt(ctx context.Context, d encrypt.Decrypter, r func opaqueBytes(fields any) ([]byte, error) { encoded, err := vcffi.MarshalStorage(fields) if err != nil { - // Not wrapped: the reason is the kind of value and where, and a - // value is never quoted, but the opaque struct is plaintext and - // nothing here needs more than the class. - return nil, fmt.Errorf("%w: the opaque value cannot be written as a storage blob (a type the storage profile has no form for, a repeated map key, or text that is not UTF-8)", encrypt.ErrEncoding) + // vcffi's error names a type or a kind (no storage form, a string + // that is not UTF-8, a repeated key, nesting too deep), never a + // value, so it is wrapped: errors.Is reads vcffi.ErrUnwritable. + return nil, fmt.Errorf("%w: the opaque value cannot be written as a storage blob: %w", encrypt.ErrEncoding, err) } return encoded, nil } @@ -418,7 +418,7 @@ func Opaque[T any](vals Values, out *T) error { if err := vcffi.UnmarshalStorage(encoded, out); err != nil { // The vcffi error names the kind found and where, never the // value; it is wrapped so errors.Is reads its class - // (vcffi.ErrUnsupportedVersion for a blob a later release + // (encrypt.ErrStorageVersion for a blob a later release // wrote, say). return fmt.Errorf("gensupport: %w: the opaque value does not read into a %T: %w", encrypt.ErrEncoding, *out, err) } diff --git a/languages/golang/encrypt/gensupport/gensupport.go b/languages/golang/encrypt/gensupport/gensupport.go index 27bf21fb8..c509dc292 100644 --- a/languages/golang/encrypt/gensupport/gensupport.go +++ b/languages/golang/encrypt/gensupport/gensupport.go @@ -2,7 +2,7 @@ // // A program never imports this package: it calls the functions the generated // file writes into its own package (users.Encrypt, users.Decrypt, -// users.Fields). The generated file names [GeneratedVersion1], builds a +// users.Fields). The generated file names [GeneratedVersion2], builds a // [Declaration] from the struct's tags, hands the library its conversions in // a [Generated] value, and prints through [Redacted] and [RedactedLog]. The // library lowers the declaration to the data plan the engine reads, sends a @@ -25,9 +25,14 @@ import ( // file names, so a file from another version does not compile. type generatedVersion uint8 -// GeneratedVersion1 is the layout of files that stashgen writes today. Every -// generated file holds `const _ = gensupport.GeneratedVersion1`. -const GeneratedVersion1 generatedVersion = 1 +// GeneratedVersion2 is the layout of files that stashgen writes today. Every +// generated file holds `const _ = gensupport.GeneratedVersion2`. +// +// Version 2 seals an opaque struct as a storage blob keyed by its declared +// field names (the shape struct's `vc` tags). A version 1 file has no such +// tags, so it would write blobs keyed by Go field names that a regenerated +// file cannot read; it does not compile against this library. +const GeneratedVersion2 generatedVersion = 2 // sealed is what a hidden field prints as. const sealed = "[sealed]" diff --git a/languages/golang/encrypt/gensupport/gensupport_internal_test.go b/languages/golang/encrypt/gensupport/gensupport_internal_test.go index 0c2655799..57c583e6e 100644 --- a/languages/golang/encrypt/gensupport/gensupport_internal_test.go +++ b/languages/golang/encrypt/gensupport/gensupport_internal_test.go @@ -307,6 +307,27 @@ func TestDecryptErrorsHoldNoPlaintext(t *testing.T) { if _, err := opaqueBytes(struct{ C chan int }{make(chan int)}); !errors.Is(err, encrypt.ErrEncoding) { t.Errorf("opaqueBytes chan: %v", err) } + // A field a blob lacks (one added to the struct since the row was + // sealed) reads as its zero value, as the generated `vc:",default"` + // tags say; a field the struct no longer has is skipped. + old := mustStorage(t, struct { + N uint32 `vc:"n,default"` + Gone string `vc:"gone,default"` + }{N: 7, Gone: "x"}) + var grown struct { + N uint32 `vc:"n,default"` + New string `vc:"new,default"` + } + if err := Opaque(Values{OpaqueField: old}, &grown); err != nil || grown.N != 7 || grown.New != "" { + t.Fatalf("a grown struct read %+v, %v", grown, err) + } + // A blob of a later storage version is encrypt.ErrStorageVersion, which + // needs no import of vitaminc's bindings to check. + later := append([]byte(nil), old...) + later[3]++ + if err := Opaque(Values{OpaqueField: later}, &grown); !errors.Is(err, encrypt.ErrStorageVersion) || !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("a later version: %v", err) + } // The storage profile writes a float by its bits: NaN and the // infinities, which encoding/json refused, are values like any other. blob, err := opaqueBytes(struct{ F float64 }{math.NaN()}) diff --git a/languages/golang/encrypt/gensupport/gensupport_test.go b/languages/golang/encrypt/gensupport/gensupport_test.go index 68bd64ea0..2f1dd08c6 100644 --- a/languages/golang/encrypt/gensupport/gensupport_test.go +++ b/languages/golang/encrypt/gensupport/gensupport_test.go @@ -64,12 +64,12 @@ func TestNoticesPrintOnceForEachType(t *testing.T) { } func TestVersionConstantIsTyped(t *testing.T) { - // A generated file holds `const _ = gensupport.GeneratedVersion1`. The + // A generated file holds `const _ = gensupport.GeneratedVersion2`. The // constant's type is unexported, so no other package can declare a value // that satisfies the same reference. accepts := func(generatedVersion) {} - accepts(GeneratedVersion1) - if GeneratedVersion1 != 1 { - t.Fatalf("GeneratedVersion1 = %d", GeneratedVersion1) + accepts(GeneratedVersion2) + if GeneratedVersion2 != 2 { + t.Fatalf("GeneratedVersion2 = %d", GeneratedVersion2) } } diff --git a/languages/golang/encrypt/internal/testpolicy/member_stash.go b/languages/golang/encrypt/internal/testpolicy/member_stash.go index 1b52480cf..9ec08574d 100644 --- a/languages/golang/encrypt/internal/testpolicy/member_stash.go +++ b/languages/golang/encrypt/internal/testpolicy/member_stash.go @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // testmember.Member prints its sealed fields in the clear, and stashgen cannot // add print methods to a type from another package. diff --git a/languages/golang/encrypt/internal/testusers/account_stash.go b/languages/golang/encrypt/internal/testusers/account_stash.go index 27cf2ba7b..5628ec3da 100644 --- a/languages/golang/encrypt/internal/testusers/account_stash.go +++ b/languages/golang/encrypt/internal/testusers/account_stash.go @@ -13,7 +13,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Account prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/birthday_stash.go b/languages/golang/encrypt/internal/testusers/birthday_stash.go index 78e3e2206..1debc9a8c 100644 --- a/languages/golang/encrypt/internal/testusers/birthday_stash.go +++ b/languages/golang/encrypt/internal/testusers/birthday_stash.go @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Birthday prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/contact_stash.go b/languages/golang/encrypt/internal/testusers/contact_stash.go index a30de7b20..ee0c321f1 100644 --- a/languages/golang/encrypt/internal/testusers/contact_stash.go +++ b/languages/golang/encrypt/internal/testusers/contact_stash.go @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Contact prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/document_stash.go b/languages/golang/encrypt/internal/testusers/document_stash.go index 3c37e969b..734f4b7e0 100644 --- a/languages/golang/encrypt/internal/testusers/document_stash.go +++ b/languages/golang/encrypt/internal/testusers/document_stash.go @@ -11,7 +11,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Document prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. @@ -39,12 +39,13 @@ type documentShape struct { } // The opaque value as it crosses the binding: one storage blob of the -// struct's fields, by their declared names. The json names read a value -// an earlier release sealed as a JSON document. +// struct's fields, by their declared names. A field a blob lacks (one +// added since it was sealed) reads as its zero value. The json names read +// a value an earlier release sealed as a JSON document. type documentOpaque struct { - Title string `vc:"title" json:"title"` - Body string `vc:"body" json:"body"` - Tags []string `vc:"tags" json:"tags"` + Title string `vc:"title,default" json:"title"` + Body string `vc:"body,default" json:"body"` + Tags []string `vc:"tags,default" json:"tags"` } var documentDeclaration = gensupport.DeclareOpaque("documents/v2/body") diff --git a/languages/golang/encrypt/internal/testusers/entry_stash.go b/languages/golang/encrypt/internal/testusers/entry_stash.go index de9ce0098..ccd83d16f 100644 --- a/languages/golang/encrypt/internal/testusers/entry_stash.go +++ b/languages/golang/encrypt/internal/testusers/entry_stash.go @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Entry prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/everything_stash.go b/languages/golang/encrypt/internal/testusers/everything_stash.go index d832eda24..47823b9b1 100644 --- a/languages/golang/encrypt/internal/testusers/everything_stash.go +++ b/languages/golang/encrypt/internal/testusers/everything_stash.go @@ -13,7 +13,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Everything prints its sealed fields in the clear: it has no String or // LogValue method. Write them, or run stashgen with -redact. @@ -66,37 +66,38 @@ type everythingShape struct { } // The opaque value as it crosses the binding: one storage blob of the -// struct's fields, by their declared names. The json names read a value -// an earlier release sealed as a JSON document. +// struct's fields, by their declared names. A field a blob lacks (one +// added since it was sealed) reads as its zero value. The json names read +// a value an earlier release sealed as a JSON document. type everythingOpaque struct { - S string `vc:"s" json:"s"` - E Email `vc:"e" json:"e"` - Bo bool `vc:"bo" json:"bo"` - I8 int8 `vc:"i8" json:"i8"` - I int `vc:"i" json:"i"` - U64 uint64 `vc:"u64" json:"u64"` - F32 float32 `vc:"f32" json:"f32"` - F64 float64 `vc:"f64" json:"f64"` - By []byte `vc:"by" json:"by"` - Bl Blob `vc:"bl" json:"bl"` - Sc Score `vc:"sc" json:"sc"` - Tags []string `vc:"tags" json:"tags"` - Ints []int `vc:"ints" json:"ints"` - F32s []float32 `vc:"f32s" json:"f32s"` - St Status `vc:"st" json:"st"` - D time.Duration `vc:"d" json:"d"` - Emails []Email `vc:"emails" json:"emails"` - Counts map[string]int `vc:"counts" json:"counts"` - Labels map[string]string `vc:"labels" json:"labels"` - ByKey map[int]string `vc:"by_key" json:"by_key"` - In Inner `vc:"in" json:"in"` - Ins []Inner `vc:"ins" json:"ins"` - P *string `vc:"p" json:"p"` - PI *Inner `vc:"pi" json:"pi"` - T time.Time `vc:"t" json:"t"` - N sql.NullTime `vc:"n" json:"n"` - Nested [][]byte `vc:"nested" json:"nested"` - Arr [2]uint8 `vc:"arr" json:"arr"` + S string `vc:"s,default" json:"s"` + E Email `vc:"e,default" json:"e"` + Bo bool `vc:"bo,default" json:"bo"` + I8 int8 `vc:"i8,default" json:"i8"` + I int `vc:"i,default" json:"i"` + U64 uint64 `vc:"u64,default" json:"u64"` + F32 float32 `vc:"f32,default" json:"f32"` + F64 float64 `vc:"f64,default" json:"f64"` + By []byte `vc:"by,default" json:"by"` + Bl Blob `vc:"bl,default" json:"bl"` + Sc Score `vc:"sc,default" json:"sc"` + Tags []string `vc:"tags,default" json:"tags"` + Ints []int `vc:"ints,default" json:"ints"` + F32s []float32 `vc:"f32s,default" json:"f32s"` + St Status `vc:"st,default" json:"st"` + D time.Duration `vc:"d,default" json:"d"` + Emails []Email `vc:"emails,default" json:"emails"` + Counts map[string]int `vc:"counts,default" json:"counts"` + Labels map[string]string `vc:"labels,default" json:"labels"` + ByKey map[int]string `vc:"by_key,default" json:"by_key"` + In Inner `vc:"in,default" json:"in"` + Ins []Inner `vc:"ins,default" json:"ins"` + P *string `vc:"p,default" json:"p"` + PI *Inner `vc:"pi,default" json:"pi"` + T time.Time `vc:"t,default" json:"t"` + N sql.NullTime `vc:"n,default" json:"n"` + Nested [][]byte `vc:"nested,default" json:"nested"` + Arr [2]uint8 `vc:"arr,default" json:"arr"` } var everythingDeclaration = gensupport.DeclareOpaque("everything") diff --git a/languages/golang/encrypt/internal/testusers/kinds_stash.go b/languages/golang/encrypt/internal/testusers/kinds_stash.go index bf9a3be67..ff96a1f3a 100644 --- a/languages/golang/encrypt/internal/testusers/kinds_stash.go +++ b/languages/golang/encrypt/internal/testusers/kinds_stash.go @@ -13,7 +13,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Kinds prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/ledger_stash.go b/languages/golang/encrypt/internal/testusers/ledger_stash.go index eb1db959d..3b25e74e0 100644 --- a/languages/golang/encrypt/internal/testusers/ledger_stash.go +++ b/languages/golang/encrypt/internal/testusers/ledger_stash.go @@ -13,7 +13,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Ledger prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/lookup_stash.go b/languages/golang/encrypt/internal/testusers/lookup_stash.go index 6b47dbb05..913596f7c 100644 --- a/languages/golang/encrypt/internal/testusers/lookup_stash.go +++ b/languages/golang/encrypt/internal/testusers/lookup_stash.go @@ -11,7 +11,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Lookup prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/memberstash_stash.go b/languages/golang/encrypt/internal/testusers/memberstash_stash.go index 9efb5414e..8a011eeb7 100644 --- a/languages/golang/encrypt/internal/testusers/memberstash_stash.go +++ b/languages/golang/encrypt/internal/testusers/memberstash_stash.go @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // testmember.Member prints its sealed fields in the clear, and stashgen cannot // add print methods to a type from another package. diff --git a/languages/golang/encrypt/internal/testusers/note_stash.go b/languages/golang/encrypt/internal/testusers/note_stash.go index e09124ffe..ab438affa 100644 --- a/languages/golang/encrypt/internal/testusers/note_stash.go +++ b/languages/golang/encrypt/internal/testusers/note_stash.go @@ -11,7 +11,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Note prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/person_stash.go b/languages/golang/encrypt/internal/testusers/person_stash.go index 840545a05..15fa29f78 100644 --- a/languages/golang/encrypt/internal/testusers/person_stash.go +++ b/languages/golang/encrypt/internal/testusers/person_stash.go @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Person prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/probe_stash.go b/languages/golang/encrypt/internal/testusers/probe_stash.go index 3be4d5959..e8b98064a 100644 --- a/languages/golang/encrypt/internal/testusers/probe_stash.go +++ b/languages/golang/encrypt/internal/testusers/probe_stash.go @@ -11,7 +11,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Probe prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/profile_stash.go b/languages/golang/encrypt/internal/testusers/profile_stash.go index 28a8a6bcc..b70f1f5d5 100644 --- a/languages/golang/encrypt/internal/testusers/profile_stash.go +++ b/languages/golang/encrypt/internal/testusers/profile_stash.go @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Profile prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/queryprofile_stash.go b/languages/golang/encrypt/internal/testusers/queryprofile_stash.go index af27d39b4..3815b8c67 100644 --- a/languages/golang/encrypt/internal/testusers/queryprofile_stash.go +++ b/languages/golang/encrypt/internal/testusers/queryprofile_stash.go @@ -11,7 +11,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // QueryProfile prints its sealed fields in the clear: it has no String or // LogValue method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/ranking_stash.go b/languages/golang/encrypt/internal/testusers/ranking_stash.go index 96c368b90..491e55c8f 100644 --- a/languages/golang/encrypt/internal/testusers/ranking_stash.go +++ b/languages/golang/encrypt/internal/testusers/ranking_stash.go @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Ranking prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/secret_stash.go b/languages/golang/encrypt/internal/testusers/secret_stash.go index 9f85d763c..c78dacf73 100644 --- a/languages/golang/encrypt/internal/testusers/secret_stash.go +++ b/languages/golang/encrypt/internal/testusers/secret_stash.go @@ -11,7 +11,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 type EncryptedSecret struct { ID int64 diff --git a/languages/golang/encrypt/internal/testusers/tally_stash.go b/languages/golang/encrypt/internal/testusers/tally_stash.go index 286d89dac..4f49349b1 100644 --- a/languages/golang/encrypt/internal/testusers/tally_stash.go +++ b/languages/golang/encrypt/internal/testusers/tally_stash.go @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Tally prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/user_stash.go b/languages/golang/encrypt/internal/testusers/user_stash.go index 9b304cd6b..61cbf88f4 100644 --- a/languages/golang/encrypt/internal/testusers/user_stash.go +++ b/languages/golang/encrypt/internal/testusers/user_stash.go @@ -11,7 +11,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // User prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/valuekinds_stash.go b/languages/golang/encrypt/internal/testusers/valuekinds_stash.go index d959931a1..0966ce8b5 100644 --- a/languages/golang/encrypt/internal/testusers/valuekinds_stash.go +++ b/languages/golang/encrypt/internal/testusers/valuekinds_stash.go @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // ValueKinds prints its sealed fields in the clear: it has no String or // LogValue method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/internal/testusers/widened_stash.go b/languages/golang/encrypt/internal/testusers/widened_stash.go index c9f163aa0..0ee732931 100644 --- a/languages/golang/encrypt/internal/testusers/widened_stash.go +++ b/languages/golang/encrypt/internal/testusers/widened_stash.go @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Widened prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/encrypt/kinds_test.go b/languages/golang/encrypt/kinds_test.go index 9370d83ee..2569233fa 100644 --- a/languages/golang/encrypt/kinds_test.go +++ b/languages/golang/encrypt/kinds_test.go @@ -16,6 +16,7 @@ import ( "github.com/cipherstash/stack/languages/golang/encrypt" "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" + "github.com/cipherstash/vitaminc/bindings/go/vcffi" ) // Everything the generator accepts round-trips: one sealed field of every @@ -324,3 +325,19 @@ func TestARedactedStructPrintsNoSealedField(t *testing.T) { t.Errorf("slog printed the sealed field: %s", buf.String()) } } + +// A value with no storage form (here a string that is not UTF-8, which the +// JSON document of earlier releases wrote as U+FFFD) refuses the batch as +// ErrEncoding naming the value, with vcffi's class wrapped and no byte of +// the value in the error. +func TestAnOpaqueStructWithNoStorageFormIsErrEncoding(t *testing.T) { + c := deterministicClient(t) + in := []testusers.Everything{{S: "fine"}, {S: "secret-4242\xff"}} + _, err := testusers.EncryptEverything(context.Background(), c.DefaultKeyset(), in) + if !errors.Is(err, encrypt.ErrEncoding) || !errors.Is(err, vcffi.ErrUnwritable) { + t.Fatalf("err = %v, want ErrEncoding wrapping vcffi.ErrUnwritable", err) + } + if !strings.Contains(err.Error(), "value 1") || strings.Contains(err.Error(), "4242") { + t.Fatalf("err = %v, want it to name value 1 and hold none of the value", err) + } +} diff --git a/languages/golang/stashgen/doc.go b/languages/golang/stashgen/doc.go index a868d1ff3..e62346085 100644 --- a/languages/golang/stashgen/doc.go +++ b/languages/golang/stashgen/doc.go @@ -11,7 +11,7 @@ // It ignores its own output file when it loads the package, so a stale // generated file does not stop it. The same input always gives the same file: // fields keep their declared order, and the file carries no version and no -// time beyond the gensupport.GeneratedVersion1 constant it names. +// time beyond the gensupport.GeneratedVersion2 constant it names. // // [Generate] writes the same file for a type that cannot carry tags, from a // policy in package encrypt/policy: a source gives the facts about each field diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index e91a6d6d3..3331d64f2 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -66,7 +66,7 @@ func (w *writer) header(f *genFile) { w.b.WriteString(f.imports.block()) w.nl() w.p("// Stops compiling when the library does not accept this version of generated file.") - w.p("const _ = gensupport.GeneratedVersion1") + w.p("const _ = gensupport.GeneratedVersion2") for _, n := range f.notices { w.nl() w.b.WriteString(wrapComment(n, 80)) @@ -175,11 +175,12 @@ func (w *writer) opaqueShape(f *genFile) { } w.nl() w.p("// The opaque value as it crosses the binding: one storage blob of the") - w.p("// struct's fields, by their declared names. The json names read a value") - w.p("// an earlier release sealed as a JSON document.") + w.p("// struct's fields, by their declared names. A field a blob lacks (one") + w.p("// added since it was sealed) reads as its zero value. The json names read") + w.p("// a value an earlier release sealed as a JSON document.") w.p("type %s struct {", f.opaqueShape) for _, g := range f.opaque { - w.p("\t%s %s `vc:%q json:%q`", g.GoName, g.typeExpr, g.Name, g.Name) + w.p("\t%s %s `vc:%q json:%q`", g.GoName, g.typeExpr, g.Name+",default", g.Name) } w.p("}") } diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index 43e3aa2ce..81048a0a2 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -1104,22 +1104,26 @@ func notJSONEncodable(t types.Type, seen map[types.Type]bool) string { return "is not a type JSON carries" } -// notStorable says why the storage profile (vcffi.MarshalStorage) cannot -// carry a type both ways when encoding/json can, or "" when it can. The -// storage profile writes a struct field by field, by name, and a value -// with MarshalText as its text; it knows nothing of MarshalJSON. So a type -// that carries itself through JSON alone, over a struct with unexported -// fields, would be written without them, and a struct with no exported -// field at all has no storage form. notJSONEncodable has refused every -// other type the two disagree on. +// notStorable says why the storage profile (vcffi.MarshalStorage and +// UnmarshalStorage) cannot carry a type both ways, or "" when it can, by +// vcffi's own rules: a scalar by its kind; a type with MarshalText and +// UnmarshalText as its text; a slice, array or pointer by its element; a +// map whose key is an integer, a string, bytes or text, by its value; a +// struct by its exported fields. It knows nothing of MarshalJSON, so a type +// that carries itself through JSON alone is walked like any other. +// notJSONEncodable is the other half: an opaque struct must still read the +// JSON documents earlier releases sealed. func notStorable(t types.Type, seen map[types.Type]bool) string { if seen[t] { return "" } seen[t] = true defer delete(seen, t) - if hasMethod(types.NewPointer(t), "UnmarshalText") && hasMethod(t, "MarshalText") { - return "" + if reason := noStorageForm(t); reason != "" { + return reason + } + if text, reason := textForm(t); text || reason != "" { + return reason } switch u := t.Underlying().(type) { case *types.Slice: @@ -1129,26 +1133,96 @@ func notStorable(t types.Type, seen map[types.Type]bool) string { case *types.Pointer: return notStorable(u.Elem(), seen) case *types.Map: + if reason := notStorableKey(u.Key()); reason != "" { + return reason + } return notStorable(u.Elem(), seen) case *types.Struct: - exported := 0 + // vcffi writes a struct by its exported fields and refuses one + // that has unexported fields only; a struct with none at all + // (struct{}) is an empty object, read back as itself. for i := range u.NumFields() { fld := u.Field(i) if !fld.Exported() { return fmt.Sprintf("has an unexported field %s that a storage blob would drop (give the type MarshalText and UnmarshalText)", fld.Name()) } - exported++ if reason := notStorable(fld.Type(), seen); reason != "" { return "has a field " + fld.Name() + " that " + reason } } - if exported == 0 { - return "has no exported field, and a storage blob writes a struct by its exported fields" - } } return "" } +// noStorageForm refuses vitaminc's logical value types, which the +// transport carries and the storage profile does not: encrypt.Date and +// encrypt.Decimal (vcvalue.Date and vcvalue.Decimal), and vcvalue's other +// logical types. Its 128-bit integers and Object have a storage form. +func noStorageForm(t types.Type) string { + switch kind, _ := valueKind(t); kind { + case KindDate: + return "is a date, which has no storage form (store it as text, or by its parts)" + case KindDecimal: + return "is a decimal, which has no storage form (store it as text, or by its parts)" + case KindInt128, KindUint128: + return "" + } + named, ok := types.Unalias(t).(*types.Named) + if !ok || named.Obj().Pkg() == nil || named.Obj().Pkg().Path() != vcvaluePath || named.Obj().Name() == "Object" { + return "" + } + return "is vcvalue." + named.Obj().Name() + ", which has no storage form" +} + +// textForm reports whether t is written as its text, and why not when it +// has only half of the pair: a type with MarshalText alone is written as +// text and read back by its kind, which fails; one with UnmarshalText +// alone is written by its kind and read back as text. +func textForm(t types.Type) (bool, string) { + marshals := hasMethod(t, "MarshalText") || hasMethod(types.NewPointer(t), "MarshalText") + unmarshals := hasMethod(types.NewPointer(t), "UnmarshalText") + switch { + case marshals && unmarshals: + return true, "" + case marshals: + return false, "has MarshalText but no UnmarshalText, so a storage blob would write it as text and could not read it back" + case unmarshals: + return false, "has UnmarshalText but no MarshalText, so a storage blob would write it by its kind and read it back as text" + } + return false, "" +} + +// notStorableKey says why a map key has no storage form: vcffi takes an +// integer, a string, bytes (a byte slice or array, a 128-bit integer) or a +// type with MarshalText and UnmarshalText. +func notStorableKey(k types.Type) string { + if kind, _ := valueKind(k); kind == KindInt128 || kind == KindUint128 { + return "" + } + if text, reason := textForm(k); text { + return "" + } else if reason != "" { + return "has a map key that " + reason + } + switch u := k.Underlying().(type) { + case *types.Basic: + if u.Info()&(types.IsInteger|types.IsString) != 0 { + return "" + } + case *types.Slice, *types.Array: + var elem types.Type + if s, ok := u.(*types.Slice); ok { + elem = s.Elem() + } else { + elem = u.(*types.Array).Elem() + } + if b, ok := elem.Underlying().(*types.Basic); ok && b.Kind() == types.Byte { + return "" + } + } + return "has a map key that is not an integer, a string, bytes or text, which a storage blob cannot key a map by" +} + // valueBaseName is the plaintext type's bare name: User for User, *pb.Individual, crm.Contact. func valueBaseName(f *genFile) string { name := strings.TrimPrefix(f.typeExpr, "*") diff --git a/languages/golang/stashgen/refusal_test.go b/languages/golang/stashgen/refusal_test.go index 043ca3852..67f161491 100644 --- a/languages/golang/stashgen/refusal_test.go +++ b/languages/golang/stashgen/refusal_test.go @@ -105,7 +105,11 @@ func TestRefusals(t *testing.T) { {"an opaque struct with an interface", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tAny any"), stashgen.Request{Type: "User"}, "Any", "is not a type JSON carries"}, {"an opaque struct with a struct JSON would truncate", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tTx hidden") + "\ntype hidden struct {\n\tA int\n\tb int\n}\n", stashgen.Request{Type: "User"}, "Tx", "JSON would drop"}, {"an opaque struct with a type that writes itself only as JSON", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tTx jsonOnly") + "\ntype jsonOnly struct {\n\tA int\n\tb int\n}\n\nfunc (j jsonOnly) MarshalJSON() ([]byte, error) { return nil, nil }\n\nfunc (j *jsonOnly) UnmarshalJSON([]byte) error { return nil }\n", stashgen.Request{Type: "User"}, "Tx", "a storage blob would drop"}, - {"an opaque struct with a struct of no exported field", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tE empty") + "\ntype empty struct{}\n", stashgen.Request{Type: "User"}, "E", "has no exported field"}, + {"an opaque struct with a decimal", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tAmount encrypt.Decimal"), stashgen.Request{Type: "User"}, "Amount", "has no storage form"}, + {"an opaque struct with a date in a slice", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tDays []encrypt.Date"), stashgen.Request{Type: "User"}, "Days", "has no storage form"}, + {"an opaque struct with a type that writes text it cannot read", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tL level") + "\ntype level int\n\nfunc (l level) MarshalText() ([]byte, error) { return nil, nil }\n", stashgen.Request{Type: "User"}, "L", "MarshalText but no UnmarshalText"}, + {"an opaque struct with a type that reads text it does not write", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tL level") + "\ntype level int\n\nfunc (l *level) UnmarshalText([]byte) error { return nil }\n", stashgen.Request{Type: "User"}, "L", "UnmarshalText but no MarshalText"}, + {"an opaque struct with a JSON-only type keyed by a struct", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tE ev") + "\ntype k struct{ A int }\n\ntype ev struct{ M map[k]int }\n\nfunc (e ev) MarshalJSON() ([]byte, error) { return nil, nil }\n\nfunc (e *ev) UnmarshalJSON([]byte) error { return nil }\n", stashgen.Request{Type: "User"}, "E", "map key that is not an integer, a string, bytes or text"}, {"an opaque struct with a map keyed by a struct", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tM map[hidden]int") + "\ntype hidden struct{ A int }\n", stashgen.Request{Type: "User"}, "M", "map key that is not a string or an integer"}, {"an EQL type the engine cannot produce yet", user(ctx + "\tAge int64 `stash:\"age,encrypt_into=BigintOrdOre\"`"), stashgen.Request{Type: "User"}, "Age", "has no EQL type BigintOrdOre"}, {"an index on a composite", user(ctx + "\tAttrs map[string]string `stash:\"attrs,encrypt,index=equality\"`"), stashgen.Request{Type: "User"}, "Attrs", "seals only as part of an opaque struct"}, @@ -198,7 +202,7 @@ func TestStaleOutputFileIsIgnored(t *testing.T) { } // A stale file: it names a type that no longer exists and redeclares // Encrypt, so the package does not type-check with it. - stale := "package users\n\nimport \"github.com/cipherstash/stack/languages/golang/encrypt/gensupport\"\n\nconst _ = gensupport.GeneratedVersion1\n\ntype EncryptedUser struct{ Gone Missing }\n\nfunc Encrypt() {}\n" + stale := "package users\n\nimport \"github.com/cipherstash/stack/languages/golang/encrypt/gensupport\"\n\nconst _ = gensupport.GeneratedVersion2\n\ntype EncryptedUser struct{ Gone Missing }\n\nfunc Encrypt() {}\n" if err := os.WriteFile(filepath.Join(dir, "user_stash.go"), []byte(stale), 0o600); err != nil { t.Fatal(err) } @@ -394,3 +398,13 @@ func TestAnIndexNoMethodQueriesGetsANotice(t *testing.T) { t.Errorf("a float field with ore has no Between:\n%s", file.Content) } } + +// The storage profile writes a struct with no fields at all as an empty +// object, so a set (map[string]struct{}) and an empty struct seal in an +// opaque struct, as they did as JSON. +func TestOpaqueStructAcceptsEmptyStructs(t *testing.T) { + src := user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tSeen map[string]struct{}\n\tMark marker") + "\ntype marker struct{}\n" + if _, err := generate(t, src, stashgen.Request{Type: "User"}); err != nil { + t.Fatal(err) + } +} diff --git a/languages/golang/stashgen/testdata/cases/accounts/account_stash.go.golden b/languages/golang/stashgen/testdata/cases/accounts/account_stash.go.golden index 502a2e3e9..6c6b0aedc 100644 --- a/languages/golang/stashgen/testdata/cases/accounts/account_stash.go.golden +++ b/languages/golang/stashgen/testdata/cases/accounts/account_stash.go.golden @@ -14,7 +14,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Not encrypted and not stored: the unexported field "cache". Tag it // `stash:"-"` to confirm that. diff --git a/languages/golang/stashgen/testdata/cases/contacts/contactstash_stash.go.golden b/languages/golang/stashgen/testdata/cases/contacts/contactstash_stash.go.golden index 366279cfe..338837fe9 100644 --- a/languages/golang/stashgen/testdata/cases/contacts/contactstash_stash.go.golden +++ b/languages/golang/stashgen/testdata/cases/contacts/contactstash_stash.go.golden @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // crm.Contact prints its sealed fields in the clear, and stashgen cannot add // print methods to a type from another package. diff --git a/languages/golang/stashgen/testdata/cases/directory/entry_stash.go.golden b/languages/golang/stashgen/testdata/cases/directory/entry_stash.go.golden index d0393bf7c..6291e9c40 100644 --- a/languages/golang/stashgen/testdata/cases/directory/entry_stash.go.golden +++ b/languages/golang/stashgen/testdata/cases/directory/entry_stash.go.golden @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Entry prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/stashgen/testdata/cases/documents/document_stash.go.golden b/languages/golang/stashgen/testdata/cases/documents/document_stash.go.golden index 97445ba49..902818d77 100644 --- a/languages/golang/stashgen/testdata/cases/documents/document_stash.go.golden +++ b/languages/golang/stashgen/testdata/cases/documents/document_stash.go.golden @@ -11,7 +11,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Document prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. @@ -39,12 +39,13 @@ type documentShape struct { } // The opaque value as it crosses the binding: one storage blob of the -// struct's fields, by their declared names. The json names read a value -// an earlier release sealed as a JSON document. +// struct's fields, by their declared names. A field a blob lacks (one +// added since it was sealed) reads as its zero value. The json names read +// a value an earlier release sealed as a JSON document. type documentOpaque struct { - Title string `vc:"title" json:"title"` - Body string `vc:"body" json:"body"` - Tags []string `vc:"tags" json:"tags"` + Title string `vc:"title,default" json:"title"` + Body string `vc:"body,default" json:"body"` + Tags []string `vc:"tags,default" json:"tags"` } var declaration = gensupport.DeclareOpaque("documents/v2/body") diff --git a/languages/golang/stashgen/testdata/cases/embedded/patient_stash.go.golden b/languages/golang/stashgen/testdata/cases/embedded/patient_stash.go.golden index 88f9c4999..7b64dd0c5 100644 --- a/languages/golang/stashgen/testdata/cases/embedded/patient_stash.go.golden +++ b/languages/golang/stashgen/testdata/cases/embedded/patient_stash.go.golden @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Patient prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/stashgen/testdata/cases/foreign/individualstash_stash.go.golden b/languages/golang/stashgen/testdata/cases/foreign/individualstash_stash.go.golden index b76f15a9d..1ac847a45 100644 --- a/languages/golang/stashgen/testdata/cases/foreign/individualstash_stash.go.golden +++ b/languages/golang/stashgen/testdata/cases/foreign/individualstash_stash.go.golden @@ -13,7 +13,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // pb.Individual prints its sealed fields in the clear, and stashgen cannot add // print methods to a type from another package. diff --git a/languages/golang/stashgen/testdata/cases/orders/order_stash.go.golden b/languages/golang/stashgen/testdata/cases/orders/order_stash.go.golden index ba0c84bf4..4b2dc4bd7 100644 --- a/languages/golang/stashgen/testdata/cases/orders/order_stash.go.golden +++ b/languages/golang/stashgen/testdata/cases/orders/order_stash.go.golden @@ -11,7 +11,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Order prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/stashgen/testdata/cases/orders/refund_stash.go.golden b/languages/golang/stashgen/testdata/cases/orders/refund_stash.go.golden index 6286d44d4..ba2d3ba83 100644 --- a/languages/golang/stashgen/testdata/cases/orders/refund_stash.go.golden +++ b/languages/golang/stashgen/testdata/cases/orders/refund_stash.go.golden @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Refund prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/stashgen/testdata/cases/tenants/note_stash.go.golden b/languages/golang/stashgen/testdata/cases/tenants/note_stash.go.golden index ffa276f75..7983edbf1 100644 --- a/languages/golang/stashgen/testdata/cases/tenants/note_stash.go.golden +++ b/languages/golang/stashgen/testdata/cases/tenants/note_stash.go.golden @@ -11,7 +11,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // Note prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/stashgen/testdata/cases/users/user_stash.go.golden b/languages/golang/stashgen/testdata/cases/users/user_stash.go.golden index 0b3c5d0fd..0a9a213e4 100644 --- a/languages/golang/stashgen/testdata/cases/users/user_stash.go.golden +++ b/languages/golang/stashgen/testdata/cases/users/user_stash.go.golden @@ -12,7 +12,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // User prints its sealed fields in the clear: it has no String or LogValue // method. Write them, or run stashgen with -redact. diff --git a/languages/golang/stashgen/testdata/policy_individual_stash.go.golden b/languages/golang/stashgen/testdata/policy_individual_stash.go.golden index b76f15a9d..1ac847a45 100644 --- a/languages/golang/stashgen/testdata/policy_individual_stash.go.golden +++ b/languages/golang/stashgen/testdata/policy_individual_stash.go.golden @@ -13,7 +13,7 @@ import ( ) // Stops compiling when the library does not accept this version of generated file. -const _ = gensupport.GeneratedVersion1 +const _ = gensupport.GeneratedVersion2 // pb.Individual prints its sealed fields in the clear, and stashgen cannot add // print methods to a type from another package. diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go index 83abf8f0f..ef1b22872 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go @@ -14,8 +14,8 @@ import ( type generatedVersion uint8 -// GeneratedVersion1 is the layout of files stashgen writes today. -const GeneratedVersion1 generatedVersion = 1 +// GeneratedVersion2 is the layout of files stashgen writes today. +const GeneratedVersion2 generatedVersion = 2 // OpaqueField is the one field of an opaque declaration. const OpaqueField = "value" From 541be400f826ae2645264db1cd78158868d296d5 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 16:13:08 +1100 Subject: [PATCH 3/4] fix(go): accept a pointer to a text type in an opaque struct stashgen refused an opaque field of type *time.Time, []*time.Time or any other pointer to a type with MarshalText and UnmarshalText as "has MarshalText but no UnmarshalText": textForm looked UnmarshalText up on a pointer to the field's type, and the method set of **time.Time holds neither method, while *time.Time still has MarshalText. Such fields were accepted before the storage blob. textForm now strips pointers first, which is what vcffi does: a pointer is written as its element and read back into a new one. The two type walkers, notJSONEncodable and notStorable, are one walk, notCarried, which traverses slices, arrays, pointers, maps and structs once and asks the JSON rule and the storage rule of every type it reaches, each stopping below a type that carries itself. The gaps the review found had slipped between the two copies; one walk has no second copy to disagree with. The call site keeps the two preambles, and every refusal case passes unchanged. Tests: an opaque struct with At *time.Time, Ats []*time.Time, a map of *time.Time and a **time.Time generates; a pointer to a MarshalText-only type is still refused; and gensupport round-trips *time.Time and a nil through the blob. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .../gensupport/gensupport_internal_test.go | 22 +++ languages/golang/stashgen/read.go | 169 +++++++++--------- languages/golang/stashgen/refusal_test.go | 13 ++ 3 files changed, 123 insertions(+), 81 deletions(-) diff --git a/languages/golang/encrypt/gensupport/gensupport_internal_test.go b/languages/golang/encrypt/gensupport/gensupport_internal_test.go index 57c583e6e..3a8200a89 100644 --- a/languages/golang/encrypt/gensupport/gensupport_internal_test.go +++ b/languages/golang/encrypt/gensupport/gensupport_internal_test.go @@ -340,6 +340,28 @@ func TestDecryptErrorsHoldNoPlaintext(t *testing.T) { } } +// A pointer to a type with MarshalText and UnmarshalText (*time.Time) is +// written as its element's text, nil as null, and read back into a new +// element: what stashgen accepts such a field on the strength of. +func TestAnOpaqueStructCarriesAPointerToATextType(t *testing.T) { + at := time.Date(2026, 10, 26, 5, 3, 16, 0, time.UTC) + type stamped struct { + At *time.Time `vc:"at,default"` + Ats []*time.Time `vc:"ats,default"` + } + blob, err := opaqueBytes(stamped{At: &at, Ats: []*time.Time{&at, nil}}) + if err != nil { + t.Fatal(err) + } + var got stamped + if err := Opaque(Values{OpaqueField: blob}, &got); err != nil { + t.Fatal(err) + } + if got.At == nil || !got.At.Equal(at) || len(got.Ats) != 2 || got.Ats[0] == nil || !got.Ats[0].Equal(at) || got.Ats[1] != nil { + t.Fatalf("*time.Time came back as %+v", got) + } +} + // mustStorage is v as a storage blob. func mustStorage(t *testing.T, v any) []byte { t.Helper() diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index 81048a0a2..c26c78163 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -663,11 +663,12 @@ func (r *reader) buildFields(c *collected) error { continue } g := genField{Field: Field{Name: cf.tag.Name, GoName: cf.goName, GoType: r.goType(cf.typ), Verb: VerbEncrypt}, typeExpr: r.typeExpr(cf.typ)} - if reason := notJSONEncodable(cf.typ, map[types.Type]bool{}); reason != "" { - return fieldErr(typeName, cf.goName, "an opaque struct is read back from the JSON document earlier releases sealed, and %s %s", g.typeExpr, reason) + asJSON, asBlob := notCarried(cf.typ, map[types.Type]bool{}, true, true) + if asJSON != "" { + return fieldErr(typeName, cf.goName, "an opaque struct is read back from the JSON document earlier releases sealed, and %s %s", g.typeExpr, asJSON) } - if reason := notStorable(cf.typ, map[types.Type]bool{}); reason != "" { - return fieldErr(typeName, cf.goName, "an opaque struct is sealed as one storage blob, and %s %s", g.typeExpr, reason) + if asBlob != "" { + return fieldErr(typeName, cf.goName, "an opaque struct is sealed as one storage blob, and %s %s", g.typeExpr, asBlob) } if prev, dup := seen[g.Name]; dup { return fieldErr(typeName, cf.goName, "two fields write the name %q: %s and %s", g.Name, prev, cf.goName) @@ -1053,105 +1054,104 @@ func itOrEach(fields []string) string { return "each" } -// notJSONEncodable says why encoding/json cannot carry a type both ways, or -// "" when it can: every field of an opaque struct must, because a row an -// earlier release sealed holds the struct as one JSON document, which -// comes back by decoding it into the generated shape struct. The struct -// is sealed as a storage blob now, which notStorable checks. A type with its own MarshalJSON and -// UnmarshalJSON (time.Time) carries itself; otherwise a scalar, a slice or -// array of an encodable type, a map with string or integer keys, a pointer, -// or a struct whose every field is exported and encodable. A struct with an -// unexported field would come back without it, an interface or a channel -// not at all. -func notJSONEncodable(t types.Type, seen map[types.Type]bool) string { - if seen[t] { - return "" +// notCarried says why an opaque struct's field type cannot be carried both +// ways by the two codecs every such field must satisfy, as a reason from +// each, or "" from one that carries it. encoding/json: a row an earlier +// release sealed holds the struct as one JSON document, which comes back +// by decoding it into the generated shape struct. The storage profile +// (vcffi.MarshalStorage and UnmarshalStorage): the struct is sealed as a +// storage blob now. One walk asks both rules of every type it reaches, so +// a case one rule covers and the other does not cannot fall between two +// walkers; byJSON and byStorage say which rules still apply, and each stops +// below a type that carries itself (a MarshalJSON and UnmarshalJSON pair +// for JSON, a MarshalText and UnmarshalText pair for storage). +// +// The JSON rule: a scalar, a slice or array of a carried type, a map with +// string or integer keys, a pointer, or a struct whose every field is +// exported and carried; a struct with an unexported field would come back +// without it, an interface or a channel not at all. The storage rule, +// vcffi's own: a scalar by its kind; a slice, array or pointer by its +// element; a map whose key is an integer, a string, bytes or text, by its +// value; a struct by its exported fields; no logical value type +// (encrypt.Decimal, encrypt.Date, which have no storage form); and no type +// with only half of the text pair. It knows nothing of MarshalJSON, so a +// type that carries itself through JSON alone is walked like any other. +func notCarried(t types.Type, seen map[types.Type]bool, byJSON, byStorage bool) (asJSON, asBlob string) { + if seen[t] || (!byJSON && !byStorage) { + return "", "" } seen[t] = true defer delete(seen, t) - if hasMethod(types.NewPointer(t), "UnmarshalJSON") && hasMethod(t, "MarshalJSON") { - return "" + if byJSON && hasMethod(types.NewPointer(t), "UnmarshalJSON") && hasMethod(t, "MarshalJSON") { + byJSON = false + } + if byStorage { + if reason := noStorageForm(t); reason != "" { + return "", reason + } + text, reason := textForm(t) + if reason != "" { + return "", reason + } + byStorage = !text + } + if !byJSON && !byStorage { + return "", "" } switch u := t.Underlying().(type) { case *types.Basic: if u.Info()&(types.IsBoolean|types.IsInteger|types.IsFloat|types.IsString) != 0 && u.Info()&types.IsUntyped == 0 { - return "" + return "", "" } - return "is not a bool, integer, float or string" + if byJSON { + return "is not a bool, integer, float or string", "" + } + return "", "is not a bool, integer, float or string, which a storage blob has no form for" case *types.Slice: - return notJSONEncodable(u.Elem(), seen) + return notCarried(u.Elem(), seen, byJSON, byStorage) case *types.Array: - return notJSONEncodable(u.Elem(), seen) + return notCarried(u.Elem(), seen, byJSON, byStorage) case *types.Pointer: - return notJSONEncodable(u.Elem(), seen) + return notCarried(u.Elem(), seen, byJSON, byStorage) case *types.Map: - if k, ok := u.Key().Underlying().(*types.Basic); !ok || k.Info()&(types.IsString|types.IsInteger) == 0 { - return "has a map key that is not a string or an integer" - } - return notJSONEncodable(u.Elem(), seen) - case *types.Struct: - for i := range u.NumFields() { - fld := u.Field(i) - if !fld.Exported() { - return fmt.Sprintf("has an unexported field %s that JSON would drop", fld.Name()) - } - if reason := notJSONEncodable(fld.Type(), seen); reason != "" { - return "has a field " + fld.Name() + " that " + reason + if byJSON { + if k, ok := u.Key().Underlying().(*types.Basic); !ok || k.Info()&(types.IsString|types.IsInteger) == 0 { + return "has a map key that is not a string or an integer", "" } } - return "" - } - return "is not a type JSON carries" -} - -// notStorable says why the storage profile (vcffi.MarshalStorage and -// UnmarshalStorage) cannot carry a type both ways, or "" when it can, by -// vcffi's own rules: a scalar by its kind; a type with MarshalText and -// UnmarshalText as its text; a slice, array or pointer by its element; a -// map whose key is an integer, a string, bytes or text, by its value; a -// struct by its exported fields. It knows nothing of MarshalJSON, so a type -// that carries itself through JSON alone is walked like any other. -// notJSONEncodable is the other half: an opaque struct must still read the -// JSON documents earlier releases sealed. -func notStorable(t types.Type, seen map[types.Type]bool) string { - if seen[t] { - return "" - } - seen[t] = true - defer delete(seen, t) - if reason := noStorageForm(t); reason != "" { - return reason - } - if text, reason := textForm(t); text || reason != "" { - return reason - } - switch u := t.Underlying().(type) { - case *types.Slice: - return notStorable(u.Elem(), seen) - case *types.Array: - return notStorable(u.Elem(), seen) - case *types.Pointer: - return notStorable(u.Elem(), seen) - case *types.Map: - if reason := notStorableKey(u.Key()); reason != "" { - return reason + if byStorage { + if reason := notStorableKey(u.Key()); reason != "" { + return "", reason + } } - return notStorable(u.Elem(), seen) + return notCarried(u.Elem(), seen, byJSON, byStorage) case *types.Struct: - // vcffi writes a struct by its exported fields and refuses one - // that has unexported fields only; a struct with none at all - // (struct{}) is an empty object, read back as itself. + // Both write a struct by its exported fields and drop the rest: + // vcffi refuses a struct that has unexported fields only, and a + // struct with none at all (struct{}) is an empty object to both, + // read back as itself. for i := range u.NumFields() { fld := u.Field(i) if !fld.Exported() { - return fmt.Sprintf("has an unexported field %s that a storage blob would drop (give the type MarshalText and UnmarshalText)", fld.Name()) + if byJSON { + return fmt.Sprintf("has an unexported field %s that JSON would drop", fld.Name()), "" + } + return "", fmt.Sprintf("has an unexported field %s that a storage blob would drop (give the type MarshalText and UnmarshalText)", fld.Name()) } - if reason := notStorable(fld.Type(), seen); reason != "" { - return "has a field " + fld.Name() + " that " + reason + asJSON, asBlob := notCarried(fld.Type(), seen, byJSON, byStorage) + if asJSON != "" { + return "has a field " + fld.Name() + " that " + asJSON, "" + } + if asBlob != "" { + return "", "has a field " + fld.Name() + " that " + asBlob } } + return "", "" } - return "" + if byJSON { + return "is not a type JSON carries", "" + } + return "", "is not a type a storage blob carries" } // noStorageForm refuses vitaminc's logical value types, which the @@ -1177,8 +1177,15 @@ func noStorageForm(t types.Type) string { // textForm reports whether t is written as its text, and why not when it // has only half of the pair: a type with MarshalText alone is written as // text and read back by its kind, which fails; one with UnmarshalText -// alone is written by its kind and read back as text. +// alone is written by its kind and read back as text. A pointer is written +// as its element and read back into a new one, so the pair is looked up on +// the element: the method set of *T holds T's methods, and that of **T +// holds none, so *time.Time asked through a pointer to it has MarshalText +// and no UnmarshalText. func textForm(t types.Type) (bool, string) { + for p, ok := t.Underlying().(*types.Pointer); ok; p, ok = t.Underlying().(*types.Pointer) { + t = p.Elem() + } marshals := hasMethod(t, "MarshalText") || hasMethod(types.NewPointer(t), "MarshalText") unmarshals := hasMethod(types.NewPointer(t), "UnmarshalText") switch { diff --git a/languages/golang/stashgen/refusal_test.go b/languages/golang/stashgen/refusal_test.go index 67f161491..6f343d23b 100644 --- a/languages/golang/stashgen/refusal_test.go +++ b/languages/golang/stashgen/refusal_test.go @@ -108,6 +108,7 @@ func TestRefusals(t *testing.T) { {"an opaque struct with a decimal", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tAmount encrypt.Decimal"), stashgen.Request{Type: "User"}, "Amount", "has no storage form"}, {"an opaque struct with a date in a slice", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tDays []encrypt.Date"), stashgen.Request{Type: "User"}, "Days", "has no storage form"}, {"an opaque struct with a type that writes text it cannot read", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tL level") + "\ntype level int\n\nfunc (l level) MarshalText() ([]byte, error) { return nil, nil }\n", stashgen.Request{Type: "User"}, "L", "MarshalText but no UnmarshalText"}, + {"an opaque struct with a pointer to a type that writes text it cannot read", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tL *level") + "\ntype level int\n\nfunc (l level) MarshalText() ([]byte, error) { return nil, nil }\n", stashgen.Request{Type: "User"}, "L", "MarshalText but no UnmarshalText"}, {"an opaque struct with a type that reads text it does not write", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tL level") + "\ntype level int\n\nfunc (l *level) UnmarshalText([]byte) error { return nil }\n", stashgen.Request{Type: "User"}, "L", "UnmarshalText but no MarshalText"}, {"an opaque struct with a JSON-only type keyed by a struct", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tE ev") + "\ntype k struct{ A int }\n\ntype ev struct{ M map[k]int }\n\nfunc (e ev) MarshalJSON() ([]byte, error) { return nil, nil }\n\nfunc (e *ev) UnmarshalJSON([]byte) error { return nil }\n", stashgen.Request{Type: "User"}, "E", "map key that is not an integer, a string, bytes or text"}, {"an opaque struct with a map keyed by a struct", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tM map[hidden]int") + "\ntype hidden struct{ A int }\n", stashgen.Request{Type: "User"}, "M", "map key that is not a string or an integer"}, @@ -408,3 +409,15 @@ func TestOpaqueStructAcceptsEmptyStructs(t *testing.T) { t.Fatal(err) } } + +// A pointer to a type with MarshalText and UnmarshalText (*time.Time, and +// a slice, a map value or a pointer to a pointer of one) is written as its +// element's text and read back into a new element. The pair is looked up on +// the element: the method set of **time.Time holds neither method, and +// asking it there refused the field as writing text it could not read. +func TestOpaqueStructAcceptsPointersToTextTypes(t *testing.T) { + src := user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tAt *time.Time\n\tAts []*time.Time\n\tByName map[string]*time.Time\n\tDeep **time.Time") + if _, err := generate(t, src, stashgen.Request{Type: "User"}); err != nil { + t.Fatal(err) + } +} From 540748c26fa869ab0b4de99b026cbdc7af588a93 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 16:19:11 +1100 Subject: [PATCH 4/4] fix(go): refuse an opaque value neither reader takes, and say how to migrate a stored Date or Decimal Opaque handed a value that begins with neither the storage header nor "{" to the JSON reader, which failed on it with a less clear error. A JSON document of the shape struct is an object, so it always begins "{"; a value beginning with neither is now refused as ErrEncoding saying so, pinned for "[]", "VC", "VC\x01", an empty value and "null". The CHANGELOG and the stashgen README say that a row an earlier release sealed with an encrypt.Decimal or encrypt.Date in an opaque struct does not decrypt under this release, because no generated file can name the field, and give the migration: decrypt with the earlier release, change the field's type, encrypt again, before upgrading. notStorableKey loses its slice branch: a Go map key is comparable, so a slice never reaches it. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 7 ++++++- languages/golang/cmd/stashgen/README.md | 1 + languages/golang/encrypt/gensupport/codec.go | 7 +++++++ .../gensupport/gensupport_internal_test.go | 9 +++++++++ languages/golang/stashgen/read.go | 15 +++++---------- 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index ce6cd16c9..6bdabe5ee 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -69,7 +69,12 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. form for them), a type with only one of `MarshalText` and `UnmarshalText`, a map key that is not an integer, a string, bytes or text, and a type that only `MarshalJSON` carries over a struct with - unexported fields. A blob a later release writes is + unexported fields. **Rows already sealed with an `encrypt.Decimal` or + `encrypt.Date` in an opaque struct do not decrypt under this release**, + since no generated file can name the field: before upgrading, decrypt + them with the earlier release, change the field to a string (or to its + parts), and encrypt them again; the earlier release writes JSON, which + this release reads. A blob a later release writes is `encrypt.ErrStorageVersion`. This takes vcffi from the unreleased cipherstash/vitaminc#526 by pseudo-version; the storage format is a compatibility commitment only once vitaminc releases it, so no Go diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 5967fda76..ca7ae7ef6 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -277,6 +277,7 @@ An `opaque` struct crosses as one storage blob and is one column, read back into The blob is vitaminc's storage profile, version 1 (`vcffi.MarshalStorage`), the format vitaminc keeps readable across releases and languages. A field may be any type that both the storage profile and `encoding/json` carry both ways: a scalar or a type defined over one, `[]byte`, slices, maps with string or integer keys, pointers, structs whose fields are all exported, and types with both `MarshalText` and `UnmarshalText`, such as `time.Time`. `encrypt.Decimal` and `encrypt.Date` have no storage form and are refused in an opaque struct: store them as text, or by their parts. +A row an earlier release sealed with one of them in an opaque struct does not decrypt under this release, because no generated file can name the field: decrypt such rows with the earlier release, change the field's type, and encrypt them again before upgrading. The generated shape struct tags each field `vc:"name,default"`, so a field added to the opaque struct reads as its zero value from a row sealed before it; a field added to a nested struct type needs a pointer type for the same. `Encrypt` refuses a string that is not UTF-8 and a value nested more than 128 levels deep, as `encrypt.ErrEncoding`. `encoding/json` still matters because a row an earlier release sealed holds a JSON document, and `Decrypt` reads it as JSON. diff --git a/languages/golang/encrypt/gensupport/codec.go b/languages/golang/encrypt/gensupport/codec.go index e7f9acdec..6b26d15ac 100644 --- a/languages/golang/encrypt/gensupport/codec.go +++ b/languages/golang/encrypt/gensupport/codec.go @@ -424,6 +424,13 @@ func Opaque[T any](vals Values, out *T) error { } return nil } + // encoding/json writes a struct as an object, so a JSON document of + // the shape struct begins "{"; a value that begins with neither is not + // one either reader can take, and is said so rather than handed to + // the JSON reader to fail on. + if !bytes.HasPrefix(encoded, []byte("{")) { + return fmt.Errorf("gensupport: %w: the opaque value is neither a storage blob nor a JSON document", encrypt.ErrEncoding) + } if err := json.Unmarshal(encoded, out); err != nil { // Its error is not wrapped: encoding/json's text quotes the value. return fmt.Errorf("gensupport: %w: the opaque value does not decode into a %T", encrypt.ErrEncoding, *out) diff --git a/languages/golang/encrypt/gensupport/gensupport_internal_test.go b/languages/golang/encrypt/gensupport/gensupport_internal_test.go index 3a8200a89..43281240c 100644 --- a/languages/golang/encrypt/gensupport/gensupport_internal_test.go +++ b/languages/golang/encrypt/gensupport/gensupport_internal_test.go @@ -181,6 +181,15 @@ func TestConvertStaysWithinAFamily(t *testing.T) { if err := Opaque(Values{OpaqueField: future}, &back); !errors.Is(err, encrypt.ErrEncoding) || !errors.Is(err, vcffi.ErrUnsupportedVersion) { t.Fatalf("Opaque of a version %d blob: %v", vcffi.StorageVersion+1, err) } + // A value that is neither a storage blob nor a JSON object (a JSON + // array, a header cut short of its "S", nothing at all) is refused as + // such, by neither reader. + for _, neither := range [][]byte{[]byte("[]"), []byte("VC"), []byte("VC\x01"), {}, []byte("null")} { + err := Opaque(Values{OpaqueField: neither}, &back) + if !errors.Is(err, encrypt.ErrEncoding) || !strings.Contains(err.Error(), "neither a storage blob nor a JSON document") { + t.Fatalf("Opaque of %q: %v", neither, err) + } + } if err := Opaque(Values{OpaqueField: "not bytes"}, &back); err == nil { t.Fatal("Opaque accepted a string") } diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index c26c78163..cfc1177a1 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -1200,8 +1200,9 @@ func textForm(t types.Type) (bool, string) { } // notStorableKey says why a map key has no storage form: vcffi takes an -// integer, a string, bytes (a byte slice or array, a 128-bit integer) or a -// type with MarshalText and UnmarshalText. +// integer, a string, bytes (a byte array, a 128-bit integer; a slice is +// not comparable, so Go never keys a map by one) or a type with MarshalText +// and UnmarshalText. func notStorableKey(k types.Type) string { if kind, _ := valueKind(k); kind == KindInt128 || kind == KindUint128 { return "" @@ -1216,14 +1217,8 @@ func notStorableKey(k types.Type) string { if u.Info()&(types.IsInteger|types.IsString) != 0 { return "" } - case *types.Slice, *types.Array: - var elem types.Type - if s, ok := u.(*types.Slice); ok { - elem = s.Elem() - } else { - elem = u.(*types.Array).Elem() - } - if b, ok := elem.Underlying().(*types.Basic); ok && b.Kind() == types.Byte { + case *types.Array: + if b, ok := u.Elem().Underlying().(*types.Basic); ok && b.Kind() == types.Byte { return "" } }