diff --git a/.github/workflows/tests-golang.yml b/.github/workflows/tests-golang.yml index 7494b4dcb..868a65fc1 100644 --- a/.github/workflows/tests-golang.yml +++ b/.github/workflows/tests-golang.yml @@ -21,7 +21,9 @@ name: Tests (Go) # the Go module's PostgreSQL tests, against PostgreSQL 17 with # the EQL v4 bundle built from this tree installed: rows the Go # SDK stores in every EQL type it produces, found by the queries -# it derives, through EQL's operators. +# it derives, through EQL's operators; and a few hundred random +# values in every ordered type, which must order as their +# plaintexts do. # live the Go live tests and the stack-encrypt examples, against real # ZeroKMS with the repository's CS_* credentials. # diff --git a/languages/golang/encrypt/eql_order_postgres_test.go b/languages/golang/encrypt/eql_order_postgres_test.go new file mode 100644 index 000000000..a7f9b302b --- /dev/null +++ b/languages/golang/encrypt/eql_order_postgres_test.go @@ -0,0 +1,725 @@ +package encrypt_test + +import ( + "cmp" + "context" + "fmt" + "hash/fnv" + "math" + "math/big" + "math/rand/v2" + "os" + "slices" + "strconv" + "strings" + "testing" + "time" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" +) + +// Order through PostgreSQL, over random values. TestEQLTypesRoundTripAndQueryInPostgres +// runs each ordering operator over four fixed values, and a comparator that +// is only partly wrong (one that reads a block ORE term's first blocks, a +// sign bit, or a decimal's scale) still orders most pairs right. Here every +// ordered EQL v4 type gets a temporary table of a few hundred random values +// and the edges of its order, sealed by the Go SDK. Random query values are +// run through `<`, `<=`, `>` and `>=` and must find exactly the rows +// plaintext comparison selects, and ORDER BY on the type's order term must +// put the rows in plaintext order. +// +// Plaintext order is the engine's order key (ADR-0002, and +// stack-encrypt's value module): text by the UTF-8 bytes of its fold, a +// float with -0.0 equal to 0.0 and every NaN one value above +Inf, a +// timestamp to the microsecond (the nanoseconds below it truncated, floor +// on the count since the epoch), and a decimal by value, whatever its scale. +// +// The seed is fixed and logged; EQL_ORDER_SEED replaces it. Each type draws +// from its own generator, seeded from the seed and the type's name, so +// `-run 'TestEQLOrderMatchesPlaintextInPostgres/TextOrdOre'` with the logged +// seed replays one type's values exactly. +// +// Like TestEQLTypesRoundTripAndQueryInPostgres it is skipped without +// DATABASE_URL, and fails when EQL v4 is not installed there. + +const ( + eqlOrderSeedEnv = "EQL_ORDER_SEED" + eqlOrderDefaultSeed = 1343 + // Random values stored per type, beside the edges and the duplicates. + eqlOrderRandom = 300 + // Query values per type, beside every edge: random draws, and stored + // values (so `<=` and `>=` meet ties). + eqlOrderQueryDraws, eqlOrderQueryStored = 8, 8 + // Rows per INSERT statement. + eqlOrderBatch = 100 +) + +// The ordering operators, by SDK name, and what each selects given the +// plaintext comparison of the stored value with the query value. +var eqlOrderOperators = []struct { + op, sql string + holds func(c int) bool +}{ + {"lt", "<", func(c int) bool { return c < 0 }}, + {"lte", "<=", func(c int) bool { return c <= 0 }}, + {"gt", ">", func(c int) bool { return c > 0 }}, + {"gte", ">=", func(c int) bool { return c >= 0 }}, +} + +// The function that extracts a column's order term, by the index that +// carries it: what an ordered index is built over and ORDER BY sorts by. +// EQL v4 puts no operator class on its domains, so a bare ORDER BY v would +// sort the jsonb. +var eqlOrderExtractors = map[string]string{ + "ope": "eql_v4.ord_term", + "ore_block": "eql_v4.ord_term_ore", +} + +func TestEQLOrderMatchesPlaintextInPostgres(t *testing.T) { + url := os.Getenv("DATABASE_URL") + if url == "" { + t.Skip("DATABASE_URL is not set") + } + seed := uint64(eqlOrderDefaultSeed) + if s := os.Getenv(eqlOrderSeedEnv); s != "" { + n, err := strconv.ParseUint(s, 10, 64) + if err != nil { + t.Fatalf("%s=%q: %v", eqlOrderSeedEnv, s, err) + } + seed = n + } + t.Logf("seed %d; replay with %s=%d", seed, eqlOrderSeedEnv, seed) + + ctx := context.Background() + c := deterministicEQLClient(t) + cipher := c.DefaultKeyset() + conn := eqlPostgresConn(t, ctx, url) + + covered := 0 + for _, typ := range eql.Types { + index := eqlOrderIndex(typ) + if !typ.Producible || index == "" { + continue + } + covered++ + t.Run(typ.Name, func(t *testing.T) { + h := eqlHarness{t: t, ctx: ctx, cipher: cipher, conn: conn, typ: typ} + o := eqlOrder{seed: seed, index: index, r: eqlOrderRand(seed, typ.Name)} + switch typ.Plaintext { + case "int16": + runEQLOrder(h, o, eqlIntOrder[int16](gensupport.Int16, 16)) + case "int32": + runEQLOrder(h, o, eqlIntOrder[int32](gensupport.Int32, 32)) + case "int64": + runEQLOrder(h, o, eqlIntOrder[int64](gensupport.Int64, 64)) + case "float32": + runEQLOrder(h, o, eqlFloat32Order()) + case "float64": + runEQLOrder(h, o, eqlFloat64Order()) + case "string": + runEQLOrder(h, o, eqlTextOrder()) + case "date": + runEQLOrder(h, o, eqlDateOrder()) + case "timestamp": + runEQLOrder(h, o, eqlTimestampOrder()) + case "decimal": + runEQLOrder(h, o, eqlDecimalOrder(t)) + default: + t.Fatalf("no ordered plaintexts for wire kind %q: add a case so %s is tested", typ.Plaintext, typ.Name) + } + }) + } + if covered == 0 { + t.Fatal("eql.Types lists no producible ordered type, so this test checked nothing") + } +} + +// eqlOrderIndex is the order index typ carries, or "" for none: an index +// that answers `lt` in eqlIndexOperators, the list the round-trip test fails +// without. So an order index added there is tested here too, and fails at +// the extractor until it has one, rather than being skipped. +func eqlOrderIndex(typ eql.Type) string { + for _, index := range typ.Indexes { + if slices.Contains(eqlIndexOperators[index], "lt") { + return index + } + } + return "" +} + +// eqlOrderRand is one type's generator: the seed and the type's name, so a +// type's values do not depend on which other types ran. +func eqlOrderRand(seed uint64, name string) *rand.Rand { + h := fnv.New64a() + _, _ = h.Write([]byte(name)) + return rand.New(rand.NewPCG(seed, h.Sum64())) +} + +type eqlOrder struct { + seed uint64 + index string + r *rand.Rand +} + +// eqlOrderGen is how one plaintext kind is drawn and ordered. +type eqlOrderGen[T any] struct { + kind gensupport.Kind + // cmp is the engine's plaintext order. + cmp func(a, b T) int + // draw draws one value. It may keep state (a pool of shared prefixes, + // say) so that values that differ only late are drawn together. + draw func(r *rand.Rand) T + // edges are stored in every run: the ends of the range, zeros, and + // neighbours. + edges []T + show func(T) string +} + +func runEQLOrder[T any](h eqlHarness, o eqlOrder, g eqlOrderGen[T]) { + t, ctx, typ := h.t, h.ctx, h.typ + t.Helper() + extractor, ok := eqlOrderExtractors[o.index] + if !ok { + t.Fatalf("no order term extractor for the %q index: add one so %s is tested", o.index, typ.Name) + } + r := o.r + + // The stored values: the edges, the random draws, and thirty of those + // again, so ties are not left to chance. Shuffled, so a row's id says + // nothing of its order. + values := slices.Clone(g.edges) + for range eqlOrderRandom { + values = append(values, g.draw(r)) + } + for range eqlOrderRandom / 10 { + values = append(values, values[r.IntN(len(values))]) + } + r.Shuffle(len(values), func(i, j int) { values[i], values[j] = values[j], values[i] }) + + decl := gensupport.Declare("go_eql_order").Passthrough("id").EncryptInto("v", g.kind, typ.Name) + if err := decl.Err(); err != nil { + t.Fatal(err) + } + rows := make([]gensupport.Values, len(values)) + for i, v := range values { + rows[i] = gensupport.Values{"id": int32(i + 1), "v": v} + } + sealed, err := eqlCodec(decl).Encrypt(ctx, h.cipher, rows) + if err != nil { + t.Fatal(err) + } + + table := "go_eql_order_" + strings.ToLower(typ.Name) + if _, err := h.conn.ExecContext(ctx, fmt.Sprintf(`CREATE TEMPORARY TABLE %s (id int PRIMARY KEY, v %s NOT NULL)`, table, typ.SQLDomain)); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _, _ = h.conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS `+table) }) + for start := 0; start < len(sealed); start += eqlOrderBatch { + batch := sealed[start:min(start+eqlOrderBatch, len(sealed))] + var stmt strings.Builder + fmt.Fprintf(&stmt, "INSERT INTO %s VALUES ", table) + args := make([]any, 0, 2*len(batch)) + for i, rec := range batch { + if i > 0 { + stmt.WriteString(", ") + } + fmt.Fprintf(&stmt, "($%d, $%d::text::jsonb::%s)", 2*i+1, 2*i+2, typ.SQLDomain) + args = append(args, start+i+1, string(rec["v"].EQL)) + } + if _, err := h.conn.ExecContext(ctx, stmt.String(), args...); err != nil { + t.Fatalf("inserting rows %d to %d: %v", start+1, start+len(batch), err) + } + } + + ids := func(query string, args ...any) []int { + t.Helper() + res, err := h.conn.QueryContext(ctx, query, args...) + if err != nil { + t.Fatalf("%s: %v", query, err) + } + defer res.Close() + var found []int + for res.Next() { + var id int + if err := res.Scan(&id); err != nil { + t.Fatal(err) + } + found = append(found, id) + } + if err := res.Err(); err != nil { + t.Fatalf("%s: %v", query, err) + } + return found + } + replay := fmt.Sprintf("seed %d; replay with %s=%d", o.seed, eqlOrderSeedEnv, o.seed) + failures := 0 + fail := func(format string, args ...any) { + t.Helper() + t.Errorf(format, args...) + if failures++; failures == 5 { + t.Fatalf("%s: stopping after %d disagreements (%s)", typ.Name, failures, replay) + } + } + + // Each query value through each operator, against plaintext comparison + // of every stored value. Every edge is a query value, not a sample of + // them: the edges hold the ties the order rules make (-0.0 and 0.0, 1.5 + // and 1.50, instants in one microsecond), and an engine that splits a tie + // shows only in a query on one of the tied values. ORDER BY cannot show + // it, since plaintext order accepts tied rows in either order. + queries := slices.Clone(g.edges) + for range eqlOrderQueryDraws { + queries = append(queries, g.draw(r)) + } + for range eqlOrderQueryStored { + queries = append(queries, values[r.IntN(len(values))]) + } + field := gensupport.NewField[T](decl, "v") + for _, q := range queries { + for _, op := range eqlOrderOperators { + term, err := field.Term(ctx, h.cipher, op.op, q) + if err != nil { + t.Fatalf("%s(%s): %v", op.op, g.show(q), err) + } + found := ids(fmt.Sprintf(`SELECT id FROM %s WHERE v %s $1::text::jsonb::%s ORDER BY id`, table, op.sql, typ.QuerySQLDomain), string(term)) + var want []int + for i, v := range values { + if op.holds(g.cmp(v, q)) { + want = append(want, i+1) + } + } + if slices.Equal(found, want) { + continue + } + // The first row the two disagree on names the value pair. + id, returned := eqlFirstDifference(found, want) + v := values[id-1] + if returned { + fail("%s: v %s q returned row %d, but in plaintext %s %s %s is false (%s)", + typ.Name, op.sql, id, g.show(v), op.sql, g.show(q), replay) + } else { + fail("%s: v %s q missed row %d, but in plaintext %s %s %s is true (%s)", + typ.Name, op.sql, id, g.show(v), op.sql, g.show(q), replay) + } + } + } + + // ORDER BY the order term, both ways: every row, in plaintext order, + // so equal values come out next to each other. + for _, dir := range []string{"ASC", "DESC"} { + order := ids(fmt.Sprintf(`SELECT id FROM %s ORDER BY %s(v) %s`, table, extractor, dir)) + if len(order) != len(values) { + t.Fatalf("%s: ORDER BY %s(v) %s returned %d rows, want %d", typ.Name, extractor, dir, len(order), len(values)) + } + for i := 1; i < len(order); i++ { + a, b := values[order[i-1]-1], values[order[i]-1] + c := g.cmp(a, b) + if (dir == "ASC" && c > 0) || (dir == "DESC" && c < 0) { + fail("%s: ORDER BY %s(v) %s put row %d (%s) right before row %d (%s), out of plaintext order (%s)", + typ.Name, extractor, dir, order[i-1], g.show(a), order[i], g.show(b), replay) + break + } + } + } +} + +// eqlFirstDifference is the lowest id in one of two ascending lists and not +// the other, and whether it is in found. +func eqlFirstDifference(found, want []int) (int, bool) { + i, j := 0, 0 + for i < len(found) && j < len(want) { + switch { + case found[i] == want[j]: + i++ + j++ + case found[i] < want[j]: + return found[i], true + default: + return want[j], false + } + } + if i < len(found) { + return found[i], true + } + return want[j], false +} + +// Integers: any bit pattern, small values around zero, values that share +// all but their low six bits (block ORE's last block), and values that +// differ only in one of their top bits (the sign among them). +func eqlIntOrder[I int16 | int32 | int64](kind gensupport.Kind, width uint) eqlOrderGen[I] { + hi := I(1)<<(width-1) - 1 + lo := -hi - 1 + var bases []I + draw := func(r *rand.Rand) I { + if bases == nil { + for range 4 { + bases = append(bases, I(r.Uint64())&^63) + } + } + switch r.IntN(4) { + case 0: + return I(r.Uint64()) + case 1: + return I(r.IntN(201) - 100) + case 2: + return bases[r.IntN(len(bases))] | I(r.IntN(64)) + default: + return bases[r.IntN(len(bases))] ^ I(1)<<(width-1-uint(r.IntN(3))) + } + } + edges := []I{lo, lo + 1, -256, -1, 0, 1, 63, 64, 255, 256, hi - 1, hi} + if width == 64 { + shift := width / 2 // 32, written so int16 compiles + edges = append(edges, -I(1)< 0 { + if len(digits) <= scale { + digits = strings.Repeat("0", scale+1-len(digits)) + digits + } + digits = digits[:len(digits)-scale] + "." + digits[len(digits)-scale:] + } + if negative { + digits = "-" + digits + } + return digits +} + +// Decimals: any mantissa and scale; small values at small scales, which +// tie across scales (1.5 and 1.50); a few large mantissas and their +// neighbours, which differ only in the last digit; and pooled values again +// with trailing zeros, the same value at a larger scale. +func eqlDecimalOrder(t *testing.T) eqlOrderGen[encrypt.Decimal] { + parse := func(s string) encrypt.Decimal { + d, err := encrypt.ParseDecimal(s) + if err != nil { + t.Fatalf("ParseDecimal(%q): %v", s, err) + } + return d + } + type base struct { + mantissa *big.Int + scale int + negative bool + } + var bases []base + draw := func(r *rand.Rand) encrypt.Decimal { + if bases == nil { + for range 4 { + // 96 bits, with room for the neighbours above it. + m := eqlRandBits(r, 94) + m.SetBit(m, 95, 1) + bases = append(bases, base{m, r.IntN(29), r.IntN(2) == 0}) + } + } + switch r.IntN(4) { + case 0: + m := eqlRandBits(r, r.IntN(97)) + return parse(eqlDecimalText(m, r.IntN(29), r.IntN(2) == 0)) + case 1: + return parse(eqlDecimalText(big.NewInt(r.Int64N(4001)), r.IntN(4), r.IntN(2) == 0)) + case 2: + b := bases[r.IntN(len(bases))] + return parse(eqlDecimalText(new(big.Int).Add(b.mantissa, big.NewInt(r.Int64N(4))), b.scale, b.negative)) + default: + // A small value at a scale it does not need. + m := big.NewInt(r.Int64N(4001)) + scale := r.IntN(4) + extra := r.IntN(6) + m.Mul(m, new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(extra)), nil)) + return parse(eqlDecimalText(m, scale+extra, r.IntN(2) == 0)) + } + } + maxText := eqlDecimalMaxMantissa.String() + var edges []encrypt.Decimal + for _, s := range []string{ + "0", "0.00", "-0.000", "1.5", "1.50", "1.500", "-1.5", "-1.50", + maxText, "-" + maxText, + eqlDecimalText(eqlDecimalMaxMantissa, 28, false), eqlDecimalText(eqlDecimalMaxMantissa, 28, true), + eqlDecimalText(new(big.Int).Sub(eqlDecimalMaxMantissa, big.NewInt(1)), 0, false), + "0.0000000000000000000000000001", "0.0000000000000000000000000002", "-0.0000000000000000000000000001", + } { + edges = append(edges, parse(s)) + } + rat := func(d encrypt.Decimal) *big.Rat { + v, ok := new(big.Rat).SetString(d.String()) + if !ok { + t.Fatalf("decimal %s does not read back as a rational", d) + } + return v + } + return eqlOrderGen[encrypt.Decimal]{ + kind: gensupport.Decimal, + cmp: func(a, b encrypt.Decimal) int { return rat(a).Cmp(rat(b)) }, + draw: draw, edges: edges, + show: func(d encrypt.Decimal) string { return d.String() }, + } +} + +// eqlRandBits is a random integer below 2^n, for n at most 128. +func eqlRandBits(r *rand.Rand, n int) *big.Int { + m := new(big.Int).SetUint64(r.Uint64()) + m.Lsh(m, 64) + m.Or(m, new(big.Int).SetUint64(r.Uint64())) + return m.Rsh(m, uint(128-n)) +} diff --git a/languages/golang/encrypt/eql_postgres_test.go b/languages/golang/encrypt/eql_postgres_test.go index aa4a07750..9772237cc 100644 --- a/languages/golang/encrypt/eql_postgres_test.go +++ b/languages/golang/encrypt/eql_postgres_test.go @@ -106,26 +106,7 @@ func TestEQLTypesRoundTripAndQueryInPostgres(t *testing.T) { ctx := context.Background() c := deterministicEQLClient(t) cipher := c.DefaultKeyset() - - db, err := sql.Open("pgx", url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = db.Close() }) - // One connection: the tables are temporary and live in this session. - conn, err := db.Conn(ctx) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = conn.Close() }) - - var installed bool - if err := conn.QueryRowContext(ctx, `SELECT to_regnamespace('eql_v4') IS NOT NULL`).Scan(&installed); err != nil { - t.Fatal(err) - } - if !installed { - t.Fatal("EQL v4 is not installed in DATABASE_URL: install the bundle built from the tree (`mise run build` in packages/eql writes release/cipherstash-encrypt-v4.sql), or run `mise run go:test:postgres`") - } + conn := eqlPostgresConn(t, ctx, url) covered := 0 for _, typ := range eql.Types { @@ -177,6 +158,32 @@ func TestEQLTypesRoundTripAndQueryInPostgres(t *testing.T) { } } +// eqlPostgresConn opens one connection to url, which must have EQL v4 +// installed. One connection, because the tests' tables are temporary and +// live in this session. +func eqlPostgresConn(t *testing.T, ctx context.Context, url string) *sql.Conn { + t.Helper() + db, err := sql.Open("pgx", url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + conn, err := db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + + var installed bool + if err := conn.QueryRowContext(ctx, `SELECT to_regnamespace('eql_v4') IS NOT NULL`).Scan(&installed); err != nil { + t.Fatal(err) + } + if !installed { + t.Fatal("EQL v4 is not installed in DATABASE_URL: install the bundle built from the tree (`mise run build` in packages/eql writes release/cipherstash-encrypt-v4.sql), or run `mise run go:test:postgres`") + } + return conn +} + type eqlHarness struct { t *testing.T ctx context.Context diff --git a/mise.toml b/mise.toml index 507787ae8..da66a306a 100644 --- a/mise.toml +++ b/mise.toml @@ -368,7 +368,7 @@ description = "Format check, vet and test the Go module (languages/golang: encry run = "scripts/go-binding-test.sh languages/golang" [tasks."go:test:postgres"] -description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names and run the Go module's PostgreSQL tests against it: rows the Go SDK stores in every EQL type it produces, found by the queries it derives; needs `wasm:guest:build:deterministic` and `wasm:guest:build:eql:deterministic` first" +description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names and run the Go module's PostgreSQL tests against it: rows the Go SDK stores in every EQL type it produces, found by the queries it derives, and random values in every ordered type, ordered as their plaintexts (EQL_ORDER_SEED replays a seed); needs `wasm:guest:build:deterministic` and `wasm:guest:build:eql:deterministic` first" # The bundle is built from packages/eql, as test:encryption:postgres builds # it: the copy under crates/eql-bindings/sql is the RELEASED bundle and moves # only at the lockstep bump, so it cannot show a SQL change in the same pull @@ -384,7 +384,7 @@ cd languages/golang # The tests skip without DATABASE_URL and without their guest builds; -v and # the grep turn either skip into a failure here, where both are provided. DATABASE_URL="$EQL_TEST_DATABASE_URL" CGO_ENABLED=0 go test -count=1 -v \ - -run '^(TestEQLTypesRoundTripAndQueryInPostgres|TestQueriesRunAgainstPostgres)$' ./encrypt \ + -run '^(TestEQLTypesRoundTripAndQueryInPostgres|TestEQLOrderMatchesPlaintextInPostgres|TestQueriesRunAgainstPostgres)$' ./encrypt \ | tee "${TMPDIR:-/tmp}/go-test-postgres.log" if grep -q -- '--- SKIP' "${TMPDIR:-/tmp}/go-test-postgres.log"; then echo "error: a PostgreSQL test skipped; build the deterministic guests first" >&2 @@ -392,7 +392,7 @@ if grep -q -- '--- SKIP' "${TMPDIR:-/tmp}/go-test-postgres.log"; then fi # A -run pattern that matches no test (one renamed, say) prints "no tests # to run" and PASS, so each test has to be seen to pass by name. -for test in TestEQLTypesRoundTripAndQueryInPostgres TestQueriesRunAgainstPostgres; do +for test in TestEQLTypesRoundTripAndQueryInPostgres TestEQLOrderMatchesPlaintextInPostgres TestQueriesRunAgainstPostgres; do if ! grep -q -- "^--- PASS: $test " "${TMPDIR:-/tmp}/go-test-postgres.log"; then echo "error: $test did not run and pass; was it renamed?" >&2 exit 1