fix(reads): read dotted names as nested paths in orderBy, lookup and indexes

AGENTS.md

@@ -11,7 +11,7 @@ The PostgreSQL implementation of `@antelopejs/interface-database` and `@antelope
 
 ## Versioning
 
-- Both implemented interfaces stay capped below the next minor (`interface-database` `>=0.3.1 <0.4.0`, `interface-database-catalog` `>=0.0.3 <0.1.0`), because a new minor can add required handlers. `pnpm lint` enforces it with `antelopejs-check-interface-ranges`.
+- Both implemented interfaces stay capped below the next minor (`interface-database` `>=0.3.3 <0.4.0`, `interface-database-catalog` `>=0.0.3 <0.1.0`), because a new minor can add required handlers. `pnpm lint` enforces it with `antelopejs-check-interface-ranges`.
 - When a dependency range changes, update the compatibility table in `README.md` too.
 - A release that needs action from users gets an `### Upgrade notes` section in `CHANGELOG.md`, above the generated entries.
 

README.md

@@ -119,7 +119,7 @@ This repository documents what is specific to the PostgreSQL backend:
 | Package                                  | Supported        | Tested with |
 | ---------------------------------------- | ---------------- | ----------- |
 | `@antelopejs/interface-core`             | `>=0.1.1 <1.0.0` | `0.4.0`     |
-| `@antelopejs/interface-database`         | `>=0.3.1 <0.4.0` | `0.3.1`     |
+| `@antelopejs/interface-database`         | `>=0.3.3 <0.4.0` | `0.3.3`     |
 | `@antelopejs/interface-database-catalog` | `>=0.0.3 <0.1.0` | `0.0.3`     |
 
 <!-- antelopejs-docs-compat:end -->

docs/errors.md

@@ -40,7 +40,8 @@ A cursor holds a pooled connection for as long as it is open, so the module recl
 | Two fields that truncate to the same column name              | `Fields '{a}' and '{b}' on '{table}' both map to column '{column}'; rename one`                                                                                         |
 | Two tables or indexes that truncate to the same name          | `The physical name '{name}' is claimed by both {a} and {b}; rename one`                                                                                                 |
 | Declared a `unique` `multi` index over an array field         | `Index '{indexId}' on table '{table}' is unique and multi over an array field, whose elements PostgreSQL cannot hold unique; drop one of the two flags`                 |
-| Indexed a reserved column                                     | `Index '{indexId}' keys on '{field}', which is a reserved column the adapter owns; index a declared field instead`                                                      |
+| Indexed a reserved column                                     | `Index '{indexId}' keys on '{field}', which reads the reserved column '{column}' the adapter owns; index a declared field instead`                                      |
+| Indexed a path below a declared value with no members         | `Index '{indexId}' keys on '{field}', but '{owner}' is declared as a {type} with no members; index a declared field instead`                                            |
 | Changed a declared type under `migrations: "safe"`            | `Column '{name}' on {table} is {actual} but the schema declares {declared}; set migrations: "destructive" to alter it`                                                  |
 | Pointed the adapter at a pre-existing table it did not create | `Table {target} has no '{column}' column and one cannot be added without a default, so the table was not created by this adapter; rename it or drop it`                 |
 

docs/storage.md

@@ -62,6 +62,7 @@ A declared index becomes a real index, and the operations that can use one do:
 - **`crossInstance: true` adds an unprefixed twin.** The adapter also maintains `<table>_<index>__g` over the same keys without `_instance`, and that twin serves a `CROSS_INSTANCE` read of the index. Without the flag, such a read stays correct but cannot use the index well. The first `getAll`, `between` or `orderBy` that names the index under `CROSS_INSTANCE` logs one warning per index per registration. A plain `filter` never warns.
 - **`unique: true` makes the `_instance`-prefixed index a `UNIQUE` one.** The key is unique within each instance, and two instances may share it. The `__g` twin of a `crossInstance` index is never unique. Rows that lack a field of the key hold NULL there and never conflict. Registration refuses a unique `multi` index over an array field, because GIN cannot be unique. When a schema declares a unique index, its writes wait for its index phase, so no write can store a duplicate before the index exists. Reads never wait. A write that the index refuses throws `DuplicateKeyError`. See [Duplicate keys](./errors.md#duplicate-keys).
 - **`multi` maps to GIN** over a single field backed by `jsonb`, which lets an index serve `@>`, `?`, `?|` and `?&`. A GIN index is keyed on the column alone, with no leading `_instance`. It already serves both kinds of read, so `crossInstance` changes nothing for it. `multi` on a scalar field falls back to btree instead of failing.
+- **A dotted name is a nested path.** An index field, an `orderBy`, `getAll` or `between` name, and the other key of a `lookup` read `title.fr` as the `fr` member of `title`, unless the table declares a field named `title.fr` itself. The path may start at a declared field or at an undeclared key, as in `_internal.data.title.fr`. The index keys on the same expression the read compiles to, so it serves that read. A `date` member below a declared field is the exception: PostgreSQL cannot index its typed read, so the index keys on the stored JSON member. It still enforces `unique`, and it serves no read.
 - **These operations reach an index:** `filter` and `orderBy` on a declared field, `get`, `getAll`, `between`, a `joinInner` predicate, and the local key of a `lookup`. The local key reaches the index also when the looked-up query only plucked its columns, because the adapter pushes the correlation under that projection onto the column itself. A projection that computed its members is not pushed under, and the lateral join then scans the other table once per row of the left.
 
 Declaring a field is therefore the most useful thing you can do for query performance. An undeclared key lives in `_extra`, where it has no declared type to compare by and no index to serve it.

package.json

@@ -52,7 +52,7 @@
   },
   "dependencies": {
     "@antelopejs/interface-core": ">=0.1.1 <1.0.0",
-    "@antelopejs/interface-database": ">=0.3.1 <0.4.0",
+    "@antelopejs/interface-database": ">=0.3.3 <0.4.0",
     "@antelopejs/interface-database-catalog": ">=0.0.3 <0.1.0",
     "pg": "^8.22.0"
   },

src/implementations/database/ddl.ts

@@ -4,13 +4,22 @@ import type {
   TableDefinition,
 } from "@antelopejs/interface-database/schema";
 
-import { JsonKey, QuoteIdentifier, QuoteLiteral } from "./sql";
+import { JSON_NULL, JsonKey, QuoteIdentifier, QuoteLiteral } from "./sql";
+import {
+  type ColumnAddress,
+  ColumnRowSource,
+  NamedField,
+  PATH_SEPARATOR,
+} from "./row-source";
 import {
   IsJsonBacked,
+  type KeyTypes,
+  KeyTypesOf,
   OPAQUE,
   PhysicalType,
   type SqlType,
   TEXT,
+  TIMESTAMP,
 } from "./types";
 import {
   EXTRA_COLUMN,
@@ -133,22 +142,71 @@ export function ColumnDdl(column: ColumnPlan): string {
   return parts.join(" ");
 }
 
-const RESERVED_INDEX_KEYS = new Map([[ID_COLUMN, QuoteIdentifier(ID_COLUMN)]]);
+/** The table's own columns, unqualified, as an index expression names them. */
+const IndexedColumns: ColumnAddress = (column, type) => ({
+  sql: QuoteIdentifier(column),
+  type,
+});
 
-function IndexKey(indexId: string, field: string, spec?: FieldSpec): string {
-  if (spec) {
-    return QuoteIdentifier(spec.column);
+/**
+ * Refuses a path through a declared value that has no members, such as `name.fr` over a `string`.
+ * Its read is always NULL, and it reads through `to_jsonb`, which PostgreSQL refuses in an index
+ * expression, so the build would fail and stop the schema's index phase.
+ */
+function AssertPathHasMembers(
+  indexId: string,
+  field: string,
+  row: ColumnRowSource,
+): void {
+  const segments = field.split(PATH_SEPARATOR);
+  let keys: KeyTypes | undefined = row.keyTypes();
+  for (const [depth, segment] of segments.slice(0, -1).entries()) {
+    if (!keys || !Object.hasOwn(keys, segment)) {
+      return;
+    }
+    const type = keys[segment];
+    if (!IsJsonBacked(type)) {
+      const owner = segments.slice(0, depth + 1).join(PATH_SEPARATOR);
+      throw new Error(
+        `Index '${indexId}' keys on '${field}', but '${owner}' is declared as a ${PhysicalType(type)} with no members; index a declared field instead`,
+      );
+    }
+    keys = KeyTypesOf(type);
   }
-  const reserved = RESERVED_INDEX_KEYS.get(field);
-  if (reserved !== undefined) {
-    return reserved;
+}
+
+/**
+ * An index key, spelled as the reads it serves spell the field: the same {@link NamedField} over
+ * the table's columns, so `orderBy`, `getAll`, `between` and `lookup` compile to its expression.
+ *
+ * A date member below a declared field is the exception. Its read parses text into `timestamptz`,
+ * which PostgreSQL refuses in an index expression, and a failed build stops the schema's index
+ * phase. Its key is the stored JSON member instead, a JSON null read as absent as the typed read
+ * does: the index builds and keeps `unique`, and serves no read.
+ */
+function IndexKey(
+  indexId: string,
+  field: string,
+  row: ColumnRowSource,
+  isDeclared: boolean,
+): string {
+  if (isDeclared) {
+    return NamedField(row, field).sql;
   }
-  if (RESERVED_COLUMNS.includes(field)) {
+  const [head] = field.split(PATH_SEPARATOR);
+  if (field !== ID_COLUMN && RESERVED_COLUMNS.includes(head)) {
     throw new Error(
-      `Index '${indexId}' keys on '${field}', which is a reserved column the adapter owns; index a declared field instead`,
+      `Index '${indexId}' keys on '${field}', which reads the reserved column '${head}' the adapter owns; index a declared field instead`,
     );
   }
-  return JsonKey(QuoteIdentifier(EXTRA_COLUMN), field);
+  AssertPathHasMembers(indexId, field, row);
+  const read = NamedField(row, field);
+  const cut = field.lastIndexOf(PATH_SEPARATOR);
+  if (read.type.kind !== TIMESTAMP.kind || cut < 0) {
+    return read.sql;
+  }
+  const parent = NamedField(row, field.slice(0, cut));
+  return `NULLIF(${JsonKey(parent.sql, field.slice(cut + 1))}, ${JSON_NULL})`;
 }
 
 /**
@@ -208,8 +266,9 @@ function BuildIndexPlans(
 ): IndexPlan[] {
   const fields = IndexFields(indexId, index);
   const specs = IndexSpecs(fields, model);
+  const row = new ColumnRowSource(IndexedColumns, model);
   const keys = fields.map((field, position) =>
-    IndexKey(indexId, field, specs[position]),
+    IndexKey(indexId, field, row, specs[position] !== undefined),
   );
   const name = IndexName(tableId, indexId);
   const isUnique = index.unique === true;

src/implementations/database/relation.ts

@@ -11,6 +11,7 @@ import {
   ExternalRowSource,
   JsonRowSource,
   LoggedColumns,
+  NamedField,
   type RowSource,
 } from "./row-source";
 
@@ -160,6 +161,8 @@ export function RequireOrigin(relation: Relation): TableOrigin {
  * same way: a second expansion would sort by one field list and match against another, and the
  * btree scan would quietly stop being served with nothing failing. A relation that no longer knows
  * its table can only read the name as a field, which is what keeps `map(…).orderBy("age")` working.
+ * Each field is a {@link NamedField}, the reading the index DDL keys on, so `title.fr` sorts by the
+ * nested value and its index serves that sort.
  *
  * Each column keeps the field it was expanded from, because a compound index is also a *document*
  * to a `group`'s mapper and its member names are the index's own. Reading them beside the columns
@@ -172,7 +175,7 @@ export function IndexColumns(
   const fields = relation.origin ? relation.origin.index(name) : [name];
   return fields.map((field) => ({
     name: field,
-    ...relation.source.field(field),
+    ...NamedField(relation.source, field),
   }));
 }
 

src/implementations/database/row-source.ts

@@ -104,6 +104,26 @@ export function BuildDocument(
   };
 }
 
+/** What separates the segments of a dotted name, `title.fr`, that {@link NamedField} reads. */
+export const PATH_SEPARATOR = ".";
+
+/**
+ * Reads a field by the name an index, a sort, a scan or a lookup's other key takes: a member by
+ * its exact name where the row declares one, and otherwise a dotted name as a path below its first
+ * segment. So `title.fr` reads the `fr` member of `title`, and the index over it holds that value.
+ */
+export function NamedField(source: RowSource, name: string): Compiled {
+  const [head, ...rest] = name.split(PATH_SEPARATOR);
+  const declared = source.keyTypes();
+  if (rest.length === 0 || (declared && Object.hasOwn(declared, name))) {
+    return source.field(name);
+  }
+  return rest.reduce(
+    (value, segment) => new JsonRowSource(value).field(segment),
+    source.field(head),
+  );
+}
+
 /** Names a key in the message {@link Defaulted} raises for a default the column cannot hold. */
 function KeySubject(name: string): string {
   return `The default for key '${name}'`;

src/implementations/database/stages-join.ts

@@ -8,7 +8,7 @@ import { UnifyOperands } from "./coerce";
 import { DecodeFunction, RecordedLambda } from "./value";
 import { AsJson, Predicate } from "./readings";
 import type { DecodingContext } from "./context";
-import { NullableRowSource, type RowSource } from "./row-source";
+import { NamedField, NullableRowSource, type RowSource } from "./row-source";
 import type { PipelineStage, PipelineStageMap } from "./pipeline";
 import { FIRST_OPERAND, RequireOperand, SECOND_OPERAND } from "./stage";
 import {
@@ -255,6 +255,7 @@ const PUSHDOWN_BLOCKERS: readonly RelationFlag[] = WHERE_BLOCKERS.filter(
 /**
  * Pushes the correlation onto the looked-up relation, comparing two declared columns.
  * Two columns of one kind take no cast; a scalar key against an array column takes containment.
+ * The other key is a {@link NamedField}, so `title.fr` matches the nested value and its index.
  */
 function Correlate(
   right: Relation,
@@ -263,7 +264,7 @@ function Correlate(
   local: Compiled,
   form: LookupForm,
 ): void {
-  const other = source.field(otherKey);
+  const other = NamedField(source, otherKey);
   if (IsArray(other.type) && !IsArray(local.type)) {
     right.where.push(`(${other.sql} ${CONTAINS} ${AsJson(local).sql})`);
     return;