Test every document metadata field through the existing Prism API

.github/workflows/document-metadata.yaml

@@ -0,0 +1,133 @@
+name: Document metadata integration
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      db_ref:
+        description: Polypheny-DB revision containing document metadata support
+        default: a35687bda0f5f13a40eda56a2ab340faf4977d2c
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  build-polypheny:
+    timeout-minutes: 30
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: polypheny/Polypheny-DB
+          ref: ${{ inputs.db_ref || 'a35687bda0f5f13a40eda56a2ab340faf4977d2c' }}
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+
+      - name: Build server and plugins and validate Java metadata
+        run: >-
+          ./gradlew :dbms:shadowJar assemblePlugins
+          :plugins:prism-interface:test
+          :plugins:mongodb-adapter:test --tests '*MongoIndexMetadataTest'
+          -PwithoutpullingUi=true --console=plain
+
+      - name: Produce final executable server JAR
+        # dbms:jar and dbms:shadowJar currently share an output path. Run the
+        # shadow task last so the uploaded artifact contains runtime classes.
+        run: ./gradlew :dbms:shadowJar -PwithoutpullingUi=true --console=plain
+
+      - name: Select executable server JAR
+        run: |
+          python - <<'PY'
+          from pathlib import Path
+          import shutil
+          import zipfile
+          candidates = [path for path in Path('dbms/build/libs').glob('dbms-*.jar')
+                        if not path.stem.endswith(('-tests', '-sources', '-javadoc'))]
+          assert len(candidates) == 1, f'Expected one executable JAR, found {candidates}'
+          with zipfile.ZipFile(candidates[0]) as archive:
+              assert 'org/slf4j/Logger.class' in archive.namelist(), 'Server JAR lacks runtime dependencies'
+          shutil.copyfile(candidates[0], 'polypheny.jar')
+          PY
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: polypheny-jar
+          path: polypheny.jar
+
+  document-metadata:
+    timeout-minutes: 15
+    runs-on: ubuntu-24.04
+    needs: build-polypheny
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/checkout@v4
+        with:
+          repository: polypheny/Polypheny-Prism-API
+          ref: ac57da2459071234c4cacc6836e3f4ca6f13f101
+          path: .metadata-prism
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+
+      - name: Install test dependencies
+        run: python -m pip install -r requirements.txt grpcio-tools==1.62.3 pymongo==4.10.1
+
+      - name: Validate Prism wire contract
+        run: python -m unittest discover -s .metadata-prism/tests -v
+
+      - name: Start MongoDB replica set
+        run: |
+          docker run --detach --name prism-metadata-mongo --publish 27017:27017 mongo:7.0 --replSet prism --bind_ip_all
+          python - <<'PY'
+          import time
+          from pymongo import MongoClient
+          from pymongo.errors import PyMongoError
+          client = MongoClient('127.0.0.1', 27017, directConnection=True, serverSelectionTimeoutMS=1000)
+          deadline = time.monotonic() + 60
+          while True:
+              try:
+                  client.admin.command('ping')
+                  break
+              except PyMongoError:
+                  if time.monotonic() >= deadline:
+                      raise
+                  time.sleep(1)
+          client.admin.command('replSetInitiate', {'_id': 'prism', 'members': [{'_id': 0, 'host': '127.0.0.1:27017'}]})
+          while not client.admin.command('hello').get('isWritablePrimary'):
+              if time.monotonic() >= deadline:
+                  raise RuntimeError('MongoDB did not elect a primary')
+              time.sleep(0.2)
+          client.close()
+          PY
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: polypheny-jar
+
+      - name: Run live document and index metadata suites
+        uses: polypheny/GitHub-Action-Run-Polypheny@v0.2.0
+        with:
+          cmd: python scripts/test_document_metadata.py --prism-api .metadata-prism --mongo
+          jar: polypheny.jar
+          autodocker: false
+        timeout-minutes: 10
+
+      - name: MongoDB logs
+        if: failure()
+        run: docker logs prism-metadata-mongo
+
+      - name: Stop MongoDB
+        if: always()
+        run: docker rm --force prism-metadata-mongo

.github/workflows/test.yaml

@@ -6,20 +6,56 @@ on:
 
 jobs:
   build-polypheny:
-    timeout-minutes: 15
-    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    runs-on: ubuntu-24.04
     steps:
-      - uses: polypheny/GitHub-Action-Build-Polypheny@main
+      - uses: actions/checkout@v4
         with:
-          branch: master
+          repository: polypheny/Polypheny-DB
+          ref: master
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+
+      - name: Build executable server and plugins
+        run: ./gradlew :dbms:shadowJar assemblePlugins -PwithoutpullingUi=true --console=plain
+
+      - name: Select executable server JAR
+        run: |
+          python - <<'PY'
+          from pathlib import Path
+          import shutil
+          import zipfile
+          candidates = [path for path in Path('dbms/build/libs').glob('dbms-*.jar')
+                        if not path.stem.endswith(('-tests', '-sources', '-javadoc'))]
+          assert len(candidates) == 1, f'Expected one executable JAR, found {candidates}'
+          with zipfile.ZipFile(candidates[0]) as archive:
+              assert 'org/slf4j/Logger.class' in archive.namelist(), 'Server JAR lacks runtime dependencies'
+          shutil.copyfile(candidates[0], 'polypheny.jar')
+          PY
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: polypheny-jar
+          path: polypheny.jar
 
   test-system-python:
     timeout-minutes: 10
     strategy:
       fail-fast: false
       matrix:
         version: ['3.9', '3.10', '3.11', '3.12', '3.13']
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-24.04, macos-latest, windows-latest]
+        # Python 3.9 has no Ubuntu 26.04 build. Python 3.9/3.10 on
+        # macOS require the Intel distribution; retain all 15 combinations.
+        exclude:
+          - {version: '3.9', os: macos-latest}
+          - {version: '3.10', os: macos-latest}
+        include:
+          - {version: '3.9', os: macos-15-intel}
+          - {version: '3.10', os: macos-15-intel}
     runs-on: ${{ matrix.os }}
     needs: build-polypheny
     steps:
@@ -74,8 +110,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        adapter: [ mongodb, hsqldb, monetdb, postgresql, file, cottontail, neo4j ]
-    runs-on: ubuntu-latest
+        adapter: [ mongodb, hsqldb, monetdb, postgresql, file, neo4j ]
+    runs-on: ubuntu-24.04
     needs: build-polypheny
     steps:
       - name: Set store env variable
@@ -129,7 +165,7 @@ jobs:
 
   build-docs:
     timeout-minutes: 2
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/setup-python@v5
         with:

README.md

@@ -83,6 +83,46 @@ Run the tests with coverage report:
 coverage run --source polypheny -m pytest && coverage report -m
 ```
 
+### Document metadata integration tests
+
+The document metadata suites cover the existing Prism `EntitiesRequest` /
+`EntitiesResponse` exchange, all `Document` and `DocumentField` properties,
+nested paths, heterogeneous and nullable values, sample JSON, collection
+filters, empty namespaces, and relational metadata regression. The MongoDB
+suite verifies every `DocumentIndex` property against real unique B-tree,
+hash, nested-field, and wildcard indexes.
+
+Until the extended Prism schema is released, use the companion API revision
+and a Polypheny server with document metadata support. The helper generates
+only the extended namespace metadata module into a temporary directory; the
+remaining wire definitions and handshake use the installed Prism API 1.9:
+
+```bash
+python -m pip install -r requirements.txt grpcio-tools==1.62.3 pymongo==4.10.1
+git clone https://github.com/polypheny/Polypheny-Prism-API.git ../Polypheny-Prism-API
+git -C ../Polypheny-Prism-API checkout 9b64b905d9dae2bb8b09592d907f0c5b1c69e2ec
+python scripts/test_document_metadata.py --prism-api ../Polypheny-Prism-API
+```
+
+The suites use plain Prism at `127.0.0.1:20590`, user `pa`, and an empty
+password. Override these with `POLYPHENY_PRISM_HOST`, `POLYPHENY_PRISM_PORT`,
+`POLYPHENY_PRISM_USER`, and `POLYPHENY_PRISM_PASSWORD` when needed.
+
+Run the real index tests with `--mongo`. They also require a MongoDB replica
+set, reachable by both the tests and Polypheny at `127.0.0.1:27017` by default.
+Set `POLYPHENY_MONGO_HOST` and `POLYPHENY_MONGO_PORT` for another endpoint.
+The tests create a private MongoDB database and a temporary Polypheny source
+adapter, then remove both. Explicit runs fail if a required service is missing.
+
+```bash
+python scripts/test_document_metadata.py --prism-api ../Polypheny-Prism-API --mongo
+```
+
+The dedicated `Document metadata integration` workflow runs both suites and
+the Prism wire-contract tests against the companion server branch; its manual
+trigger accepts a different `db_ref`. The ordinary connector matrix continues
+to exercise the released Prism API.
+
 
 ## Roadmap
 See the [open issues](https://github.com/polypheny/Polypheny-DB/labels/A-python) for a list of proposed features (and known issues).

scripts/test_document_metadata.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""Generate companion Prism Python bindings and run the live metadata tests.
+
+The generated bindings live in a temporary directory. This neither overwrites
+the installed Prism package nor changes the connector's release dependencies.
+"""
+
+import argparse
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--prism-api", required=True, type=Path, help="Checkout of the companion Polypheny-Prism-API revision")
+    parser.add_argument("--mongo", action="store_true", help="Also test indexes using a live MongoDB replica set")
+    arguments = parser.parse_args()
+    api_root = arguments.prism_api.resolve()
+    namespace_proto = api_root / "org/polypheny/prism/namespace_meta_responses.proto"
+    if not namespace_proto.is_file():
+        parser.error("--prism-api must contain org/polypheny/prism/namespace_meta_responses.proto")
+
+    connector_root = Path(__file__).resolve().parents[1]
+    with tempfile.TemporaryDirectory(prefix="prism-document-metadata-") as output:
+        subprocess.run([
+            sys.executable, "-m", "grpc_tools.protoc", "-I", str(api_root),
+            "--python_out", output,
+            str(namespace_proto),
+        ], check=True)
+        # Overlay only this additive metadata module. Other definitions on the
+        # API's main branch may differ from the released 1.9 wire protocol;
+        # value serialization and version negotiation keep using that release.
+
+        environment = os.environ.copy()
+        environment["PYTHONPATH"] = os.pathsep.join(filter(None, [
+            output, str(connector_root), environment.get("PYTHONPATH"),
+        ]))
+        tests = ["tests/document_metadata.py"]
+        if arguments.mongo:
+            tests.append("tests/document_metadata_mongo.py")
+        return subprocess.call(
+            [sys.executable, "-m", "pytest", "-v", *tests],
+            cwd=connector_root, env=environment,
+        )
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())