From c8dd7ecdcd98cbd8711767af377af4d77afd3f2a Mon Sep 17 00:00:00 2001 From: Marco Vogt Date: Fri, 6 Nov 2026 10:05:48 +0100 Subject: [PATCH 1/4] Test document entity metadata end to end over Prism --- .github/workflows/document-metadata.yaml | 96 +++++++ README.md | 40 +++ scripts/test_document_metadata.py | 51 ++++ tests/document_metadata.py | 333 +++++++++++++++++++++++ tests/document_metadata_mongo.py | 104 +++++++ 5 files changed, 624 insertions(+) create mode 100644 .github/workflows/document-metadata.yaml create mode 100644 scripts/test_document_metadata.py create mode 100644 tests/document_metadata.py create mode 100644 tests/document_metadata_mongo.py diff --git a/.github/workflows/document-metadata.yaml b/.github/workflows/document-metadata.yaml new file mode 100644 index 0000000..a715ac4 --- /dev/null +++ b/.github/workflows/document-metadata.yaml @@ -0,0 +1,96 @@ +name: Document metadata integration + +on: + push: + pull_request: + workflow_dispatch: + inputs: + db_ref: + description: Polypheny-DB revision containing document metadata support + default: 24797eb5548e3ca07f15a1a12ecf236327751de1 + required: true + +permissions: + contents: read + +jobs: + build-polypheny: + timeout-minutes: 30 + runs-on: ubuntu-24.04 + steps: + - uses: polypheny/GitHub-Action-Build-Polypheny@main + with: + branch: ${{ inputs.db_ref || '24797eb5548e3ca07f15a1a12ecf236327751de1' }} + + document-metadata: + timeout-minutes: 15 + runs-on: ubuntu-24.04 + needs: build-polypheny + steps: + - uses: actions/checkout@v4 + + - uses: actions/checkout@v4 + with: + repository: polypheny/Polypheny-Prism-API + ref: ac57da2459071234c4cacc6836e3f4ca6f13f101 + path: .metadata-prism + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + + - name: Install test dependencies + run: python -m pip install -r requirements.txt grpcio-tools==1.62.3 pymongo==4.10.1 + + - name: Validate Prism wire contract + run: python -m unittest discover -s .metadata-prism/tests -v + + - name: Start MongoDB replica set + run: | + docker run --detach --name prism-metadata-mongo --publish 27017:27017 mongo:7.0 --replSet prism --bind_ip_all + python - <<'PY' + import time + from pymongo import MongoClient + from pymongo.errors import PyMongoError + client = MongoClient('127.0.0.1', 27017, directConnection=True, serverSelectionTimeoutMS=1000) + deadline = time.monotonic() + 60 + while True: + try: + client.admin.command('ping') + break + except PyMongoError: + if time.monotonic() >= deadline: + raise + time.sleep(1) + client.admin.command('replSetInitiate', {'_id': 'prism', 'members': [{'_id': 0, 'host': '127.0.0.1:27017'}]}) + while not client.admin.command('hello').get('isWritablePrimary'): + if time.monotonic() >= deadline: + raise RuntimeError('MongoDB did not elect a primary') + time.sleep(0.2) + client.close() + PY + + - uses: actions/download-artifact@v4 + with: + name: polypheny-jar + + - name: Run live document and index metadata suites + uses: polypheny/GitHub-Action-Run-Polypheny@v0.2.0 + with: + cmd: python scripts/test_document_metadata.py --prism-api .metadata-prism --mongo + jar: polypheny.jar + autodocker: false + timeout-minutes: 10 + + - name: MongoDB logs + if: failure() + run: docker logs prism-metadata-mongo + + - name: Stop MongoDB + if: always() + run: docker rm --force prism-metadata-mongo diff --git a/README.md b/README.md index 51e34c7..6a009fb 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,46 @@ Run the tests with coverage report: coverage run --source polypheny -m pytest && coverage report -m ``` +### Document metadata integration tests + +The document metadata suites cover the existing Prism `EntitiesRequest` / +`EntitiesResponse` exchange, all `Document` and `DocumentField` properties, +nested paths, heterogeneous and nullable values, sample JSON, collection +filters, empty namespaces, and relational metadata regression. The MongoDB +suite verifies every `DocumentIndex` property against real unique B-tree, +hash, nested-field, and wildcard indexes. + +Until the extended Prism schema is released, use the companion API revision +and a Polypheny server with document metadata support. The helper generates +only the extended namespace metadata module into a temporary directory; the +remaining wire definitions and handshake use the installed Prism API 1.9: + +```bash +python -m pip install -r requirements.txt grpcio-tools==1.62.3 pymongo==4.10.1 +git clone https://github.com/polypheny/Polypheny-Prism-API.git ../Polypheny-Prism-API +git -C ../Polypheny-Prism-API checkout 9b64b905d9dae2bb8b09592d907f0c5b1c69e2ec +python scripts/test_document_metadata.py --prism-api ../Polypheny-Prism-API +``` + +The suites use plain Prism at `127.0.0.1:20590`, user `pa`, and an empty +password. Override these with `POLYPHENY_PRISM_HOST`, `POLYPHENY_PRISM_PORT`, +`POLYPHENY_PRISM_USER`, and `POLYPHENY_PRISM_PASSWORD` when needed. + +Run the real index tests with `--mongo`. They also require a MongoDB replica +set, reachable by both the tests and Polypheny at `127.0.0.1:27017` by default. +Set `POLYPHENY_MONGO_HOST` and `POLYPHENY_MONGO_PORT` for another endpoint. +The tests create a private MongoDB database and a temporary Polypheny source +adapter, then remove both. Explicit runs fail if a required service is missing. + +```bash +python scripts/test_document_metadata.py --prism-api ../Polypheny-Prism-API --mongo +``` + +The dedicated `Document metadata integration` workflow runs both suites and +the Prism wire-contract tests against the companion server branch; its manual +trigger accepts a different `db_ref`. The ordinary connector matrix continues +to exercise the released Prism API. + ## Roadmap See the [open issues](https://github.com/polypheny/Polypheny-DB/labels/A-python) for a list of proposed features (and known issues). diff --git a/scripts/test_document_metadata.py b/scripts/test_document_metadata.py new file mode 100644 index 0000000..05b1065 --- /dev/null +++ b/scripts/test_document_metadata.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Generate companion Prism Python bindings and run the live metadata tests. + +The generated bindings live in a temporary directory. This neither overwrites +the installed Prism package nor changes the connector's release dependencies. +""" + +import argparse +import os +from pathlib import Path +import subprocess +import sys +import tempfile + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prism-api", required=True, type=Path, help="Checkout of the companion Polypheny-Prism-API revision") + parser.add_argument("--mongo", action="store_true", help="Also test indexes using a live MongoDB replica set") + arguments = parser.parse_args() + api_root = arguments.prism_api.resolve() + namespace_proto = api_root / "org/polypheny/prism/namespace_meta_responses.proto" + if not namespace_proto.is_file(): + parser.error("--prism-api must contain org/polypheny/prism/namespace_meta_responses.proto") + + connector_root = Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory(prefix="prism-document-metadata-") as output: + subprocess.run([ + sys.executable, "-m", "grpc_tools.protoc", "-I", str(api_root), + "--python_out", output, + str(namespace_proto), + ], check=True) + # Overlay only this additive metadata module. Other definitions on the + # API's main branch may differ from the released 1.9 wire protocol; + # value serialization and version negotiation keep using that release. + + environment = os.environ.copy() + environment["PYTHONPATH"] = os.pathsep.join(filter(None, [ + output, str(connector_root), environment.get("PYTHONPATH"), + ])) + tests = ["tests/document_metadata.py"] + if arguments.mongo: + tests.append("tests/document_metadata_mongo.py") + return subprocess.call( + [sys.executable, "-m", "pytest", "-v", *tests], + cwd=connector_root, env=environment, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/document_metadata.py b/tests/document_metadata.py new file mode 100644 index 0000000..f498b8a --- /dev/null +++ b/tests/document_metadata.py @@ -0,0 +1,333 @@ +# Copyright 2026 The Polypheny Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Live regression tests for document metadata over the ordinary Prism RPC. + +Run explicitly with ``pytest tests/document_metadata.py``. These tests need +Polypheny with the companion document metadata change and Python bindings +generated from the corresponding Prism definitions. They connect to the +ordinary plain Prism endpoint, just like the connector's Windows tests. +""" + +import json +import os +import uuid + +import pytest +import polypheny +from org.polypheny.prism import namespace_meta_requests_pb2 +from org.polypheny.prism import namespace_meta_responses_pb2 + +DOCUMENTS = [ + { + "_id": "patient-1", + "patient_id": "p-001", + "age": 23, + "viral_load_day1": 6.25, + "active": True, + "symptoms": ["fever", "fatigue"], + "progression": [{"day": 1, "viral_load": 6.25}], + "complications": {"respiratory": {"onset_day": 2}, "renal": None}, + "notes": None, + "score": 1, + "mixed": 1, + "optional_field": 42, + "empty_object": {}, + "nullable": "present", + "nullable_parent": {"child": 1}, + }, + { + "_id": "patient-2", + "patient_id": "p-002", + "age": 64, + "viral_load_day1": 3.5, + "active": False, + "symptoms": [], + "progression": [], + "complications": {"respiratory": {"onset_day": 4}, "renal": "mild"}, + "notes": None, + "score": 1.5, + "mixed": "unknown", + "empty_object": {}, + "nullable": None, + "nullable_parent": None, + }, + { + "_id": "patient-3", + "patient_id": "p-003", + "age": 7, + "viral_load_day1": 5.75, + "active": True, + "symptoms": ["pain"], + "progression": [{"day": 1, "viral_load": 5.75}], + "complications": {}, + "notes": None, + "score": 2, + "mixed": True, + "empty_object": {}, + "nullable": "again", + "nullable_parent": {"child": 2}, + }, +] + + +@pytest.fixture +def con(): + connection = polypheny.connect( + (os.environ.get("POLYPHENY_PRISM_HOST", "127.0.0.1"), + int(os.environ.get("POLYPHENY_PRISM_PORT", "20590"))), + username=os.environ.get("POLYPHENY_PRISM_USER", "pa"), + password=os.environ.get("POLYPHENY_PRISM_PASSWORD", ""), + transport="plain", + ) + try: + yield connection + finally: + connection.close() + + +def get_entities(connection, namespace, pattern=None): + """Use the same request wrapper and RPC call as relational metadata.""" + message = connection.con.new_request() + request = namespace_meta_requests_pb2.EntitiesRequest(namespace_name=namespace) + if pattern is not None: + request.entity_pattern = pattern + message.entities_request.CopyFrom(request) + response = connection.con.call(message) + assert response.HasField("entities_response") + return response.entities_response + + +@pytest.fixture(autouse=True) +def document_metadata_schema(): + fields = namespace_meta_responses_pb2.Document.DESCRIPTOR.fields_by_name + assert "collection_name" in fields, ( + "Document metadata requires the companion Prism schema. Run " + "python scripts/test_document_metadata.py --prism-api /path/to/Polypheny-Prism-API" + ) + + +def get_document(connection, namespace, collection): + response = get_entities(connection, namespace, collection) + assert len(response.entities) == 1 + entity = response.entities[0] + assert entity.WhichOneof("Entity") == "document" + assert not entity.HasField("table") + assert not entity.HasField("graph") + return entity.document + + +@pytest.fixture +def document_namespace(con): + # A private namespace avoids modifying user data or colliding with parallel runs. + namespace = "prism_doc_meta_" + uuid.uuid4().hex + cursor = con.cursor() + cursor.execute("CREATE DOCUMENT NAMESPACE " + namespace) + try: + cursor.executeany("mongo", 'db.createCollection("symptoms")', namespace=namespace) + cursor.executeany("mongo", 'db.createCollection("empty")', namespace=namespace) + for document in DOCUMENTS: + cursor.executeany( + "mongo", "db.symptoms.insertOne(" + json.dumps(document) + ")", + namespace=namespace, + ) + con.commit() + yield namespace + finally: + con.rollback() + cursor.execute("DROP NAMESPACE " + namespace) + cursor.close() + + +def test_document_identity_and_entity_wrapper(con, document_namespace): + response = get_entities(con, document_namespace) + assert {entity.document.collection_name for entity in response.entities} == {"symptoms", "empty"} + for entity in response.entities: + assert entity.WhichOneof("Entity") == "document" + assert entity.document.namespace_name == document_namespace + + +def test_all_document_field_metadata(con, document_namespace): + document = get_document(con, document_namespace, "symptoms") + # Every DocumentField property is checked against input data. In particular, + # false is_optional values must survive protobuf's default-value encoding. + expected = { + "_id": ("string", False), + "patient_id": ("string", False), + "age": ("int", False), + "viral_load_day1": ("double", False), + "active": ("bool", False), + "symptoms": ("array", False), + "progression": ("array", False), + "complications": ("object", False), + "complications.respiratory": ("object", True), + "complications.respiratory.onset_day": ("int", True), + "complications.renal": ("string", True), + "notes": ("null", True), + "score": ("double", False), + "mixed": ("mixed", False), + "optional_field": ("int", True), + "empty_object": ("object", False), + "nullable": ("string", True), + "nullable_parent": ("object", True), + "nullable_parent.child": ("int", True), + } + actual = { + field.field_path: (field.type_name, field.is_optional) + for field in document.fields + } + assert len(actual) == len(document.fields), "Field paths must not be duplicated" + assert actual == expected + + +def test_sample_document_is_actual_json_document(con, document_namespace): + document = get_document(con, document_namespace, "symptoms") + assert document.HasField("sample_document_json") + sample = json.loads(document.sample_document_json) + assert sample in DOCUMENTS + # Sampling metadata must not consume or modify the collection. + cursor = con.cursor() + try: + cursor.executeany("mongo", "db.symptoms.find({})", namespace=document_namespace) + assert sorted(cursor.fetchall(), key=lambda item: item["_id"]) == DOCUMENTS + finally: + cursor.close() + + +def test_document_field_paths_escape_literal_separators(con, document_namespace): + cursor = con.cursor() + original = { + "_id": "escaped", + "literal.key": 1, + "literal": {"key": "nested"}, + "slash\\key": True, + } + try: + cursor.executeany("mongo", 'db.createCollection("escapedpaths")', namespace=document_namespace) + cursor.executeany( + "mongo", "db.escapedpaths.insertOne(" + json.dumps(original) + ")", + namespace=document_namespace, + ) + con.commit() + document = get_document(con, document_namespace, "escapedpaths") + actual = { + field.field_path: (field.type_name, field.is_optional) + for field in document.fields + } + assert actual == { + "_id": ("string", False), + r"literal\.key": ("int", False), + "literal": ("object", False), + "literal.key": ("string", False), + r"slash\\key": ("bool", False), + } + assert len(actual) == len(document.fields), "Escaping must preserve distinct field paths" + assert json.loads(document.sample_document_json) == original + finally: + cursor.close() + + +def test_empty_collection_has_valid_metadata(con, document_namespace): + document = get_document(con, document_namespace, "empty") + assert document.namespace_name == document_namespace + assert document.collection_name == "empty" + assert list(document.fields) == [] + assert not document.HasField("sample_document_json") + # HSQLDB does not expose document indexes. Absence must be an empty list, + # rather than invented metadata for an implicit MongoDB index. + if os.environ.get("DEFAULT_STORE", "hsqldb").lower() == "hsqldb": + assert list(document.document_indexes) == [] + + +@pytest.mark.parametrize("pattern, expected", [ + ("symptoms", {"symptoms"}), + ("sym%", {"symptoms"}), + ("empt_", {"empty"}), + ("%", {"symptoms", "empty"}), + ("missing", set()), +]) +def test_document_entity_patterns(con, document_namespace, pattern, expected): + response = get_entities(con, document_namespace, pattern) + assert {entity.document.collection_name for entity in response.entities} == expected + + +def test_metadata_reflects_committed_document_updates(con, document_namespace): + before = get_document(con, document_namespace, "empty") + assert not before.HasField("sample_document_json") + cursor = con.cursor() + try: + cursor.executeany( + "mongo", 'db.empty.insertOne({"_id":"new", "new_field":17})', + namespace=document_namespace, + ) + con.commit() + after = get_document(con, document_namespace, "empty") + fields = {field.field_path: field for field in after.fields} + assert fields["new_field"].type_name == "int" + assert not fields["new_field"].is_optional + assert json.loads(after.sample_document_json) == {"_id": "new", "new_field": 17} + finally: + cursor.close() + + +def test_metadata_reads_pending_writes_without_committing(con, document_namespace): + cursor = con.cursor() + try: + cursor.executeany( + "mongo", 'db.empty.insertOne({"_id":"pending", "uncommitted":true})', + namespace=document_namespace, + ) + document = get_document(con, document_namespace, "empty") + assert json.loads(document.sample_document_json) == {"_id": "pending", "uncommitted": True} + field = next(item for item in document.fields if item.field_path == "uncommitted") + assert (field.type_name, field.is_optional) == ("bool", False) + con.rollback() + after = get_document(con, document_namespace, "empty") + assert list(after.fields) == [] + assert not after.HasField("sample_document_json") + finally: + cursor.close() + + +def test_empty_document_namespace_returns_empty_entities(con): + namespace = "prism_doc_empty_" + uuid.uuid4().hex + cursor = con.cursor() + cursor.execute("CREATE DOCUMENT NAMESPACE " + namespace) + try: + assert list(get_entities(con, namespace).entities) == [] + finally: + con.rollback() + cursor.execute("DROP NAMESPACE " + namespace) + cursor.close() + + +def test_relational_metadata_still_works_after_document_request(con, document_namespace): + namespace = "prism_rel_meta_" + uuid.uuid4().hex + cursor = con.cursor() + cursor.execute("CREATE RELATIONAL NAMESPACE " + namespace) + try: + cursor.execute("CREATE TABLE " + namespace + ".patients(id INTEGER NOT NULL, name VARCHAR(50), PRIMARY KEY(id))") + get_entities(con, document_namespace) + response = get_entities(con, namespace, "patients") + assert len(response.entities) == 1 + assert response.entities[0].WhichOneof("Entity") == "table" + table = response.entities[0].table + assert table.namespace_name == namespace + assert table.table_name == "patients" + assert {column.column_name for column in table.columns} == {"id", "name"} + assert table.HasField("primary_key") + finally: + con.rollback() + cursor.execute("DROP NAMESPACE " + namespace) + cursor.close() diff --git a/tests/document_metadata_mongo.py b/tests/document_metadata_mongo.py new file mode 100644 index 0000000..4a2ec35 --- /dev/null +++ b/tests/document_metadata_mongo.py @@ -0,0 +1,104 @@ +# Copyright 2026 The Polypheny Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Explicit live MongoDB source tests, including real DocumentIndex metadata. + +Run with ``pytest -v tests/document_metadata_mongo.py`` against a MongoDB +replica set (defaults to 127.0.0.1:27017). This file is intentionally outside +pytest's default ``test_*.py`` selection, since ordinary connector tests need +only Polypheny. An explicit run fails if either service is unavailable. +""" + +import json +import os +import time +import uuid + +import pytest +from pymongo import MongoClient + +from document_metadata import con, get_document, document_metadata_schema + + +@pytest.fixture +def mongo_source(con): + suffix = uuid.uuid4().hex + database = "prism_metadata_" + suffix + adapter = "prism_mongo_" + suffix + host = os.environ.get("POLYPHENY_MONGO_HOST", "127.0.0.1") + port = int(os.environ.get("POLYPHENY_MONGO_PORT", "27017")) + client = MongoClient(host, port, directConnection=True, serverSelectionTimeoutMS=5000) + cursor = con.cursor() + registered = False + try: + client.admin.command("ping") + collection = client[database]["symptoms"] + collection.insert_many([ + {"_id": "p1", "patient_id": "p1", "age": 23, "complications": {"renal": "mild"}}, + {"_id": "p2", "patient_id": "p2", "age": 64, "complications": {"renal": "severe"}}, + ]) + collection.create_index("patient_id", unique=True) + collection.create_index([("age", "hashed")]) + collection.create_index("complications.renal") + collection.create_index([("$**", 1)]) + + settings = { + "mode": "remote", "host": host, "port": str(port), + "database": database, "username": "", "password": "", "authSource": "", + } + cursor.execute( + 'ALTER ADAPTERS ADD "' + adapter + '" USING \'MongoDB\' AS \'Source\' WITH \'' + + json.dumps(settings) + "'" + ) + registered = True + + # Source discovery may run asynchronously. Poll the public namespaces + # endpoint instead of assuming a fixed startup delay. + deadline = time.monotonic() + 30 + while True: + message = con.con.new_request() + message.namespaces_request.namespace_pattern = adapter + response = con.con.call(message) + if any(item.namespace_name == adapter for item in response.namespaces_response.namespaces): + break + assert time.monotonic() < deadline, "MongoDB source namespace was not discovered" + time.sleep(0.1) + + yield adapter + finally: + try: + con.rollback() + if registered: + cursor.execute('ALTER ADAPTERS DROP "' + adapter + '"') + finally: + cursor.close() + client.drop_database(database) + client.close() + + +def test_real_document_indexes_cover_every_index_field(con, mongo_source): + metadata = get_document(con, mongo_source, "symptoms") + actual = {(index.field_path, index.index_type, index.unique) for index in metadata.document_indexes} + assert actual == { + ("_id", "btree", True), + ("patient_id", "btree", True), + ("age", "hash", False), + ("complications.renal", "btree", False), + ("$**", "wildcard", False), + } + assert len(metadata.document_indexes) == len(actual), "Indexes must not be duplicated" + assert metadata.namespace_name == mongo_source + assert metadata.collection_name == "symptoms" + assert metadata.HasField("sample_document_json") + assert json.loads(metadata.sample_document_json)["patient_id"] in {"p1", "p2"} From 14479f24be3ed36449a720c5b0fbce1b35edd943 Mon Sep 17 00:00:00 2001 From: Marco Vogt Date: Fri, 6 Nov 2026 10:08:51 +0100 Subject: [PATCH 2/4] Build and validate the paired server directly in metadata CI --- .github/workflows/document-metadata.yaml | 41 ++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/.github/workflows/document-metadata.yaml b/.github/workflows/document-metadata.yaml index a715ac4..a996f9d 100644 --- a/.github/workflows/document-metadata.yaml +++ b/.github/workflows/document-metadata.yaml @@ -18,9 +18,46 @@ jobs: timeout-minutes: 30 runs-on: ubuntu-24.04 steps: - - uses: polypheny/GitHub-Action-Build-Polypheny@main + - uses: actions/checkout@v4 + with: + repository: polypheny/Polypheny-DB + ref: ${{ inputs.db_ref || '24797eb5548e3ca07f15a1a12ecf236327751de1' }} + + - uses: actions/setup-java@v4 with: - branch: ${{ inputs.db_ref || '24797eb5548e3ca07f15a1a12ecf236327751de1' }} + distribution: temurin + java-version: '17' + + - name: Build server and plugins and validate Java metadata + run: >- + ./gradlew :dbms:shadowJar assemblePlugins + :plugins:prism-interface:test --tests '*DocumentMetaRetrieverTest' + :plugins:mongodb-adapter:test --tests '*MongoIndexMetadataTest' + -PwithoutpullingUi=true --console=plain + + - name: Produce final executable server JAR + # dbms:jar and dbms:shadowJar currently share an output path. Run the + # shadow task last so the uploaded artifact contains runtime classes. + run: ./gradlew :dbms:shadowJar -PwithoutpullingUi=true --console=plain + + - name: Select executable server JAR + run: | + python - <<'PY' + from pathlib import Path + import shutil + import zipfile + candidates = [path for path in Path('dbms/build/libs').glob('dbms-*.jar') + if not path.stem.endswith(('-tests', '-sources', '-javadoc'))] + assert len(candidates) == 1, f'Expected one executable JAR, found {candidates}' + with zipfile.ZipFile(candidates[0]) as archive: + assert 'org/slf4j/Logger.class' in archive.namelist(), 'Server JAR lacks runtime dependencies' + shutil.copyfile(candidates[0], 'polypheny.jar') + PY + + - uses: actions/upload-artifact@v4 + with: + name: polypheny-jar + path: polypheny.jar document-metadata: timeout-minutes: 15 From 190febc4e679a68d7ebd8743d3804202a86be6e5 Mon Sep 17 00:00:00 2001 From: Marco Vogt Date: Fri, 6 Nov 2026 10:16:27 +0100 Subject: [PATCH 3/4] Repair Python CI environments and preserve optional metadata test dependencies --- .github/workflows/test.yaml | 52 +++++++++++++++++++++++++++----- tests/document_metadata_mongo.py | 5 ++- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d10fa45..3008998 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,12 +6,40 @@ on: jobs: build-polypheny: - timeout-minutes: 15 - runs-on: ubuntu-latest + timeout-minutes: 30 + runs-on: ubuntu-24.04 steps: - - uses: polypheny/GitHub-Action-Build-Polypheny@main + - uses: actions/checkout@v4 with: - branch: master + repository: polypheny/Polypheny-DB + ref: master + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + + - name: Build executable server and plugins + run: ./gradlew :dbms:shadowJar assemblePlugins -PwithoutpullingUi=true --console=plain + + - name: Select executable server JAR + run: | + python - <<'PY' + from pathlib import Path + import shutil + import zipfile + candidates = [path for path in Path('dbms/build/libs').glob('dbms-*.jar') + if not path.stem.endswith(('-tests', '-sources', '-javadoc'))] + assert len(candidates) == 1, f'Expected one executable JAR, found {candidates}' + with zipfile.ZipFile(candidates[0]) as archive: + assert 'org/slf4j/Logger.class' in archive.namelist(), 'Server JAR lacks runtime dependencies' + shutil.copyfile(candidates[0], 'polypheny.jar') + PY + + - uses: actions/upload-artifact@v4 + with: + name: polypheny-jar + path: polypheny.jar test-system-python: timeout-minutes: 10 @@ -19,7 +47,15 @@ jobs: fail-fast: false matrix: version: ['3.9', '3.10', '3.11', '3.12', '3.13'] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-24.04, macos-latest, windows-latest] + # Python 3.9 has no Ubuntu 26.04 build. Python 3.9/3.10 on + # macOS require the Intel distribution; retain all 15 combinations. + exclude: + - {version: '3.9', os: macos-latest} + - {version: '3.10', os: macos-latest} + include: + - {version: '3.9', os: macos-15-intel} + - {version: '3.10', os: macos-15-intel} runs-on: ${{ matrix.os }} needs: build-polypheny steps: @@ -74,8 +110,8 @@ jobs: strategy: fail-fast: false matrix: - adapter: [ mongodb, hsqldb, monetdb, postgresql, file, cottontail, neo4j ] - runs-on: ubuntu-latest + adapter: [ mongodb, hsqldb, monetdb, postgresql, file, neo4j ] + runs-on: ubuntu-24.04 needs: build-polypheny steps: - name: Set store env variable @@ -129,7 +165,7 @@ jobs: build-docs: timeout-minutes: 2 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/setup-python@v5 with: diff --git a/tests/document_metadata_mongo.py b/tests/document_metadata_mongo.py index 4a2ec35..af60d9f 100644 --- a/tests/document_metadata_mongo.py +++ b/tests/document_metadata_mongo.py @@ -26,13 +26,16 @@ import uuid import pytest -from pymongo import MongoClient from document_metadata import con, get_document, document_metadata_schema @pytest.fixture def mongo_source(con): + # Ordinary --doctest-modules collection also imports this opt-in module. + # Require pymongo only when its live MongoDB fixture is actually executed. + from pymongo import MongoClient + suffix = uuid.uuid4().hex database = "prism_metadata_" + suffix adapter = "prism_mongo_" + suffix From 52b6d1ccb8364457063e01f52ee0ad19ee559834 Mon Sep 17 00:00:00 2001 From: Marco Vogt Date: Fri, 6 Nov 2026 10:21:34 +0100 Subject: [PATCH 4/4] Validate the full Prism suite with the corrected integration lifecycle --- .github/workflows/document-metadata.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/document-metadata.yaml b/.github/workflows/document-metadata.yaml index a996f9d..17b6c79 100644 --- a/.github/workflows/document-metadata.yaml +++ b/.github/workflows/document-metadata.yaml @@ -7,7 +7,7 @@ on: inputs: db_ref: description: Polypheny-DB revision containing document metadata support - default: 24797eb5548e3ca07f15a1a12ecf236327751de1 + default: a35687bda0f5f13a40eda56a2ab340faf4977d2c required: true permissions: @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 with: repository: polypheny/Polypheny-DB - ref: ${{ inputs.db_ref || '24797eb5548e3ca07f15a1a12ecf236327751de1' }} + ref: ${{ inputs.db_ref || 'a35687bda0f5f13a40eda56a2ab340faf4977d2c' }} - uses: actions/setup-java@v4 with: @@ -31,7 +31,7 @@ jobs: - name: Build server and plugins and validate Java metadata run: >- ./gradlew :dbms:shadowJar assemblePlugins - :plugins:prism-interface:test --tests '*DocumentMetaRetrieverTest' + :plugins:prism-interface:test :plugins:mongodb-adapter:test --tests '*MongoIndexMetadataTest' -PwithoutpullingUi=true --console=plain