From 375e2a6216b7e64fd37f6429f76406742d3f5f79 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 20:21:34 +0900 Subject: [PATCH 01/33] Rename the JVM distribution to arcadedb-jvm The JVM build is published as arcadedb-jvm (environment pypi-jvm); the name arcadedb is reserved for the native build. The import name stays arcadedb. Wheel files are arcadedb_jvm--..., so every glob, the wheel-count checks, the name guard in the publish job, the check_version.py call, after_tag_verify.sh and the pip hints for the extras follow. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/release.yml | 21 ++++++++----------- .github/workflows/test-python-examples.yml | 2 +- CLAUDE.md | 2 +- README.md | 6 +++--- docs/antora.yml | 6 +++--- .../ROOT/pages/development/release.adoc | 4 ++-- examples/07_stackoverflow_tables_oltp.py | 2 +- examples/08_stackoverflow_tables_olap.py | 2 +- examples/09_stackoverflow_graph_oltp.py | 2 +- examples/10_stackoverflow_graph_olap.py | 2 +- examples/11_vector_index_build.py | 2 +- examples/12_vector_search.py | 2 +- examples/22_numpy_bulk_io.py | 4 ++-- examples/27_dataframe_arrow_ingest.py | 2 +- examples/README.md | 2 +- pyproject.toml | 8 +++---- scripts/Dockerfile.build | 4 ++-- scripts/after_tag_verify.sh | 2 +- scripts/build-native.sh | 4 ++-- scripts/build.sh | 6 +++--- scripts/check_no_embedded_name.py | 4 ++-- src/arcadedb/__init__.py | 2 +- src/arcadedb/dataframe.py | 2 +- src/arcadedb/jvm.py | 2 +- src/arcadedb/results.py | 6 +++--- tests/test_check_version.py | 2 +- tests/test_no_embedded_name.py | 2 +- tests/test_runtime_cache.py | 10 ++++----- tests/test_wheel_platform_tag.py | 10 ++++----- 29 files changed, 61 insertions(+), 64 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 38364e4..dc84ffe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,20 +8,17 @@ name: Release to PyPI # # PyPI keys a trusted publisher on this FILENAME and on the job's environment, so do not rename # either without changing the publisher on PyPI at the same time: -# project arcadedb <- ArcadeData/arcadedb-python, workflow release.yml, environment pypi -# project arcadedb-native <- same repository and workflow, environment pypi-native (not used yet) +# project arcadedb-jvm <- ArcadeData/arcadedb-python, workflow release.yml, environment pypi-jvm +# project arcadedb <- ArcadeData/arcadedb-python, workflow native.yml, environment pypi (the native build) # Both GitHub environments only accept deployments from tags that match [0-9]*. # # Nothing here stores a token: the publish job authenticates with PyPI trusted publishing (OIDC), # and only that job has the id-token permission. # -# pyproject.toml names the distribution arcadedb (tracker #19). -# The guard in the publish job refuses to publish anything that is not `arcadedb`, so this workflow -# cannot publish another distribution from here by accident. -# -# TODO (native-wheel port, tracker #19): a second publish job for arcadedb-native, in environment -# pypi-native with its own id-token permission and the same kind of guard (distribution name must -# be arcadedb-native). There is no native job yet. +# pyproject.toml names the distribution arcadedb-jvm (tracker #19). The wheel file is arcadedb_jvm--... +# (PEP 427 turns the dash into an underscore). The guard in the publish job refuses to publish anything +# that is not `arcadedb-jvm`, so this workflow cannot publish the native `arcadedb` by accident; the +# native build has its own workflow, native.yml, with the same kind of guard. on: push: @@ -38,7 +35,7 @@ concurrency: env: # The only distribution this workflow may publish. See the guard in the publish job. - DIST_NAME: arcadedb + DIST_NAME: arcadedb-jvm jobs: # The tag must be a bare PEP 440 version of the allowed kinds, and the next number of its base. @@ -250,7 +247,7 @@ jobs: needs: [validate-version, verify-wheels] runs-on: ubuntu-24.04 timeout-minutes: 30 - environment: pypi + environment: pypi-jvm permissions: id-token: write steps: @@ -260,7 +257,7 @@ jobs: name: release-dist path: dist/ - # GUARD: this workflow publishes the distribution `arcadedb` and nothing else. A wheel with + # GUARD: this workflow publishes the distribution `arcadedb-jvm` and nothing else. A wheel with # another name stops the job here, before anything reaches PyPI. Two checks, so neither can drift alone: # the wheel file name and the Name field of its METADATA, both must be DIST_NAME. - name: Refuse unless every wheel is the distribution ${{ env.DIST_NAME }} diff --git a/.github/workflows/test-python-examples.yml b/.github/workflows/test-python-examples.yml index 245f98a..65b08a5 100644 --- a/.github/workflows/test-python-examples.yml +++ b/.github/workflows/test-python-examples.yml @@ -260,7 +260,7 @@ jobs: - name: Install ArcadeDB Python bindings shell: bash run: | - uv pip install --system dist/arcadedb-*.whl + uv pip install --system dist/arcadedb_jvm-*.whl - name: Install example dependencies shell: bash diff --git a/CLAUDE.md b/CLAUDE.md index 933e1a1..720b5ef 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What this is -`arcadedb`: Python bindings for ArcadeDB that run the Java engine in-process through JPype. Each wheel is platform-specific and bundles the ArcadeDB jars plus a jlink'd Java 25 JRE, so users need no Java install. Moved here from `bindings/python` in ArcadeData/arcadedb (history kept; issue numbers in older commit messages refer to that repo). +This repository builds two distributions that both import as `arcadedb`: `arcadedb` (the native build: the engine as a GraalVM native-image shared library loaded with ctypes) and `arcadedb-jvm` (the JVM build: the Java engine in-process through JPype). `arcadedb-jvm` is described first below. Each JVM wheel is platform-specific and bundles the ArcadeDB jars plus a jlink'd Java 25 JRE, so users need no Java install. Moved here from `bindings/python` in ArcadeData/arcadedb (history kept; issue numbers in older commit messages refer to that repo). ## Working rules diff --git a/README.md b/README.md index 2cd234b..4f274cf 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # ArcadeDB Python -[![PyPI](https://img.shields.io/pypi/v/arcadedb)](https://pypi.org/project/arcadedb/) -[![Python versions](https://img.shields.io/pypi/pyversions/arcadedb)](https://pypi.org/project/arcadedb/) +[![PyPI](https://img.shields.io/pypi/v/arcadedb-jvm)](https://pypi.org/project/arcadedb-jvm/) +[![Python versions](https://img.shields.io/pypi/pyversions/arcadedb-jvm)](https://pypi.org/project/arcadedb-jvm/) [![Tests](https://github.com/ArcadeData/arcadedb-python/actions/workflows/test-python-bindings.yml/badge.svg)](https://github.com/ArcadeData/arcadedb-python/actions/workflows/test-python-bindings.yml) [![License](https://img.shields.io/badge/license-Apache--2.0-blue)](https://github.com/ArcadeData/arcadedb-python/blob/main/LICENSE) @@ -10,7 +10,7 @@ Python bindings for [ArcadeDB](https://arcadedb.com), the multi-model database ( ## Install ```bash -pip install arcadedb # or: uv add arcadedb +pip install arcadedb-jvm # or: uv add arcadedb-jvm ``` This package was published as arcadedb-embedded until 26.10.x; install arcadedb now. diff --git a/docs/antora.yml b/docs/antora.yml index cb59a7c..1c06e68 100644 --- a/docs/antora.yml +++ b/docs/antora.yml @@ -7,10 +7,10 @@ nav: asciidoc: attributes: # Package names. Every page takes them from here. - pkg-name: arcadedb + pkg-name: arcadedb-jvm import-name: arcadedb import-stmt: import arcadedb - native-pkg-name: arcadedb-native + native-pkg-name: arcadedb native-import-name: arcadedb # Other names a reader meets on PyPI (the pages that compare packages use these). jvm-pkg-name: arcadedb-jvm @@ -19,4 +19,4 @@ asciidoc: community-driver-pkg-name: arcadedb-python community-repo-url: https://github.com/stevereiner/arcadedb-python repo-url: https://github.com/ArcadeData/arcadedb-python - pypi-url: https://pypi.org/project/arcadedb/ + pypi-url: https://pypi.org/project/arcadedb-jvm/ diff --git a/docs/modules/ROOT/pages/development/release.adoc b/docs/modules/ROOT/pages/development/release.adoc index cada48b..5067337 100644 --- a/docs/modules/ROOT/pages/development/release.adoc +++ b/docs/modules/ROOT/pages/development/release.adoc @@ -1,5 +1,5 @@ = Release -:description: How a release of the arcadedb package is numbered, tagged, checked and published to PyPI. +:description: How a release of the arcadedb-jvm package is numbered, tagged, checked and published to PyPI. A release is a Git tag. Pushing the tag starts the `release.yml` workflow. It builds the wheels, tests them, checks them and publishes them to PyPI with trusted publishing. The workflow stores no token. @@ -37,7 +37,7 @@ The base is the version of the engine. The workflow compares the tag with the ve [source,bash] ---- -python3 scripts/check_version.py 26.11.1.dev0 --git-remote origin --pypi arcadedb +python3 scripts/check_version.py 26.11.1.dev0 --git-remote origin --pypi arcadedb-jvm ---- It exits 0 when the version is the correct next one, 1 when it is not, and 2 when a source cannot be read. Exit 2 is a failure, not a pass: the check fails closed. diff --git a/examples/07_stackoverflow_tables_oltp.py b/examples/07_stackoverflow_tables_oltp.py index 648ed2e..3d0fa3e 100644 --- a/examples/07_stackoverflow_tables_oltp.py +++ b/examples/07_stackoverflow_tables_oltp.py @@ -2148,7 +2148,7 @@ def run_in_docker(args): arcadedb_wheel_mount_path = None if args.db == "arcadedb_sql": - wheel_candidates = sorted((repo_root / "dist").glob("arcadedb-*.whl")) + wheel_candidates = sorted((repo_root / "dist").glob("arcadedb_jvm-*.whl")) if not wheel_candidates: if os.environ.get("GITHUB_ACTIONS", "").lower() == "true": raise RuntimeError( diff --git a/examples/08_stackoverflow_tables_olap.py b/examples/08_stackoverflow_tables_olap.py index 11ee0c7..d693d97 100644 --- a/examples/08_stackoverflow_tables_olap.py +++ b/examples/08_stackoverflow_tables_olap.py @@ -1449,7 +1449,7 @@ def run_in_docker(args): arcadedb_wheel_mount_path = None if args.db == "arcadedb_sql": - wheel_candidates = sorted((repo_root / "dist").glob("arcadedb-*.whl")) + wheel_candidates = sorted((repo_root / "dist").glob("arcadedb_jvm-*.whl")) if not wheel_candidates: if os.environ.get("GITHUB_ACTIONS", "").lower() == "true": raise RuntimeError( diff --git a/examples/09_stackoverflow_graph_oltp.py b/examples/09_stackoverflow_graph_oltp.py index 9da67ea..0042366 100644 --- a/examples/09_stackoverflow_graph_oltp.py +++ b/examples/09_stackoverflow_graph_oltp.py @@ -9747,7 +9747,7 @@ def run_in_docker(args) -> bool: arcadedb_wheel_mount_path = None if args.db.startswith("arcadedb_"): - wheel_candidates = sorted((repo_root / "dist").glob("arcadedb-*.whl")) + wheel_candidates = sorted((repo_root / "dist").glob("arcadedb_jvm-*.whl")) if not wheel_candidates: if os.environ.get("GITHUB_ACTIONS", "").lower() == "true": raise RuntimeError( diff --git a/examples/10_stackoverflow_graph_olap.py b/examples/10_stackoverflow_graph_olap.py index 5640cfd..1eb4dff 100644 --- a/examples/10_stackoverflow_graph_olap.py +++ b/examples/10_stackoverflow_graph_olap.py @@ -6520,7 +6520,7 @@ def run_in_docker(args) -> bool: arcadedb_wheel_mount_path = None if args.db == "arcadedb_cypher": - wheel_candidates = sorted((repo_root / "dist").glob("arcadedb-*.whl")) + wheel_candidates = sorted((repo_root / "dist").glob("arcadedb_jvm-*.whl")) if not wheel_candidates: if os.environ.get("GITHUB_ACTIONS", "").lower() == "true": raise RuntimeError( diff --git a/examples/11_vector_index_build.py b/examples/11_vector_index_build.py index 7da4bbe..6beb006 100644 --- a/examples/11_vector_index_build.py +++ b/examples/11_vector_index_build.py @@ -1688,7 +1688,7 @@ def run_in_docker(args) -> bool: arcadedb_wheel_mount_path = None if args.backend == "arcadedb_sql": - wheel_candidates = sorted((repo_root / "dist").glob("arcadedb-*.whl")) + wheel_candidates = sorted((repo_root / "dist").glob("arcadedb_jvm-*.whl")) if not wheel_candidates: if os.environ.get("GITHUB_ACTIONS", "").lower() == "true": raise RuntimeError( diff --git a/examples/12_vector_search.py b/examples/12_vector_search.py index 4b909f9..c1f3edb 100644 --- a/examples/12_vector_search.py +++ b/examples/12_vector_search.py @@ -1955,7 +1955,7 @@ def _to_container_path(value: str) -> str: arcadedb_wheel_mount_path = None if args.backend == "arcadedb_sql": - wheel_candidates = sorted((repo_root / "dist").glob("arcadedb-*.whl")) + wheel_candidates = sorted((repo_root / "dist").glob("arcadedb_jvm-*.whl")) if not wheel_candidates: if os.environ.get("GITHUB_ACTIONS", "").lower() == "true": raise RuntimeError( diff --git a/examples/22_numpy_bulk_io.py b/examples/22_numpy_bulk_io.py index aaa59e0..36cd334 100644 --- a/examples/22_numpy_bulk_io.py +++ b/examples/22_numpy_bulk_io.py @@ -22,7 +22,7 @@ Requirements: - numpy (pip install numpy) - pyarrow, optional, only for the to_arrow section - (pip install 'arcadedb[arrow]') + (pip install 'arcadedb-jvm[arrow]') """ from __future__ import annotations @@ -167,7 +167,7 @@ def nulls_to_arrow(db, n_rows: int) -> None: if table is None: print( "to_arrow : pyarrow not installed, skipping " - "(pip install 'arcadedb[arrow]')" + "(pip install 'arcadedb-jvm[arrow]')" ) return diff --git a/examples/27_dataframe_arrow_ingest.py b/examples/27_dataframe_arrow_ingest.py index 2cdb1d6..cccfc05 100644 --- a/examples/27_dataframe_arrow_ingest.py +++ b/examples/27_dataframe_arrow_ingest.py @@ -17,7 +17,7 @@ Requirements: - pandas, polars and pyarrow - (pip install 'arcadedb[pandas,polars]') + (pip install 'arcadedb-jvm[pandas,polars]') """ from __future__ import annotations diff --git a/examples/README.md b/examples/README.md index 999802a..6cec86e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -401,7 +401,7 @@ DataFrames and Arrow tables loaded straight into a type, and a result back out a column refused before a row is written - `to_polars()` on the result - synthetic data; flags: `--rows` (default 100,000) and `--db-path` -- requires `pip install 'arcadedb[pandas,polars]'` +- requires `pip install 'arcadedb-jvm[pandas,polars]'` **Learn:** How a DataFrame becomes documents or vertices without a row loop, and what each Arrow type is stored as diff --git a/pyproject.toml b/pyproject.toml index 4beab33..f6b000c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,16 +10,16 @@ build-backend = "setuptools.build_meta" # scripts/extract_version.py [project] -name = "arcadedb" +name = "arcadedb-jvm" version = "0.0.0" -description = "ArcadeDB embedded multi-model database with bundled JRE - no Java installation required" +description = "ArcadeDB embedded multi-model database, JVM build with a bundled JRE (no Java installation needed). The native build is the package arcadedb; both import as arcadedb" readme = "README.md" requires-python = ">=3.11" license = { text = "Apache-2.0" } authors = [ { name = "Taewoon Kim", email = "taewoon@humem.ai" } ] -keywords = ["database", "graph", "document", "multi-model", "nosql", "arcadedb", "embedded", "jre", "java"] +keywords = ["database", "graph", "document", "multi-model", "nosql", "arcadedb", "embedded", "jre", "java", "jvm"] classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", @@ -110,7 +110,7 @@ pandas = [ ] # Optional: ResultSet.to_polars() and insert_dataframe() with a Polars frame. Both go # through Arrow, so pyarrow comes with it. insert_dataframe() with a pandas frame needs -# `pip install 'arcadedb[arrow,pandas]'` (the pandas extra alone has no pyarrow, +# `pip install 'arcadedb-jvm[arrow,pandas]'` (the pandas extra alone has no pyarrow, # because to_dataframe() does not need it). polars = [ "polars>=1.0", diff --git a/scripts/Dockerfile.build b/scripts/Dockerfile.build index 16bc549..dd11e84 100644 --- a/scripts/Dockerfile.build +++ b/scripts/Dockerfile.build @@ -7,12 +7,12 @@ # # REQUIRED BUILD ARGS (passed from build.sh): # PYTHON_VERSION - Python version for wheel (e.g., 3.11, 3.12, 3.13, 3.14) -# PACKAGE_NAME - always: arcadedb +# PACKAGE_NAME - always: arcadedb-jvm # PACKAGE_DESCRIPTION - package description # TARGET_PLATFORM - Linux target platform for bundled JRE (linux-x64, linux-arm64) ARG PYTHON_VERSION=3.12 -ARG PACKAGE_NAME=arcadedb +ARG PACKAGE_NAME=arcadedb-jvm ARG PACKAGE_DESCRIPTION="ArcadeDB embedded multi-model database with bundled JRE - no Java installation required" # Deterministic placeholder. The wrapper build script must pass the real ArcadeDB tag. ARG ARCADEDB_TAG=build-arg-required diff --git a/scripts/after_tag_verify.sh b/scripts/after_tag_verify.sh index 675032b..6a4e16f 100755 --- a/scripts/after_tag_verify.sh +++ b/scripts/after_tag_verify.sh @@ -18,7 +18,7 @@ RUN="${3:-}" ENG_SHA="${4:-}" ENG_BUILD="${5:-}" R=ArcadeData/arcadedb-python -DIST=arcadedb +DIST=arcadedb-jvm W="${WORK:-$(mktemp -d "${TMPDIR:-$HOME}/after-tag-XXXXXX")}" FAILS=0 pass() { printf 'PASS %s\n' "$*"; } diff --git a/scripts/build-native.sh b/scripts/build-native.sh index 7df18e3..9d9b775 100755 --- a/scripts/build-native.sh +++ b/scripts/build-native.sh @@ -25,7 +25,7 @@ JRE_MODULE_EXCLUSIONS_FILE="$SCRIPT_DIR/jre_module_exclusions.txt" # Parameters PLATFORM="${1:-}" -PACKAGE_NAME="${2:-arcadedb}" +PACKAGE_NAME="${2:-arcadedb-jvm}" PACKAGE_DESCRIPTION="${3:-ArcadeDB embedded multi-model database with bundled JRE}" ARCADEDB_TAG="${4:-}" BUILD_VERSION="${5:-}" @@ -378,7 +378,7 @@ WHEEL_FILE=$(ls "$DIST_DIR"/*.whl | head -n1) if [[ -n "$WHEEL_FILE" ]]; then # Extract components from wheel filename WHEEL_NAME=$(basename "$WHEEL_FILE") - # arcadedb-25.10.1-py3-none-any.whl -> arcadedb-25.10.1-py3-none-PLAT_NAME.whl + # arcadedb_jvm-25.10.1-py3-none-any.whl -> arcadedb_jvm-25.10.1-py3-none-PLAT_NAME.whl NEW_WHEEL_NAME=$(echo "$WHEEL_NAME" | sed "s|-py3-none-any\.whl|-py3-none-${PLAT_NAME}.whl|") if [[ "$WHEEL_NAME" != "$NEW_WHEEL_NAME" ]]; then mv "$WHEEL_FILE" "$DIST_DIR/$NEW_WHEEL_NAME" diff --git a/scripts/build.sh b/scripts/build.sh index cdbb10a..2e4efb1 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -270,7 +270,7 @@ echo -e " Build Method: ${YELLOW}${BUILD_METHOD}${NC}" echo "" # Package configuration -PACKAGE_NAME="arcadedb" +PACKAGE_NAME="arcadedb-jvm" DESCRIPTION="ArcadeDB embedded multi-model database with bundled JRE - no Java installation required" echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" @@ -447,10 +447,10 @@ def _is_wanted(wheel_path): _new = [Path(p) for p in os.environ.get("NEW_WHEELS", "").split() if p.endswith(".whl")] -wheels = _new or sorted((p for p in Path("dist").glob("arcadedb-*.whl") if _is_wanted(p)), +wheels = _new or sorted((p for p in Path("dist").glob("arcadedb_jvm-*.whl") if _is_wanted(p)), key=lambda p: p.stat().st_mtime) if not wheels: - print(f"❌ no arcadedb-{_want} wheel in dist/", file=sys.stderr) + print(f"❌ no arcadedb_jvm-{_want} wheel in dist/", file=sys.stderr) sys.exit(1) wheel = wheels[-1] if not _is_wanted(wheel): diff --git a/scripts/check_no_embedded_name.py b/scripts/check_no_embedded_name.py index 8ad7d0c..65ba0c8 100644 --- a/scripts/check_no_embedded_name.py +++ b/scripts/check_no_embedded_name.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Fail when the old package name comes back. -The distribution is `arcadedb` (JVM wheel) and `arcadedb-native` (native wheel), and both import +The distributions are `arcadedb` (native wheel) and `arcadedb-jvm` (JVM wheel), and both import as `arcadedb`. The names `arcadedb_embedded` and `arcadedb-embedded` belong to the old PyPI packages, which get a deprecation shim from the humemai repository, not from this one. This script scans every tracked (and every untracked, not ignored) text file and every path in the repository @@ -17,7 +17,7 @@ documents under docs/superpowers/, a changelog, and this script. * References to issues of the old repository, written `humemai/arcadedb-embedded-python#123`. They point at real threads, so they keep their text. -* The sentence "This package was published as arcadedb-embedded until 26.10.x; install arcadedb now." +* The sentence "This package was published as arcadedb-embedded until 26.10.x; install arcadedb or arcadedb-jvm now." and lines that say "formerly arcadedb-embedded". * Any line that carries the marker `allow-old-name`, for the rare mention that needs it. diff --git a/src/arcadedb/__init__.py b/src/arcadedb/__init__.py index d99e6be..8a7c5a9 100644 --- a/src/arcadedb/__init__.py +++ b/src/arcadedb/__init__.py @@ -15,7 +15,7 @@ try: from importlib.metadata import version as _dist_version - __version__ = _dist_version("arcadedb") + __version__ = _dist_version("arcadedb-jvm") except Exception: # not installed: fall back to the build-time file try: from ._version import __version__ diff --git a/src/arcadedb/dataframe.py b/src/arcadedb/dataframe.py index 527f44f..a38a170 100644 --- a/src/arcadedb/dataframe.py +++ b/src/arcadedb/dataframe.py @@ -19,7 +19,7 @@ _PYARROW_HINT = ( "pyarrow is required for {what}. Install with: pip install pyarrow " - "(or pip install 'arcadedb[arrow]')" + "(or pip install 'arcadedb-jvm[arrow]')" ) # Declared property types (the engine's Type names) that an Arrow column kind diff --git a/src/arcadedb/jvm.py b/src/arcadedb/jvm.py index c4fb5b0..d13750f 100644 --- a/src/arcadedb/jvm.py +++ b/src/arcadedb/jvm.py @@ -91,7 +91,7 @@ def _extract_runtime_resource(resource_name: str) -> Path: project_dir = _project_dir() dist_dir = project_dir / "dist" wheels = sorted( - dist_dir.glob("arcadedb-*.whl"), + dist_dir.glob("arcadedb_jvm-*.whl"), key=lambda p: p.stat().st_mtime_ns, reverse=True, ) diff --git a/src/arcadedb/results.py b/src/arcadedb/results.py index 1dcfca8..9c09dba 100644 --- a/src/arcadedb/results.py +++ b/src/arcadedb/results.py @@ -660,7 +660,7 @@ def to_polars( Reads :meth:`to_arrow` and hands the table to Polars, so nulls keep their type (a nullable integer stays an integer column) and a DECIMAL column is a Polars ``Decimal``. Requires polars and pyarrow - (``pip install 'arcadedb[polars]'``). + (``pip install 'arcadedb-jvm[polars]'``). Args: batch_size: Rows per bridge crossing, as in :meth:`to_arrow`. @@ -682,14 +682,14 @@ def to_polars( except ImportError as exc: raise ImportError( "polars is required for to_polars(). " - "Install with: pip install 'arcadedb[polars]'" + "Install with: pip install 'arcadedb-jvm[polars]'" ) from exc table = self.to_arrow(batch_size=batch_size, columns=columns) if table is None: raise ImportError( "to_polars() reads through Arrow, which needs pyarrow and numpy " "and the bundled bridge jar. " - "Install with: pip install 'arcadedb[polars]'" + "Install with: pip install 'arcadedb-jvm[polars]'" ) return pl.from_arrow(table) diff --git a/tests/test_check_version.py b/tests/test_check_version.py index 6d9d27e..cb712b7 100644 --- a/tests/test_check_version.py +++ b/tests/test_check_version.py @@ -260,4 +260,4 @@ def test_release_workflow_checks_the_tag_before_anything_is_built(): r'python3 -I scripts/check_version\.py "\$TAG" --git-remote origin --pypi "\$DIST_NAME"', first_job, ) - assert re.search(r"^ DIST_NAME: arcadedb$", workflow, re.MULTILINE) + assert re.search(r"^ DIST_NAME: arcadedb-jvm$", workflow, re.MULTILINE) diff --git a/tests/test_no_embedded_name.py b/tests/test_no_embedded_name.py index e44d11f..c41e6bd 100644 --- a/tests/test_no_embedded_name.py +++ b/tests/test_no_embedded_name.py @@ -34,6 +34,6 @@ def test_wildcards_for_the_old_wheel_name_are_caught_but_markdown_bold_is_not(): for ok in ( "**embedding** models", "embedded multi-model database", - "dist/arcadedb-*.whl", + "dist/arcadedb_jvm-*.whl", ): assert not mod.OLD_NAME.search(ok), ok diff --git a/tests/test_runtime_cache.py b/tests/test_runtime_cache.py index 5ac3f2b..dc00b7b 100644 --- a/tests/test_runtime_cache.py +++ b/tests/test_runtime_cache.py @@ -41,8 +41,8 @@ def project(tmp_path, monkeypatch): def test_newest_wheel_is_chosen_by_build_time_not_by_name(project): """A reverse string sort ranks 26.9.1 above 26.10.1 because "9" > "1".""" dist = project / "dist" - _wheel(dist, "arcadedb-26.9.1-py3-none-any.whl", {"v.txt": "old"}, 1_000) - _wheel(dist, "arcadedb-26.10.1-py3-none-any.whl", {"v.txt": "new"}, 2_000) + _wheel(dist, "arcadedb_jvm-26.9.1-py3-none-any.whl", {"v.txt": "old"}, 1_000) + _wheel(dist, "arcadedb_jvm-26.10.1-py3-none-any.whl", {"v.txt": "new"}, 2_000) out = jvm._extract_runtime_resource(RES) @@ -52,7 +52,7 @@ def test_newest_wheel_is_chosen_by_build_time_not_by_name(project): def test_a_rebuilt_wheel_replaces_the_cache_and_leaves_nothing_behind(project): """The June 4 failure: a newer wheel appears and the old extraction stays.""" dist = project / "dist" - name = "arcadedb-26.10.1.dev0-py3-none-any.whl" + name = "arcadedb_jvm-26.10.1.dev0-py3-none-any.whl" _wheel(dist, name, {"engine-26.6.1.jar": "june", "only-in-old.jar": "x"}, 1_000) first = jvm._extract_runtime_resource(RES) assert (first / "engine-26.6.1.jar").exists() @@ -70,7 +70,7 @@ def test_a_rebuilt_wheel_replaces_the_cache_and_leaves_nothing_behind(project): def test_an_unchanged_wheel_is_not_re_extracted(project): """The cache is still a cache: the same wheel does not pay extraction twice.""" dist = project / "dist" - _wheel(dist, "arcadedb-26.10.1-py3-none-any.whl", {"a.jar": "a"}, 1_000) + _wheel(dist, "arcadedb_jvm-26.10.1-py3-none-any.whl", {"a.jar": "a"}, 1_000) out = jvm._extract_runtime_resource(RES) (out / "marker").write_text("still here") @@ -87,7 +87,7 @@ def test_a_cache_from_before_stamping_is_treated_as_stale(project): (legacy / "engine-26.6.1.jar").write_text("june") _wheel( dist, - "arcadedb-26.10.1-py3-none-any.whl", + "arcadedb_jvm-26.10.1-py3-none-any.whl", {"engine-26.10.1.jar": "sep"}, 2_000, ) diff --git a/tests/test_wheel_platform_tag.py b/tests/test_wheel_platform_tag.py index 8997f0f..ab874b0 100644 --- a/tests/test_wheel_platform_tag.py +++ b/tests/test_wheel_platform_tag.py @@ -57,7 +57,7 @@ def test_max_glibc_in_dir_picks_highest(tmp_path: Path) -> None: def test_parse_wheel_tag_extracts_version_and_arch(tmp_path: Path) -> None: verifier = _load_verifier() - wheel = tmp_path / "arcadedb-26.4.2-py3-none-manylinux_2_34_x86_64.whl" + wheel = tmp_path / "arcadedb_jvm-26.4.2-py3-none-manylinux_2_34_x86_64.whl" wheel.write_bytes(b"") glibc, arch = verifier.parse_wheel_tag(wheel) @@ -73,7 +73,7 @@ def test_main_succeeds_when_tag_matches( jre_dir.mkdir() _make_elf_with_glibc_symbols(jre_dir / "libjvm.so", ["2.17", "2.34"]) - wheel = tmp_path / "arcadedb-26.4.2-py3-none-manylinux_2_34_x86_64.whl" + wheel = tmp_path / "arcadedb_jvm-26.4.2-py3-none-manylinux_2_34_x86_64.whl" wheel.write_bytes(b"") rc = _run_main(verifier, [str(wheel), str(jre_dir)]) @@ -90,7 +90,7 @@ def test_main_fails_when_tag_higher_than_jre( jre_dir.mkdir() _make_elf_with_glibc_symbols(jre_dir / "libjvm.so", ["2.17", "2.34"]) - wheel = tmp_path / "arcadedb-26.4.2-py3-none-manylinux_2_35_x86_64.whl" + wheel = tmp_path / "arcadedb_jvm-26.4.2-py3-none-manylinux_2_35_x86_64.whl" wheel.write_bytes(b"") rc = _run_main(verifier, [str(wheel), str(jre_dir)]) @@ -109,7 +109,7 @@ def test_main_fails_when_tag_lower_than_jre( jre_dir.mkdir() _make_elf_with_glibc_symbols(jre_dir / "libjvm.so", ["2.17", "2.36"]) - wheel = tmp_path / "arcadedb-26.4.2-py3-none-manylinux_2_28_x86_64.whl" + wheel = tmp_path / "arcadedb_jvm-26.4.2-py3-none-manylinux_2_28_x86_64.whl" wheel.write_bytes(b"") rc = _run_main(verifier, [str(wheel), str(jre_dir)]) @@ -151,7 +151,7 @@ def test_dunder_version_matches_distribution_metadata(): import arcadedb try: - expected = dist_version("arcadedb") + expected = dist_version("arcadedb-jvm") except PackageNotFoundError: # source checkout, nothing to compare against return assert arcadedb.__version__ == expected From d6022cafb0709794c8e0f645f447ce9ed721e2ee Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 20:22:31 +0900 Subject: [PATCH 02/33] Add arcadedb.__backend__ and refuse to import next to the other distribution arcadedb.__backend__ is 'jvm' or 'native'. The import-time check that only warned when both distributions were installed now raises DistributionConflictError (an ImportError) in both builds: arcadedb-jvm refuses to import when arcadedb (native) is installed, and the other way round. The message names both packages, says how to fix it (pip uninstall arcadedb arcadedb-jvm, then install one) and links the Which package? page. Unreadable metadata still never breaks an import. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/__init__.py | 19 ++++--- src/arcadedb/_dist_conflict.py | 84 ++++++++++++++++++------------ tests/test_dist_conflict.py | 93 +++++++++++++++++++++------------- 3 files changed, 124 insertions(+), 72 deletions(-) diff --git a/src/arcadedb/__init__.py b/src/arcadedb/__init__.py index 8a7c5a9..c5a98d7 100644 --- a/src/arcadedb/__init__.py +++ b/src/arcadedb/__init__.py @@ -5,6 +5,17 @@ directly in the Python process using JPype. """ +# Which build this is: 'jvm' (the arcadedb-jvm distribution, the Java engine through JPype) or 'native' (the +# arcadedb distribution, the engine as a GraalVM native-image library). The import name is arcadedb for both. +__backend__ = "jvm" + +# Both distributions install a package named arcadedb into the same directory; pip overwrites one with the other +# silently. Refuse to import when the other one is installed too, before anything else runs. +from ._dist_conflict import DISTRIBUTIONS as _DISTRIBUTIONS +from ._dist_conflict import check as _check_distributions + +_check_distributions(__backend__) + # The installed distribution's version is the single source of truth: the # wheel is versioned from the release tag, while the generated _version.py # is derived from the engine jar's version (scripts/write_version.py), so the @@ -15,18 +26,13 @@ try: from importlib.metadata import version as _dist_version - __version__ = _dist_version("arcadedb-jvm") + __version__ = _dist_version(_DISTRIBUTIONS[__backend__]) except Exception: # not installed: fall back to the build-time file try: from ._version import __version__ except ModuleNotFoundError: __version__ = "0.0.0" -# The JVM and the native wheel both import as `arcadedb`; pip overwrites one with the other silently. -from ._dist_conflict import warn_if_both_installed as _warn_if_both_installed - -_warn_if_both_installed(stacklevel=3) # blame the importing line - # Import async execution from .async_executor import AsyncExecutor @@ -87,6 +93,7 @@ __all__ = [ "__version__", + "__backend__", # Exceptions "ArcadeDBError", # Core classes diff --git a/src/arcadedb/_dist_conflict.py b/src/arcadedb/_dist_conflict.py index c318f41..0b253b0 100644 --- a/src/arcadedb/_dist_conflict.py +++ b/src/arcadedb/_dist_conflict.py @@ -1,31 +1,49 @@ -"""Warn when both distributions that provide the `arcadedb` import are installed. +"""Refuse to import when both distributions that provide the `arcadedb` import are installed. -The JVM wheel (`arcadedb`) and the native wheel (`arcadedb-native`) both import as `arcadedb`. Their -files land in the same directory, so installing one after the other overwrites files of the first with -files of the second, and pip says nothing. The result is a mix of two builds. Package metadata still -lists both, so this module looks there. +This repository builds two distributions. `arcadedb` is the native build (the engine as a GraalVM native-image +library, loaded with ctypes) and `arcadedb-jvm` is the JVM build (the Java engine in-process through JPype). +Both put a package named `arcadedb` into the same site-packages directory, so installing one after the other +overwrites files of the first with files of the second, and pip says nothing. What would be imported is a mix +of two builds. Package metadata still lists both, so this module looks there and raises an ImportError that +names both packages and says how to fix it, instead of letting the mix run. + +`check(backend)` is called first thing in `arcadedb/__init__.py`. `backend` is the build this code belongs to: +"jvm" or "native". """ from __future__ import annotations -import warnings from importlib import metadata from typing import Callable -# Distributions that put an `arcadedb` package into site-packages. -DISTRIBUTIONS = ("arcadedb", "arcadedb-native") +# backend -> the distribution that carries it +DISTRIBUTIONS = {"native": "arcadedb", "jvm": "arcadedb-jvm"} + +# The page that explains the two packages (the "Which package?" page of the docs, linked from the README). +DOCS_URL = "https://github.com/ArcadeData/arcadedb-python/blob/main/docs/modules/ROOT/pages/which-package.adoc" + + +class DistributionConflictError(ImportError): + """Both the native `arcadedb` and the JVM `arcadedb-jvm` distribution are installed.""" + +def other_backend(backend: str) -> str: + if backend not in DISTRIBUTIONS: + raise ValueError(f"backend must be 'jvm' or 'native', not {backend!r}") + return "jvm" if backend == "native" else "native" -def installed_distributions( - names: tuple[str, ...] = DISTRIBUTIONS, + +def installed( + backends: tuple[str, ...] = ("native", "jvm"), lookup: Callable[[str], str] = metadata.version, ) -> list[str]: - """The names from `names` that are installed, in the order given. + """The distribution names of `backends` that are installed, in the order given. `lookup` raises `importlib.metadata.PackageNotFoundError` for a name that is not installed. """ found = [] - for name in names: + for backend in backends: + name = DISTRIBUTIONS[backend] try: lookup(name) except metadata.PackageNotFoundError: @@ -34,26 +52,30 @@ def installed_distributions( return found -def warn_if_both_installed( - lookup: Callable[[str], str] = metadata.version, stacklevel: int = 2 -) -> bool: - """Emit a RuntimeWarning when both distributions are installed. True when it did. +def message(backend: str) -> str: + own, other = DISTRIBUTIONS[backend], DISTRIBUTIONS[other_backend(backend)] + kind = {"native": "native", "jvm": "JVM"} + return ( + f"The distribution '{own}' ({kind[backend]} build) cannot be imported while '{other}' " + f"({kind[other_backend(backend)]} build) is installed. " + f"Both provide the Python package 'arcadedb', so each install overwrote files of the other " + f"without a message, and what you would import is a mix of the two builds. " + f"Keep one: run 'pip uninstall {DISTRIBUTIONS['native']} {DISTRIBUTIONS['jvm']}', then install " + f"'{DISTRIBUTIONS['native']}' (native, the default for most embedded use) or " + f"'{DISTRIBUTIONS['jvm']}' (JVM, for heavy long-running loads). " + f"Which one to pick: {DOCS_URL}" + ) + - Never raises: an import must not fail because the metadata could not be read. +def check(backend: str, lookup: Callable[[str], str] = metadata.version) -> None: + """Raise DistributionConflictError when the distribution of the other backend is installed too. + + Never raises for anything else: an import must not fail because the metadata could not be read. """ + other = other_backend(backend) try: - found = installed_distributions(lookup=lookup) + found = installed((other,), lookup) except Exception: # unreadable metadata, an odd sys.path entry: say nothing - return False - if len(found) < len(DISTRIBUTIONS): - return False - warnings.warn( - "Both the 'arcadedb' and the 'arcadedb-native' distributions are installed. " - "Both provide the Python package 'arcadedb', so each install overwrote files of the " - "other without a message, and what you import now is a mix of the two builds. " - "Install only one: pip uninstall arcadedb arcadedb-native, then pip install arcadedb " - "(or arcadedb-native).", - RuntimeWarning, - stacklevel=stacklevel, - ) - return True + return + if found: + raise DistributionConflictError(message(backend)) diff --git a/tests/test_dist_conflict.py b/tests/test_dist_conflict.py index 78735e7..de7b870 100644 --- a/tests/test_dist_conflict.py +++ b/tests/test_dist_conflict.py @@ -1,7 +1,8 @@ -"""The `arcadedb` and `arcadedb-native` distributions both import as `arcadedb`. +"""The `arcadedb` (native) and `arcadedb-jvm` (JVM) distributions both import as `arcadedb`. -pip overwrites one with the other without a message, so the package warns at import time when it -finds both distributions in the installed metadata. These tests need no engine and no JVM. +pip overwrites one with the other without a message, so each build refuses to import when the metadata of +the other distribution is installed. `arcadedb.__backend__` says which build is running. These tests need no +engine and no JVM. """ import os @@ -12,6 +13,7 @@ import pytest +import arcadedb from arcadedb import _dist_conflict @@ -27,37 +29,51 @@ def lookup(name): def test_distribution_names(): - assert _dist_conflict.DISTRIBUTIONS == ("arcadedb", "arcadedb-native") + assert _dist_conflict.DISTRIBUTIONS == {"native": "arcadedb", "jvm": "arcadedb-jvm"} -def test_both_installed_warns(): - with pytest.warns(RuntimeWarning, match="arcadedb-native") as record: - assert _dist_conflict.warn_if_both_installed( - lookup_with("arcadedb", "arcadedb-native") - ) - text = str(record[0].message) - assert "'arcadedb'" in text - assert "Install only one" in text +def test_backend_is_one_of_the_two_builds(): + assert arcadedb.__backend__ in ("jvm", "native") + assert "__backend__" in arcadedb.__all__ @pytest.mark.parametrize( - "installed", - [("arcadedb",), ("arcadedb-native",), ()], - ids=["jvm", "native", "none"], + "backend,own,other", + [("jvm", "arcadedb-jvm", "arcadedb"), ("native", "arcadedb", "arcadedb-jvm")], ) -def test_one_or_none_installed_is_silent(installed): +def test_the_other_distribution_installed_refuses_the_import(backend, own, other): + with pytest.raises(ImportError) as info: + _dist_conflict.check(backend, lookup_with(own, other)) + assert isinstance(info.value, _dist_conflict.DistributionConflictError) + text = str(info.value) + # names both packages, how to fix it, and where to read more + assert f"'{own}'" in text and f"'{other}'" in text + assert "pip uninstall arcadedb arcadedb-jvm" in text + assert "which-package" in text + + +@pytest.mark.parametrize("backend", ["jvm", "native"]) +def test_own_distribution_alone_or_none_is_silent(backend): + own = _dist_conflict.DISTRIBUTIONS[backend] with warnings.catch_warnings(): warnings.simplefilter("error") - assert not _dist_conflict.warn_if_both_installed(lookup_with(*installed)) + _dist_conflict.check(backend, lookup_with(own)) + _dist_conflict.check( + backend, lookup_with() + ) # a source checkout: neither is installed -def test_unreadable_metadata_never_breaks_the_import(): +@pytest.mark.parametrize("backend", ["jvm", "native"]) +def test_unreadable_metadata_never_breaks_the_import(backend): def broken(name): raise OSError("unreadable site-packages") - with warnings.catch_warnings(): - warnings.simplefilter("error") - assert not _dist_conflict.warn_if_both_installed(broken) + _dist_conflict.check(backend, broken) + + +def test_an_unknown_backend_is_a_programming_error(): + with pytest.raises(ValueError, match="'jvm' or 'native'"): + _dist_conflict.check("python", lookup_with()) def fake_dist_info(site, name, version): @@ -70,32 +86,39 @@ def fake_dist_info(site, name, version): def import_arcadedb(extra_site): - """Import arcadedb in a fresh interpreter with `extra_site` ahead on sys.path; return stderr.""" + """Import arcadedb in a fresh interpreter with `extra_site` ahead on sys.path; return the CompletedProcess.""" env = dict(os.environ) # sys.path of this run (installed wheel or source checkout) follows the fake site env["PYTHONPATH"] = os.pathsep.join([str(extra_site)] + [p for p in sys.path if p]) - result = subprocess.run( # nosec B603 - fixed argv, no shell + return subprocess.run( # nosec B603 - fixed argv, no shell [sys.executable, "-W", "default", "-c", "import arcadedb"], capture_output=True, text=True, env=env, check=False, ) - assert result.returncode == 0, result.stderr - return result.stderr -def test_import_warns_when_both_distributions_are_installed(tmp_path): +def sibling(): + """The distribution of the other build than the one under test.""" + return _dist_conflict.DISTRIBUTIONS[ + _dist_conflict.other_backend(arcadedb.__backend__) + ] + + +def test_import_fails_when_the_other_distribution_is_installed_too(tmp_path): site = tmp_path / "site" - fake_dist_info(site, "arcadedb", "26.11.1") - fake_dist_info(site, "arcadedb-native", "26.11.1") - stderr = import_arcadedb(site) - assert "RuntimeWarning" in stderr - assert "arcadedb-native" in stderr - assert "Install only one" in stderr + fake_dist_info(site, sibling(), "26.11.1") + result = import_arcadedb(site) + assert result.returncode != 0 + assert "DistributionConflictError" in result.stderr + assert "pip uninstall arcadedb arcadedb-jvm" in result.stderr + assert "'arcadedb'" in result.stderr and "'arcadedb-jvm'" in result.stderr -def test_import_is_silent_with_the_jvm_distribution_alone(tmp_path): +def test_import_is_silent_when_only_this_distribution_is_installed(tmp_path): site = tmp_path / "site" - fake_dist_info(site, "arcadedb", "26.11.1") - assert "Install only one" not in import_arcadedb(site) + fake_dist_info(site, _dist_conflict.DISTRIBUTIONS[arcadedb.__backend__], "26.11.1") + result = import_arcadedb(site) + assert result.returncode == 0, result.stderr + assert "pip uninstall" not in result.stderr From 7189d6ab3edd98bdd6da61ec552a1038a1fda54a Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 20:35:00 +0900 Subject: [PATCH 03/33] Port the native build: Java glue, native wheel builder, Python native backend, native tests The distribution `arcadedb` is the native build: the engine compiled by GraalVM Community Edition into a shared library (libarcadedb) that Python loads with ctypes. No JVM, no JPype. - native/: the Java glue (C entry points: ArcadeNative, NativeCodec, NativeExtras, BulkOps), the Maven wrapper, and scripts/build-image.sh (GraalVM CE 25.4.4.1.1, -O2, -march=compatibility, engine jars of a pinned release from Maven Central). - native/wheel/: build_wheel.py copies src/arcadedb, rewrites the JPype imports to a stub, flips the default backend to native, adds the library and the licence texts, checks THIRD_PARTY_NOTICES.md against what is compiled in (gen_notices.py), computes the platform tag from the library and builds a py3-none- wheel named arcadedb. parity_wheel.py runs the test suite against the INSTALLED wheel, file by file. - src/arcadedb/_native/: the Python native path, plus the hooks in core.py, server.py, vector.py, dataframe.py, exporter.py and exceptions.py. arcadedb.__backend__ is 'native' when it is active. The JVM wheel ships these files too (setup lists the subpackage; test_wheel_packages.py proves it); the library is only in the native wheel. - tests: test_native_*.py (collected on the native build only), conftest.py (NATIVE_NOT_APPLICABLE files are not collected on the native build; tests that need a Java object or the JVM are NOTYET / NA-JVM skips that the parity run counts apart; any other skip fails), the parity-aware child scripts in test_bridge_fixes.py and test_sigint.py. Ported from the humemai/arcadedb-embedded-python branch native-preview at 91a6ad719b0308c8ccbf569bf130412619292925 (open native-preview PRs #514 and #520-#538 are not merged there and are not in this port). Co-Authored-By: Claude Sonnet 5.5 --- native/.gitignore | 1 + native/.mvn/wrapper/maven-wrapper.properties | 3 + native/mvnw | 295 + native/mvnw.cmd | 189 + native/pom.xml | 347 + native/scripts/build-image.sh | 66 + .../jvector/vector/cnative/FastMode.java | 16 + .../jvector/vector/cnative/FastOps.java | 58 + .../jvector/vector/cnative/FfmKernels.java | 45 + .../jvector/vector/cnative/LibraryLoader.java | 72 + .../jvector/vector/cnative/NativeSimdOps.java | 3654 ++++++++++ .../java/org/humemai/nativelib/ArcadeLib.java | 153 + .../org/humemai/nativelib/ArcadeNative.java | 691 ++ .../java/org/humemai/nativelib/BulkOps.java | 132 + .../main/java/org/humemai/nativelib/Json.java | 462 ++ .../org/humemai/nativelib/NativeCodec.java | 385 + .../org/humemai/nativelib/NativeExtras.java | 681 ++ .../org/humemai/nativelib/Prefetched.java | 57 + .../org/wildfly/common/Substitutions.java | 45 + .../native-image.properties | 7 + .../reachability-metadata.json | 6307 +++++++++++++++++ native/wheel/ARCADEDB_ENGINE_NOTICE | 201 + native/wheel/NOTICE | 25 + native/wheel/THIRD_PARTY_NOTICES.md | 3629 ++++++++++ native/wheel/_jpype_stub.py | 44 + native/wheel/build_wheel.py | 292 + native/wheel/gen_notices.py | 406 ++ .../java/antlr4-runtime-4.13.2.LICENSE.txt | 28 + ...064d0bb52952888a14236bec75c104.LICENSE.txt | 371 + .../java/jts-core-1.20.0.LICENSE_EDLv1.txt | 30 + .../java/jvector-4.0.2.NOTICE.txt | 2 + native/wheel/parity_wheel.py | 171 + native/wheel/pyproject.toml | 47 + native/wheel/setup.py | 35 + pyproject.toml | 7 +- scripts/build.sh | 4 +- src/arcadedb/__init__.py | 17 +- src/arcadedb/_native/__init__.py | 41 + src/arcadedb/_native/bulk.py | 96 + src/arcadedb/_native/codec.py | 229 + src/arcadedb/_native/columns.py | 167 + src/arcadedb/_native/database.py | 1361 ++++ src/arcadedb/_native/extras.py | 819 +++ src/arcadedb/_native/lib.py | 338 + src/arcadedb/_native/records.py | 206 + src/arcadedb/_native/schema_objects.py | 374 + src/arcadedb/_native/server.py | 118 + src/arcadedb/core.py | 10 + src/arcadedb/dataframe.py | 143 +- src/arcadedb/exceptions.py | 8 + src/arcadedb/exporter.py | 79 +- src/arcadedb/server.py | 16 + src/arcadedb/vector.py | 10 + tests/conftest.py | 165 +- tests/test_bridge_fixes.py | 22 +- tests/test_dataframe_ingest.py | 2 +- tests/test_dist_conflict.py | 11 +- tests/test_docs_examples.py | 5 + tests/test_free_threading.py | 2 +- tests/test_interrupt_hygiene.py | 16 +- tests/test_native_startup.py | 29 + tests/test_native_startup_runtime.py | 25 + tests/test_native_udf.py | 125 + tests/test_native_wheel_template.py | 49 + tests/test_select_native_matrix.py | 75 + tests/test_sigint.py | 6 +- tests/test_vector_sql.py | 19 +- tests/test_wheel_packages.py | 69 + 68 files changed, 23545 insertions(+), 65 deletions(-) create mode 100644 native/.gitignore create mode 100644 native/.mvn/wrapper/maven-wrapper.properties create mode 100755 native/mvnw create mode 100644 native/mvnw.cmd create mode 100644 native/pom.xml create mode 100755 native/scripts/build-image.sh create mode 100644 native/src/main/java/io/github/jbellis/jvector/vector/cnative/FastMode.java create mode 100644 native/src/main/java/io/github/jbellis/jvector/vector/cnative/FastOps.java create mode 100644 native/src/main/java/io/github/jbellis/jvector/vector/cnative/FfmKernels.java create mode 100644 native/src/main/java/io/github/jbellis/jvector/vector/cnative/LibraryLoader.java create mode 100644 native/src/main/java/io/github/jbellis/jvector/vector/cnative/NativeSimdOps.java create mode 100644 native/src/main/java/org/humemai/nativelib/ArcadeLib.java create mode 100644 native/src/main/java/org/humemai/nativelib/ArcadeNative.java create mode 100644 native/src/main/java/org/humemai/nativelib/BulkOps.java create mode 100644 native/src/main/java/org/humemai/nativelib/Json.java create mode 100644 native/src/main/java/org/humemai/nativelib/NativeCodec.java create mode 100644 native/src/main/java/org/humemai/nativelib/NativeExtras.java create mode 100644 native/src/main/java/org/humemai/nativelib/Prefetched.java create mode 100644 native/src/main/java/org/wildfly/common/Substitutions.java create mode 100644 native/src/main/resources/META-INF/native-image/org.humemai/arcadedb-native-lib/native-image.properties create mode 100644 native/src/main/resources/META-INF/native-image/org.humemai/arcadedb-native-lib/reachability-metadata.json create mode 100644 native/wheel/ARCADEDB_ENGINE_NOTICE create mode 100644 native/wheel/NOTICE create mode 100644 native/wheel/THIRD_PARTY_NOTICES.md create mode 100644 native/wheel/_jpype_stub.py create mode 100644 native/wheel/build_wheel.py create mode 100644 native/wheel/gen_notices.py create mode 100644 native/wheel/notice-texts/java/antlr4-runtime-4.13.2.LICENSE.txt create mode 100644 native/wheel/notice-texts/java/highway-16f5f5c768064d0bb52952888a14236bec75c104.LICENSE.txt create mode 100644 native/wheel/notice-texts/java/jts-core-1.20.0.LICENSE_EDLv1.txt create mode 100644 native/wheel/notice-texts/java/jvector-4.0.2.NOTICE.txt create mode 100644 native/wheel/parity_wheel.py create mode 100644 native/wheel/pyproject.toml create mode 100644 native/wheel/setup.py create mode 100644 src/arcadedb/_native/__init__.py create mode 100644 src/arcadedb/_native/bulk.py create mode 100644 src/arcadedb/_native/codec.py create mode 100644 src/arcadedb/_native/columns.py create mode 100644 src/arcadedb/_native/database.py create mode 100644 src/arcadedb/_native/extras.py create mode 100644 src/arcadedb/_native/lib.py create mode 100644 src/arcadedb/_native/records.py create mode 100644 src/arcadedb/_native/schema_objects.py create mode 100644 src/arcadedb/_native/server.py create mode 100644 tests/test_native_startup.py create mode 100644 tests/test_native_startup_runtime.py create mode 100644 tests/test_native_udf.py create mode 100644 tests/test_native_wheel_template.py create mode 100644 tests/test_select_native_matrix.py create mode 100644 tests/test_wheel_packages.py diff --git a/native/.gitignore b/native/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/native/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/native/.mvn/wrapper/maven-wrapper.properties b/native/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..bb58b3c --- /dev/null +++ b/native/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,3 @@ +wrapperVersion=3.3.4 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.10.0/apache-maven-3.10.0-bin.zip diff --git a/native/mvnw b/native/mvnw new file mode 100755 index 0000000..db9f6f7 --- /dev/null +++ b/native/mvnw @@ -0,0 +1,295 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.4 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in + CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2> /dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2> /dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +scriptDir="$(dirname "$0")" +scriptName="$(basename "$0")" + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done < "$scriptDir/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in + maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; + maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; + *) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in + *?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; + *) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip > /dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in + '') MVNW_USERNAME='' MVNW_PASSWORD='' ;; + has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget > /dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl > /dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat > "$javaSource" <<- END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum > /dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - > /dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum > /dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c > /dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip > /dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +actualDistributionDir="" + +# First try the expected directory name (for regular distributions) +if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then + if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then + actualDistributionDir="$distributionUrlNameMain" + fi +fi + +# If not found, search for any directory with the Maven executable (for snapshots) +if [ -z "$actualDistributionDir" ]; then + # enable globbing to iterate over items + set +f + for dir in "$TMP_DOWNLOAD_DIR"/*; do + if [ -d "$dir" ]; then + if [ -f "$dir/bin/$MVN_CMD" ]; then + actualDistributionDir="$(basename "$dir")" + break + fi + fi + done + set -f +fi + +if [ -z "$actualDistributionDir" ]; then + verbose "Contents of $TMP_DOWNLOAD_DIR:" + verbose "$(ls -la "$TMP_DOWNLOAD_DIR")" + die "Could not find Maven distribution directory in extracted archive" +fi + +verbose "Found extracted Maven distribution directory: $actualDistributionDir" +printf %s\\n "$distributionUrl" > "$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/native/mvnw.cmd b/native/mvnw.cmd new file mode 100644 index 0000000..5761d94 --- /dev/null +++ b/native/mvnw.cmd @@ -0,0 +1,189 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.4 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' + +$MAVEN_M2_PATH = "$HOME/.m2" +if ($env:MAVEN_USER_HOME) { + $MAVEN_M2_PATH = "$env:MAVEN_USER_HOME" +} + +if (-not (Test-Path -Path $MAVEN_M2_PATH)) { + New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null +} + +$MAVEN_WRAPPER_DISTS = $null +if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) { + $MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists" +} else { + $MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists" +} + +$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain" +$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +$actualDistributionDir = "" + +# First try the expected directory name (for regular distributions) +$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain" +$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD" +if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) { + $actualDistributionDir = $distributionUrlNameMain +} + +# If not found, search for any directory with the Maven executable (for snapshots) +if (!$actualDistributionDir) { + Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object { + $testPath = Join-Path $_.FullName "bin/$MVN_CMD" + if (Test-Path -Path $testPath -PathType Leaf) { + $actualDistributionDir = $_.Name + } + } +} + +if (!$actualDistributionDir) { + Write-Error "Could not find Maven distribution directory in extracted archive" +} + +Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir" +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/native/pom.xml b/native/pom.xml new file mode 100644 index 0000000..cd0d1c1 --- /dev/null +++ b/native/pom.xml @@ -0,0 +1,347 @@ + + + + + 4.0.0 + org.humemai.nativelib + arcadedb-native-lib + 0.0.1 + jar + + + 26.10.2 + + 25.0.2 + + -Os + 1.1.14 + 25 + UTF-8 + + + + + + io.github.jbellisjvector4.0.2 + + org.jboss.threadsjboss-threads3.10.1 + io.smallrye.commonsmallrye-common-annotation2.20.0 + io.smallrye.commonsmallrye-common-constraint2.20.0 + io.smallrye.commonsmallrye-common-cpu2.20.0 + io.smallrye.commonsmallrye-common-function2.20.0 + + + + + + com.arcadedb + arcadedb-engine + ${arcadedb.version} + + com.arcadedbarcadedb-gremlin + com.arcadedbarcadedb-grpcw + org.graalvm.js* + org.graalvm.shadowedicu4j + org.graalvm.truffle* + org.graalvm.regex* + com.arcadedbarcadedb-ha-raft + com.arcadedbarcadedb-metrics + com.arcadedbarcadedb-mongodbw + com.fasterxml.jackson.core* + com.fasterxml.jackson.dataformat* + com.fasterxml.jackson.datatype* + org.graalvm.polyglotjs + org.xerial.snappy* + com.arcadedbarcadedb-tracing + org.jline* + org.apache.commonscommons-math3 + + + + com.arcadedb + arcadedb-server + ${arcadedb.version} + + com.arcadedbarcadedb-gremlin + com.arcadedbarcadedb-grpcw + org.graalvm.js* + org.graalvm.shadowedicu4j + org.graalvm.truffle* + org.graalvm.regex* + com.arcadedbarcadedb-ha-raft + com.arcadedbarcadedb-metrics + com.arcadedbarcadedb-mongodbw + com.fasterxml.jackson.core* + com.fasterxml.jackson.dataformat* + com.fasterxml.jackson.datatype* + org.graalvm.polyglotjs + org.xerial.snappy* + com.arcadedbarcadedb-tracing + org.jline* + org.apache.commonscommons-math3 + + + + com.arcadedb + arcadedb-network + ${arcadedb.version} + + com.arcadedbarcadedb-gremlin + com.arcadedbarcadedb-grpcw + org.graalvm.js* + org.graalvm.shadowedicu4j + org.graalvm.truffle* + org.graalvm.regex* + com.arcadedbarcadedb-ha-raft + com.arcadedbarcadedb-metrics + com.arcadedbarcadedb-mongodbw + com.fasterxml.jackson.core* + com.fasterxml.jackson.dataformat* + com.fasterxml.jackson.datatype* + org.graalvm.polyglotjs + org.xerial.snappy* + com.arcadedbarcadedb-tracing + org.jline* + org.apache.commonscommons-math3 + + + + com.arcadedb + arcadedb-integration + ${arcadedb.version} + + com.arcadedbarcadedb-gremlin + com.arcadedbarcadedb-grpcw + org.graalvm.js* + org.graalvm.shadowedicu4j + org.graalvm.truffle* + org.graalvm.regex* + com.arcadedbarcadedb-ha-raft + com.arcadedbarcadedb-metrics + com.arcadedbarcadedb-mongodbw + com.fasterxml.jackson.core* + com.fasterxml.jackson.dataformat* + com.fasterxml.jackson.datatype* + org.graalvm.polyglotjs + org.xerial.snappy* + com.arcadedbarcadedb-tracing + org.jline* + org.apache.commonscommons-math3 + + + + com.arcadedb + arcadedb-console + ${arcadedb.version} + + com.arcadedbarcadedb-gremlin + com.arcadedbarcadedb-grpcw + org.graalvm.js* + org.graalvm.shadowedicu4j + org.graalvm.truffle* + org.graalvm.regex* + com.arcadedbarcadedb-ha-raft + com.arcadedbarcadedb-metrics + com.arcadedbarcadedb-mongodbw + com.fasterxml.jackson.core* + com.fasterxml.jackson.dataformat* + com.fasterxml.jackson.datatype* + org.graalvm.polyglotjs + org.xerial.snappy* + com.arcadedbarcadedb-tracing + org.jline* + org.apache.commonscommons-math3 + + + + + com.arcadedb + arcadedb-postgresw + ${arcadedb.version} + + com.arcadedbarcadedb-gremlin + com.arcadedbarcadedb-grpcw + org.graalvm.js* + org.graalvm.shadowedicu4j + org.graalvm.truffle* + org.graalvm.regex* + com.arcadedbarcadedb-ha-raft + com.arcadedbarcadedb-metrics + com.arcadedbarcadedb-mongodbw + com.fasterxml.jackson.core* + com.fasterxml.jackson.dataformat* + com.fasterxml.jackson.datatype* + org.graalvm.polyglotjs + org.xerial.snappy* + com.arcadedbarcadedb-tracing + org.jline* + org.apache.commonscommons-math3 + + + + com.arcadedb + arcadedb-redisw + ${arcadedb.version} + + com.arcadedbarcadedb-gremlin + com.arcadedbarcadedb-grpcw + org.graalvm.js* + org.graalvm.shadowedicu4j + org.graalvm.truffle* + org.graalvm.regex* + com.arcadedbarcadedb-ha-raft + com.arcadedbarcadedb-metrics + com.arcadedbarcadedb-mongodbw + com.fasterxml.jackson.core* + com.fasterxml.jackson.dataformat* + com.fasterxml.jackson.datatype* + org.graalvm.polyglotjs + org.xerial.snappy* + com.arcadedbarcadedb-tracing + org.jline* + org.apache.commonscommons-math3 + + + + com.arcadedb + arcadedb-bolt + ${arcadedb.version} + + com.arcadedbarcadedb-gremlin + com.arcadedbarcadedb-grpcw + org.graalvm.js* + org.graalvm.shadowedicu4j + org.graalvm.truffle* + org.graalvm.regex* + com.arcadedbarcadedb-ha-raft + com.arcadedbarcadedb-metrics + com.arcadedbarcadedb-mongodbw + com.fasterxml.jackson.core* + com.fasterxml.jackson.dataformat* + com.fasterxml.jackson.datatype* + org.graalvm.polyglotjs + org.xerial.snappy* + com.arcadedbarcadedb-tracing + org.jline* + org.apache.commonscommons-math3 + + + + com.arcadedb + arcadedb-graphql + ${arcadedb.version} + + com.arcadedbarcadedb-gremlin + com.arcadedbarcadedb-grpcw + org.graalvm.js* + org.graalvm.shadowedicu4j + org.graalvm.truffle* + org.graalvm.regex* + com.arcadedbarcadedb-ha-raft + com.arcadedbarcadedb-metrics + com.arcadedbarcadedb-mongodbw + com.fasterxml.jackson.core* + com.fasterxml.jackson.dataformat* + com.fasterxml.jackson.datatype* + org.graalvm.polyglotjs + org.xerial.snappy* + com.arcadedbarcadedb-tracing + org.jline* + org.apache.commonscommons-math3 + + + + com.arcadedb + arcadedb-mcp + ${arcadedb.version} + + com.arcadedbarcadedb-gremlin + com.arcadedbarcadedb-grpcw + org.graalvm.js* + org.graalvm.shadowedicu4j + org.graalvm.truffle* + org.graalvm.regex* + com.arcadedbarcadedb-ha-raft + com.arcadedbarcadedb-metrics + com.arcadedbarcadedb-mongodbw + com.fasterxml.jackson.core* + com.fasterxml.jackson.dataformat* + com.fasterxml.jackson.datatype* + org.graalvm.polyglotjs + org.xerial.snappy* + com.arcadedbarcadedb-tracing + org.jline* + org.apache.commonscommons-math3 + + + + org.graalvm.sdk + nativeimage + ${graalvm.version} + provided + + + org.slf4j + slf4j-jdk14 + 2.0.20 + runtime + + + + + + + org.graalvm.buildtools + native-maven-plugin + ${native-maven-plugin.version} + + + build-native + compile-no-fork + package + + + + libarcadedb + true + true + + ${native.opt} + --no-fallback + -H:+ReportExceptionStackTraces + --enable-url-protocols=http,https + --add-modules=jdk.incubator.vector + --enable-native-access=ALL-UNNAMED + --initialize-at-run-time=org.wildfly.common.net.HostName + --initialize-at-build-time=io.github.jbellis.jvector.vector.cnative.FastMode,io.github.jbellis.jvector.vector.cnative.FfmKernels + --initialize-at-build-time=org.slf4j,org.slf4j.helpers,org.slf4j.spi,org.slf4j.event,org.slf4j.jul + + + + + + + + + + report + + + + org.graalvm.buildtools + native-maven-plugin + + + --emit=build-report + + + + + + + + diff --git a/native/scripts/build-image.sh b/native/scripts/build-image.sh new file mode 100755 index 0000000..3ccdbb0 --- /dev/null +++ b/native/scripts/build-image.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Build the native image (the shared library the `arcadedb` wheel ships): GraalVM CE 25.4.4.1.1, -O2, FFM unbound Highway +# handle (fastops.mode 1), -H:-VectorAPISupport (CE crash oracle/graal#14751), engine jars of the pinned RELEASE (arcadedb.version, from +# Maven Central, never a local SNAPSHOT), local symbols deleted (the default: a smaller image than the phase 7/8 profiling builds). +# MARCH (default compatibility = baseline x86-64, what a manylinux wheel promises, PEP 600; x86-64-v3 is the GraalVM CE 25.4 default and the faster reference). +# usage: [MARCH=x86-64-v3] [XMX=12g] [NI_EXTRA="-H:NumberOfThreads=2"] build-image.sh [OUTDIR] A laptop with other jobs runs it through its job gate; the CI calls it directly. +# Platforms: Linux (libarcadedb.so; on x86-64 also libjvector-x86_64.so, jvector ships the Highway library for x86-64 only here), macOS +# (libarcadedb.dylib) and Windows in Git Bash with the MSVC environment loaded (libarcadedb.dll). The CI (native.yml) caps the +# builder heap and threads on the 7 GB macOS runner (XMX=5g, NI_EXTRA=-H:NumberOfThreads=2). +# e.g. NEED_GB=13 JOB_MEM_GB=14 quiet_build.sh bash native/scripts/build-image.sh +set -eu +WT=$(cd "$(dirname "$0")/../.." && pwd) +G=${GRAALVM_HOME_P5:-$HOME/build-scratch/graalvm-dl-25.4.4.1.1/graalvm-community-25.4.4.1.1+1.1} +OUT=${1:-$WT/native/target/lib} +mkdir -p "$OUT" +ENGINE=${ARCADEDB_ENGINE_VERSION:-26.10.2} +MARCH=${MARCH:-compatibility} +XMX=${XMX:-12g} +case "$(uname -s)" in + Linux) EXT=so ;; + Darwin) EXT=dylib ;; + MINGW* | MSYS* | CYGWIN*) + EXT=dll + G=$(cygpath -u "$G") + ;; + *) + echo "unsupported platform $(uname -s)" >&2 + exit 1 + ;; +esac +LIBFILE=libarcadedb.$EXT # not LIB: the MSVC environment on Windows exports LIB (library search path) +sha256() { if command -v sha256sum > /dev/null; then sha256sum "$@"; else shasum -a 256 "$@"; fi; } +export JAVA_HOME=$G PATH=$G/bin:$PATH +export NATIVE_IMAGE_OPTIONS="-J-Dfastops.mode=1 -J-Xmx$XMX -H:-VectorAPISupport -march=$MARCH${NI_EXTRA:+ $NI_EXTRA}" +cd "$WT/native" +rm -f "target/$LIBFILE" "$OUT/$LIBFILE" +./mvnw -B --no-transfer-progress -f pom.xml -DskipTests -Darcadedb.version="$ENGINE" -Dgraalvm.version=25.4.4.1.1 -Dnative.opt=-O2 package > "$OUT/build.log" 2>&1 +# Windows: the Maven plugin hands native-image its arguments in an @args file that it deletes at exit, so build.log has no +# "native-image -cp --shared" line, which gen_notices.py reads. Append one: the glue jar plus the runtime classpath that +# dependency:build-classpath resolves with the same properties (checked on Linux: the same 59 jars as the logged classpath, plus the glue jar). +if [ "$EXT" = dll ]; then + ./mvnw -B -q --no-transfer-progress -f pom.xml -Darcadedb.version="$ENGINE" -Dgraalvm.version=25.4.4.1.1 -DincludeScope=runtime \ + dependency:build-classpath -Dmdep.outputFile=target/runtime.cp >> "$OUT/build.log" 2>&1 + echo "native-image -cp target/arcadedb-native-lib-0.0.1.jar;$(tr -d '\r\n' < target/runtime.cp) --shared" >> "$OUT/build.log" +fi +# the jvector force-inline filters (native-image.properties in the glue jar) must be picked up: the build output lists the option and its origin (Windows prints the path with backslashes) +grep -a -E -q 'PriorityForceInline. \(origin\(s\): .META-INF.native-image.org\.humemai.arcadedb-native-lib.native-image\.properties' "$OUT/build.log" || + { + echo "-H:PriorityForceInline from native-image.properties is not in the build output: the jvector force-inline hints were not applied" >&2 + exit 1 + } +INLINE_LOG="PriorityForceInline picked up from native-image.properties ($(grep -a -c 'PriorityForceInline' "$OUT/build.log") log lines)" +cp "target/$LIBFILE" "$OUT/" +LIBS=("$OUT/$LIBFILE") +# Highway (jvector's native provider), found next to libarcadedb.so by the image (ARCADEDB_NATIVE_LIBDIR); Linux x86-64 only: the other +# platforms ship no jvector library and the image falls back to the Java providers +if [ "$EXT" = so ] && [ "$(uname -m)" = x86_64 ]; then + J=$(unzip -l "$HOME/.m2/repository/io/github/jbellis/jvector/4.0.2/jvector-4.0.2.jar" | awk '/libjvector-x86_64.so/{print $4}' | head -1) + if [ -n "$J" ]; then unzip -p "$HOME/.m2/repository/io/github/jbellis/jvector/4.0.2/jvector-4.0.2.jar" "$J" > "$OUT/libjvector-x86_64.so"; else + echo "libjvector-x86_64.so is not in the jvector 4.0.2 jar" >&2 + exit 1 + fi + LIBS+=("$OUT/libjvector-x86_64.so") +fi +sha256 "${LIBS[@]}" > "$OUT/sha256.txt" +echo "native image done: engine $ENGINE; march $MARCH; $INLINE_LOG; $(grep -a 'Finished generating' "$OUT/build.log")" | tee "$OUT/build.done" diff --git a/native/src/main/java/io/github/jbellis/jvector/vector/cnative/FastMode.java b/native/src/main/java/io/github/jbellis/jvector/vector/cnative/FastMode.java new file mode 100644 index 0000000..825825a --- /dev/null +++ b/native/src/main/java/io/github/jbellis/jvector/vector/cnative/FastMode.java @@ -0,0 +1,16 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Which route the three distance kernels take. Initialized at build time (pom --initialize-at-build-time), so the value is the + * builder's -J-Dfastops.mode and the branch in FastOps folds away. 0 = C function pointers (phase 4, the default), + * 1 = FFM downcall through an UNBOUND handle built at build time (symbol address passed as the first argument), + * 2 = FFM downcall through a handle bound to the symbol at run time (what jextract generates; the control). + */ +package io.github.jbellis.jvector.vector.cnative; + +final class FastMode { + static final int MODE = Integer.getInteger("fastops.mode", 0); + + private FastMode() { + } +} diff --git a/native/src/main/java/io/github/jbellis/jvector/vector/cnative/FastOps.java b/native/src/main/java/io/github/jbellis/jvector/vector/cnative/FastOps.java new file mode 100644 index 0000000..bf47b50 --- /dev/null +++ b/native/src/main/java/io/github/jbellis/jvector/vector/cnative/FastOps.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Phase 4: the hot libjvector distance kernels called through GraalVM's C interop (@InvokeCFunctionPointer, no thread-state + * transition) instead of FFM downcall handles. The function address comes from the symbol lookup of the loaded library, once. + * Heap segments (float[]-backed vectors) are pinned for the duration of the call. + */ +package io.github.jbellis.jvector.vector.cnative; + +import org.graalvm.nativeimage.PinnedObject; +import org.graalvm.nativeimage.c.function.CFunctionPointer; +import org.graalvm.nativeimage.c.function.InvokeCFunctionPointer; +import org.graalvm.nativeimage.c.function.CFunction.Transition; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; +import org.graalvm.word.WordFactory; + +import java.lang.foreign.MemorySegment; + +final class FastOps { + static final int COSINE = 0, DOT = 1, EUCLIDEAN = 2; + private static final long[] ADDRESS = new long[3]; + // what MemorySegment.address() of a float[]-backed segment starts at: the JDK's array base offset, which the image's own layout does not share + private static final long FLOAT_BASE = MemorySegment.ofArray(new float[0]).address(); + + interface Kernel extends CFunctionPointer { + @InvokeCFunctionPointer(transition = Transition.NO_TRANSITION) + float call(PointerBase a, long aoffset, PointerBase b, long boffset, long length); + } + + private FastOps() { + } + + static float call(final int which, final MemorySegment symbol, final MemorySegment a, final long aoffset, + final MemorySegment b, final long boffset, final long length) { + if (FastMode.MODE != 0) + return FfmKernels.call(which, symbol, a, aoffset, b, boffset, length); + long fn = ADDRESS[which]; + if (fn == 0) + ADDRESS[which] = fn = symbol.address(); + final Object baseA = a.isNative() ? null : a.heapBase().orElse(null); + final Object baseB = b.isNative() ? null : b.heapBase().orElse(null); + if (!(baseA == null || baseA instanceof float[]) || !(baseB == null || baseB instanceof float[])) + throw new UnsupportedOperationException("the image's distance kernels take float[]-backed or native segments"); + final PinnedObject pinA = baseA == null ? null : PinnedObject.create(baseA); + final PinnedObject pinB = baseB == null ? null : PinnedObject.create(baseB); + try { + final Pointer pa = pinA == null ? WordFactory.pointer(a.address()) : ((Pointer) pinA.addressOfArrayElement(0)).add(WordFactory.signed(a.address() - FLOAT_BASE)); + final Pointer pb = pinB == null ? WordFactory.pointer(b.address()) : ((Pointer) pinB.addressOfArrayElement(0)).add(WordFactory.signed(b.address() - FLOAT_BASE)); + return ((Kernel) WordFactory.pointer(fn)).call(pa, aoffset, pb, boffset, length); + } finally { + if (pinA != null) + pinA.close(); + if (pinB != null) + pinB.close(); + } + } +} diff --git a/native/src/main/java/io/github/jbellis/jvector/vector/cnative/FfmKernels.java b/native/src/main/java/io/github/jbellis/jvector/vector/cnative/FfmKernels.java new file mode 100644 index 0000000..9a0777b --- /dev/null +++ b/native/src/main/java/io/github/jbellis/jvector/vector/cnative/FfmKernels.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Phase 5: the distance kernels through FFM downcalls on GraalVM CE 25.4. Mode 1 uses an unbound handle created once in this + * class, which is initialized at build time; the address of the libjvector symbol is the first argument of every call (the + * pattern oracle/graal#12219 reports as fast on 25.1+). Mode 2 binds a handle to the symbol at run time, as jextract does. + */ +package io.github.jbellis.jvector.vector.cnative; + +import java.lang.foreign.AddressLayout; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemoryLayout; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; + +final class FfmKernels { + private static final AddressLayout PTR = ValueLayout.ADDRESS + .withTargetLayout(MemoryLayout.sequenceLayout(Long.MAX_VALUE, ValueLayout.JAVA_BYTE)); + private static final FunctionDescriptor DESC = FunctionDescriptor.of(ValueLayout.JAVA_FLOAT, PTR, ValueLayout.JAVA_LONG, PTR, + ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG); + + /** built at build time: no symbol yet, the address is the first call argument */ + static final MethodHandle UNBOUND = FastMode.MODE == 1 ? Linker.nativeLinker().downcallHandle(DESC, Linker.Option.critical(true)) : null; + + private static final MethodHandle[] BOUND = new MethodHandle[3]; + + private FfmKernels() { + } + + static float call(final int which, final MemorySegment symbol, final MemorySegment a, final long aoffset, + final MemorySegment b, final long boffset, final long length) { + try { + if (FastMode.MODE == 1) + return (float) UNBOUND.invokeExact(symbol, a, aoffset, b, boffset, length); + MethodHandle h = BOUND[which]; + if (h == null) + BOUND[which] = h = Linker.nativeLinker().downcallHandle(symbol, DESC, Linker.Option.critical(true)); + return (float) h.invokeExact(a, aoffset, b, boffset, length); + } catch (final Throwable t) { + throw new AssertionError("should not reach here", t); + } + } +} diff --git a/native/src/main/java/io/github/jbellis/jvector/vector/cnative/LibraryLoader.java b/native/src/main/java/io/github/jbellis/jvector/vector/cnative/LibraryLoader.java new file mode 100644 index 0000000..7c6c731 --- /dev/null +++ b/native/src/main/java/io/github/jbellis/jvector/vector/cnative/LibraryLoader.java @@ -0,0 +1,72 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Image-side shadow of jvector 4.0.2's cnative.LibraryLoader (it precedes jvector on the image class path, as the wildfly + * Substitutions shadow does). The original extracts libjvector-.so from the jar into java.io.tmpdir through a class path + * resource; in a native image that resource is not in the image. This version looks for the library where the Python backend + * ships it, next to libarcadedb.so (directory in the environment variable ARCADEDB_NATIVE_LIBDIR, or java.library.path), and + * then falls back to a resource copy of the library. Same contract: true when the library is loaded, false when it is unavailable. + */ +package io.github.jbellis.jvector.vector.cnative; + +import java.io.File; +import java.io.InputStream; +import java.nio.file.Files; + +public class LibraryLoader { + private LibraryLoader() { + } + + static String resourceNameForArch() { + final String arch = System.getProperty("os.arch", ""); + if (arch.equals("aarch64") || arch.equals("arm64")) + return "/libjvector-aarch64.so"; + if (arch.equals("amd64") || arch.equals("x86_64")) + return "/libjvector-x86_64.so"; + return null; + } + + public static boolean loadJvector() { + try { + System.loadLibrary("jvector"); + return true; + } catch (final UnsatisfiedLinkError e) { + // not on java.library.path + } catch (final IllegalCallerException | SecurityException e) { + return false; + } + final String resource = resourceNameForArch(); + if (resource == null) + return false; + final String file = resource.substring(1); + final String dir = System.getenv("ARCADEDB_NATIVE_LIBDIR"); + if (dir != null && !dir.isEmpty()) { + final File f = new File(dir, file); + if (f.isFile()) { + try { + System.load(f.getAbsolutePath()); + return true; + } catch (final UnsatisfiedLinkError | RuntimeException e) { + System.err.println("jvector: loading " + f + " failed: " + e); + } + } + } + for (final String name : new String[] { resource, "/META-INF/versions/22" + resource }) { + try (InputStream in = LibraryLoader.class.getResourceAsStream(name)) { + if (in == null) + continue; + final File tmp = File.createTempFile(file.substring(0, file.lastIndexOf('.')), file.substring(file.lastIndexOf('.'))); + try { + Files.copy(in, tmp.toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING); + System.load(tmp.getAbsolutePath()); + return true; + } finally { + Files.deleteIfExists(tmp.toPath()); + } + } catch (final Exception | UnsatisfiedLinkError e) { + System.err.println("jvector: loading the bundled " + name + " failed: " + e); + } + } + return false; + } +} diff --git a/native/src/main/java/io/github/jbellis/jvector/vector/cnative/NativeSimdOps.java b/native/src/main/java/io/github/jbellis/jvector/vector/cnative/NativeSimdOps.java new file mode 100644 index 0000000..dce29e8 --- /dev/null +++ b/native/src/main/java/io/github/jbellis/jvector/vector/cnative/NativeSimdOps.java @@ -0,0 +1,3654 @@ +// Image-side shadow of jvector 4.0.2's generated cnative.NativeSimdOps (jextract output, Apache-2.0, DataStax). Only the bodies of +// cosine_f32, dot_product_f32 and euclidean_f32 differ: they call libjvector through GraalVM C function pointers (FastOps) +// instead of an FFM method handle, which a native image runs through the method-handle interpreter (about 20 us per call). +// Generated by jextract + +package io.github.jbellis.jvector.vector.cnative; + +import java.lang.invoke.*; +import java.lang.foreign.*; +import java.nio.ByteOrder; +import java.util.*; +import java.util.function.*; +import java.util.stream.*; + +import static java.lang.foreign.ValueLayout.*; +import static java.lang.foreign.MemoryLayout.PathElement.*; + +public class NativeSimdOps { + + NativeSimdOps() { + // Should not be called directly + } + + static final Arena LIBRARY_ARENA = Arena.ofAuto(); + static final boolean TRACE_DOWNCALLS = Boolean.getBoolean("jextract.trace.downcalls"); + + static void traceDowncall(String name, Object... args) { + String traceArgs = Arrays.stream(args) + .map(Object::toString) + .collect(Collectors.joining(", ")); + System.out.printf("%s(%s)\n", name, traceArgs); + } + + static MemorySegment findOrThrow(String symbol) { + return SYMBOL_LOOKUP.find(symbol) + .orElseThrow(() -> new UnsatisfiedLinkError("unresolved symbol: " + symbol)); + } + + static MethodHandle upcallHandle(Class fi, String name, FunctionDescriptor fdesc) { + try { + return MethodHandles.lookup().findVirtual(fi, name, fdesc.toMethodType()); + } catch (ReflectiveOperationException ex) { + throw new AssertionError(ex); + } + } + + static MemoryLayout align(MemoryLayout layout, long align) { + return switch (layout) { + case PaddingLayout p -> p; + case ValueLayout v -> v.withByteAlignment(align); + case GroupLayout g -> { + MemoryLayout[] alignedMembers = g.memberLayouts().stream() + .map(m -> align(m, align)).toArray(MemoryLayout[]::new); + yield g instanceof StructLayout ? + MemoryLayout.structLayout(alignedMembers) : MemoryLayout.unionLayout(alignedMembers); + } + case SequenceLayout s -> MemoryLayout.sequenceLayout(s.elementCount(), align(s.elementLayout(), align)); + }; + } + + static final SymbolLookup SYMBOL_LOOKUP = SymbolLookup.loaderLookup() + .or(Linker.nativeLinker().defaultLookup()); + + public static final ValueLayout.OfBoolean C_BOOL = ValueLayout.JAVA_BOOLEAN; + public static final ValueLayout.OfByte C_CHAR = ValueLayout.JAVA_BYTE; + public static final ValueLayout.OfShort C_SHORT = ValueLayout.JAVA_SHORT; + public static final ValueLayout.OfInt C_INT = ValueLayout.JAVA_INT; + public static final ValueLayout.OfLong C_LONG_LONG = ValueLayout.JAVA_LONG; + public static final ValueLayout.OfFloat C_FLOAT = ValueLayout.JAVA_FLOAT; + public static final ValueLayout.OfDouble C_DOUBLE = ValueLayout.JAVA_DOUBLE; + public static final AddressLayout C_POINTER = ValueLayout.ADDRESS + .withTargetLayout(MemoryLayout.sequenceLayout(java.lang.Long.MAX_VALUE, JAVA_BYTE)); + public static final ValueLayout.OfLong C_LONG = ValueLayout.JAVA_LONG; + private static final int true_ = (int)1L; + /** + * {@snippet lang=c : + * #define true 1 + * } + */ + public static int true_() { + return true_; + } + private static final int false_ = (int)0L; + /** + * {@snippet lang=c : + * #define false 0 + * } + */ + public static int false_() { + return false_; + } + private static final int __bool_true_false_are_defined = (int)1L; + /** + * {@snippet lang=c : + * #define __bool_true_false_are_defined 1 + * } + */ + public static int __bool_true_false_are_defined() { + return __bool_true_false_are_defined; + } + private static final int _STDINT_H = (int)1L; + /** + * {@snippet lang=c : + * #define _STDINT_H 1 + * } + */ + public static int _STDINT_H() { + return _STDINT_H; + } + private static final int _FEATURES_H = (int)1L; + /** + * {@snippet lang=c : + * #define _FEATURES_H 1 + * } + */ + public static int _FEATURES_H() { + return _FEATURES_H; + } + private static final int _DEFAULT_SOURCE = (int)1L; + /** + * {@snippet lang=c : + * #define _DEFAULT_SOURCE 1 + * } + */ + public static int _DEFAULT_SOURCE() { + return _DEFAULT_SOURCE; + } + private static final int __GLIBC_USE_ISOC2X = (int)0L; + /** + * {@snippet lang=c : + * #define __GLIBC_USE_ISOC2X 0 + * } + */ + public static int __GLIBC_USE_ISOC2X() { + return __GLIBC_USE_ISOC2X; + } + private static final int __USE_ISOC11 = (int)1L; + /** + * {@snippet lang=c : + * #define __USE_ISOC11 1 + * } + */ + public static int __USE_ISOC11() { + return __USE_ISOC11; + } + private static final int __USE_ISOC99 = (int)1L; + /** + * {@snippet lang=c : + * #define __USE_ISOC99 1 + * } + */ + public static int __USE_ISOC99() { + return __USE_ISOC99; + } + private static final int __USE_ISOC95 = (int)1L; + /** + * {@snippet lang=c : + * #define __USE_ISOC95 1 + * } + */ + public static int __USE_ISOC95() { + return __USE_ISOC95; + } + private static final int __USE_POSIX_IMPLICITLY = (int)1L; + /** + * {@snippet lang=c : + * #define __USE_POSIX_IMPLICITLY 1 + * } + */ + public static int __USE_POSIX_IMPLICITLY() { + return __USE_POSIX_IMPLICITLY; + } + private static final int _POSIX_SOURCE = (int)1L; + /** + * {@snippet lang=c : + * #define _POSIX_SOURCE 1 + * } + */ + public static int _POSIX_SOURCE() { + return _POSIX_SOURCE; + } + private static final int __USE_POSIX = (int)1L; + /** + * {@snippet lang=c : + * #define __USE_POSIX 1 + * } + */ + public static int __USE_POSIX() { + return __USE_POSIX; + } + private static final int __USE_POSIX2 = (int)1L; + /** + * {@snippet lang=c : + * #define __USE_POSIX2 1 + * } + */ + public static int __USE_POSIX2() { + return __USE_POSIX2; + } + private static final int __USE_POSIX199309 = (int)1L; + /** + * {@snippet lang=c : + * #define __USE_POSIX199309 1 + * } + */ + public static int __USE_POSIX199309() { + return __USE_POSIX199309; + } + private static final int __USE_POSIX199506 = (int)1L; + /** + * {@snippet lang=c : + * #define __USE_POSIX199506 1 + * } + */ + public static int __USE_POSIX199506() { + return __USE_POSIX199506; + } + private static final int __USE_XOPEN2K = (int)1L; + /** + * {@snippet lang=c : + * #define __USE_XOPEN2K 1 + * } + */ + public static int __USE_XOPEN2K() { + return __USE_XOPEN2K; + } + private static final int __USE_XOPEN2K8 = (int)1L; + /** + * {@snippet lang=c : + * #define __USE_XOPEN2K8 1 + * } + */ + public static int __USE_XOPEN2K8() { + return __USE_XOPEN2K8; + } + private static final int _ATFILE_SOURCE = (int)1L; + /** + * {@snippet lang=c : + * #define _ATFILE_SOURCE 1 + * } + */ + public static int _ATFILE_SOURCE() { + return _ATFILE_SOURCE; + } + private static final int __WORDSIZE = (int)64L; + /** + * {@snippet lang=c : + * #define __WORDSIZE 64 + * } + */ + public static int __WORDSIZE() { + return __WORDSIZE; + } + private static final int __WORDSIZE_TIME64_COMPAT32 = (int)1L; + /** + * {@snippet lang=c : + * #define __WORDSIZE_TIME64_COMPAT32 1 + * } + */ + public static int __WORDSIZE_TIME64_COMPAT32() { + return __WORDSIZE_TIME64_COMPAT32; + } + private static final int __SYSCALL_WORDSIZE = (int)64L; + /** + * {@snippet lang=c : + * #define __SYSCALL_WORDSIZE 64 + * } + */ + public static int __SYSCALL_WORDSIZE() { + return __SYSCALL_WORDSIZE; + } + private static final int __USE_MISC = (int)1L; + /** + * {@snippet lang=c : + * #define __USE_MISC 1 + * } + */ + public static int __USE_MISC() { + return __USE_MISC; + } + private static final int __USE_ATFILE = (int)1L; + /** + * {@snippet lang=c : + * #define __USE_ATFILE 1 + * } + */ + public static int __USE_ATFILE() { + return __USE_ATFILE; + } + private static final int __USE_FORTIFY_LEVEL = (int)0L; + /** + * {@snippet lang=c : + * #define __USE_FORTIFY_LEVEL 0 + * } + */ + public static int __USE_FORTIFY_LEVEL() { + return __USE_FORTIFY_LEVEL; + } + private static final int __GLIBC_USE_DEPRECATED_GETS = (int)0L; + /** + * {@snippet lang=c : + * #define __GLIBC_USE_DEPRECATED_GETS 0 + * } + */ + public static int __GLIBC_USE_DEPRECATED_GETS() { + return __GLIBC_USE_DEPRECATED_GETS; + } + private static final int __GLIBC_USE_DEPRECATED_SCANF = (int)0L; + /** + * {@snippet lang=c : + * #define __GLIBC_USE_DEPRECATED_SCANF 0 + * } + */ + public static int __GLIBC_USE_DEPRECATED_SCANF() { + return __GLIBC_USE_DEPRECATED_SCANF; + } + private static final int __GLIBC_USE_C2X_STRTOL = (int)0L; + /** + * {@snippet lang=c : + * #define __GLIBC_USE_C2X_STRTOL 0 + * } + */ + public static int __GLIBC_USE_C2X_STRTOL() { + return __GLIBC_USE_C2X_STRTOL; + } + private static final int _STDC_PREDEF_H = (int)1L; + /** + * {@snippet lang=c : + * #define _STDC_PREDEF_H 1 + * } + */ + public static int _STDC_PREDEF_H() { + return _STDC_PREDEF_H; + } + private static final int __STDC_IEC_559__ = (int)1L; + /** + * {@snippet lang=c : + * #define __STDC_IEC_559__ 1 + * } + */ + public static int __STDC_IEC_559__() { + return __STDC_IEC_559__; + } + private static final int __STDC_IEC_559_COMPLEX__ = (int)1L; + /** + * {@snippet lang=c : + * #define __STDC_IEC_559_COMPLEX__ 1 + * } + */ + public static int __STDC_IEC_559_COMPLEX__() { + return __STDC_IEC_559_COMPLEX__; + } + private static final int __GNU_LIBRARY__ = (int)6L; + /** + * {@snippet lang=c : + * #define __GNU_LIBRARY__ 6 + * } + */ + public static int __GNU_LIBRARY__() { + return __GNU_LIBRARY__; + } + private static final int __GLIBC__ = (int)2L; + /** + * {@snippet lang=c : + * #define __GLIBC__ 2 + * } + */ + public static int __GLIBC__() { + return __GLIBC__; + } + private static final int __GLIBC_MINOR__ = (int)39L; + /** + * {@snippet lang=c : + * #define __GLIBC_MINOR__ 39 + * } + */ + public static int __GLIBC_MINOR__() { + return __GLIBC_MINOR__; + } + private static final int _SYS_CDEFS_H = (int)1L; + /** + * {@snippet lang=c : + * #define _SYS_CDEFS_H 1 + * } + */ + public static int _SYS_CDEFS_H() { + return _SYS_CDEFS_H; + } + private static final int __glibc_c99_flexarr_available = (int)1L; + /** + * {@snippet lang=c : + * #define __glibc_c99_flexarr_available 1 + * } + */ + public static int __glibc_c99_flexarr_available() { + return __glibc_c99_flexarr_available; + } + private static final int __LDOUBLE_REDIRECTS_TO_FLOAT128_ABI = (int)0L; + /** + * {@snippet lang=c : + * #define __LDOUBLE_REDIRECTS_TO_FLOAT128_ABI 0 + * } + */ + public static int __LDOUBLE_REDIRECTS_TO_FLOAT128_ABI() { + return __LDOUBLE_REDIRECTS_TO_FLOAT128_ABI; + } + private static final int __HAVE_GENERIC_SELECTION = (int)1L; + /** + * {@snippet lang=c : + * #define __HAVE_GENERIC_SELECTION 1 + * } + */ + public static int __HAVE_GENERIC_SELECTION() { + return __HAVE_GENERIC_SELECTION; + } + private static final int __GLIBC_USE_LIB_EXT2 = (int)0L; + /** + * {@snippet lang=c : + * #define __GLIBC_USE_LIB_EXT2 0 + * } + */ + public static int __GLIBC_USE_LIB_EXT2() { + return __GLIBC_USE_LIB_EXT2; + } + private static final int __GLIBC_USE_IEC_60559_BFP_EXT = (int)0L; + /** + * {@snippet lang=c : + * #define __GLIBC_USE_IEC_60559_BFP_EXT 0 + * } + */ + public static int __GLIBC_USE_IEC_60559_BFP_EXT() { + return __GLIBC_USE_IEC_60559_BFP_EXT; + } + private static final int __GLIBC_USE_IEC_60559_BFP_EXT_C2X = (int)0L; + /** + * {@snippet lang=c : + * #define __GLIBC_USE_IEC_60559_BFP_EXT_C2X 0 + * } + */ + public static int __GLIBC_USE_IEC_60559_BFP_EXT_C2X() { + return __GLIBC_USE_IEC_60559_BFP_EXT_C2X; + } + private static final int __GLIBC_USE_IEC_60559_EXT = (int)0L; + /** + * {@snippet lang=c : + * #define __GLIBC_USE_IEC_60559_EXT 0 + * } + */ + public static int __GLIBC_USE_IEC_60559_EXT() { + return __GLIBC_USE_IEC_60559_EXT; + } + private static final int __GLIBC_USE_IEC_60559_FUNCS_EXT = (int)0L; + /** + * {@snippet lang=c : + * #define __GLIBC_USE_IEC_60559_FUNCS_EXT 0 + * } + */ + public static int __GLIBC_USE_IEC_60559_FUNCS_EXT() { + return __GLIBC_USE_IEC_60559_FUNCS_EXT; + } + private static final int __GLIBC_USE_IEC_60559_FUNCS_EXT_C2X = (int)0L; + /** + * {@snippet lang=c : + * #define __GLIBC_USE_IEC_60559_FUNCS_EXT_C2X 0 + * } + */ + public static int __GLIBC_USE_IEC_60559_FUNCS_EXT_C2X() { + return __GLIBC_USE_IEC_60559_FUNCS_EXT_C2X; + } + private static final int __GLIBC_USE_IEC_60559_TYPES_EXT = (int)0L; + /** + * {@snippet lang=c : + * #define __GLIBC_USE_IEC_60559_TYPES_EXT 0 + * } + */ + public static int __GLIBC_USE_IEC_60559_TYPES_EXT() { + return __GLIBC_USE_IEC_60559_TYPES_EXT; + } + private static final int _BITS_TYPES_H = (int)1L; + /** + * {@snippet lang=c : + * #define _BITS_TYPES_H 1 + * } + */ + public static int _BITS_TYPES_H() { + return _BITS_TYPES_H; + } + private static final int _BITS_TYPESIZES_H = (int)1L; + /** + * {@snippet lang=c : + * #define _BITS_TYPESIZES_H 1 + * } + */ + public static int _BITS_TYPESIZES_H() { + return _BITS_TYPESIZES_H; + } + private static final int __OFF_T_MATCHES_OFF64_T = (int)1L; + /** + * {@snippet lang=c : + * #define __OFF_T_MATCHES_OFF64_T 1 + * } + */ + public static int __OFF_T_MATCHES_OFF64_T() { + return __OFF_T_MATCHES_OFF64_T; + } + private static final int __INO_T_MATCHES_INO64_T = (int)1L; + /** + * {@snippet lang=c : + * #define __INO_T_MATCHES_INO64_T 1 + * } + */ + public static int __INO_T_MATCHES_INO64_T() { + return __INO_T_MATCHES_INO64_T; + } + private static final int __RLIM_T_MATCHES_RLIM64_T = (int)1L; + /** + * {@snippet lang=c : + * #define __RLIM_T_MATCHES_RLIM64_T 1 + * } + */ + public static int __RLIM_T_MATCHES_RLIM64_T() { + return __RLIM_T_MATCHES_RLIM64_T; + } + private static final int __STATFS_MATCHES_STATFS64 = (int)1L; + /** + * {@snippet lang=c : + * #define __STATFS_MATCHES_STATFS64 1 + * } + */ + public static int __STATFS_MATCHES_STATFS64() { + return __STATFS_MATCHES_STATFS64; + } + private static final int __KERNEL_OLD_TIMEVAL_MATCHES_TIMEVAL64 = (int)1L; + /** + * {@snippet lang=c : + * #define __KERNEL_OLD_TIMEVAL_MATCHES_TIMEVAL64 1 + * } + */ + public static int __KERNEL_OLD_TIMEVAL_MATCHES_TIMEVAL64() { + return __KERNEL_OLD_TIMEVAL_MATCHES_TIMEVAL64; + } + private static final int __FD_SETSIZE = (int)1024L; + /** + * {@snippet lang=c : + * #define __FD_SETSIZE 1024 + * } + */ + public static int __FD_SETSIZE() { + return __FD_SETSIZE; + } + private static final int _BITS_TIME64_H = (int)1L; + /** + * {@snippet lang=c : + * #define _BITS_TIME64_H 1 + * } + */ + public static int _BITS_TIME64_H() { + return _BITS_TIME64_H; + } + private static final int _BITS_WCHAR_H = (int)1L; + /** + * {@snippet lang=c : + * #define _BITS_WCHAR_H 1 + * } + */ + public static int _BITS_WCHAR_H() { + return _BITS_WCHAR_H; + } + private static final int _BITS_STDINT_INTN_H = (int)1L; + /** + * {@snippet lang=c : + * #define _BITS_STDINT_INTN_H 1 + * } + */ + public static int _BITS_STDINT_INTN_H() { + return _BITS_STDINT_INTN_H; + } + private static final int _BITS_STDINT_UINTN_H = (int)1L; + /** + * {@snippet lang=c : + * #define _BITS_STDINT_UINTN_H 1 + * } + */ + public static int _BITS_STDINT_UINTN_H() { + return _BITS_STDINT_UINTN_H; + } + private static final int _BITS_STDINT_LEAST_H = (int)1L; + /** + * {@snippet lang=c : + * #define _BITS_STDINT_LEAST_H 1 + * } + */ + public static int _BITS_STDINT_LEAST_H() { + return _BITS_STDINT_LEAST_H; + } + /** + * {@snippet lang=c : + * typedef long ptrdiff_t + * } + */ + public static final OfLong ptrdiff_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long size_t + * } + */ + public static final OfLong size_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef int wchar_t + * } + */ + public static final OfInt wchar_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef unsigned char __u_char + * } + */ + public static final OfByte __u_char = NativeSimdOps.C_CHAR; + /** + * {@snippet lang=c : + * typedef unsigned short __u_short + * } + */ + public static final OfShort __u_short = NativeSimdOps.C_SHORT; + /** + * {@snippet lang=c : + * typedef unsigned int __u_int + * } + */ + public static final OfInt __u_int = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef unsigned long __u_long + * } + */ + public static final OfLong __u_long = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef signed char __int8_t + * } + */ + public static final OfByte __int8_t = NativeSimdOps.C_CHAR; + /** + * {@snippet lang=c : + * typedef unsigned char __uint8_t + * } + */ + public static final OfByte __uint8_t = NativeSimdOps.C_CHAR; + /** + * {@snippet lang=c : + * typedef short __int16_t + * } + */ + public static final OfShort __int16_t = NativeSimdOps.C_SHORT; + /** + * {@snippet lang=c : + * typedef unsigned short __uint16_t + * } + */ + public static final OfShort __uint16_t = NativeSimdOps.C_SHORT; + /** + * {@snippet lang=c : + * typedef int __int32_t + * } + */ + public static final OfInt __int32_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef unsigned int __uint32_t + * } + */ + public static final OfInt __uint32_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef long __int64_t + * } + */ + public static final OfLong __int64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long __uint64_t + * } + */ + public static final OfLong __uint64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef __int8_t __int_least8_t + * } + */ + public static final OfByte __int_least8_t = NativeSimdOps.C_CHAR; + /** + * {@snippet lang=c : + * typedef __uint8_t __uint_least8_t + * } + */ + public static final OfByte __uint_least8_t = NativeSimdOps.C_CHAR; + /** + * {@snippet lang=c : + * typedef __int16_t __int_least16_t + * } + */ + public static final OfShort __int_least16_t = NativeSimdOps.C_SHORT; + /** + * {@snippet lang=c : + * typedef __uint16_t __uint_least16_t + * } + */ + public static final OfShort __uint_least16_t = NativeSimdOps.C_SHORT; + /** + * {@snippet lang=c : + * typedef __int32_t __int_least32_t + * } + */ + public static final OfInt __int_least32_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef __uint32_t __uint_least32_t + * } + */ + public static final OfInt __uint_least32_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef __int64_t __int_least64_t + * } + */ + public static final OfLong __int_least64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef __uint64_t __uint_least64_t + * } + */ + public static final OfLong __uint_least64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef long __quad_t + * } + */ + public static final OfLong __quad_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long __u_quad_t + * } + */ + public static final OfLong __u_quad_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef long __intmax_t + * } + */ + public static final OfLong __intmax_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long __uintmax_t + * } + */ + public static final OfLong __uintmax_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long __dev_t + * } + */ + public static final OfLong __dev_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned int __uid_t + * } + */ + public static final OfInt __uid_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef unsigned int __gid_t + * } + */ + public static final OfInt __gid_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef unsigned long __ino_t + * } + */ + public static final OfLong __ino_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long __ino64_t + * } + */ + public static final OfLong __ino64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned int __mode_t + * } + */ + public static final OfInt __mode_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef unsigned long __nlink_t + * } + */ + public static final OfLong __nlink_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef long __off_t + * } + */ + public static final OfLong __off_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef long __off64_t + * } + */ + public static final OfLong __off64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef int __pid_t + * } + */ + public static final OfInt __pid_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef long __clock_t + * } + */ + public static final OfLong __clock_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long __rlim_t + * } + */ + public static final OfLong __rlim_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long __rlim64_t + * } + */ + public static final OfLong __rlim64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned int __id_t + * } + */ + public static final OfInt __id_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef long __time_t + * } + */ + public static final OfLong __time_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned int __useconds_t + * } + */ + public static final OfInt __useconds_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef long __suseconds_t + * } + */ + public static final OfLong __suseconds_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef long __suseconds64_t + * } + */ + public static final OfLong __suseconds64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef int __daddr_t + * } + */ + public static final OfInt __daddr_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef int __key_t + * } + */ + public static final OfInt __key_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef int __clockid_t + * } + */ + public static final OfInt __clockid_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef void *__timer_t + * } + */ + public static final AddressLayout __timer_t = NativeSimdOps.C_POINTER; + /** + * {@snippet lang=c : + * typedef long __blksize_t + * } + */ + public static final OfLong __blksize_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef long __blkcnt_t + * } + */ + public static final OfLong __blkcnt_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef long __blkcnt64_t + * } + */ + public static final OfLong __blkcnt64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long __fsblkcnt_t + * } + */ + public static final OfLong __fsblkcnt_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long __fsblkcnt64_t + * } + */ + public static final OfLong __fsblkcnt64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long __fsfilcnt_t + * } + */ + public static final OfLong __fsfilcnt_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long __fsfilcnt64_t + * } + */ + public static final OfLong __fsfilcnt64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef long __fsword_t + * } + */ + public static final OfLong __fsword_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef long __ssize_t + * } + */ + public static final OfLong __ssize_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef long __syscall_slong_t + * } + */ + public static final OfLong __syscall_slong_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long __syscall_ulong_t + * } + */ + public static final OfLong __syscall_ulong_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef __off64_t __loff_t + * } + */ + public static final OfLong __loff_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef char *__caddr_t + * } + */ + public static final AddressLayout __caddr_t = NativeSimdOps.C_POINTER; + /** + * {@snippet lang=c : + * typedef long __intptr_t + * } + */ + public static final OfLong __intptr_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned int __socklen_t + * } + */ + public static final OfInt __socklen_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef int __sig_atomic_t + * } + */ + public static final OfInt __sig_atomic_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef __int8_t int8_t + * } + */ + public static final OfByte int8_t = NativeSimdOps.C_CHAR; + /** + * {@snippet lang=c : + * typedef __int16_t int16_t + * } + */ + public static final OfShort int16_t = NativeSimdOps.C_SHORT; + /** + * {@snippet lang=c : + * typedef __int32_t int32_t + * } + */ + public static final OfInt int32_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef __int64_t int64_t + * } + */ + public static final OfLong int64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef __uint8_t uint8_t + * } + */ + public static final OfByte uint8_t = NativeSimdOps.C_CHAR; + /** + * {@snippet lang=c : + * typedef __uint16_t uint16_t + * } + */ + public static final OfShort uint16_t = NativeSimdOps.C_SHORT; + /** + * {@snippet lang=c : + * typedef __uint32_t uint32_t + * } + */ + public static final OfInt uint32_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef __uint64_t uint64_t + * } + */ + public static final OfLong uint64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef __int_least8_t int_least8_t + * } + */ + public static final OfByte int_least8_t = NativeSimdOps.C_CHAR; + /** + * {@snippet lang=c : + * typedef __int_least16_t int_least16_t + * } + */ + public static final OfShort int_least16_t = NativeSimdOps.C_SHORT; + /** + * {@snippet lang=c : + * typedef __int_least32_t int_least32_t + * } + */ + public static final OfInt int_least32_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef __int_least64_t int_least64_t + * } + */ + public static final OfLong int_least64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef __uint_least8_t uint_least8_t + * } + */ + public static final OfByte uint_least8_t = NativeSimdOps.C_CHAR; + /** + * {@snippet lang=c : + * typedef __uint_least16_t uint_least16_t + * } + */ + public static final OfShort uint_least16_t = NativeSimdOps.C_SHORT; + /** + * {@snippet lang=c : + * typedef __uint_least32_t uint_least32_t + * } + */ + public static final OfInt uint_least32_t = NativeSimdOps.C_INT; + /** + * {@snippet lang=c : + * typedef __uint_least64_t uint_least64_t + * } + */ + public static final OfLong uint_least64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef signed char int_fast8_t + * } + */ + public static final OfByte int_fast8_t = NativeSimdOps.C_CHAR; + /** + * {@snippet lang=c : + * typedef long int_fast16_t + * } + */ + public static final OfLong int_fast16_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef long int_fast32_t + * } + */ + public static final OfLong int_fast32_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef long int_fast64_t + * } + */ + public static final OfLong int_fast64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned char uint_fast8_t + * } + */ + public static final OfByte uint_fast8_t = NativeSimdOps.C_CHAR; + /** + * {@snippet lang=c : + * typedef unsigned long uint_fast16_t + * } + */ + public static final OfLong uint_fast16_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long uint_fast32_t + * } + */ + public static final OfLong uint_fast32_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long uint_fast64_t + * } + */ + public static final OfLong uint_fast64_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef long intptr_t + * } + */ + public static final OfLong intptr_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef unsigned long uintptr_t + * } + */ + public static final OfLong uintptr_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef __intmax_t intmax_t + * } + */ + public static final OfLong intmax_t = NativeSimdOps.C_LONG; + /** + * {@snippet lang=c : + * typedef __uintmax_t uintmax_t + * } + */ + public static final OfLong uintmax_t = NativeSimdOps.C_LONG; + + private static class jvector_supports_abi { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_INT, + NativeSimdOps.C_INT + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("jvector_supports_abi"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * int32_t jvector_supports_abi(int32_t abi) + * } + */ + public static FunctionDescriptor jvector_supports_abi$descriptor() { + return jvector_supports_abi.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * int32_t jvector_supports_abi(int32_t abi) + * } + */ + public static MethodHandle jvector_supports_abi$handle() { + return jvector_supports_abi.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * int32_t jvector_supports_abi(int32_t abi) + * } + */ + public static MemorySegment jvector_supports_abi$address() { + return jvector_supports_abi.ADDR; + } + + /** + * {@snippet lang=c : + * int32_t jvector_supports_abi(int32_t abi) + * } + */ + public static int jvector_supports_abi(int abi) { + var mh$ = jvector_supports_abi.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("jvector_supports_abi", abi); + } + return (int)mh$.invokeExact(abi); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class cosine_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("cosine_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * float cosine_f32(const float *a, size_t aoffset, const float *b, size_t boffset, size_t length) + * } + */ + public static FunctionDescriptor cosine_f32$descriptor() { + return cosine_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * float cosine_f32(const float *a, size_t aoffset, const float *b, size_t boffset, size_t length) + * } + */ + public static MethodHandle cosine_f32$handle() { + return cosine_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * float cosine_f32(const float *a, size_t aoffset, const float *b, size_t boffset, size_t length) + * } + */ + public static MemorySegment cosine_f32$address() { + return cosine_f32.ADDR; + } + + /** + * {@snippet lang=c : + * float cosine_f32(const float *a, size_t aoffset, const float *b, size_t boffset, size_t length) + * } + */ + public static float cosine_f32(MemorySegment a, long aoffset, MemorySegment b, long boffset, long length) { + return FastOps.call(FastOps.COSINE, cosine_f32$address(), a, aoffset, b, boffset, length); + } + + private static class dot_product_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("dot_product_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * float dot_product_f32(const float *a, size_t aoffset, const float *b, size_t boffset, size_t length) + * } + */ + public static FunctionDescriptor dot_product_f32$descriptor() { + return dot_product_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * float dot_product_f32(const float *a, size_t aoffset, const float *b, size_t boffset, size_t length) + * } + */ + public static MethodHandle dot_product_f32$handle() { + return dot_product_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * float dot_product_f32(const float *a, size_t aoffset, const float *b, size_t boffset, size_t length) + * } + */ + public static MemorySegment dot_product_f32$address() { + return dot_product_f32.ADDR; + } + + /** + * {@snippet lang=c : + * float dot_product_f32(const float *a, size_t aoffset, const float *b, size_t boffset, size_t length) + * } + */ + public static float dot_product_f32(MemorySegment a, long aoffset, MemorySegment b, long boffset, long length) { + return FastOps.call(FastOps.DOT, dot_product_f32$address(), a, aoffset, b, boffset, length); + } + + private static class euclidean_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("euclidean_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * float euclidean_f32(const float *a, size_t aoffset, const float *b, size_t boffset, size_t length) + * } + */ + public static FunctionDescriptor euclidean_f32$descriptor() { + return euclidean_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * float euclidean_f32(const float *a, size_t aoffset, const float *b, size_t boffset, size_t length) + * } + */ + public static MethodHandle euclidean_f32$handle() { + return euclidean_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * float euclidean_f32(const float *a, size_t aoffset, const float *b, size_t boffset, size_t length) + * } + */ + public static MemorySegment euclidean_f32$address() { + return euclidean_f32.ADDR; + } + + /** + * {@snippet lang=c : + * float euclidean_f32(const float *a, size_t aoffset, const float *b, size_t boffset, size_t length) + * } + */ + public static float euclidean_f32(MemorySegment a, long aoffset, MemorySegment b, long boffset, long length) { + return FastOps.call(FastOps.EUCLIDEAN, euclidean_f32$address(), a, aoffset, b, boffset, length); + } + + private static class closest_centroid_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_INT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_INT, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("closest_centroid_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * int closest_centroid_f32(const float *point, size_t pointOffset, const float *centroids, int centroidCount, size_t dimension) + * } + */ + public static FunctionDescriptor closest_centroid_f32$descriptor() { + return closest_centroid_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * int closest_centroid_f32(const float *point, size_t pointOffset, const float *centroids, int centroidCount, size_t dimension) + * } + */ + public static MethodHandle closest_centroid_f32$handle() { + return closest_centroid_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * int closest_centroid_f32(const float *point, size_t pointOffset, const float *centroids, int centroidCount, size_t dimension) + * } + */ + public static MemorySegment closest_centroid_f32$address() { + return closest_centroid_f32.ADDR; + } + + /** + * {@snippet lang=c : + * int closest_centroid_f32(const float *point, size_t pointOffset, const float *centroids, int centroidCount, size_t dimension) + * } + */ + public static int closest_centroid_f32(MemorySegment point, long pointOffset, MemorySegment centroids, int centroidCount, long dimension) { + var mh$ = closest_centroid_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("closest_centroid_f32", point, pointOffset, centroids, centroidCount, dimension); + } + return (int)mh$.invokeExact(point, pointOffset, centroids, centroidCount, dimension); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class add_in_place_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("add_in_place_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * void add_in_place_f32(float *v1, const float *v2, size_t length) + * } + */ + public static FunctionDescriptor add_in_place_f32$descriptor() { + return add_in_place_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * void add_in_place_f32(float *v1, const float *v2, size_t length) + * } + */ + public static MethodHandle add_in_place_f32$handle() { + return add_in_place_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * void add_in_place_f32(float *v1, const float *v2, size_t length) + * } + */ + public static MemorySegment add_in_place_f32$address() { + return add_in_place_f32.ADDR; + } + + /** + * {@snippet lang=c : + * void add_in_place_f32(float *v1, const float *v2, size_t length) + * } + */ + public static void add_in_place_f32(MemorySegment v1, MemorySegment v2, long length) { + var mh$ = add_in_place_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("add_in_place_f32", v1, v2, length); + } + mh$.invokeExact(v1, v2, length); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class add_scalar_in_place_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("add_scalar_in_place_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * void add_scalar_in_place_f32(float *v1, float value, size_t length) + * } + */ + public static FunctionDescriptor add_scalar_in_place_f32$descriptor() { + return add_scalar_in_place_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * void add_scalar_in_place_f32(float *v1, float value, size_t length) + * } + */ + public static MethodHandle add_scalar_in_place_f32$handle() { + return add_scalar_in_place_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * void add_scalar_in_place_f32(float *v1, float value, size_t length) + * } + */ + public static MemorySegment add_scalar_in_place_f32$address() { + return add_scalar_in_place_f32.ADDR; + } + + /** + * {@snippet lang=c : + * void add_scalar_in_place_f32(float *v1, float value, size_t length) + * } + */ + public static void add_scalar_in_place_f32(MemorySegment v1, float value, long length) { + var mh$ = add_scalar_in_place_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("add_scalar_in_place_f32", v1, value, length); + } + mh$.invokeExact(v1, value, length); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class scale_to_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, NativeSimdOps.C_POINTER, NativeSimdOps.C_LONG, NativeSimdOps.C_FLOAT, NativeSimdOps.C_LONG); + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("scale_to_f32"); + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + public static FunctionDescriptor scale_to_f32$descriptor() { return scale_to_f32.DESC; } + public static MethodHandle scale_to_f32$handle() { return scale_to_f32.HANDLE; } + public static MemorySegment scale_to_f32$address() { return scale_to_f32.ADDR; } + + public static void scale_to_f32(MemorySegment source, MemorySegment destination, long destinationOffset, float multiplier, long length) { + var mh$ = scale_to_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { traceDowncall("scale_to_f32", source, destination, destinationOffset, multiplier, length); } + mh$.invokeExact(source, destination, destinationOffset, multiplier, length); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class subtract_add_in_place_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, NativeSimdOps.C_POINTER, NativeSimdOps.C_POINTER, NativeSimdOps.C_LONG); + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("subtract_add_in_place_f32"); + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + public static FunctionDescriptor subtract_add_in_place_f32$descriptor() { return subtract_add_in_place_f32.DESC; } + public static MethodHandle subtract_add_in_place_f32$handle() { return subtract_add_in_place_f32.HANDLE; } + public static MemorySegment subtract_add_in_place_f32$address() { return subtract_add_in_place_f32.ADDR; } + + public static void subtract_add_in_place_f32(MemorySegment oldSum, MemorySegment newSum, MemorySegment point, long length) { + var mh$ = subtract_add_in_place_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { traceDowncall("subtract_add_in_place_f32", oldSum, newSum, point, length); } + mh$.invokeExact(oldSum, newSum, point, length); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class sub_in_place_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("sub_in_place_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * void sub_in_place_f32(float *v1, const float *v2, size_t length) + * } + */ + public static FunctionDescriptor sub_in_place_f32$descriptor() { + return sub_in_place_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * void sub_in_place_f32(float *v1, const float *v2, size_t length) + * } + */ + public static MethodHandle sub_in_place_f32$handle() { + return sub_in_place_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * void sub_in_place_f32(float *v1, const float *v2, size_t length) + * } + */ + public static MemorySegment sub_in_place_f32$address() { + return sub_in_place_f32.ADDR; + } + + /** + * {@snippet lang=c : + * void sub_in_place_f32(float *v1, const float *v2, size_t length) + * } + */ + public static void sub_in_place_f32(MemorySegment v1, MemorySegment v2, long length) { + var mh$ = sub_in_place_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("sub_in_place_f32", v1, v2, length); + } + mh$.invokeExact(v1, v2, length); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class sub_scalar_in_place_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("sub_scalar_in_place_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * void sub_scalar_in_place_f32(float *v1, float value, size_t length) + * } + */ + public static FunctionDescriptor sub_scalar_in_place_f32$descriptor() { + return sub_scalar_in_place_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * void sub_scalar_in_place_f32(float *v1, float value, size_t length) + * } + */ + public static MethodHandle sub_scalar_in_place_f32$handle() { + return sub_scalar_in_place_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * void sub_scalar_in_place_f32(float *v1, float value, size_t length) + * } + */ + public static MemorySegment sub_scalar_in_place_f32$address() { + return sub_scalar_in_place_f32.ADDR; + } + + /** + * {@snippet lang=c : + * void sub_scalar_in_place_f32(float *v1, float value, size_t length) + * } + */ + public static void sub_scalar_in_place_f32(MemorySegment v1, float value, long length) { + var mh$ = sub_scalar_in_place_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("sub_scalar_in_place_f32", v1, value, length); + } + mh$.invokeExact(v1, value, length); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class sub_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("sub_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * void sub_f32(const float *a, size_t aoffset, const float *b, size_t boffset, float *res, size_t length) + * } + */ + public static FunctionDescriptor sub_f32$descriptor() { + return sub_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * void sub_f32(const float *a, size_t aoffset, const float *b, size_t boffset, float *res, size_t length) + * } + */ + public static MethodHandle sub_f32$handle() { + return sub_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * void sub_f32(const float *a, size_t aoffset, const float *b, size_t boffset, float *res, size_t length) + * } + */ + public static MemorySegment sub_f32$address() { + return sub_f32.ADDR; + } + + /** + * {@snippet lang=c : + * void sub_f32(const float *a, size_t aoffset, const float *b, size_t boffset, float *res, size_t length) + * } + */ + public static void sub_f32(MemorySegment a, long aoffset, MemorySegment b, long boffset, MemorySegment res, long length) { + var mh$ = sub_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("sub_f32", a, aoffset, b, boffset, res, length); + } + mh$.invokeExact(a, aoffset, b, boffset, res, length); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class sub_scalar_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("sub_scalar_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * void sub_scalar_f32(const float *a, size_t aoffset, float value, float *res, size_t length) + * } + */ + public static FunctionDescriptor sub_scalar_f32$descriptor() { + return sub_scalar_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * void sub_scalar_f32(const float *a, size_t aoffset, float value, float *res, size_t length) + * } + */ + public static MethodHandle sub_scalar_f32$handle() { + return sub_scalar_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * void sub_scalar_f32(const float *a, size_t aoffset, float value, float *res, size_t length) + * } + */ + public static MemorySegment sub_scalar_f32$address() { + return sub_scalar_f32.ADDR; + } + + /** + * {@snippet lang=c : + * void sub_scalar_f32(const float *a, size_t aoffset, float value, float *res, size_t length) + * } + */ + public static void sub_scalar_f32(MemorySegment a, long aoffset, float value, MemorySegment res, long length) { + var mh$ = sub_scalar_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("sub_scalar_f32", a, aoffset, value, res, length); + } + mh$.invokeExact(a, aoffset, value, res, length); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class scale_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("scale_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * void scale_f32(float *v, float multiplier, size_t length) + * } + */ + public static FunctionDescriptor scale_f32$descriptor() { + return scale_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * void scale_f32(float *v, float multiplier, size_t length) + * } + */ + public static MethodHandle scale_f32$handle() { + return scale_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * void scale_f32(float *v, float multiplier, size_t length) + * } + */ + public static MemorySegment scale_f32$address() { + return scale_f32.ADDR; + } + + /** + * {@snippet lang=c : + * void scale_f32(float *v, float multiplier, size_t length) + * } + */ + public static void scale_f32(MemorySegment v, float multiplier, long length) { + var mh$ = scale_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("scale_f32", v, multiplier, length); + } + mh$.invokeExact(v, multiplier, length); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class max_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("max_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * float max_f32(const float *v, size_t length) + * } + */ + public static FunctionDescriptor max_f32$descriptor() { + return max_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * float max_f32(const float *v, size_t length) + * } + */ + public static MethodHandle max_f32$handle() { + return max_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * float max_f32(const float *v, size_t length) + * } + */ + public static MemorySegment max_f32$address() { + return max_f32.ADDR; + } + + /** + * {@snippet lang=c : + * float max_f32(const float *v, size_t length) + * } + */ + public static float max_f32(MemorySegment v, long length) { + var mh$ = max_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("max_f32", v, length); + } + return (float)mh$.invokeExact(v, length); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class min_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("min_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * float min_f32(const float *v, size_t length) + * } + */ + public static FunctionDescriptor min_f32$descriptor() { + return min_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * float min_f32(const float *v, size_t length) + * } + */ + public static MethodHandle min_f32$handle() { + return min_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * float min_f32(const float *v, size_t length) + * } + */ + public static MemorySegment min_f32$address() { + return min_f32.ADDR; + } + + /** + * {@snippet lang=c : + * float min_f32(const float *v, size_t length) + * } + */ + public static float min_f32(MemorySegment v, long length) { + var mh$ = min_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("min_f32", v, length); + } + return (float)mh$.invokeExact(v, length); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class min_in_place_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("min_in_place_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * void min_in_place_f32(float *v1, const float *v2, size_t length) + * } + */ + public static FunctionDescriptor min_in_place_f32$descriptor() { + return min_in_place_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * void min_in_place_f32(float *v1, const float *v2, size_t length) + * } + */ + public static MethodHandle min_in_place_f32$handle() { + return min_in_place_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * void min_in_place_f32(float *v1, const float *v2, size_t length) + * } + */ + public static MemorySegment min_in_place_f32$address() { + return min_in_place_f32.ADDR; + } + + /** + * {@snippet lang=c : + * void min_in_place_f32(float *v1, const float *v2, size_t length) + * } + */ + public static void min_in_place_f32(MemorySegment v1, MemorySegment v2, long length) { + var mh$ = min_in_place_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("min_in_place_f32", v1, v2, length); + } + mh$.invokeExact(v1, v2, length); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class sum_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("sum_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * float sum_f32(const float *v, size_t length) + * } + */ + public static FunctionDescriptor sum_f32$descriptor() { + return sum_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * float sum_f32(const float *v, size_t length) + * } + */ + public static MethodHandle sum_f32$handle() { + return sum_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * float sum_f32(const float *v, size_t length) + * } + */ + public static MemorySegment sum_f32$address() { + return sum_f32.ADDR; + } + + /** + * {@snippet lang=c : + * float sum_f32(const float *v, size_t length) + * } + */ + public static float sum_f32(MemorySegment v, long length) { + var mh$ = sum_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("sum_f32", v, length); + } + return (float)mh$.invokeExact(v, length); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class assemble_and_sum_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_INT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_INT, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("assemble_and_sum_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * float assemble_and_sum_f32(const float *data, int dataBase, const unsigned char *baseOffsets, int baseOffsetsOffset, size_t baseOffsetsLength) + * } + */ + public static FunctionDescriptor assemble_and_sum_f32$descriptor() { + return assemble_and_sum_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * float assemble_and_sum_f32(const float *data, int dataBase, const unsigned char *baseOffsets, int baseOffsetsOffset, size_t baseOffsetsLength) + * } + */ + public static MethodHandle assemble_and_sum_f32$handle() { + return assemble_and_sum_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * float assemble_and_sum_f32(const float *data, int dataBase, const unsigned char *baseOffsets, int baseOffsetsOffset, size_t baseOffsetsLength) + * } + */ + public static MemorySegment assemble_and_sum_f32$address() { + return assemble_and_sum_f32.ADDR; + } + + /** + * {@snippet lang=c : + * float assemble_and_sum_f32(const float *data, int dataBase, const unsigned char *baseOffsets, int baseOffsetsOffset, size_t baseOffsetsLength) + * } + */ + public static float assemble_and_sum_f32(MemorySegment data, int dataBase, MemorySegment baseOffsets, int baseOffsetsOffset, long baseOffsetsLength) { + var mh$ = assemble_and_sum_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("assemble_and_sum_f32", data, dataBase, baseOffsets, baseOffsetsOffset, baseOffsetsLength); + } + return (float)mh$.invokeExact(data, dataBase, baseOffsets, baseOffsetsOffset, baseOffsetsLength); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class assemble_and_sum_pq_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_INT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_INT, + NativeSimdOps.C_INT + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("assemble_and_sum_pq_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * float assemble_and_sum_pq_f32(const float *data, size_t subspaceCount, const unsigned char *baseOffsets1, int baseOffsetsOffset1, const unsigned char *baseOffsets2, int baseOffsetsOffset2, int clusterCount) + * } + */ + public static FunctionDescriptor assemble_and_sum_pq_f32$descriptor() { + return assemble_and_sum_pq_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * float assemble_and_sum_pq_f32(const float *data, size_t subspaceCount, const unsigned char *baseOffsets1, int baseOffsetsOffset1, const unsigned char *baseOffsets2, int baseOffsetsOffset2, int clusterCount) + * } + */ + public static MethodHandle assemble_and_sum_pq_f32$handle() { + return assemble_and_sum_pq_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * float assemble_and_sum_pq_f32(const float *data, size_t subspaceCount, const unsigned char *baseOffsets1, int baseOffsetsOffset1, const unsigned char *baseOffsets2, int baseOffsetsOffset2, int clusterCount) + * } + */ + public static MemorySegment assemble_and_sum_pq_f32$address() { + return assemble_and_sum_pq_f32.ADDR; + } + + /** + * {@snippet lang=c : + * float assemble_and_sum_pq_f32(const float *data, size_t subspaceCount, const unsigned char *baseOffsets1, int baseOffsetsOffset1, const unsigned char *baseOffsets2, int baseOffsetsOffset2, int clusterCount) + * } + */ + public static float assemble_and_sum_pq_f32(MemorySegment data, long subspaceCount, MemorySegment baseOffsets1, int baseOffsetsOffset1, MemorySegment baseOffsets2, int baseOffsetsOffset2, int clusterCount) { + var mh$ = assemble_and_sum_pq_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("assemble_and_sum_pq_f32", data, subspaceCount, baseOffsets1, baseOffsetsOffset1, baseOffsets2, baseOffsetsOffset2, clusterCount); + } + return (float)mh$.invokeExact(data, subspaceCount, baseOffsets1, baseOffsetsOffset1, baseOffsets2, baseOffsetsOffset2, clusterCount); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class pq_decoded_cosine_similarity_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_INT, + NativeSimdOps.C_LONG, + NativeSimdOps.C_INT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_FLOAT + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("pq_decoded_cosine_similarity_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * float pq_decoded_cosine_similarity_f32(const unsigned char *baseOffsets, int baseOffsetsOffset, size_t baseOffsetsLength, int clusterCount, const float *partialSums, const float *aMagnitude, float bMagnitude) + * } + */ + public static FunctionDescriptor pq_decoded_cosine_similarity_f32$descriptor() { + return pq_decoded_cosine_similarity_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * float pq_decoded_cosine_similarity_f32(const unsigned char *baseOffsets, int baseOffsetsOffset, size_t baseOffsetsLength, int clusterCount, const float *partialSums, const float *aMagnitude, float bMagnitude) + * } + */ + public static MethodHandle pq_decoded_cosine_similarity_f32$handle() { + return pq_decoded_cosine_similarity_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * float pq_decoded_cosine_similarity_f32(const unsigned char *baseOffsets, int baseOffsetsOffset, size_t baseOffsetsLength, int clusterCount, const float *partialSums, const float *aMagnitude, float bMagnitude) + * } + */ + public static MemorySegment pq_decoded_cosine_similarity_f32$address() { + return pq_decoded_cosine_similarity_f32.ADDR; + } + + /** + * {@snippet lang=c : + * float pq_decoded_cosine_similarity_f32(const unsigned char *baseOffsets, int baseOffsetsOffset, size_t baseOffsetsLength, int clusterCount, const float *partialSums, const float *aMagnitude, float bMagnitude) + * } + */ + public static float pq_decoded_cosine_similarity_f32(MemorySegment baseOffsets, int baseOffsetsOffset, long baseOffsetsLength, int clusterCount, MemorySegment partialSums, MemorySegment aMagnitude, float bMagnitude) { + var mh$ = pq_decoded_cosine_similarity_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("pq_decoded_cosine_similarity_f32", baseOffsets, baseOffsetsOffset, baseOffsetsLength, clusterCount, partialSums, aMagnitude, bMagnitude); + } + return (float)mh$.invokeExact(baseOffsets, baseOffsetsOffset, baseOffsetsLength, clusterCount, partialSums, aMagnitude, bMagnitude); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class calculate_partial_sums_dot_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, + NativeSimdOps.C_INT, + NativeSimdOps.C_LONG, + NativeSimdOps.C_INT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_INT, + NativeSimdOps.C_POINTER + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("calculate_partial_sums_dot_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * void calculate_partial_sums_dot_f32(const float *codebook, int codebookIndex, size_t size, int clusterCount, const float *query, int queryOffset, float *partialSums) + * } + */ + public static FunctionDescriptor calculate_partial_sums_dot_f32$descriptor() { + return calculate_partial_sums_dot_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * void calculate_partial_sums_dot_f32(const float *codebook, int codebookIndex, size_t size, int clusterCount, const float *query, int queryOffset, float *partialSums) + * } + */ + public static MethodHandle calculate_partial_sums_dot_f32$handle() { + return calculate_partial_sums_dot_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * void calculate_partial_sums_dot_f32(const float *codebook, int codebookIndex, size_t size, int clusterCount, const float *query, int queryOffset, float *partialSums) + * } + */ + public static MemorySegment calculate_partial_sums_dot_f32$address() { + return calculate_partial_sums_dot_f32.ADDR; + } + + /** + * {@snippet lang=c : + * void calculate_partial_sums_dot_f32(const float *codebook, int codebookIndex, size_t size, int clusterCount, const float *query, int queryOffset, float *partialSums) + * } + */ + public static void calculate_partial_sums_dot_f32(MemorySegment codebook, int codebookIndex, long size, int clusterCount, MemorySegment query, int queryOffset, MemorySegment partialSums) { + var mh$ = calculate_partial_sums_dot_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("calculate_partial_sums_dot_f32", codebook, codebookIndex, size, clusterCount, query, queryOffset, partialSums); + } + mh$.invokeExact(codebook, codebookIndex, size, clusterCount, query, queryOffset, partialSums); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class calculate_partial_sums_euclidean_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, + NativeSimdOps.C_INT, + NativeSimdOps.C_LONG, + NativeSimdOps.C_INT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_INT, + NativeSimdOps.C_POINTER + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("calculate_partial_sums_euclidean_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * void calculate_partial_sums_euclidean_f32(const float *codebook, int codebookIndex, size_t size, int clusterCount, const float *query, int queryOffset, float *partialSums) + * } + */ + public static FunctionDescriptor calculate_partial_sums_euclidean_f32$descriptor() { + return calculate_partial_sums_euclidean_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * void calculate_partial_sums_euclidean_f32(const float *codebook, int codebookIndex, size_t size, int clusterCount, const float *query, int queryOffset, float *partialSums) + * } + */ + public static MethodHandle calculate_partial_sums_euclidean_f32$handle() { + return calculate_partial_sums_euclidean_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * void calculate_partial_sums_euclidean_f32(const float *codebook, int codebookIndex, size_t size, int clusterCount, const float *query, int queryOffset, float *partialSums) + * } + */ + public static MemorySegment calculate_partial_sums_euclidean_f32$address() { + return calculate_partial_sums_euclidean_f32.ADDR; + } + + /** + * {@snippet lang=c : + * void calculate_partial_sums_euclidean_f32(const float *codebook, int codebookIndex, size_t size, int clusterCount, const float *query, int queryOffset, float *partialSums) + * } + */ + public static void calculate_partial_sums_euclidean_f32(MemorySegment codebook, int codebookIndex, long size, int clusterCount, MemorySegment query, int queryOffset, MemorySegment partialSums) { + var mh$ = calculate_partial_sums_euclidean_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("calculate_partial_sums_euclidean_f32", codebook, codebookIndex, size, clusterCount, query, queryOffset, partialSums); + } + mh$.invokeExact(codebook, codebookIndex, size, clusterCount, query, queryOffset, partialSums); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class calculate_partial_sums_self_magnitude_f32 { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, + NativeSimdOps.C_INT, + NativeSimdOps.C_LONG, + NativeSimdOps.C_INT, + NativeSimdOps.C_POINTER + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("calculate_partial_sums_self_magnitude_f32"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * void calculate_partial_sums_self_magnitude_f32(const float *codebook, int codebookIndex, size_t size, int clusterCount, float *partialSums) + * } + */ + public static FunctionDescriptor calculate_partial_sums_self_magnitude_f32$descriptor() { + return calculate_partial_sums_self_magnitude_f32.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * void calculate_partial_sums_self_magnitude_f32(const float *codebook, int codebookIndex, size_t size, int clusterCount, float *partialSums) + * } + */ + public static MethodHandle calculate_partial_sums_self_magnitude_f32$handle() { + return calculate_partial_sums_self_magnitude_f32.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * void calculate_partial_sums_self_magnitude_f32(const float *codebook, int codebookIndex, size_t size, int clusterCount, float *partialSums) + * } + */ + public static MemorySegment calculate_partial_sums_self_magnitude_f32$address() { + return calculate_partial_sums_self_magnitude_f32.ADDR; + } + + /** + * {@snippet lang=c : + * void calculate_partial_sums_self_magnitude_f32(const float *codebook, int codebookIndex, size_t size, int clusterCount, float *partialSums) + * } + */ + public static void calculate_partial_sums_self_magnitude_f32(MemorySegment codebook, int codebookIndex, long size, int clusterCount, MemorySegment partialSums) { + var mh$ = calculate_partial_sums_self_magnitude_f32.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("calculate_partial_sums_self_magnitude_f32", codebook, codebookIndex, size, clusterCount, partialSums); + } + mh$.invokeExact(codebook, codebookIndex, size, clusterCount, partialSums); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class nvq_quantize_8bit { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("nvq_quantize_8bit"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * void nvq_quantize_8bit(const float *vector, size_t length, float alpha, float x0, float minValue, float maxValue, unsigned char *destination) + * } + */ + public static FunctionDescriptor nvq_quantize_8bit$descriptor() { + return nvq_quantize_8bit.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * void nvq_quantize_8bit(const float *vector, size_t length, float alpha, float x0, float minValue, float maxValue, unsigned char *destination) + * } + */ + public static MethodHandle nvq_quantize_8bit$handle() { + return nvq_quantize_8bit.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * void nvq_quantize_8bit(const float *vector, size_t length, float alpha, float x0, float minValue, float maxValue, unsigned char *destination) + * } + */ + public static MemorySegment nvq_quantize_8bit$address() { + return nvq_quantize_8bit.ADDR; + } + + /** + * {@snippet lang=c : + * void nvq_quantize_8bit(const float *vector, size_t length, float alpha, float x0, float minValue, float maxValue, unsigned char *destination) + * } + */ + public static void nvq_quantize_8bit(MemorySegment vector, long length, float alpha, float x0, float minValue, float maxValue, MemorySegment destination) { + var mh$ = nvq_quantize_8bit.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("nvq_quantize_8bit", vector, length, alpha, x0, minValue, maxValue, destination); + } + mh$.invokeExact(vector, length, alpha, x0, minValue, maxValue, destination); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class nvq_loss { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_INT + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("nvq_loss"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * float nvq_loss(const float *vector, size_t length, float alpha, float x0, float minValue, float maxValue, int nBits) + * } + */ + public static FunctionDescriptor nvq_loss$descriptor() { + return nvq_loss.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * float nvq_loss(const float *vector, size_t length, float alpha, float x0, float minValue, float maxValue, int nBits) + * } + */ + public static MethodHandle nvq_loss$handle() { + return nvq_loss.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * float nvq_loss(const float *vector, size_t length, float alpha, float x0, float minValue, float maxValue, int nBits) + * } + */ + public static MemorySegment nvq_loss$address() { + return nvq_loss.ADDR; + } + + /** + * {@snippet lang=c : + * float nvq_loss(const float *vector, size_t length, float alpha, float x0, float minValue, float maxValue, int nBits) + * } + */ + public static float nvq_loss(MemorySegment vector, long length, float alpha, float x0, float minValue, float maxValue, int nBits) { + var mh$ = nvq_loss.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("nvq_loss", vector, length, alpha, x0, minValue, maxValue, nBits); + } + return (float)mh$.invokeExact(vector, length, alpha, x0, minValue, maxValue, nBits); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class nvq_uniform_loss { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_INT + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("nvq_uniform_loss"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * float nvq_uniform_loss(const float *vector, size_t length, float minValue, float maxValue, int nBits) + * } + */ + public static FunctionDescriptor nvq_uniform_loss$descriptor() { + return nvq_uniform_loss.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * float nvq_uniform_loss(const float *vector, size_t length, float minValue, float maxValue, int nBits) + * } + */ + public static MethodHandle nvq_uniform_loss$handle() { + return nvq_uniform_loss.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * float nvq_uniform_loss(const float *vector, size_t length, float minValue, float maxValue, int nBits) + * } + */ + public static MemorySegment nvq_uniform_loss$address() { + return nvq_uniform_loss.ADDR; + } + + /** + * {@snippet lang=c : + * float nvq_uniform_loss(const float *vector, size_t length, float minValue, float maxValue, int nBits) + * } + */ + public static float nvq_uniform_loss(MemorySegment vector, long length, float minValue, float maxValue, int nBits) { + var mh$ = nvq_uniform_loss.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("nvq_uniform_loss", vector, length, minValue, maxValue, nBits); + } + return (float)mh$.invokeExact(vector, length, minValue, maxValue, nBits); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class nvq_growth_rate { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_INT + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("nvq_growth_rate"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * float nvq_growth_rate(const float *vector, size_t length, float minValue, float maxValue, int nBits) + * } + */ + public static FunctionDescriptor nvq_growth_rate$descriptor() { + return nvq_growth_rate.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * float nvq_growth_rate(const float *vector, size_t length, float minValue, float maxValue, int nBits) + * } + */ + public static MethodHandle nvq_growth_rate$handle() { + return nvq_growth_rate.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * float nvq_growth_rate(const float *vector, size_t length, float minValue, float maxValue, int nBits) + * } + */ + public static MemorySegment nvq_growth_rate$address() { + return nvq_growth_rate.ADDR; + } + + /** + * {@snippet lang=c : + * float nvq_growth_rate(const float *vector, size_t length, float minValue, float maxValue, int nBits) + * } + */ + public static float nvq_growth_rate(MemorySegment vector, long length, float minValue, float maxValue, int nBits) { + var mh$ = nvq_growth_rate.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("nvq_growth_rate", vector, length, minValue, maxValue, nBits); + } + return (float)mh$.invokeExact(vector, length, minValue, maxValue, nBits); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class nvq_square_l2_distance_8bit { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("nvq_square_l2_distance_8bit"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * float nvq_square_l2_distance_8bit(const float *vector, const unsigned char *quantized, size_t length, float alpha, float x0, float minValue, float maxValue) + * } + */ + public static FunctionDescriptor nvq_square_l2_distance_8bit$descriptor() { + return nvq_square_l2_distance_8bit.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * float nvq_square_l2_distance_8bit(const float *vector, const unsigned char *quantized, size_t length, float alpha, float x0, float minValue, float maxValue) + * } + */ + public static MethodHandle nvq_square_l2_distance_8bit$handle() { + return nvq_square_l2_distance_8bit.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * float nvq_square_l2_distance_8bit(const float *vector, const unsigned char *quantized, size_t length, float alpha, float x0, float minValue, float maxValue) + * } + */ + public static MemorySegment nvq_square_l2_distance_8bit$address() { + return nvq_square_l2_distance_8bit.ADDR; + } + + /** + * {@snippet lang=c : + * float nvq_square_l2_distance_8bit(const float *vector, const unsigned char *quantized, size_t length, float alpha, float x0, float minValue, float maxValue) + * } + */ + public static float nvq_square_l2_distance_8bit(MemorySegment vector, MemorySegment quantized, long length, float alpha, float x0, float minValue, float maxValue) { + var mh$ = nvq_square_l2_distance_8bit.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("nvq_square_l2_distance_8bit", vector, quantized, length, alpha, x0, minValue, maxValue); + } + return (float)mh$.invokeExact(vector, quantized, length, alpha, x0, minValue, maxValue); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class nvq_dot_product_8bit { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("nvq_dot_product_8bit"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * float nvq_dot_product_8bit(const float *vector, const unsigned char *quantized, size_t length, float alpha, float x0, float minValue, float maxValue) + * } + */ + public static FunctionDescriptor nvq_dot_product_8bit$descriptor() { + return nvq_dot_product_8bit.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * float nvq_dot_product_8bit(const float *vector, const unsigned char *quantized, size_t length, float alpha, float x0, float minValue, float maxValue) + * } + */ + public static MethodHandle nvq_dot_product_8bit$handle() { + return nvq_dot_product_8bit.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * float nvq_dot_product_8bit(const float *vector, const unsigned char *quantized, size_t length, float alpha, float x0, float minValue, float maxValue) + * } + */ + public static MemorySegment nvq_dot_product_8bit$address() { + return nvq_dot_product_8bit.ADDR; + } + + /** + * {@snippet lang=c : + * float nvq_dot_product_8bit(const float *vector, const unsigned char *quantized, size_t length, float alpha, float x0, float minValue, float maxValue) + * } + */ + public static float nvq_dot_product_8bit(MemorySegment vector, MemorySegment quantized, long length, float alpha, float x0, float minValue, float maxValue) { + var mh$ = nvq_dot_product_8bit.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("nvq_dot_product_8bit", vector, quantized, length, alpha, x0, minValue, maxValue); + } + return (float)mh$.invokeExact(vector, quantized, length, alpha, x0, minValue, maxValue); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class nvq_cosine_8bit_packed { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_LONG, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_FLOAT, + NativeSimdOps.C_POINTER + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("nvq_cosine_8bit_packed"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * int64_t nvq_cosine_8bit_packed(const float *vector, const unsigned char *quantized, size_t length, float alpha, float x0, float minValue, float maxValue, const float *centroid) + * } + */ + public static FunctionDescriptor nvq_cosine_8bit_packed$descriptor() { + return nvq_cosine_8bit_packed.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * int64_t nvq_cosine_8bit_packed(const float *vector, const unsigned char *quantized, size_t length, float alpha, float x0, float minValue, float maxValue, const float *centroid) + * } + */ + public static MethodHandle nvq_cosine_8bit_packed$handle() { + return nvq_cosine_8bit_packed.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * int64_t nvq_cosine_8bit_packed(const float *vector, const unsigned char *quantized, size_t length, float alpha, float x0, float minValue, float maxValue, const float *centroid) + * } + */ + public static MemorySegment nvq_cosine_8bit_packed$address() { + return nvq_cosine_8bit_packed.ADDR; + } + + /** + * {@snippet lang=c : + * int64_t nvq_cosine_8bit_packed(const float *vector, const unsigned char *quantized, size_t length, float alpha, float x0, float minValue, float maxValue, const float *centroid) + * } + */ + public static long nvq_cosine_8bit_packed(MemorySegment vector, MemorySegment quantized, long length, float alpha, float x0, float minValue, float maxValue, MemorySegment centroid) { + var mh$ = nvq_cosine_8bit_packed.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("nvq_cosine_8bit_packed", vector, quantized, length, alpha, x0, minValue, maxValue, centroid); + } + return (long)mh$.invokeExact(vector, quantized, length, alpha, x0, minValue, maxValue, centroid); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class nvq_shuffle_query_in_place_8bit { + public static final FunctionDescriptor DESC = FunctionDescriptor.ofVoid( + NativeSimdOps.C_POINTER, + NativeSimdOps.C_LONG + ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("nvq_shuffle_query_in_place_8bit"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * void nvq_shuffle_query_in_place_8bit(float *vector, size_t length) + * } + */ + public static FunctionDescriptor nvq_shuffle_query_in_place_8bit$descriptor() { + return nvq_shuffle_query_in_place_8bit.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * void nvq_shuffle_query_in_place_8bit(float *vector, size_t length) + * } + */ + public static MethodHandle nvq_shuffle_query_in_place_8bit$handle() { + return nvq_shuffle_query_in_place_8bit.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * void nvq_shuffle_query_in_place_8bit(float *vector, size_t length) + * } + */ + public static MemorySegment nvq_shuffle_query_in_place_8bit$address() { + return nvq_shuffle_query_in_place_8bit.ADDR; + } + + /** + * {@snippet lang=c : + * void nvq_shuffle_query_in_place_8bit(float *vector, size_t length) + * } + */ + public static void nvq_shuffle_query_in_place_8bit(MemorySegment vector, long length) { + var mh$ = nvq_shuffle_query_in_place_8bit.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("nvq_shuffle_query_in_place_8bit", vector, length); + } + mh$.invokeExact(vector, length); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class jvector_simd_get_active_isa { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_POINTER ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("jvector_simd_get_active_isa"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * const char *jvector_simd_get_active_isa() + * } + */ + public static FunctionDescriptor jvector_simd_get_active_isa$descriptor() { + return jvector_simd_get_active_isa.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * const char *jvector_simd_get_active_isa() + * } + */ + public static MethodHandle jvector_simd_get_active_isa$handle() { + return jvector_simd_get_active_isa.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * const char *jvector_simd_get_active_isa() + * } + */ + public static MemorySegment jvector_simd_get_active_isa$address() { + return jvector_simd_get_active_isa.ADDR; + } + + /** + * {@snippet lang=c : + * const char *jvector_simd_get_active_isa() + * } + */ + public static MemorySegment jvector_simd_get_active_isa() { + var mh$ = jvector_simd_get_active_isa.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("jvector_simd_get_active_isa"); + } + return (MemorySegment)mh$.invokeExact(); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + + private static class jvector_simd_get_max_isa_env { + public static final FunctionDescriptor DESC = FunctionDescriptor.of( + NativeSimdOps.C_POINTER ); + + public static final MemorySegment ADDR = NativeSimdOps.findOrThrow("jvector_simd_get_max_isa_env"); + + public static final MethodHandle HANDLE = Linker.nativeLinker().downcallHandle(ADDR, DESC, Linker.Option.critical(true)); + } + + /** + * Function descriptor for: + * {@snippet lang=c : + * const char *jvector_simd_get_max_isa_env() + * } + */ + public static FunctionDescriptor jvector_simd_get_max_isa_env$descriptor() { + return jvector_simd_get_max_isa_env.DESC; + } + + /** + * Downcall method handle for: + * {@snippet lang=c : + * const char *jvector_simd_get_max_isa_env() + * } + */ + public static MethodHandle jvector_simd_get_max_isa_env$handle() { + return jvector_simd_get_max_isa_env.HANDLE; + } + + /** + * Address for: + * {@snippet lang=c : + * const char *jvector_simd_get_max_isa_env() + * } + */ + public static MemorySegment jvector_simd_get_max_isa_env$address() { + return jvector_simd_get_max_isa_env.ADDR; + } + + /** + * {@snippet lang=c : + * const char *jvector_simd_get_max_isa_env() + * } + */ + public static MemorySegment jvector_simd_get_max_isa_env() { + var mh$ = jvector_simd_get_max_isa_env.HANDLE; + try { + if (TRACE_DOWNCALLS) { + traceDowncall("jvector_simd_get_max_isa_env"); + } + return (MemorySegment)mh$.invokeExact(); + } catch (Throwable ex$) { + throw new AssertionError("should not reach here", ex$); + } + } + private static final MemorySegment NULL = MemorySegment.ofAddress(0L); + /** + * {@snippet lang=c : + * #define NULL (void*) 0 + * } + */ + public static MemorySegment NULL() { + return NULL; + } + private static final long _POSIX_C_SOURCE = 200809L; + /** + * {@snippet lang=c : + * #define _POSIX_C_SOURCE 200809 + * } + */ + public static long _POSIX_C_SOURCE() { + return _POSIX_C_SOURCE; + } + private static final int __TIMESIZE = (int)64L; + /** + * {@snippet lang=c : + * #define __TIMESIZE 64 + * } + */ + public static int __TIMESIZE() { + return __TIMESIZE; + } + private static final long __STDC_IEC_60559_BFP__ = 201404L; + /** + * {@snippet lang=c : + * #define __STDC_IEC_60559_BFP__ 201404 + * } + */ + public static long __STDC_IEC_60559_BFP__() { + return __STDC_IEC_60559_BFP__; + } + private static final long __STDC_IEC_60559_COMPLEX__ = 201404L; + /** + * {@snippet lang=c : + * #define __STDC_IEC_60559_COMPLEX__ 201404 + * } + */ + public static long __STDC_IEC_60559_COMPLEX__() { + return __STDC_IEC_60559_COMPLEX__; + } + private static final long __STDC_ISO_10646__ = 201706L; + /** + * {@snippet lang=c : + * #define __STDC_ISO_10646__ 201706 + * } + */ + public static long __STDC_ISO_10646__() { + return __STDC_ISO_10646__; + } + private static final int __WCHAR_MAX = (int)2147483647L; + /** + * {@snippet lang=c : + * #define __WCHAR_MAX 2147483647 + * } + */ + public static int __WCHAR_MAX() { + return __WCHAR_MAX; + } + private static final int __WCHAR_MIN = (int)-2147483648L; + /** + * {@snippet lang=c : + * #define __WCHAR_MIN -2147483648 + * } + */ + public static int __WCHAR_MIN() { + return __WCHAR_MIN; + } + private static final int INT8_MIN = (int)-128L; + /** + * {@snippet lang=c : + * #define INT8_MIN -128 + * } + */ + public static int INT8_MIN() { + return INT8_MIN; + } + private static final int INT16_MIN = (int)-32768L; + /** + * {@snippet lang=c : + * #define INT16_MIN -32768 + * } + */ + public static int INT16_MIN() { + return INT16_MIN; + } + private static final int INT32_MIN = (int)-2147483648L; + /** + * {@snippet lang=c : + * #define INT32_MIN -2147483648 + * } + */ + public static int INT32_MIN() { + return INT32_MIN; + } + private static final long INT64_MIN = -9223372036854775808L; + /** + * {@snippet lang=c : + * #define INT64_MIN -9223372036854775808 + * } + */ + public static long INT64_MIN() { + return INT64_MIN; + } + private static final int INT8_MAX = (int)127L; + /** + * {@snippet lang=c : + * #define INT8_MAX 127 + * } + */ + public static int INT8_MAX() { + return INT8_MAX; + } + private static final int INT16_MAX = (int)32767L; + /** + * {@snippet lang=c : + * #define INT16_MAX 32767 + * } + */ + public static int INT16_MAX() { + return INT16_MAX; + } + private static final int INT32_MAX = (int)2147483647L; + /** + * {@snippet lang=c : + * #define INT32_MAX 2147483647 + * } + */ + public static int INT32_MAX() { + return INT32_MAX; + } + private static final long INT64_MAX = 9223372036854775807L; + /** + * {@snippet lang=c : + * #define INT64_MAX 9223372036854775807 + * } + */ + public static long INT64_MAX() { + return INT64_MAX; + } + private static final int UINT8_MAX = (int)255L; + /** + * {@snippet lang=c : + * #define UINT8_MAX 255 + * } + */ + public static int UINT8_MAX() { + return UINT8_MAX; + } + private static final int UINT16_MAX = (int)65535L; + /** + * {@snippet lang=c : + * #define UINT16_MAX 65535 + * } + */ + public static int UINT16_MAX() { + return UINT16_MAX; + } + private static final int UINT32_MAX = (int)4294967295L; + /** + * {@snippet lang=c : + * #define UINT32_MAX 4294967295 + * } + */ + public static int UINT32_MAX() { + return UINT32_MAX; + } + private static final long UINT64_MAX = -1L; + /** + * {@snippet lang=c : + * #define UINT64_MAX -1 + * } + */ + public static long UINT64_MAX() { + return UINT64_MAX; + } + private static final int INT_LEAST8_MIN = (int)-128L; + /** + * {@snippet lang=c : + * #define INT_LEAST8_MIN -128 + * } + */ + public static int INT_LEAST8_MIN() { + return INT_LEAST8_MIN; + } + private static final int INT_LEAST16_MIN = (int)-32768L; + /** + * {@snippet lang=c : + * #define INT_LEAST16_MIN -32768 + * } + */ + public static int INT_LEAST16_MIN() { + return INT_LEAST16_MIN; + } + private static final int INT_LEAST32_MIN = (int)-2147483648L; + /** + * {@snippet lang=c : + * #define INT_LEAST32_MIN -2147483648 + * } + */ + public static int INT_LEAST32_MIN() { + return INT_LEAST32_MIN; + } + private static final long INT_LEAST64_MIN = -9223372036854775808L; + /** + * {@snippet lang=c : + * #define INT_LEAST64_MIN -9223372036854775808 + * } + */ + public static long INT_LEAST64_MIN() { + return INT_LEAST64_MIN; + } + private static final int INT_LEAST8_MAX = (int)127L; + /** + * {@snippet lang=c : + * #define INT_LEAST8_MAX 127 + * } + */ + public static int INT_LEAST8_MAX() { + return INT_LEAST8_MAX; + } + private static final int INT_LEAST16_MAX = (int)32767L; + /** + * {@snippet lang=c : + * #define INT_LEAST16_MAX 32767 + * } + */ + public static int INT_LEAST16_MAX() { + return INT_LEAST16_MAX; + } + private static final int INT_LEAST32_MAX = (int)2147483647L; + /** + * {@snippet lang=c : + * #define INT_LEAST32_MAX 2147483647 + * } + */ + public static int INT_LEAST32_MAX() { + return INT_LEAST32_MAX; + } + private static final long INT_LEAST64_MAX = 9223372036854775807L; + /** + * {@snippet lang=c : + * #define INT_LEAST64_MAX 9223372036854775807 + * } + */ + public static long INT_LEAST64_MAX() { + return INT_LEAST64_MAX; + } + private static final int UINT_LEAST8_MAX = (int)255L; + /** + * {@snippet lang=c : + * #define UINT_LEAST8_MAX 255 + * } + */ + public static int UINT_LEAST8_MAX() { + return UINT_LEAST8_MAX; + } + private static final int UINT_LEAST16_MAX = (int)65535L; + /** + * {@snippet lang=c : + * #define UINT_LEAST16_MAX 65535 + * } + */ + public static int UINT_LEAST16_MAX() { + return UINT_LEAST16_MAX; + } + private static final int UINT_LEAST32_MAX = (int)4294967295L; + /** + * {@snippet lang=c : + * #define UINT_LEAST32_MAX 4294967295 + * } + */ + public static int UINT_LEAST32_MAX() { + return UINT_LEAST32_MAX; + } + private static final long UINT_LEAST64_MAX = -1L; + /** + * {@snippet lang=c : + * #define UINT_LEAST64_MAX -1 + * } + */ + public static long UINT_LEAST64_MAX() { + return UINT_LEAST64_MAX; + } + private static final int INT_FAST8_MIN = (int)-128L; + /** + * {@snippet lang=c : + * #define INT_FAST8_MIN -128 + * } + */ + public static int INT_FAST8_MIN() { + return INT_FAST8_MIN; + } + private static final long INT_FAST16_MIN = -9223372036854775808L; + /** + * {@snippet lang=c : + * #define INT_FAST16_MIN -9223372036854775808 + * } + */ + public static long INT_FAST16_MIN() { + return INT_FAST16_MIN; + } + private static final long INT_FAST32_MIN = -9223372036854775808L; + /** + * {@snippet lang=c : + * #define INT_FAST32_MIN -9223372036854775808 + * } + */ + public static long INT_FAST32_MIN() { + return INT_FAST32_MIN; + } + private static final long INT_FAST64_MIN = -9223372036854775808L; + /** + * {@snippet lang=c : + * #define INT_FAST64_MIN -9223372036854775808 + * } + */ + public static long INT_FAST64_MIN() { + return INT_FAST64_MIN; + } + private static final int INT_FAST8_MAX = (int)127L; + /** + * {@snippet lang=c : + * #define INT_FAST8_MAX 127 + * } + */ + public static int INT_FAST8_MAX() { + return INT_FAST8_MAX; + } + private static final long INT_FAST16_MAX = 9223372036854775807L; + /** + * {@snippet lang=c : + * #define INT_FAST16_MAX 9223372036854775807 + * } + */ + public static long INT_FAST16_MAX() { + return INT_FAST16_MAX; + } + private static final long INT_FAST32_MAX = 9223372036854775807L; + /** + * {@snippet lang=c : + * #define INT_FAST32_MAX 9223372036854775807 + * } + */ + public static long INT_FAST32_MAX() { + return INT_FAST32_MAX; + } + private static final long INT_FAST64_MAX = 9223372036854775807L; + /** + * {@snippet lang=c : + * #define INT_FAST64_MAX 9223372036854775807 + * } + */ + public static long INT_FAST64_MAX() { + return INT_FAST64_MAX; + } + private static final int UINT_FAST8_MAX = (int)255L; + /** + * {@snippet lang=c : + * #define UINT_FAST8_MAX 255 + * } + */ + public static int UINT_FAST8_MAX() { + return UINT_FAST8_MAX; + } + private static final long UINT_FAST16_MAX = -1L; + /** + * {@snippet lang=c : + * #define UINT_FAST16_MAX -1 + * } + */ + public static long UINT_FAST16_MAX() { + return UINT_FAST16_MAX; + } + private static final long UINT_FAST32_MAX = -1L; + /** + * {@snippet lang=c : + * #define UINT_FAST32_MAX -1 + * } + */ + public static long UINT_FAST32_MAX() { + return UINT_FAST32_MAX; + } + private static final long UINT_FAST64_MAX = -1L; + /** + * {@snippet lang=c : + * #define UINT_FAST64_MAX -1 + * } + */ + public static long UINT_FAST64_MAX() { + return UINT_FAST64_MAX; + } + private static final long INTPTR_MIN = -9223372036854775808L; + /** + * {@snippet lang=c : + * #define INTPTR_MIN -9223372036854775808 + * } + */ + public static long INTPTR_MIN() { + return INTPTR_MIN; + } + private static final long INTPTR_MAX = 9223372036854775807L; + /** + * {@snippet lang=c : + * #define INTPTR_MAX 9223372036854775807 + * } + */ + public static long INTPTR_MAX() { + return INTPTR_MAX; + } + private static final long UINTPTR_MAX = -1L; + /** + * {@snippet lang=c : + * #define UINTPTR_MAX -1 + * } + */ + public static long UINTPTR_MAX() { + return UINTPTR_MAX; + } + private static final long INTMAX_MIN = -9223372036854775808L; + /** + * {@snippet lang=c : + * #define INTMAX_MIN -9223372036854775808 + * } + */ + public static long INTMAX_MIN() { + return INTMAX_MIN; + } + private static final long INTMAX_MAX = 9223372036854775807L; + /** + * {@snippet lang=c : + * #define INTMAX_MAX 9223372036854775807 + * } + */ + public static long INTMAX_MAX() { + return INTMAX_MAX; + } + private static final long UINTMAX_MAX = -1L; + /** + * {@snippet lang=c : + * #define UINTMAX_MAX -1 + * } + */ + public static long UINTMAX_MAX() { + return UINTMAX_MAX; + } + private static final long PTRDIFF_MIN = -9223372036854775808L; + /** + * {@snippet lang=c : + * #define PTRDIFF_MIN -9223372036854775808 + * } + */ + public static long PTRDIFF_MIN() { + return PTRDIFF_MIN; + } + private static final long PTRDIFF_MAX = 9223372036854775807L; + /** + * {@snippet lang=c : + * #define PTRDIFF_MAX 9223372036854775807 + * } + */ + public static long PTRDIFF_MAX() { + return PTRDIFF_MAX; + } + private static final int SIG_ATOMIC_MIN = (int)-2147483648L; + /** + * {@snippet lang=c : + * #define SIG_ATOMIC_MIN -2147483648 + * } + */ + public static int SIG_ATOMIC_MIN() { + return SIG_ATOMIC_MIN; + } + private static final int SIG_ATOMIC_MAX = (int)2147483647L; + /** + * {@snippet lang=c : + * #define SIG_ATOMIC_MAX 2147483647 + * } + */ + public static int SIG_ATOMIC_MAX() { + return SIG_ATOMIC_MAX; + } + private static final long SIZE_MAX = -1L; + /** + * {@snippet lang=c : + * #define SIZE_MAX -1 + * } + */ + public static long SIZE_MAX() { + return SIZE_MAX; + } + private static final int WCHAR_MIN = (int)-2147483648L; + /** + * {@snippet lang=c : + * #define WCHAR_MIN -2147483648 + * } + */ + public static int WCHAR_MIN() { + return WCHAR_MIN; + } + private static final int WCHAR_MAX = (int)2147483647L; + /** + * {@snippet lang=c : + * #define WCHAR_MAX 2147483647 + * } + */ + public static int WCHAR_MAX() { + return WCHAR_MAX; + } + private static final int WINT_MIN = (int)0L; + /** + * {@snippet lang=c : + * #define WINT_MIN 0 + * } + */ + public static int WINT_MIN() { + return WINT_MIN; + } + private static final int WINT_MAX = (int)4294967295L; + /** + * {@snippet lang=c : + * #define WINT_MAX 4294967295 + * } + */ + public static int WINT_MAX() { + return WINT_MAX; + } +} diff --git a/native/src/main/java/org/humemai/nativelib/ArcadeLib.java b/native/src/main/java/org/humemai/nativelib/ArcadeLib.java new file mode 100644 index 0000000..2fa32f9 --- /dev/null +++ b/native/src/main/java/org/humemai/nativelib/ArcadeLib.java @@ -0,0 +1,153 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Feasibility spike: C entry points over the ArcadeDB embedded engine and server, for a + * GraalVM native-image shared library. Every string result is malloc'd and must be released + * with arcadedb_free. Every result is a JSON text: {"ok":true,"result":...} or {"ok":false,"error":"..."}. + */ +package org.humemai.nativelib; + +import com.arcadedb.ContextConfiguration; +import com.arcadedb.database.Database; +import com.arcadedb.database.DatabaseFactory; +import com.arcadedb.query.sql.executor.Result; +import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.serializer.json.JSONArray; +import com.arcadedb.serializer.json.JSONObject; +import com.arcadedb.server.ArcadeDBServer; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.UnmanagedMemory; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.type.CCharPointer; +import org.graalvm.nativeimage.c.type.CTypeConversion; +import org.graalvm.word.PointerBase; +import org.graalvm.word.WordFactory; + +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +public final class ArcadeLib { + private static final Map DATABASES = new ConcurrentHashMap<>(); + private static final AtomicLong NEXT = new AtomicLong(1); + private static volatile ArcadeDBServer server; + + private ArcadeLib() { + } + + /** Opens (creating if absent) the database at path. Returns a handle, or 0 with the error in the result of arcadedb_last_error. */ + @CEntryPoint(name = "arcadedb_open") + public static long open(final IsolateThread thread, final CCharPointer path) { + try { + final DatabaseFactory factory = new DatabaseFactory(CTypeConversion.toJavaString(path)); + final Database db = factory.exists() ? factory.open() : factory.create(); + final long handle = NEXT.getAndIncrement(); + DATABASES.put(handle, db); + return handle; + } catch (final Throwable t) { + lastError = describe(t); + return 0; + } + } + + private static volatile String lastError = ""; + + @CEntryPoint(name = "arcadedb_last_error") + public static CCharPointer lastError(final IsolateThread thread) { + return toC(lastError); + } + + /** Runs one command (any language: sql, sqlscript, opencypher...) and returns its rows as JSON. */ + @CEntryPoint(name = "arcadedb_command") + public static CCharPointer command(final IsolateThread thread, final long handle, final CCharPointer language, + final CCharPointer text, final CCharPointer paramsJson) { + try { + final Database db = DATABASES.get(handle); + if (db == null) + throw new IllegalArgumentException("unknown handle " + handle); + final String paramsText = CTypeConversion.toJavaString(paramsJson); + final Map params = paramsText == null || paramsText.isEmpty() ? Map.of() : new JSONObject(paramsText).toMap(); + final JSONArray rows = new JSONArray(); + try (final ResultSet rs = db.command(CTypeConversion.toJavaString(language), CTypeConversion.toJavaString(text), params)) { + while (rs.hasNext()) { + final Result r = rs.next(); + rows.put(r.toJSON()); + } + } + return toC(new JSONObject().put("ok", true).put("result", rows).toString()); + } catch (final Throwable t) { + return toC(new JSONObject().put("ok", false).put("error", describe(t)).toString()); + } + } + + @CEntryPoint(name = "arcadedb_close") + public static int close(final IsolateThread thread, final long handle) { + try { + final Database db = DATABASES.remove(handle); + if (db == null) + return 1; + db.close(); + return 0; + } catch (final Throwable t) { + lastError = describe(t); + return 2; + } + } + + /** Starts the in-process HTTP server (rootPath holds databases/). Returns 0 on success. */ + @CEntryPoint(name = "arcadedb_server_start") + public static int serverStart(final IsolateThread thread, final CCharPointer rootPath, final int httpPort, + final CCharPointer rootPassword) { + try { + final ContextConfiguration config = new ContextConfiguration(); + config.setValue("arcadedb.server.rootPath", CTypeConversion.toJavaString(rootPath)); + config.setValue("arcadedb.server.rootPassword", CTypeConversion.toJavaString(rootPassword)); + config.setValue("arcadedb.server.mode", "development"); + config.setValue("arcadedb.server.httpIncomingHost", "127.0.0.1"); + config.setValue("arcadedb.server.httpIncomingPort", httpPort); + server = new ArcadeDBServer(config); + server.start(); + return 0; + } catch (final Throwable t) { + lastError = describe(t); + return 1; + } + } + + @CEntryPoint(name = "arcadedb_server_stop") + public static int serverStop(final IsolateThread thread) { + try { + if (server != null) + server.stop(); + server = null; + return 0; + } catch (final Throwable t) { + lastError = describe(t); + return 1; + } + } + + @CEntryPoint(name = "arcadedb_free") + public static void free(final IsolateThread thread, final PointerBase ptr) { + if (ptr.isNonNull()) + UnmanagedMemory.free(ptr); + } + + private static String describe(final Throwable t) { + final StringBuilder sb = new StringBuilder(t.getClass().getName()).append(": ").append(t.getMessage()); + Throwable c = t.getCause(); + for (int i = 0; c != null && i < 3; i++, c = c.getCause()) + sb.append(" <- ").append(c.getClass().getName()).append(": ").append(c.getMessage()); + return sb.toString(); + } + + private static CCharPointer toC(final String s) { + final byte[] b = s.getBytes(StandardCharsets.UTF_8); + final CCharPointer p = UnmanagedMemory.malloc(b.length + 1); + for (int i = 0; i < b.length; i++) + p.write(i, b[i]); + p.write(b.length, (byte) 0); + return p; + } +} diff --git a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java new file mode 100644 index 0000000..49a102f --- /dev/null +++ b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java @@ -0,0 +1,691 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Phase 2: the C entry points behind the Python native backend (arcadedb/_native). Every function returns a + * malloc'd UTF-8 JSON text that the caller releases with arcadedb_free: {"ok":true,...} or + * {"ok":false,"error":msg,"cls":java class,"chain":[[class,msg],..],"retry":"need_retry"|"duplicated_key"|null,"retry_ms":n}. + * Databases, result sets and servers are long handles. Any thread attached to the isolate may call any function; a + * transaction belongs to the (Java) thread that began it, as in the embedded API. + */ +package org.humemai.nativelib; + +import com.arcadedb.ContextConfiguration; +import com.arcadedb.GlobalConfiguration; +import com.arcadedb.database.Database; +import com.arcadedb.database.DatabaseFactory; +import com.arcadedb.database.DatabaseInternal; +import com.arcadedb.database.LocalDatabase; +import com.arcadedb.database.MutableDocument; +import com.arcadedb.database.TransactionContext; +import com.arcadedb.engine.ComponentFile; +import com.arcadedb.exception.DuplicatedKeyException; +import com.arcadedb.exception.NeedRetryException; +import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.schema.VertexType; +import com.arcadedb.server.ArcadeDBServer; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.UnmanagedMemory; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.type.CCharPointer; +import org.graalvm.nativeimage.c.type.CFloatPointer; +import org.graalvm.nativeimage.c.type.CLongPointer; +import org.graalvm.word.WordFactory; +import org.graalvm.nativeimage.c.type.CTypeConversion; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +public final class ArcadeNative { + static final Map DBS = new ConcurrentHashMap<>(); + private static final Map RESULT_SETS = new ConcurrentHashMap<>(); + private static final Map SERVERS = new ConcurrentHashMap<>(); + static final AtomicLong IDS = new AtomicLong(1); + + static { + // The image carries no GraalVM polyglot languages by design, so the engine's polyglot registration can only find none and + // would log "GraalVM Polyglot Engine: no languages found" at WARNING in every process (#477). Turn the engine off before the + // first database is opened (QueryEngineManager reads this when it first loads); no other warning is touched. + GlobalConfiguration.POLYGLOT_ENGINE_ENABLED.setValue(false); + } + + static { + // ARCADEDB_VECTOR_PROVIDER = native | panama | default (jvector's VectorizationProvider, forced); unset: jvector chooses + // (Highway through the FFM downcalls when the bundled libjvector loads, else the Vector API classes). + final String forced = System.getenv("ARCADEDB_VECTOR_PROVIDER"); + if (forced != null && !forced.isEmpty() && System.getProperty("jvector.vectorization_provider") == null) + System.setProperty("jvector.vectorization_provider", forced); + } + + private ArcadeNative() { + } + + // ---- phase 6: stack-sampling profiler (ARCADEDB_NATIVE_SAMPLE=, a CPU profile of the image without perf/JFR). A daemon thread + // takes Thread.getAllStackTraces every , counts the top frame per runnable thread ("self") and every distinct frame in the top + // 40 ("incl"); arcadedb_n_sample_dump returns the counts as JSON. Safepoint-biased, like any Java sampler: a direction, not a measurement. + private static final Map SELF = new ConcurrentHashMap<>(), INCL = new ConcurrentHashMap<>(); + private static volatile long samples; + + static { + final String ms = System.getenv("ARCADEDB_NATIVE_SAMPLE"); + if (ms != null && !ms.isEmpty()) { + final long every = Long.parseLong(ms); + final Thread t = new Thread(() -> { + final Thread me = Thread.currentThread(); + while (true) { + try { + Thread.sleep(every); + for (final Map.Entry e : Thread.getAllStackTraces().entrySet()) { + final StackTraceElement[] st = e.getValue(); + if (e.getKey() == me || st.length == 0 || e.getKey().getState() != Thread.State.RUNNABLE) + continue; + SELF.computeIfAbsent(st[0].getClassName() + "." + st[0].getMethodName(), k -> new long[1])[0]++; + final java.util.HashSet seen = new java.util.HashSet<>(); + for (int i = 0; i < st.length && i < 40; i++) + if (seen.add(st[i].getClassName() + "." + st[i].getMethodName())) + INCL.computeIfAbsent(st[i].getClassName() + "." + st[i].getMethodName(), k -> new long[1])[0]++; + samples++; + } + } catch (final Throwable ignored) { + // keep sampling + } + } + }, "native-sampler"); + t.setDaemon(true); + t.start(); + } + } + + @CEntryPoint(name = "arcadedb_n_sample_dump") + public static CCharPointer sampleDump(final IsolateThread thread) { + final StringBuilder sb = new StringBuilder("{\"ok\":true,\"samples\":").append(samples); + for (final String which : new String[] { "self", "incl" }) { + sb.append(",\"").append(which).append("\":["); + final java.util.List> l = new java.util.ArrayList<>((which.equals("self") ? SELF : INCL).entrySet()); + l.sort((a, b) -> Long.compare(b.getValue()[0], a.getValue()[0])); + for (int i = 0; i < l.size() && i < 70; i++) { + if (i > 0) + sb.append(','); + sb.append('['); + NativeCodec.quote(sb, l.get(i).getKey()); + sb.append(',').append(l.get(i).getValue()[0]).append(']'); + } + sb.append(']'); + } + return out(sb.append('}').toString()); + } + + // ---- databases + + /** mode 0 create, 1 open read-write, 2 open read-only, 3 exists. */ + @CEntryPoint(name = "arcadedb_n_open") + public static CCharPointer open(final IsolateThread thread, final CCharPointer path, final int mode) { + try { + final DatabaseFactory factory = new DatabaseFactory(str(path)); + if (mode == 3) + return out("{\"ok\":true,\"exists\":" + factory.exists() + "}"); + final Database db = switch (mode) { + case 0 -> factory.create(); + case 1 -> factory.open(); + default -> factory.open(ComponentFile.MODE.READ_ONLY); + }; + return out(registered(db)); + } catch (final Throwable t) { + return err(t); + } + } + + private static String registered(final Database db) { + final long h = IDS.getAndIncrement(); + DBS.put(h, db); + return "{\"ok\":true,\"h\":" + h + "}"; + } + + /** drop 0: close, 1: drop (close and delete the files). A database owned by a server reports its refusal. */ + @CEntryPoint(name = "arcadedb_n_close") + public static CCharPointer close(final IsolateThread thread, final long h, final int drop) { + try { + final Database db = db(h); + if (drop != 0 && db instanceof LocalDatabase local) + local.drop(); + else + db.close(); + DBS.remove(h); + return OK(); + } catch (final Throwable t) { + return err(t); + } + } + + @CEntryPoint(name = "arcadedb_n_info") + public static CCharPointer info(final IsolateThread thread, final long h) { + try { + final Database db = db(h); + final StringBuilder sb = new StringBuilder("{\"ok\":true,\"open\":").append(db.isOpen()); + if (db.isOpen()) { + sb.append(",\"name\":"); + NativeCodec.quote(sb, db.getName()); + sb.append(",\"path\":"); + NativeCodec.quote(sb, db.getDatabasePath()); + sb.append(",\"mode\":"); + NativeCodec.quote(sb, String.valueOf(db.getMode())); + } + return out(sb.append('}').toString()); + } catch (final Throwable t) { + return err(t); + } + } + + /** op 0 begin, 1 commit, 2 rollback, 3 mark. Every reply carries {"active":b,"depth":n,"commits":n}. */ + @CEntryPoint(name = "arcadedb_n_tx") + public static CCharPointer tx(final IsolateThread thread, final long h, final int op) { + try { + final Database db = db(h); + switch (op) { + case 0 -> db.begin(); + case 1 -> db.commit(); + case 2 -> db.rollback(); + default -> { + } + } + return out(txState(db)); + } catch (final Throwable t) { + return err(t); + } + } + + private static String txState(final Database db) { + int depth = 0; + long commits = 0; + if (db instanceof LocalDatabase local) + depth = local.getNestedTransactions(); + if (db instanceof DatabaseInternal internal) { + final TransactionContext ctx = internal.getTransactionIfExists(); + if (ctx != null) + commits = ctx.getCommitCount(); + } + return "{\"ok\":true,\"active\":" + db.isTransactionActive() + ",\"depth\":" + depth + ",\"commits\":" + commits + "}"; + } + + // ---- statements + + /** + * flags bit 0: command (else query). params: "" or {"named":{..}} or {"pos":[..]} (tagged values, see Json). Runs the + * statement and returns its first batch of up to max rows; {"rs":id} is the open cursor (0 once drained and closed). + */ + @CEntryPoint(name = "arcadedb_n_exec") + public static CCharPointer exec(final IsolateThread thread, final long h, final int flags, final CCharPointer language, + final CCharPointer text, final CCharPointer params, final int max) { + return execImpl(h, flags, language, text, params, max, WordFactory.nullPointer(), 0); + } + + /** Same as arcadedb_n_exec plus a float32 side buffer (pointer + count of floats) that params values {"\u0001":["B",offset,count]} read from. */ + @CEntryPoint(name = "arcadedb_n_exec_bin") + public static CCharPointer execBin(final IsolateThread thread, final long h, final int flags, final CCharPointer language, + final CCharPointer text, final CCharPointer params, final int max, final CFloatPointer buf, final long bufFloats) { + return execImpl(h, flags, language, text, params, max, buf, bufFloats); + } + + /** Timers (ns) of the exec path, read through arcadedb_n_diag: decode of the text arguments, JSON parse, engine call, reply encoding. */ + static final long[] T = new long[5]; + + @SuppressWarnings("unchecked") + private static CCharPointer execImpl(final long h, final int flags, final CCharPointer language, final CCharPointer text, + final CCharPointer params, final int max, final CFloatPointer buf, final long bufFloats) { + ResultSet rs = null; + try { + final long t0 = System.nanoTime(); + final Database db = db(h); + final String lang = str(language); + final String sql = str(text); + final String p = str(params); + final boolean command = (flags & 1) != 0; + final long t1 = System.nanoTime(); + Map named = null; + Object[] pos = null; + if (p != null && !p.isEmpty()) { + final Map env = (Map) Json.parse(p, buf.rawValue(), bufFloats); + final Object n = env.get("named"); + if (n instanceof Map m) + named = (Map) m; + else + pos = ((List) env.get("pos")).toArray(); + } + final long t2 = System.nanoTime(); + if (named != null) + rs = command ? db.command(lang, sql, named) : db.query(lang, sql, named); + else if (pos != null) + rs = command ? db.command(lang, sql, pos) : db.query(lang, sql, pos); + else + rs = command ? db.command(lang, sql) : db.query(lang, sql); + final long t3 = System.nanoTime(); + final CCharPointer o = replyOf(rs, max, flags); + final long t4 = System.nanoTime(); + T[0]++; + T[1] += t1 - t0; + T[2] += t2 - t1; + T[3] += t3 - t2; + T[4] += t4 - t3; + return o; + } catch (final Throwable t) { + closeQuietly(rs); + return err(t); + } + } + + @CEntryPoint(name = "arcadedb_n_next") + public static CCharPointer next(final IsolateThread thread, final long rsId, final int max) { + ResultSet rs = null; + try { + rs = RESULT_SETS.get(rsId); + if (rs == null) + throw new IllegalArgumentException("unknown result set " + rsId); + return out(batchOf(rsId, rs, max)); + } catch (final Throwable t) { + RESULT_SETS.remove(rsId); + closeQuietly(rs); + return err(t); + } + } + + @CEntryPoint(name = "arcadedb_n_rs_close") + public static CCharPointer rsClose(final IsolateThread thread, final long rsId) { + closeQuietly(RESULT_SETS.remove(rsId)); + return OK(); + } + + private static String batch(final ResultSet rs, final int max) { + if (rs == null) + return "{\"ok\":true,\"rs\":0,\"rows\":[],\"meta\":[],\"drained\":true}"; + final long id = IDS.getAndIncrement(); + RESULT_SETS.put(id, rs); + return batchOf(id, rs, max); + } + + private static String batchOf(final long id, final ResultSet rs, final int max) { + final StringBuilder sb = new StringBuilder("{\"ok\":true,\"rs\":"); + final int mark = sb.length(); + sb.append(id).append(','); + final boolean drained = NativeCodec.appendBatch(sb, rs, max); + if (drained) { + RESULT_SETS.remove(id); + closeQuietly(rs); + sb.replace(mark, mark + String.valueOf(id).length(), "0"); + } + return sb.append('}').toString(); + } + + private static void closeQuietly(final ResultSet rs) { + if (rs != null) + try { + rs.close(); + } catch (final Throwable ignored) { + // closing a failed cursor must not hide the error that is being reported + } + } + + // ---- server + + /** + * config: {"settings":{"arcadedb.x.y":value,..},"host":"localhost"}: every listener host setting that defaults to all + * interfaces and is absent from settings is set to host (the bindings never expose a listener unasked). + */ + @CEntryPoint(name = "arcadedb_n_server_start") + @SuppressWarnings("unchecked") + public static CCharPointer serverStart(final IsolateThread thread, final CCharPointer configJson) { + ArcadeDBServer server = null; + try { + final Map cfg = (Map) Json.parse(str(configJson)); + final Map settings = (Map) cfg.get("settings"); + final String host = (String) cfg.get("host"); + final ContextConfiguration config = new ContextConfiguration(); + if (host != null) + for (final GlobalConfiguration setting : GlobalConfiguration.values()) { + final String key = setting.getKey(); + final Object def = setting.getDefValue(); + if (key.endsWith(".host") && ("0.0.0.0".equals(def) || "::".equals(def)) && !settings.containsKey(key)) + config.setValue(key, host); + } + for (final Map.Entry e : settings.entrySet()) + config.setValue(e.getKey(), e.getValue()); + server = new ArcadeDBServer(config); + server.start(); + final long id = IDS.getAndIncrement(); + SERVERS.put(id, server); + return out("{\"ok\":true,\"h\":" + id + "}"); + } catch (final Throwable t) { + if (server != null) + try { + server.stop(); + } catch (final Throwable ignored) { + // the start error is what the caller needs + } + return err(t); + } + } + + @CEntryPoint(name = "arcadedb_n_server_stop") + public static CCharPointer serverStop(final IsolateThread thread, final long id) { + try { + final ArcadeDBServer server = SERVERS.remove(id); + if (server != null) + server.stop(); + return OK(); + } catch (final Throwable t) { + return err(t); + } + } + + /** create 0: getDatabase(name); 1: getDatabase(name, true, true). */ + @CEntryPoint(name = "arcadedb_n_server_db") + public static CCharPointer serverDb(final IsolateThread thread, final long id, final CCharPointer name, final int create) { + try { + final ArcadeDBServer server = SERVERS.get(id); + if (server == null) + throw new IllegalArgumentException("unknown server " + id); + final Database db = create != 0 ? server.getDatabase(str(name), true, true) : server.getDatabase(str(name)); + return out(registered(db)); + } catch (final Throwable t) { + return err(t); + } + } + + /** Which vector math the image runs: jvector's VectorizationProvider and VectorUtilSupport classes (no logging needed). */ + @CEntryPoint(name = "arcadedb_n_diag") + public static CCharPointer diag(final IsolateThread thread) { + try { + final StringBuilder sb = new StringBuilder("{\"ok\":true,\"java\":"); + NativeCodec.quote(sb, Runtime.version().toString()); + sb.append(",\"vector_module\":").append(ModuleLayer.boot().findModule("jdk.incubator.vector").isPresent()); + sb.append(",\"cpus\":").append(Runtime.getRuntime().availableProcessors()); + sb.append(",\"max_heap\":").append(Runtime.getRuntime().maxMemory()); + sb.append(",\"exec_timers_ns\":{\"calls\":").append(T[0]).append(",\"args\":").append(T[1]).append(",\"json\":").append(T[2]) + .append(",\"engine\":").append(T[3]).append(",\"reply\":").append(T[4]).append('}'); + final io.github.jbellis.jvector.vector.VectorizationProvider vp = io.github.jbellis.jvector.vector.VectorizationProvider.getInstance(); + sb.append(",\"provider\":"); + NativeCodec.quote(sb, vp.getClass().getName()); + sb.append(",\"util_support\":"); + NativeCodec.quote(sb, vp.getVectorUtilSupport().getClass().getName()); + sb.append(",\"isa\":"); + String isa = null; + try { + final Object support = vp.getVectorUtilSupport(); + isa = (String) support.getClass().getMethod("getActiveIsa").invoke(support); + } catch (final ReflectiveOperationException | RuntimeException e) { + // the Panama and default providers have no ISA tier + } + NativeCodec.quote(sb, String.valueOf(isa)); + return out(sb.append('}').toString()); + } catch (final Throwable t) { + return err(t); + } + } + + // ---- plumbing + + static Database db(final long h) { + final Database db = DBS.get(h); + if (db == null) + throw new IllegalArgumentException("unknown database handle " + h); + return db; + } + + static String str(final CCharPointer p) { + return CTypeConversion.toJavaString(p); + } + + static CCharPointer OK() { + return out("{\"ok\":true}"); + } + + static CCharPointer err(final Throwable t) { + return out(errorJson(t)); + } + + static String errorJson(final Throwable t) { + final StringBuilder sb = new StringBuilder("{\"ok\":false,\"error\":"); + NativeCodec.quote(sb, String.valueOf(t.getMessage())); + sb.append(",\"cls\":"); + NativeCodec.quote(sb, t.getClass().getName()); + sb.append(",\"chain\":["); + String retry = null; + long retryMs = 0; + Throwable c = t; + for (int i = 0; c != null && i < 16; i++, c = c.getCause()) { + if (i > 0) + sb.append(','); + sb.append('['); + NativeCodec.quote(sb, c.getClass().getName()); + sb.append(','); + NativeCodec.quote(sb, String.valueOf(c.getMessage())); + sb.append(']'); + if (retry == null && c instanceof NeedRetryException nre) { + retry = "need_retry"; + retryMs = nre.getRetryAfterMs(); + } else if (retry == null && c instanceof DuplicatedKeyException) + retry = "duplicated_key"; + } + sb.append("],\"retry\":").append(retry == null ? "null" : "\"" + retry + "\"").append(",\"retry_ms\":").append(retryMs).append('}'); + return sb.toString(); + } + + // ---- phase 7: lazy command rows, binary float reply, columnar insert (small and separate: BulkOps, Prefetched, NativeCodec.SINK) + + /** + * flags bit 1 (2): the statement has run and its first rows are pulled, but not encoded; arcadedb_n_next encodes them when + * read. Bit 2 (4): float[] values of 8 or more elements in the reply go to a float32 tail of the reply ({"\u0001":["B",off,n]}, + * the JSON ends with ,"bin":total and the floats follow the NUL, 4-byte aligned) instead of number text. + */ + private static CCharPointer replyOf(final ResultSet rs, final int max, final int flags) { + if ((flags & 2) != 0 && rs != null) { + final ResultSet lazy = new Prefetched(rs, max); + final long id = IDS.getAndIncrement(); + RESULT_SETS.put(id, lazy); + return out("{\"ok\":true,\"rs\":" + id + ",\"rows\":[],\"meta\":[],\"drained\":false}"); + } + if ((flags & 4) == 0) + return out(batch(rs, max)); + final NativeCodec.FloatSink sink = new NativeCodec.FloatSink(); + NativeCodec.SINK.set(sink); + final String reply; + try { + reply = batch(rs, max); + } finally { + NativeCodec.SINK.remove(); + } + if (sink.n == 0) + return out(reply); + final byte[] b = (reply.substring(0, reply.length() - 1) + ",\"bin\":" + sink.n + "}").getBytes(StandardCharsets.UTF_8); + final int off = (b.length + 1 + 3) & ~3; + final CCharPointer p = UnmanagedMemory.malloc(off + 4 * sink.n); + for (int i = 0; i < b.length; i++) + p.write(i, b[i]); + for (int i = b.length; i < off; i++) + p.write(i, (byte) 0); + final CFloatPointer fp = (CFloatPointer) ((org.graalvm.word.Pointer) p).add(off); + for (int i = 0; i < sink.n; i++) + fp.write(i, sink.a[i]); + return p; + } + + /** Columnar insert (see BulkOps): header JSON, one buffer holding the columns. Returns {"ok":true,"n":rows}. */ + @CEntryPoint(name = "arcadedb_n_insert_cols") + public static CCharPointer insertCols(final IsolateThread thread, final long h, final CCharPointer type, final CCharPointer header, + final CCharPointer buf, final long bufBytes) { + try { + final java.nio.ByteBuffer bb = CTypeConversion.asByteBuffer(buf, (int) bufBytes); + return out("{\"ok\":true,\"n\":" + BulkOps.insertColumns(db(h), str(type), str(header), bb) + "}"); + } catch (final Throwable t) { + return err(t); + } + } + + // ---- phase 8: fewer crossings and no reply text for a command whose rows are read late or never; batches of statements and rows per crossing + + private static final ThreadLocal LAST_ERROR = new ThreadLocal<>(); + + /** + * A command (flags bit 0 selects command, else query) whose first rows are pulled but not encoded (as flags bit 1 of arcadedb_n_exec). + * Returns the cursor id (> 0, read with arcadedb_n_next, closed with arcadedb_n_rs_close or arcadedb_n_rs_close_many), 0 when the + * statement gave no cursor, -1 on error: the error text is then read with arcadedb_n_last_error on the same thread. No reply text, so + * no malloc, no arcadedb_free and no JSON decode in the caller. + */ + @CEntryPoint(name = "arcadedb_n_cmd") + @SuppressWarnings("unchecked") + public static long cmd(final IsolateThread thread, final long h, final int flags, final CCharPointer language, final CCharPointer text, + final CCharPointer params, final CFloatPointer buf, final long bufFloats, final int max) { + ResultSet rs = null; + try { + final long t0 = System.nanoTime(); + final Database db = db(h); + final String lang = str(language); + final String sql = str(text); + final String p = str(params); + final boolean command = (flags & 1) != 0; + final long t1 = System.nanoTime(); + Map named = null; + Object[] pos = null; + if (p != null && !p.isEmpty()) { + final Map env = (Map) Json.parse(p, buf.rawValue(), bufFloats); + final Object n = env.get("named"); + if (n instanceof Map m) + named = (Map) m; + else + pos = ((List) env.get("pos")).toArray(); + } + final long t2 = System.nanoTime(); + if (named != null) + rs = command ? db.command(lang, sql, named) : db.query(lang, sql, named); + else if (pos != null) + rs = command ? db.command(lang, sql, pos) : db.query(lang, sql, pos); + else + rs = command ? db.command(lang, sql) : db.query(lang, sql); + final long t3 = System.nanoTime(); + long id = 0; + if (rs != null) { + id = IDS.getAndIncrement(); + RESULT_SETS.put(id, new Prefetched(rs, max)); + } + final long t4 = System.nanoTime(); + T[0]++; + T[1] += t1 - t0; + T[2] += t2 - t1; + T[3] += t3 - t2; + T[4] += t4 - t3; + return id; + } catch (final Throwable t) { + closeQuietly(rs); + LAST_ERROR.set(errorJson(t)); + return -1; + } + } + + /** The error of the last failed arcadedb_n_cmd on this thread, in the usual {"ok":false,...} form. */ + @CEntryPoint(name = "arcadedb_n_last_error") + public static CCharPointer lastError(final IsolateThread thread) { + final String e = LAST_ERROR.get(); + return out(e != null ? e : "{\"ok\":false,\"error\":\"no error recorded\",\"cls\":\"java.lang.IllegalStateException\",\"chain\":[],\"retry\":null,\"retry_ms\":0}"); + } + + /** Closes n cursors in one crossing (ids is an array of n longs). */ + @CEntryPoint(name = "arcadedb_n_rs_close_many") + public static void rsCloseMany(final IsolateThread thread, final CLongPointer ids, final int n) { + try { + for (int i = 0; i < n; i++) + closeQuietly(RESULT_SETS.remove(ids.read(i))); + } catch (final Throwable ignored) { + // closing is best effort, as in arcadedb_n_rs_close + } + } + + /** + * One statement run once per row of rowsJson, all in this crossing and in the caller's transaction (none is opened here). rowsJson is + * [[positional values],..] or [{named},..] (tagged values, see Json). The cursors are drained and closed. On a failure the reply is the + * usual error plus "at": the index of the failed row; the rows before it stay done. + */ + @CEntryPoint(name = "arcadedb_n_exec_many") + @SuppressWarnings("unchecked") + public static CCharPointer execMany(final IsolateThread thread, final long h, final CCharPointer language, final CCharPointer text, + final CCharPointer rowsJson) { + int i = 0; + try { + final Database db = db(h); + final String lang = str(language); + final String sql = str(text); + final List rows = (List) Json.parse(str(rowsJson)); + for (; i < rows.size(); i++) { + final Object row = rows.get(i); + final ResultSet rs = row instanceof Map m ? db.command(lang, sql, (Map) m) + : db.command(lang, sql, ((List) row).toArray()); + if (rs != null) { + while (rs.hasNext()) + rs.next(); + rs.close(); + } + } + return out("{\"ok\":true,\"n\":" + rows.size() + "}"); + } catch (final Throwable t) { + final String e = errorJson(t); + return out(e.substring(0, e.length() - 1) + ",\"at\":" + i + "}"); + } + } + + /** + * Documents (or vertices of a vertex type) from rows [{property: value},..], one record per row through newDocument/newVertex, set per + * property and save, as DocumentBatcher.insertManyJson does on the JPype path; no transaction is opened or closed here. + */ + @CEntryPoint(name = "arcadedb_n_insert_rows") + public static CCharPointer insertRows(final IsolateThread thread, final long h, final CCharPointer type, final CCharPointer rowsJson) { + try { + final Database db = db(h); + final String t = str(type); + final Chars c = chars(rowsJson); + return out("{\"ok\":true,\"n\":" + BulkOps.insertRows(db, t, c.b, c.n) + "}"); + } catch (final Throwable t) { + return err(t); + } + } + + /** The bytes of a C string in a per-thread scratch array (grown on demand, zero-terminated at n): no String and no new array per call. */ + static final class Chars { + byte[] b = new byte[1 << 16]; + int n; + } + + private static final int SCRATCH_KEEP = 32 << 20; // a payload above this is read into a one-off array, not kept by the thread + private static final ThreadLocal CHARS = ThreadLocal.withInitial(Chars::new); + + static Chars chars(final CCharPointer p) { + int n = 0; + while (p.read(n) != 0) + n++; + final Chars c = CHARS.get(); + byte[] b = c.b; + if (b.length <= n) { + b = new byte[Math.max(n + 1, b.length * 2)]; + if (b.length <= SCRATCH_KEEP) + c.b = b; + } + CTypeConversion.asByteBuffer(p, n).get(b, 0, n); + b[n] = 0; + if (b != c.b) { + final Chars once = new Chars(); + once.b = b; + once.n = n; + return once; + } + c.n = n; + return c; + } + + static CCharPointer out(final String s) { + final byte[] b = s.getBytes(StandardCharsets.UTF_8); + final CCharPointer p = UnmanagedMemory.malloc(b.length + 1); + for (int i = 0; i < b.length; i++) + p.write(i, b[i]); + p.write(b.length, (byte) 0); + return p; + } +} diff --git a/native/src/main/java/org/humemai/nativelib/BulkOps.java b/native/src/main/java/org/humemai/nativelib/BulkOps.java new file mode 100644 index 0000000..a3042bb --- /dev/null +++ b/native/src/main/java/org/humemai/nativelib/BulkOps.java @@ -0,0 +1,132 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Phase 7: columnar bulk insert through one crossing. The documents are built here from whole columns that Python lays out + * in one native-endian buffer (int64, float64, bool bytes, float32 vector matrix), as DocumentBatcher.insertColumns does on + * the JPype path from long[], double[], boolean[] and Object[] columns. + */ +package org.humemai.nativelib; + +import com.arcadedb.database.Database; +import com.arcadedb.database.MutableDocument; +import com.arcadedb.schema.VertexType; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.List; +import java.util.Map; + +final class BulkOps { + private BulkOps() { + } + + /** + * header: {"n":rows,"commit":every,"cols":[[name,kind,dim],..]} with kind q (int64), d (float64), b (bool byte) or f + * (float32 matrix, dim floats per row); the buffer holds the columns in that order. Same transaction contract as + * DocumentBatcher.insertColumns: batch commits and the rollback on failure apply only to a transaction opened here. + */ + @SuppressWarnings("unchecked") + static long insertColumns(final Database db, final String type, final String header, final ByteBuffer raw) { + final Map h = (Map) Json.parse(header); + final int n = ((Number) h.get("n")).intValue(); + final int commitEvery = ((Number) h.get("commit")).intValue(); + final List cols = (List) h.get("cols"); + final int k = cols.size(); + final String[] names = new String[k]; + final char[] kinds = new char[k]; + final Object[] data = new Object[k]; + final int[] dims = new int[k]; + raw.order(ByteOrder.nativeOrder()); + int pos = 0; + for (int c = 0; c < k; c++) { + final List col = (List) cols.get(c); + names[c] = (String) col.get(0); + kinds[c] = ((String) col.get(1)).charAt(0); + dims[c] = ((Number) col.get(2)).intValue(); + final int bytes = switch (kinds[c]) { + case 'q', 'd' -> 8 * n; + case 'b' -> n; + case 'f' -> 4 * n * dims[c]; + default -> throw new IllegalArgumentException("unknown column kind " + kinds[c]); + }; + final ByteBuffer part = raw.slice(pos, bytes).order(ByteOrder.nativeOrder()); + pos += bytes; + switch (kinds[c]) { + case 'q' -> { + final long[] a = new long[n]; + part.asLongBuffer().get(a); + data[c] = a; + } + case 'd' -> { + final double[] a = new double[n]; + part.asDoubleBuffer().get(a); + data[c] = a; + } + case 'b' -> { + final boolean[] a = new boolean[n]; + for (int i = 0; i < n; i++) + a[i] = part.get(i) != 0; + data[c] = a; + } + default -> data[c] = part.asFloatBuffer(); + } + } + final boolean vertex = db.getSchema().getType(type) instanceof VertexType; + final boolean wasActive = db.isTransactionActive(); + if (!wasActive) + db.begin(); + try { + for (int i = 0; i < n; i++) { + final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); + for (int c = 0; c < k; c++) { + switch (kinds[c]) { + case 'q' -> doc.set(names[c], ((long[]) data[c])[i]); + case 'd' -> doc.set(names[c], ((double[]) data[c])[i]); + case 'b' -> doc.set(names[c], ((boolean[]) data[c])[i]); + default -> { + final float[] v = new float[dims[c]]; + ((java.nio.FloatBuffer) data[c]).get(i * dims[c], v); + doc.set(names[c], v); + } + } + } + doc.save(); + if (!wasActive && commitEvery > 0 && (i + 1) % commitEvery == 0) { + db.commit(); + db.begin(); + } + } + if (!wasActive) + db.commit(); + } catch (final Throwable e) { + if (!wasActive && db.isTransactionActive()) { + try { + db.rollback(); + } catch (final Throwable rollbackError) { + e.addSuppressed(rollbackError); + } + } + throw e; + } + return n; + } + + /** + * Documents (or vertices of a vertex type) from rows [{property: value},..] in the UTF-8 JSON text b[0, n): one record per row through + * newDocument/newVertex, set per property and save, as DocumentBatcher.insertManyJson does on the JPype path; no transaction is opened + * or closed here. The rows are parsed one at a time into one reused map (the property order of a fresh HashMap), so no tree of the + * chunk and no String of the whole text is built. + */ + static int insertRows(final Database db, final String type, final byte[] b, final int n) { + final boolean vertex = db.getSchema().getType(type) instanceof VertexType; + final int[] count = new int[1]; + Json.rows(b, n, row -> { + final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); + for (final Map.Entry e : row.entrySet()) + doc.set(e.getKey(), e.getValue()); + doc.save(); + count[0]++; + }); + return count[0]; + } +} diff --git a/native/src/main/java/org/humemai/nativelib/Json.java b/native/src/main/java/org/humemai/nativelib/Json.java new file mode 100644 index 0000000..809d115 --- /dev/null +++ b/native/src/main/java/org/humemai/nativelib/Json.java @@ -0,0 +1,462 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Minimal strict JSON reader for the native entry points. Integers without a fraction or exponent come back as Long + * (BigInteger past 64 bits), other numbers as Double, objects as HashMap (the iteration order of the JPype path's java.util.HashMap). NaN/Infinity/-Infinity are accepted + * because Python's json module writes them. An object {"\u0001": [kind, payload]} is a tagged value (see decodeTag). + */ +package org.humemai.nativelib; + +import org.graalvm.nativeimage.c.type.CFloatPointer; +import org.graalvm.word.WordFactory; + + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +final class Json { + static final String TAG = "\u0001"; + + /** One row of a streamed top-level array; the map is reused for the next row, so a sink must not keep it. */ + interface RowSink { + void row(Map row); + } + + private static final double[] POW10 = { 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, + 1e20, 1e21, 1e22 }; + private static final int CACHE_MIN_INPUT = 4096; // below this a string cache costs more than it saves + private static final int CACHE_SLOTS = 1024; + private static final int CACHE_MAX_LEN = 24; + private static final int DEFAULT_TABLE_ENTRIES = 12; // a HashMap of 16 buckets resizes on the 13th entry + + private final byte[] s; + private final int n; + private final long buf; // address of the float32 side buffer, 0 for none (a plain long: no Word value flows through this shared parser) + private final long bufFloats; + private int i; + private String[] cache; // direct-mapped: short ASCII keys and values shared across rows instead of one new String per occurrence + private Map rowMap = new HashMap<>(); + + private Json(final byte[] s, final int n, final long buf, final long bufFloats) { + this.s = s; + this.n = n; + this.buf = buf; + this.bufFloats = bufFloats; + if (n >= CACHE_MIN_INPUT) + cache = new String[CACHE_SLOTS]; + } + + static Object parse(final String text) { + return parse(text, 0L, 0); + } + + /** buf: the float32 side buffer that {"\u0001":["B",offset,count]} values point into (offset and count in floats); may be null. */ + static Object parse(final String text, final long buf, final long bufFloats) { + final byte[] b = text.getBytes(StandardCharsets.UTF_8); + return parse(b, b.length, buf, bufFloats); + } + + static Object parse(final byte[] b, final int len, final long buf, final long bufFloats) { + final Json j = new Json(b, len, buf, bufFloats); + final Object v = j.value(); + j.end(); + return v; + } + + /** Streams the elements of a top-level array of objects to the sink (one reused map, so no tree of the whole payload is built). */ + @SuppressWarnings("unchecked") + static void rows(final byte[] b, final int len, final RowSink sink) { + final Json j = new Json(b, len, 0L, 0); + j.rowsOf(sink); + j.end(); + } + + /** + * Parses a top-level object whose value for streamKey is an array of objects: those rows go to the sink one by one and the key maps to + * the row count (Integer) instead of the list. Every other key is parsed as usual. A streamKey value that is not an array stays as parsed. + */ + @SuppressWarnings("unchecked") + static Map object(final byte[] b, final int len, final String streamKey, final RowSink sink) { + final Json j = new Json(b, len, 0L, 0); + j.ws(); + if (j.i >= j.n || j.s[j.i] != '{') + throw new IllegalArgumentException("expected a JSON object"); + j.i++; + final Map out = new HashMap<>(); + j.ws(); + if (j.s[j.i] == '}') + j.i++; + else + while (true) { + j.ws(); + final String k = j.string(); + j.ws(); + if (j.s[j.i++] != ':') + throw new IllegalArgumentException("bad JSON object at " + (j.i - 1)); + j.ws(); + if (k.equals(streamKey) && j.i < j.n && j.s[j.i] == '[') + out.put(k, j.rowsOf(sink)); + else + out.put(k, j.value()); + j.ws(); + final char c = (char) j.s[j.i++]; + if (c == '}') + break; + if (c != ',') + throw new IllegalArgumentException("bad JSON object at " + (j.i - 1)); + } + j.end(); + return out; + } + + @SuppressWarnings("unchecked") + private Integer rowsOf(final RowSink sink) { + ws(); + if (i >= n || s[i] != '[') + throw new IllegalArgumentException("expected a JSON array"); + i++; + int count = 0; + ws(); + if (s[i] == ']') { + i++; + return 0; + } + while (true) { + ws(); + if (i < n && s[i] == '{') { + final Object r = object(rowMap); + final boolean grown = rowMap.size() > DEFAULT_TABLE_ENTRIES; // a bigger table would iterate a later row in another order than a fresh map + sink.row((Map) r); + count++; + if (grown) + rowMap = new HashMap<>(); + else + rowMap.clear(); + } else { + sink.row((Map) value()); // null or a non-object: the cast fails as it did for the whole-tree parse + count++; + } + ws(); + final char c = (char) s[i++]; + if (c == ']') + return count; + if (c != ',') + throw new IllegalArgumentException("bad JSON array at " + (i - 1)); + } + } + + private void end() { + ws(); + if (i != n) + throw new IllegalArgumentException("trailing characters in JSON at " + i); + } + + private void ws() { + while (i < n) { + final byte c = s[i]; + if (c == ' ' || c == '\n' || c == '\t' || c == '\r') + i++; + else + break; + } + } + + private Object value() { + ws(); + if (i >= n) + throw new IllegalArgumentException("unexpected end of JSON"); + final char c = (char) s[i]; + switch (c) { + case '{': + return object(null); + case '[': + return array(); + case '"': + return string(); + case 't': + expect("true"); + return Boolean.TRUE; + case 'f': + expect("false"); + return Boolean.FALSE; + case 'n': + expect("null"); + return null; + case 'N': + expect("NaN"); + return Double.NaN; + case 'I': + expect("Infinity"); + return Double.POSITIVE_INFINITY; + default: + return number(); + } + } + + private boolean startsWith(final String word) { + if (i + word.length() > n) + return false; + for (int k = 0; k < word.length(); k++) + if (s[i + k] != word.charAt(k)) + return false; + return true; + } + + private void expect(final String word) { + if (!startsWith(word)) + throw new IllegalArgumentException("bad JSON literal at " + i); + i += word.length(); + } + + private Object number() { + final int start = i; + if (s[i] == '-' && startsWith("-Infinity")) { + i += 9; + return Double.NEGATIVE_INFINITY; + } + boolean fractional = false; + boolean plain = true; // an optional '-' and digits (one '.' for a double) only: summed up without a String + long acc = 0; + int digits = 0; + int dots = 0, fracDigits = 0; + while (i < n) { + final byte c = s[i]; + if (c == '.' || c == 'e' || c == 'E') { + fractional = true; + if (c == '.') + dots++; + else + plain = false; + i++; + } else if (c >= '0' && c <= '9') { + if (++digits <= 18) + acc = acc * 10 + (c - '0'); + if (dots > 0) + fracDigits++; + i++; + } else if (c == '-' || c == '+') { + if (i != start || c == '+') + plain = false; + i++; + } else + break; + } + if (i == start) + throw new IllegalArgumentException("bad JSON value at " + start); + if (fractional && plain && dots == 1 && digits <= 18 && fracDigits >= 1 && fracDigits <= 22 && acc <= (1L << 53) + && s[i - 1] != '.' && s[start] != '.') { + // m / 10^k with m <= 2^53 and k <= 22: both exactly representable, so the one division is the correctly rounded decimal value + // (what Double.parseDouble returns), without the String and the big-number scratch that parseDouble allocates + final double d = (double) acc / POW10[fracDigits]; + return s[start] == '-' ? -d : d; + } + if (fractional) + return Double.parseDouble(new String(s, start, i - start, StandardCharsets.ISO_8859_1)); + if (plain && digits > 0 && digits <= 18) + return s[start] == '-' ? -acc : acc; + final String t = new String(s, start, i - start, StandardCharsets.ISO_8859_1); + try { + return Long.parseLong(t); + } catch (final NumberFormatException e) { + // the JPype path raises OverflowError for a Python int beyond 64 bits (query parameters and the bulk paths alike): + // refuse it here instead of letting the engine wrap or widen it + new BigInteger(t); + throw new IllegalArgumentException("integer out of range for a 64-bit parameter: " + t); + } + } + + /** The text of s[start, end): a short pure-ASCII one comes from the cache when an equal one was read before. */ + private String text(final int start, final int len, final boolean ascii) { + if (len == 0) + return ""; + if (!ascii) + return new String(s, start, len, StandardCharsets.UTF_8); + final String[] c = cache; + if (c == null || len > CACHE_MAX_LEN) + return new String(s, start, len, StandardCharsets.ISO_8859_1); + int h = len; + for (int k = 0; k < len; k++) + h = h * 31 + s[start + k]; + final int slot = (h ^ (h >>> 15)) & (CACHE_SLOTS - 1); + final String hit = c[slot]; + if (hit != null && hit.length() == len) { + int k = 0; + while (k < len && hit.charAt(k) == s[start + k]) + k++; + if (k == len) + return hit; + } + final String made = new String(s, start, len, StandardCharsets.ISO_8859_1); + c[slot] = made; + return made; + } + + private String string() { + i++; + final int start = i; + int p = i; + boolean ascii = true; + while (true) { + if (p >= n) + throw new IllegalArgumentException("unterminated JSON string"); + final byte c = s[p]; + if (c == '"') { + i = p + 1; + return text(start, p - start, ascii); + } + if (c == '\\') + break; + if (c < 0) + ascii = false; + p++; + } + // an escape in the string: a builder for this one + final StringBuilder sb = new StringBuilder(); + int seg = start; + i = p; + while (true) { + if (i >= n) + throw new IllegalArgumentException("unterminated JSON string"); + final byte c = s[i]; + if (c == '"') { + sb.append(new String(s, seg, i - seg, StandardCharsets.UTF_8)); + i++; + return sb.toString(); + } + if (c == '\\') { + sb.append(new String(s, seg, i - seg, StandardCharsets.UTF_8)); + i++; + final char e = (char) s[i++]; + switch (e) { + case 'n' -> sb.append('\n'); + case 't' -> sb.append('\t'); + case 'r' -> sb.append('\r'); + case 'b' -> sb.append('\b'); + case 'f' -> sb.append('\f'); + case 'u' -> { + int v = 0; + for (int k = 0; k < 4; k++) { + final int d = Character.digit((char) s[i + k], 16); + if (d < 0) + throw new NumberFormatException("bad \\u escape in JSON at " + i); + v = v * 16 + d; + } + sb.append((char) v); + i += 4; + } + default -> sb.append(e); + } + seg = i; + } else + i++; + } + } + + private List array() { + i++; + final List out = new ArrayList<>(); + ws(); + if (s[i] == ']') { + i++; + return out; + } + while (true) { + out.add(value()); + ws(); + final char c = (char) s[i++]; + if (c == ']') + return out; + if (c != ',') + throw new IllegalArgumentException("bad JSON array at " + (i - 1)); + } + } + + /** into: a map to fill (a streamed row, reused by the caller), or null for a fresh one. */ + private Object object(final Map into) { + i++; + final Map out = into != null ? into : new HashMap<>(); + ws(); + if (s[i] == '}') { + i++; + return out; + } + while (true) { + ws(); + final String k = string(); + ws(); + if (s[i++] != ':') + throw new IllegalArgumentException("bad JSON object at " + (i - 1)); + out.put(k, value()); + ws(); + final char c = (char) s[i++]; + if (c == '}') + break; + if (c != ',') + throw new IllegalArgumentException("bad JSON object at " + (i - 1)); + } + if (out.size() == 1 && out.get(TAG) instanceof List tag && tag.size() >= 2 && tag.get(0) instanceof String kind) + return decodeTag(kind, tag); + return out; + } + + /** Python -> Java tagged values: n Decimal, i big int, m map with non-str keys, D date, T datetime (UTC wall clock), s set, b bytes (base64), F float[] (as a list), B float[] (from the side buffer), R RID. */ + @SuppressWarnings("unchecked") + private Object decodeTag(final String kind, final List tag) { + final Object payload = tag.get(1); + switch (kind) { + case "B": { + // binary side buffer: float32 values [offset, offset + count) of the buffer passed beside the JSON, no text round trip + final long off = ((Number) payload).longValue(); + final int count = ((Number) tag.get(2)).intValue(); + if (buf == 0 || off < 0 || count < 0 || off + count > bufFloats) + throw new IllegalArgumentException("binary parameter outside the side buffer"); + final float[] f = new float[count]; + final CFloatPointer fp = WordFactory.pointer(buf); + for (int k = 0; k < count; k++) + f[k] = fp.read((int) off + k); + return f; + } + case "n": + return new BigDecimal((String) payload); + case "i": + return new BigInteger((String) payload); + case "m": { + // a Python dict with a non-str key: [[k, v], ..] (the plain JSON object form can only carry str keys) + final Map m = new HashMap<>(); + for (final Object pair : (List) payload) { + final List kv = (List) pair; + m.put(kv.get(0), kv.get(1)); + } + return m; + } + case "D": + return LocalDate.parse((String) payload); + case "T": + return LocalDateTime.parse((String) payload); + case "s": + return new HashSet<>((List) payload); + case "b": + return Base64.getDecoder().decode((String) payload); + case "R": + return new com.arcadedb.database.RID((String) payload); + case "F": { + final List l = (List) payload; + final float[] f = new float[l.size()]; + for (int k = 0; k < f.length; k++) + f[k] = ((Number) l.get(k)).floatValue(); + return f; + } + default: + throw new IllegalArgumentException("unknown parameter tag " + kind); + } + } +} diff --git a/native/src/main/java/org/humemai/nativelib/NativeCodec.java b/native/src/main/java/org/humemai/nativelib/NativeCodec.java new file mode 100644 index 0000000..189dab7 --- /dev/null +++ b/native/src/main/java/org/humemai/nativelib/NativeCodec.java @@ -0,0 +1,385 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Result rows as type-faithful JSON for the Python native backend. Same tagging scheme as the JPype bridge's + * TypedRows (bindings/python/src/java), but with no side array of Java objects (there is no JVM to hold them): every + * value is plain JSON or a tagged value that Python restores. + * + * {"\u0001": ["n", "1.25"]} BigDecimal {"\u0001": ["i", "123"]} BigInteger + * {"\u0001": ["D", "2020-01-01"]} LocalDate {"\u0001": ["T", "2020-01-01T12:00:00.000000"]} LocalDateTime, Date, Calendar (UTC wall clock) + * {"\u0001": ["Z", "...+00:00"]} Instant, ZonedDateTime, OffsetDateTime (UTC) + * {"\u0001": ["s", [..]]} Set {"\u0001": ["f", "NaN"]} non-finite Float/Double + * {"\u0001": ["m", [[k, v]..]]} Map with non-String keys + * {"\u0001": ["R", "#12:3"]} RID (a record link value; Python restores a RID object that goes back out as a RID) + * + * A record nested in a value is an object with "@rid" and "@type" members next to its + * properties; float[] and double[] are JSON number lists (a float widens to double, as it does through the bridge); + * byte[], short[], int[], long[] and boolean[] are lists. + */ +package org.humemai.nativelib; + +import com.arcadedb.database.Document; +import com.arcadedb.database.Identifiable; +import com.arcadedb.graph.Edge; +import com.arcadedb.graph.Vertex; +import com.arcadedb.query.sql.executor.Result; +import com.arcadedb.query.sql.executor.ResultSet; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Calendar; +import java.util.Collection; +import java.util.Date; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +final class NativeCodec { + static final int MAX_BATCH_CHARS = 4 * 1024 * 1024; + + private NativeCodec() { + } + + /** + * Appends {@code "rows":[..],"meta":[..],"drained":b} (no braces) with up to max rows. meta has one entry per row: 0 + * for a projection, or [rid, typeName, kind] for a record row (kind v, e or d). + */ + static boolean appendBatch(final StringBuilder sb, final ResultSet rs, final int max) { + final StringBuilder meta = new StringBuilder("["); + sb.append("\"rows\":["); + int n = 0; + boolean drained = false; + String readError = null; + try { + while (n < max) { + if (!rs.hasNext()) { + drained = true; + break; + } + final Result row = rs.next(); + final int rowStart = sb.length(); + final int metaStart = meta.length(); + try { + if (n > 0) { + sb.append(','); + meta.append(','); + } + appendRow(sb, meta, row); + } catch (final Throwable t) { + sb.setLength(rowStart); + meta.setLength(metaStart); + throw t; + } + n++; + if (sb.length() >= MAX_BATCH_CHARS) + break; + } + } catch (final Throwable t) { + // The engine plans and computes lazily: a statement's error can surface while its rows are read. The rows read + // before it still go to the caller; the error is raised when they have been consumed. The cursor is finished. + readError = ArcadeNative.errorJson(t); + drained = true; + } + sb.append("],\"meta\":").append(meta).append("],\"drained\":").append(drained); + if (readError != null) + sb.append(",\"rerr\":").append(readError); + return drained; + } + + private static void appendRow(final StringBuilder sb, final StringBuilder meta, final Result row) { + sb.append('{'); + boolean first = true; + for (final String name : row.getPropertyNames()) { + if (!first) + sb.append(','); + first = false; + quote(sb, name); + sb.append(':'); + append(sb, row.getProperty(name)); + } + sb.append('}'); + if (row.isElement()) { + final Optional element = row.getElement(); + if (element.isPresent()) { + final Document d = element.get(); + meta.append('['); + quote(meta, d.getIdentity() == null ? "" : d.getIdentity().toString()); + meta.append(','); + quote(meta, d.getTypeName()); + meta.append(",\"").append(d instanceof Vertex ? 'v' : d instanceof Edge ? 'e' : 'd').append("\"]"); + return; + } + } + meta.append('0'); + } + + /** phase 7: the float32 tail of a reply while a binary-reply batch is being encoded (see ArcadeNative.replyOf) */ + static final ThreadLocal SINK = new ThreadLocal<>(); + + static final class FloatSink { + float[] a = new float[1024]; + int n; + + int add(final float[] v) { + if (n + v.length > a.length) + a = java.util.Arrays.copyOf(a, Math.max(a.length * 2, n + v.length)); + System.arraycopy(v, 0, a, n, v.length); + final int off = n; + n += v.length; + return off; + } + } + + static void append(final StringBuilder sb, final Object value) { + if (value == null) { + sb.append("null"); + } else if (value instanceof String s) { + quote(sb, s); + } else if (value instanceof Boolean b) { + sb.append(b.booleanValue() ? "true" : "false"); + } else if (value instanceof Integer || value instanceof Long || value instanceof Short || value instanceof Byte) { + sb.append(value); + } else if (value instanceof Double || value instanceof Float) { + final double d = ((Number) value).doubleValue(); + if (Double.isFinite(d)) + sb.append(d); + else + tagged(sb, "f").append('"').append(Double.isNaN(d) ? "nan" : d > 0 ? "inf" : "-inf").append("\"]}"); + } else if (value instanceof Character c) { + quote(sb, String.valueOf(c)); + } else if (value instanceof BigDecimal bd) { + tagged(sb, "n").append('"').append(bd.toString()).append("\"]}"); + } else if (value instanceof BigInteger bi) { + tagged(sb, "i").append('"').append(bi.toString()).append("\"]}"); + } else if (value instanceof LocalDate date) { + if (yearFits(date.getYear())) { + tagged(sb, "D").append('"'); + appendDate(sb, date.getYear(), date.getMonthValue(), date.getDayOfMonth()); + sb.append("\"]}"); + } else + quote(sb, date.toString()); + } else if (value instanceof LocalDateTime dt) { + appendDateTime(sb, "T", dt, "", value); + } else if (value instanceof Instant instant) { + appendDateTime(sb, "Z", LocalDateTime.ofEpochSecond(instant.getEpochSecond(), instant.getNano(), ZoneOffset.UTC), "+00:00", + value); + } else if (value instanceof ZonedDateTime zdt) { + final Instant instant = zdt.toInstant(); + appendDateTime(sb, "Z", LocalDateTime.ofEpochSecond(instant.getEpochSecond(), instant.getNano(), ZoneOffset.UTC), "+00:00", + value); + } else if (value instanceof OffsetDateTime odt) { + final Instant instant = odt.toInstant(); + appendDateTime(sb, "Z", LocalDateTime.ofEpochSecond(instant.getEpochSecond(), instant.getNano(), ZoneOffset.UTC), "+00:00", + value); + } else if (value instanceof Date date) { + appendEpochMillis(sb, date.getTime(), value); + } else if (value instanceof Calendar calendar) { + appendEpochMillis(sb, calendar.getTimeInMillis(), value); + } else if (value instanceof Document doc) { + appendDocument(sb, doc); + } else if (value instanceof Identifiable id) { + tagged(sb, "R").append('"').append(id.getIdentity().toString()).append("\"]}"); + } else if (value instanceof Map map) { + boolean stringKeys = true; + for (final Object key : map.keySet()) + if (!(key instanceof String k) || Json.TAG.equals(k)) { + stringKeys = false; + break; + } + if (stringKeys) { + sb.append('{'); + boolean first = true; + for (final Map.Entry entry : map.entrySet()) { + if (!first) + sb.append(','); + first = false; + quote(sb, (String) entry.getKey()); + sb.append(':'); + append(sb, entry.getValue()); + } + sb.append('}'); + } else { + tagged(sb, "m").append('['); + boolean first = true; + for (final Map.Entry entry : map.entrySet()) { + if (!first) + sb.append(','); + first = false; + sb.append('['); + append(sb, entry.getKey()); + sb.append(','); + append(sb, entry.getValue()); + sb.append(']'); + } + sb.append("]]}"); + } + } else if (value instanceof Set set) { + tagged(sb, "s").append('['); + appendElements(sb, set); + sb.append("]]}"); + } else if (value instanceof Collection collection) { + sb.append('['); + appendElements(sb, collection); + sb.append(']'); + } else if (value instanceof float[] a && a.length >= 8 && SINK.get() != null) { + final FloatSink sink = SINK.get(); + tagged(sb, "B").append(sink.add(a)).append(',').append(a.length).append("]}"); + } else if (value instanceof float[] a) { + sb.append('['); + for (int i = 0; i < a.length; i++) { + if (i > 0) + sb.append(','); + appendFloating(sb, a[i]); + } + sb.append(']'); + } else if (value instanceof double[] a) { + sb.append('['); + for (int i = 0; i < a.length; i++) { + if (i > 0) + sb.append(','); + appendFloating(sb, a[i]); + } + sb.append(']'); + } else if (value instanceof byte[] a) { + sb.append('['); + for (int i = 0; i < a.length; i++) + sb.append(i > 0 ? "," : "").append(a[i]); + sb.append(']'); + } else if (value instanceof short[] a) { + sb.append('['); + for (int i = 0; i < a.length; i++) + sb.append(i > 0 ? "," : "").append(a[i]); + sb.append(']'); + } else if (value instanceof int[] a) { + sb.append('['); + for (int i = 0; i < a.length; i++) + sb.append(i > 0 ? "," : "").append(a[i]); + sb.append(']'); + } else if (value instanceof long[] a) { + sb.append('['); + for (int i = 0; i < a.length; i++) + sb.append(i > 0 ? "," : "").append(a[i]); + sb.append(']'); + } else if (value instanceof boolean[] a) { + sb.append('['); + for (int i = 0; i < a.length; i++) + sb.append(i > 0 ? "," : "").append(a[i]); + sb.append(']'); + } else if (value instanceof Object[] a) { + sb.append('['); + for (int i = 0; i < a.length; i++) { + if (i > 0) + sb.append(','); + append(sb, a[i]); + } + sb.append(']'); + } else + quote(sb, String.valueOf(value)); + } + + private static void appendFloating(final StringBuilder sb, final double d) { + if (Double.isFinite(d)) + sb.append(d); + else + tagged(sb, "f").append('"').append(Double.isNaN(d) ? "nan" : d > 0 ? "inf" : "-inf").append("\"]}"); + } + + private static void appendDocument(final StringBuilder sb, final Document doc) { + sb.append('{'); + if (doc.getIdentity() != null) { + sb.append("\"@rid\":"); + quote(sb, doc.getIdentity().toString()); + sb.append(','); + } + sb.append("\"@type\":"); + quote(sb, doc.getTypeName()); + for (final String name : doc.getPropertyNames()) { + sb.append(','); + quote(sb, name); + sb.append(':'); + append(sb, doc.get(name)); + } + sb.append('}'); + } + + private static void appendElements(final StringBuilder sb, final Collection elements) { + boolean first = true; + for (final Object element : elements) { + if (!first) + sb.append(','); + first = false; + append(sb, element); + } + } + + private static StringBuilder tagged(final StringBuilder sb, final String kind) { + return sb.append("{\"\\u0001\":[\"").append(kind).append("\","); + } + + private static boolean yearFits(final int year) { + return year >= 1 && year <= 9999; + } + + private static void appendDateTime(final StringBuilder sb, final String kind, final LocalDateTime dt, final String suffix, + final Object original) { + if (!yearFits(dt.getYear())) { + quote(sb, String.valueOf(original)); + return; + } + tagged(sb, kind).append('"'); + appendDate(sb, dt.getYear(), dt.getMonthValue(), dt.getDayOfMonth()); + sb.append('T'); + pad(sb, dt.getHour(), 2).append(':'); + pad(sb, dt.getMinute(), 2).append(':'); + pad(sb, dt.getSecond(), 2).append('.'); + pad(sb, dt.getNano() / 1000, 6).append(suffix).append("\"]}"); + } + + private static void appendEpochMillis(final StringBuilder sb, final long millis, final Object original) { + appendDateTime(sb, "T", + LocalDateTime.ofEpochSecond(Math.floorDiv(millis, 1000L), Math.floorMod(millis, 1000) * 1_000_000, ZoneOffset.UTC), "", + original); + } + + private static void appendDate(final StringBuilder sb, final int year, final int month, final int day) { + pad(sb, year, 4).append('-'); + pad(sb, month, 2).append('-'); + pad(sb, day, 2); + } + + private static StringBuilder pad(final StringBuilder sb, final int value, final int width) { + int digits = 1; + for (int v = value; v >= 10; v /= 10) + digits++; + for (int i = digits; i < width; i++) + sb.append('0'); + return sb.append(value); + } + + static void quote(final StringBuilder sb, final String s) { + sb.append('"'); + final int length = s.length(); + for (int i = 0; i < length; i++) { + final char c = s.charAt(i); + if (c == '"') + sb.append("\\\""); + else if (c == '\\') + sb.append("\\\\"); + else if (c < 0x20 || (c >= 0xD800 && c <= 0xDFFF)) { + sb.append("\\u"); + final String hex = Integer.toHexString(c); + for (int k = hex.length(); k < 4; k++) + sb.append('0'); + sb.append(hex); + } else + sb.append(c); + } + sb.append('"'); + } +} diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java new file mode 100644 index 0000000..dfc04d2 --- /dev/null +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -0,0 +1,681 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Coverage entry points of the Python native backend that are not the DSL core (ArcadeNative): GraphBatch, the async executor, + * database settings, import and export, and the Java->Python upcall (async callbacks, Python functions in SQL). + * Same reply convention as ArcadeNative: a malloc'd JSON text, {"ok":true,...} or {"ok":false,"error":..,"cls":..,"chain":..}. + * + * Upcalls. Python registers ONE ctypes function pointer with arcadedb_n_set_upcall: CCharPointer fn(long id, CCharPointer json). + * `id` names a Python-side callback, `json` is the message, the return value is a JSON reply that Python keeps alive until its + * next call on the same thread (Java copies it at once). The call is a transition-to-native C function pointer call, so the + * isolate can run a safepoint or a GC while the Python callback holds the GIL and calls back into the library on its thread. + * ctypes takes the GIL itself (PyGILState_Ensure) when a foreign Java thread (an async writer) or the engine thread inside a + * query calls the pointer. + */ +package org.humemai.nativelib; + +import com.arcadedb.database.Database; +import com.arcadedb.database.DatabaseInternal; +import com.arcadedb.database.Document; +import com.arcadedb.database.DocumentCallback; +import com.arcadedb.database.Identifiable; +import com.arcadedb.database.MutableDocument; +import com.arcadedb.database.RID; +import com.arcadedb.database.async.DatabaseAsyncExecutor; +import com.arcadedb.database.async.ErrorCallback; +import com.arcadedb.database.async.NewEdgeCallback; +import com.arcadedb.engine.ErrorRecordCallback; +import com.arcadedb.engine.WALFile; +import com.arcadedb.function.sql.DefaultSQLFunctionFactory; +import com.arcadedb.function.sql.SQLFunctionAbstract; +import com.arcadedb.graph.GraphBatch; +import com.arcadedb.graph.MutableVertex; +import com.arcadedb.graph.Vertex; +import com.arcadedb.integration.exporter.Exporter; +import com.arcadedb.integration.importer.Importer; +import com.arcadedb.query.sql.executor.CommandContext; +import com.arcadedb.query.sql.executor.ResultSet; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CFunctionPointer; +import org.graalvm.nativeimage.c.function.InvokeCFunctionPointer; +import org.graalvm.nativeimage.c.type.CCharPointer; +import org.graalvm.nativeimage.c.type.CTypeConversion; +import org.graalvm.word.WordFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +public final class NativeExtras { + private NativeExtras() { + } + + // ---- the upcall + + interface PyCallback extends CFunctionPointer { + @InvokeCFunctionPointer + CCharPointer invoke(long id, CCharPointer json); + } + + private static volatile long upcallPointer; + + @CEntryPoint(name = "arcadedb_n_set_upcall") + public static void setUpcall(final IsolateThread thread, final PyCallback fn) { + upcallPointer = fn.rawValue(); + } + + /** Calls the Python callback `id` with `json`; the reply text, or null when Python returned NULL. */ + static String upcall(final long id, final String json) { + final long raw = upcallPointer; + if (raw == 0) + throw new IllegalStateException("no Python callback is registered (arcadedb_n_set_upcall)"); + final PyCallback fn = WordFactory.pointer(raw); + try (CTypeConversion.CCharPointerHolder in = CTypeConversion.toCString(json)) { + final CCharPointer reply = fn.invoke(id, in.get()); + return reply.isNull() ? null : CTypeConversion.toJavaString(reply); + } + } + + /** A callback failure that must not stop an async writer: reported on stderr, not lost. */ + static void swallowed(final String where, final Throwable t) { + System.err.println("[arcadedb-native] " + where + " failed: " + t); + } + + /** Calls Python with an event whose reply nobody reads; a Python exception (reported in the reply) is logged, not lost. */ + static void fire(final String where, final long cb, final String message) { + final String reply = upcall(cb, message); + if (reply != null && reply.startsWith("{\"error\"")) + swallowed(where, new RuntimeException(reply)); + } + + /** Calls Python with an event whose reply is {"v":bool}: true (or no answer) goes on, false stops. */ + @SuppressWarnings("unchecked") + static boolean goOn(final String where, final long cb, final String message) { + final String reply = upcall(cb, message); + if (reply == null) + return true; + final Map r = (Map) Json.parse(reply); + if (r.get("error") != null) { + swallowed(where, new RuntimeException(reply)); + return false; + } + return !Boolean.FALSE.equals(r.get("v")); + } + + private static String event(final String kind, final String body) { + return "{\"k\":\"" + kind + "\"" + (body.isEmpty() ? "" : "," + body) + "}"; + } + + private static String errorBody(final Throwable t) { + final StringBuilder sb = new StringBuilder("\"err\":"); + sb.append(ArcadeNative.errorJson(t)); + return sb.toString(); + } + + // ---- small helpers + + @SuppressWarnings("unchecked") + private static Map args(final CCharPointer json) { + final String text = ArcadeNative.str(json); + if (text == null || text.isEmpty()) + return new HashMap<>(); + return (Map) Json.parse(text); + } + + private static int intOf(final Object v, final int def) { + return v instanceof Number n ? n.intValue() : def; + } + + private static final Object[] NO_PROPS = new Object[0]; // never written to: one shared array for every property-less edge and vertex + + /** + * Edges without properties from two ';'-joined RID texts ("#12:34;#12:35"), as EdgeBatcher.newEdgesJoined does on the JPype path, but + * each RID is read straight from the text into the engine's RID(bucket, position): no substring, split array or per-RID String. + */ + static void newEdgesJoined(final GraphBatch batch, final String src, final String type, final String dst) { + if (src.isEmpty() && dst.isEmpty()) + return; + int a = 0, b = 0; + while (true) { + int ae = src.indexOf(';', a), be = dst.indexOf(';', b); + if (ae < 0) + ae = src.length(); + if (be < 0) + be = dst.length(); + batch.newEdge(rid(src, a, ae), type, rid(dst, b, be), NO_PROPS); + final boolean lastSrc = ae >= src.length(), lastDst = be >= dst.length(); + if (lastSrc != lastDst) + throw new IllegalArgumentException("source_rids and destination_rids differ in length"); + if (lastSrc) + return; + a = ae + 1; + b = be + 1; + } + } + + /** The RID of "#bucket:position" in text[from, to) (the '#' is optional); anything else goes through RID(String), which refuses it. */ + private static RID rid(final String text, final int from, final int to) { + int p = from; + if (p < to && text.charAt(p) == '#') + p++; + long bucket = 0, position = 0; + int digits = p; + while (p < to && text.charAt(p) >= '0' && text.charAt(p) <= '9' && p - digits < 9) + bucket = bucket * 10 + (text.charAt(p++) - '0'); + if (p == digits || p >= to || text.charAt(p) != ':') + return new RID(text.substring(from, to)); + digits = ++p; + while (p < to && text.charAt(p) >= '0' && text.charAt(p) <= '9' && p - digits < 18) + position = position * 10 + (text.charAt(p++) - '0'); + if (p == digits || p != to) + return new RID(text.substring(from, to)); + return new RID((int) bucket, position); + } + + static Object[] varargs(final Object props) { + if (!(props instanceof Map m) || m.isEmpty()) + return NO_PROPS; + final Object[] out = new Object[m.size() * 2]; + int i = 0; + for (final Map.Entry e : m.entrySet()) { + out[i++] = e.getKey(); + out[i++] = e.getValue(); + } + return out; + } + + static String ridList(final RID[] rids) { + final StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < rids.length; i++) { + if (i > 0) + sb.append(','); + sb.append('"').append(rids[i]).append('"'); + } + return sb.append(']').toString(); + } + + private static WALFile.FlushType flushType(final String name) { + return WALFile.FlushType.valueOf(name); + } + + // ---- database settings: op wal_flush | read_your_writes | auto_transaction ; value "" reads (where the engine can) + + @CEntryPoint(name = "arcadedb_n_dbset") + public static CCharPointer dbSet(final IsolateThread thread, final long h, final CCharPointer op, final CCharPointer value) { + try { + final Database db = ArcadeNative.db(h); + final String o = ArcadeNative.str(op); + final String v = ArcadeNative.str(value); + switch (o) { + case "wal_flush" -> db.setWALFlush(flushType(v)); + case "read_your_writes" -> { + if (v == null || v.isEmpty()) + return ArcadeNative.out("{\"ok\":true,\"v\":" + db.isReadYourWrites() + "}"); + db.setReadYourWrites(Boolean.parseBoolean(v)); + } + case "auto_transaction" -> ((DatabaseInternal) db).setAutoTransaction(Boolean.parseBoolean(v)); + default -> throw new IllegalArgumentException("unknown database setting " + o); + } + return ArcadeNative.OK(); + } catch (final Throwable t) { + return ArcadeNative.err(t); + } + } + + // ---- GraphBatch + + private record BatchEntry(GraphBatch batch, Database db) { + } + + private static final Map BATCHES = new ConcurrentHashMap<>(); + + /** options: the Python keyword names of Database.graph_batch (absent = engine default). */ + @CEntryPoint(name = "arcadedb_n_gb_create") + public static CCharPointer gbCreate(final IsolateThread thread, final long dbh, final CCharPointer options) { + try { + final Database db = ArcadeNative.db(dbh); + final Map o = args(options); + GraphBatch.Builder b = GraphBatch.builder(db); + for (final Map.Entry e : o.entrySet()) { + final Object v = e.getValue(); + final int iv = intOf(v, 0); + switch (e.getKey()) { + case "batch_size" -> b = b.withBatchSize(iv); + case "expected_edge_count" -> b = b.withExpectedEdgeCount(iv); + case "edge_list_initial_size" -> b = b.withEdgeListInitialSize(iv); + case "light_edges" -> b = b.withLightEdges((Boolean) v); + case "bidirectional" -> b = b.withBidirectional((Boolean) v); + case "commit_every" -> b = b.withCommitEvery(iv); + case "use_wal" -> b = b.withWAL((Boolean) v); + case "wal_flush" -> b = b.withWALFlush(flushType((String) v)); + case "pre_allocate_edge_chunks" -> b = b.withPreAllocateEdgeChunks((Boolean) v); + case "parallel_flush" -> b = b.withParallelFlush((Boolean) v); + case "commit_retries" -> b = b.withCommitRetries(iv); + case "commit_retry_delay_ms" -> b = b.withCommitRetryDelay(((Number) v).longValue()); + case "chunk_cache_capacity" -> b = b.withChunkCacheCapacity(iv); + case "max_deferred_incoming_edges" -> b = b.withMaxDeferredIncomingEdges(iv); + default -> throw new IllegalArgumentException("unknown GraphBatch option " + e.getKey()); + } + } + final long id = ArcadeNative.IDS.getAndIncrement(); + BATCHES.put(id, new BatchEntry(b.build(), db)); + return ArcadeNative.out("{\"ok\":true,\"h\":" + id + "}"); + } catch (final Throwable t) { + return ArcadeNative.err(t); + } + } + + /** + * ops: create_vertex {type, props} -> rid; create_vertices {type, count} | {type, rows} -> rids; new_edge {src, type, dst, props}; + * new_edges {src:[..], type, dst:[..], props:[..]|null}; flush; close; stats. + */ + @CEntryPoint(name = "arcadedb_n_gb") + @SuppressWarnings("unchecked") + public static CCharPointer gb(final IsolateThread thread, final long h, final CCharPointer op, final CCharPointer argsJson) { + try { + final BatchEntry entry = BATCHES.get(h); + if (entry == null) + throw new IllegalArgumentException("unknown GraphBatch " + h); + final GraphBatch batch = entry.batch(); + final String opName = ArcadeNative.str(op); + final ArcadeNative.Chars chars = ArcadeNative.chars(argsJson); + final List flats = new ArrayList<>(); // create_vertices: one property array per streamed row + final Map a; + if (chars.n == 0) + a = new HashMap<>(); + else if (opName.equals("create_vertices")) + a = Json.object(chars.b, chars.n, "rows", row -> flats.add(varargs(row))); + else + a = (Map) Json.parse(chars.b, chars.n, 0L, 0); + switch (opName) { + case "create_vertex" -> { + final MutableVertex v = batch.createVertex((String) a.get("type"), varargs(a.get("props"))); + return ArcadeNative.out("{\"ok\":true,\"rid\":\"" + v.getIdentity() + "\"}"); + } + case "create_vertices" -> { + final RID[] rids; + if (!(a.get("rows") instanceof Integer)) + rids = batch.createVertices((String) a.get("type"), intOf(a.get("count"), 0)); + else + rids = batch.createVertices((String) a.get("type"), flats.toArray(new Object[0][])); + return ArcadeNative.out("{\"ok\":true,\"rids\":" + ridList(rids) + "}"); + } + case "new_edge" -> batch.newEdge(new RID((String) a.get("src")), (String) a.get("type"), new RID((String) a.get("dst")), + varargs(a.get("props"))); + case "new_edges" -> { + if (a.get("src") instanceof String srcText && a.get("dst") instanceof String dstText) { + newEdgesJoined(batch, srcText, (String) a.get("type"), dstText); + return ArcadeNative.OK(); + } + final List src = (List) a.get("src"), dst = (List) a.get("dst"); + final List props = (List) a.get("props"); + for (int i = 0; i < src.size(); i++) + batch.newEdge(new RID((String) src.get(i)), (String) a.get("type"), new RID((String) dst.get(i)), + props == null ? NO_PROPS : varargs(props.get(i))); + } + case "flush" -> batch.flush(); + case "close" -> { + batch.close(); + BATCHES.remove(h); + } + case "stats" -> { + return ArcadeNative.out("{\"ok\":true,\"total_edges\":" + batch.getTotalEdgesCreated() + ",\"buffered\":" + batch.getBufferedEdgeCount() + + ",\"deferred\":" + batch.getDeferredIncomingEdgeCount() + "}"); + } + default -> throw new IllegalArgumentException("unknown GraphBatch operation " + ArcadeNative.str(op)); + } + return ArcadeNative.OK(); + } catch (final Throwable t) { + return ArcadeNative.err(t); + } + } + + // ---- async executor (one per database) + + private static final Map ASYNC_FAILS = new ConcurrentHashMap<>(); + private static final Map> ASYNC_FIRST = new ConcurrentHashMap<>(); + + private static String replyRows(final ResultSet rs) { + final StringBuilder sb = new StringBuilder("{\"ok\":true,\"rs\":0,"); + NativeCodec.appendBatch(sb, rs, Integer.MAX_VALUE); + return sb.append('}').toString(); + } + + private static ErrorCallback errorCallback(final long cb) { + return t -> { + try { + fire("error callback", cb, event("error", errorBody(t))); + } catch (final Throwable e) { + swallowed("error callback", e); // a failing Python error handler must not take down an async writer + } + }; + } + + private static NewEdgeCallback edgeCallback(final long cb) { + if (cb == 0) + return null; + return (edge, createdSource, createdDestination) -> { + try { + final StringBuilder sb = new StringBuilder("\"edge\":"); + NativeCodec.append(sb, edge); + sb.append(",\"src\":").append(createdSource).append(",\"dst\":").append(createdDestination); + fire("edge callback", cb, event("edge", sb.toString())); + } catch (final Throwable e) { + swallowed("edge callback", e); + } + }; + } + + @SuppressWarnings("unchecked") + private static String[] strings(final Object list) { + final List in = (List) list; + final String[] out = new String[in.size()]; + for (int i = 0; i < out.length; i++) + out[i] = (String) in.get(i); + return out; + } + + @SuppressWarnings("unchecked") + private static Object[] objects(final Object list) { + return ((List) list).toArray(); + } + + /** + * ops: get | set {key,value} | wait {timeout_ms} | kill | create_record {type, kind, props, cb, err} | command / query {lang, text, params, + * cb, err} | on_ok {cb} | on_error {cb} | append_samples {type, ts, cols} | insert_parallel {type, rows} | failures {fid}. + */ + @CEntryPoint(name = "arcadedb_n_async") + @SuppressWarnings("unchecked") + public static CCharPointer async(final IsolateThread thread, final long dbh, final CCharPointer op, final CCharPointer argsJson) { + try { + final DatabaseInternal db = (DatabaseInternal) ArcadeNative.db(dbh); + final DatabaseAsyncExecutor ex = db.async(); + final Map a = args(argsJson); + final long cb = a.get("cb") instanceof Number n ? n.longValue() : 0L; + final long errCb = a.get("err") instanceof Number n ? n.longValue() : 0L; + switch (ArcadeNative.str(op)) { + case "get" -> { + return ArcadeNative.out("{\"ok\":true,\"parallel\":" + ex.getParallelLevel() + ",\"back_pressure\":" + ex.getBackPressure() + + ",\"commit_every\":" + ex.getCommitEvery() + ",\"wal\":" + ex.isTransactionUseWAL() + ",\"sync\":\"" + ex.getTransactionSync() + + "\",\"threads\":" + ex.getThreadCount() + ",\"processing\":" + ex.isProcessing() + "}"); + } + case "set" -> { + final Object v = a.get("value"); + switch ((String) a.get("key")) { + case "parallel" -> ex.setParallelLevel(intOf(v, 1)); + case "back_pressure" -> ex.setBackPressure(intOf(v, 0)); + case "commit_every" -> ex.setCommitEvery(intOf(v, 0)); + case "wal" -> ex.setTransactionUseWAL((Boolean) v); + case "sync" -> ex.setTransactionSync(flushType((String) v)); + default -> throw new IllegalArgumentException("unknown async setting " + a.get("key")); + } + } + case "wait" -> { + final boolean done; + if (a.get("timeout_ms") instanceof Number n) + done = ex.waitCompletion(n.longValue()); + else { + ex.waitCompletion(); + done = true; + } + return ArcadeNative.out("{\"ok\":true,\"done\":" + done + "}"); + } + case "kill" -> ex.kill(); + case "close" -> { + if (ex instanceof com.arcadedb.database.async.DatabaseAsyncExecutorImpl impl) + impl.close(); + } + case "create_record" -> { + final String type = (String) a.get("type"); + final MutableDocument doc = "v".equals(a.get("kind")) ? db.newVertex(type) : db.newDocument(type); + for (final Map.Entry e : ((Map) a.get("props")).entrySet()) + doc.set(e.getKey(), e.getValue()); + final com.arcadedb.database.async.NewRecordCallback ok = cb == 0 ? null : rec -> { + try { + final StringBuilder sb = new StringBuilder("\"rec\":"); + NativeCodec.append(sb, rec); + fire("record callback", cb, event("record", sb.toString())); + } catch (final Throwable e) { + swallowed("record callback", e); // the writer goes on + } + }; + if (errCb == 0) + ex.createRecord(doc, ok); + else + ex.createRecord(doc, ok, errorCallback(errCb)); + } + case "command", "query" -> { + final boolean command = "command".equals(ArcadeNative.str(op)); + final com.arcadedb.database.async.AsyncResultsetCallback callback = cb == 0 && errCb == 0 ? null : new com.arcadedb.database.async.AsyncResultsetCallback() { + @Override + public void onComplete(final ResultSet rs) { + if (cb == 0) + return; + try { + fire("result callback", cb, event("result", "\"reply\":" + replyRows(rs))); + } catch (final Throwable e) { + swallowed("result callback", e); // the writer goes on + } + } + + @Override + public void onError(final Exception e) { + if (errCb != 0) + errorCallback(errCb).call(e); + } + }; + final String lang = (String) a.get("lang"), text = (String) a.get("text"); + final Object params = a.get("params"); + if (params instanceof Map m) { + if (command) + ex.command(lang, text, callback, (Map) m); + else + ex.query(lang, text, callback, (Map) m); + } else if (params instanceof List l) { + if (command) + ex.command(lang, text, callback, l.toArray()); + else + ex.query(lang, text, callback, l.toArray()); + } else if (command) + ex.command(lang, text, callback); + else + ex.query(lang, text, callback); + } + case "on_ok" -> ex.onOk(() -> { + try { + fire("on_ok callback", cb, event("ok", "")); + } catch (final Throwable e) { + swallowed("on_ok callback", e); // the writer goes on + } + }); + case "on_error" -> ex.onError(errorCallback(cb)); + case "new_edge" -> { + final Vertex src = db.lookupByRID(new RID((String) a.get("src")), true).asVertex(); + ex.newEdge(src, (String) a.get("type"), new RID((String) a.get("dst")), Boolean.TRUE.equals(a.get("light")), edgeCallback(cb), + varargs(a.get("props"))); + } + case "new_edge_by_keys" -> ex.newEdgeByKeys((String) a.get("src_type"), strings(a.get("src_names")), objects(a.get("src_values")), + (String) a.get("dst_type"), strings(a.get("dst_names")), objects(a.get("dst_values")), Boolean.TRUE.equals(a.get("create")), + (String) a.get("type"), Boolean.TRUE.equals(a.get("bidirectional")), Boolean.TRUE.equals(a.get("light")), edgeCallback(cb), + varargs(a.get("props"))); + case "scan_type" -> { + final DocumentCallback onRecord = rec -> { + final StringBuilder sb = new StringBuilder("\"rec\":"); + NativeCodec.append(sb, rec); + return goOn("scan callback", cb, event("record", sb.toString())); + }; + final boolean polymorphic = !Boolean.FALSE.equals(a.get("polymorphic")); + if (errCb == 0) + ex.scanType((String) a.get("type"), polymorphic, onRecord); + else + ex.scanType((String) a.get("type"), polymorphic, onRecord, (rid, t) -> goOn("scan error callback", errCb, + event("scan_error", "\"rid\":\"" + rid + "\"," + errorBody(t)))); + } + case "append_samples" -> { + final List ts = (List) a.get("ts"); + final long[] stamps = new long[ts.size()]; + for (int i = 0; i < stamps.length; i++) + stamps[i] = ((Number) ts.get(i)).longValue(); + final List cols = (List) a.get("cols"); + final Object[][] columns = new Object[cols.size()][]; + for (int i = 0; i < columns.length; i++) + columns[i] = ((List) cols.get(i)).toArray(); + ex.appendSamples((String) a.get("type"), stamps, columns); + } + case "insert_parallel" -> { + final long fid = ArcadeNative.IDS.getAndIncrement(); + final AtomicLong count = new AtomicLong(); + final AtomicReference first = new AtomicReference<>(); + ASYNC_FAILS.put(fid, count); + ASYNC_FIRST.put(fid, first); + final ErrorCallback onError = t -> { + count.incrementAndGet(); + first.compareAndSet(null, t); + }; + final String type = (String) a.get("type"); + try { + for (final Object row : (List) a.get("rows")) { + final MutableDocument doc = db.newDocument(type); + for (final Map.Entry e : ((Map) row).entrySet()) + doc.set(e.getKey(), e.getValue()); + ex.createRecord(doc, null, onError); + } + } catch (final Throwable t) { + ASYNC_FAILS.remove(fid); // the caller gets the error, not a token to read failures from + ASYNC_FIRST.remove(fid); + throw t; + } + return ArcadeNative.out("{\"ok\":true,\"fid\":" + fid + "}"); + } + case "failures" -> { + final long fid = ((Number) a.get("fid")).longValue(); + final AtomicLong count = ASYNC_FAILS.remove(fid); + final AtomicReference firstRef = ASYNC_FIRST.remove(fid); + final Throwable first = firstRef == null ? null : firstRef.get(); + final StringBuilder sb = new StringBuilder("{\"ok\":true,\"count\":").append(count == null ? 0 : count.get()).append(",\"first\":"); + if (first == null) + sb.append("null"); + else + NativeCodec.quote(sb, first.toString()); + return ArcadeNative.out(sb.append('}').toString()); + } + default -> throw new IllegalArgumentException("unknown async operation " + ArcadeNative.str(op)); + } + return ArcadeNative.OK(); + } catch (final Throwable t) { + return ArcadeNative.err(t); + } + } + + // ---- import and export + + /** settings: {key: string} as the Python importer builds them; the reply is {"ok":true,"stats":{..}|null}. */ + @CEntryPoint(name = "arcadedb_n_import") + public static CCharPointer importDocuments(final IsolateThread thread, final long dbh, final CCharPointer settingsJson) { + try { + final Database db = ArcadeNative.db(dbh); + final Map settings = new HashMap<>(); + for (final Map.Entry e : args(settingsJson).entrySet()) + settings.put(e.getKey(), String.valueOf(e.getValue())); + final Importer importer = new Importer(db, null); + importer.setSettings(settings); + final Map stats = importer.load(); + final StringBuilder sb = new StringBuilder("{\"ok\":true,\"stats\":"); + if (stats == null) + sb.append("null"); + else + NativeCodec.append(sb, stats); + return ArcadeNative.out(sb.append('}').toString()); + } catch (final Throwable t) { + return ArcadeNative.err(t); + } + } + + @CEntryPoint(name = "arcadedb_n_export") + public static CCharPointer exportDatabase(final IsolateThread thread, final long dbh, final CCharPointer path, final CCharPointer format, + final int overwrite, final CCharPointer settingsJson) { + try { + final Database db = ArcadeNative.db(dbh); + final Exporter exporter = new Exporter(db, ArcadeNative.str(path)); + exporter.setFormat(ArcadeNative.str(format)); + exporter.setOverwrite(overwrite != 0); + final Map settings = new HashMap<>(); + for (final Map.Entry e : args(settingsJson).entrySet()) + settings.put(e.getKey(), String.valueOf(e.getValue())); + if (!settings.isEmpty()) + exporter.setSettings(settings); + final Map stats = exporter.exportDatabase(); + final StringBuilder sb = new StringBuilder("{\"ok\":true,\"stats\":"); + NativeCodec.append(sb, stats == null ? new HashMap() : stats); + return ArcadeNative.out(sb.append('}').toString()); + } catch (final Throwable t) { + return ArcadeNative.err(t); + } + } + + // ---- Python functions in SQL: arcadedb_n_udf_register(name, callbackId, minArgs, maxArgs) + + /** + * A SQL function whose body is a Python callable. The engine thread that runs the statement calls the upcall with + * {"k":"fn","args":[..]} (tagged values, as parameters) and expects {"v":} or {"error":"text"}; an error is thrown + * as a CommandExecutionException from the function, so the statement fails with the Python message. + */ + static final class PythonFunction extends SQLFunctionAbstract { + private final long cb; + private final int min, max; + + PythonFunction(final String name, final long cb, final int min, final int max) { + super(name); + this.cb = cb; + this.min = min; + this.max = max; + } + + @Override + public Object execute(final Object self, final Identifiable record, final Object result, final Object[] params, final CommandContext ctx) { + final StringBuilder sb = new StringBuilder("{\"k\":\"fn\",\"args\":["); + for (int i = 0; i < params.length; i++) { + if (i > 0) + sb.append(','); + NativeCodec.append(sb, params[i]); + } + final String reply = upcall(cb, sb.append("]}").toString()); + if (reply == null) + return null; + @SuppressWarnings("unchecked") final Map r = (Map) Json.parse(reply); + if (r.get("error") != null) + throw new com.arcadedb.exception.CommandExecutionException(String.valueOf(r.get("error"))); + return r.get("v"); + } + + @Override + public int getMinArgs() { + return min; + } + + @Override + public int getMaxArgs() { + return max; + } + + @Override + public String getSyntax() { + return getName() + "(*)"; + } + } + + @CEntryPoint(name = "arcadedb_n_udf_register") + public static CCharPointer udfRegister(final IsolateThread thread, final CCharPointer name, final long cb, final int min, final int max) { + try { + DefaultSQLFunctionFactory.getInstance().register(new PythonFunction(ArcadeNative.str(name), cb, min, max)); + return ArcadeNative.OK(); + } catch (final Throwable t) { + return ArcadeNative.err(t); + } + } +} diff --git a/native/src/main/java/org/humemai/nativelib/Prefetched.java b/native/src/main/java/org/humemai/nativelib/Prefetched.java new file mode 100644 index 0000000..6929c02 --- /dev/null +++ b/native/src/main/java/org/humemai/nativelib/Prefetched.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Phase 7: a ResultSet that has pulled its first rows from the engine (the statement has run) without encoding them. The + * rows are encoded when the caller reads them (arcadedb_n_next); a caller that only closes the result (an INSERT whose + * created record nobody reads) never pays for the encoding of the record, the vector in it included. + */ +package org.humemai.nativelib; + +import com.arcadedb.query.sql.executor.Result; +import com.arcadedb.query.sql.executor.ResultSet; + +import java.util.ArrayDeque; + +final class Prefetched implements ResultSet { + private final ArrayDeque head = new ArrayDeque<>(); + private final ResultSet tail; + private Throwable pending; + + Prefetched(final ResultSet tail, final int max) { + this.tail = tail; + try { + while (head.size() < max && tail.hasNext()) + head.add(tail.next()); + } catch (final Throwable t) { + // met while the engine computed the first rows: raised when the rows before it have been read (as the eager path did) + pending = t; + } + } + + @Override + public boolean hasNext() { + if (!head.isEmpty()) + return true; + if (pending != null) { + final Throwable t = pending; + pending = null; + if (t instanceof RuntimeException r) + throw r; + if (t instanceof Error e) + throw e; + throw new RuntimeException(t); + } + return tail.hasNext(); + } + + @Override + public Result next() { + final Result r = head.poll(); + return r != null ? r : tail.next(); + } + + @Override + public void close() { + tail.close(); + } +} diff --git a/native/src/main/java/org/wildfly/common/Substitutions.java b/native/src/main/java/org/wildfly/common/Substitutions.java new file mode 100644 index 0000000..e092d30 --- /dev/null +++ b/native/src/main/java/org/wildfly/common/Substitutions.java @@ -0,0 +1,45 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package org.wildfly.common; + +/** + * Build-time shadow of {@code org.wildfly.common.Substitutions} shipped in wildfly-common 1.5.4.Final + * (pulled transitively by Undertow/XNIO). That class carries GraalVM native-image {@code @TargetClass} + * substitutions ({@code Target_GraalDirectives}, {@code Target_Branch}) that inject Graal branch-probability + * intrinsics. The substitution target {@code org.graalvm.compiler.api.directives.GraalDirectives} was renamed + * to the {@code jdk.graal.compiler.*} namespace in the GraalVM shipped for JDK 21+, so on GraalVM CE 25 the + * native-image builder aborts with "Substitution target ... is not loaded". + * + *

The arcadedb-native jar precedes wildfly-common on the image classpath, so these stripped, un-annotated + * classes shadow the broken originals and remove the substitutions entirely. They are pure optimization hints + * (branch-probability predictions); dropping them has zero behavioral effect - the substituted methods simply + * run their normal Java bodies. The unrelated wildfly substitutions (net/os/lock, including the run-time + * HostName handling) live in different classes and are untouched. + */ +final class Substitutions { + + static final class Target_GraalDirectives { + } + + static final class Target_Branch { + } + + private Substitutions() { + } +} diff --git a/native/src/main/resources/META-INF/native-image/org.humemai/arcadedb-native-lib/native-image.properties b/native/src/main/resources/META-INF/native-image/org.humemai/arcadedb-native-lib/native-image.properties new file mode 100644 index 0000000..021c1ee --- /dev/null +++ b/native/src/main/resources/META-INF/native-image/org.humemai/arcadedb-native-lib/native-image.properties @@ -0,0 +1,7 @@ +# Force-inline the scalar distance path of jvector 4.0.2 (the provider every native image uses: -H:-VectorAPISupport, see +# oracle/graal#14751). Measured with untimed profiles on the baseline image (-march=compatibility), 50k x 128-d L2 graph: ArrayVectorFloat.length, +# a one-line getter, stayed a call and took 31% of the samples; with these filters it takes 0%, the graph build went from 94.2 s to 76.1 s +# and a search from 1.07 ms to 0.89 ms. The dotProduct and cosine entries are the same scalar counterparts for DOT_PRODUCT and COSINE +# indexes (not profiled; the same getters sit in their loops). A filter without a signature matches every overload. This file is read from +# the glue jar on the image build classpath; preview-build.sh fails the build when the log does not show the option as picked up. +Args = -H:PriorityForceInline=io.github.jbellis.jvector.vector.ArrayVectorFloat.length,io.github.jbellis.jvector.vector.ArrayVectorFloat.get,io.github.jbellis.jvector.vector.VectorUtil.squareL2Distance,io.github.jbellis.jvector.vector.VectorUtil.dotProduct,io.github.jbellis.jvector.vector.VectorUtil.cosine,io.github.jbellis.jvector.vector.DefaultVectorUtilSupport.squareDistance,io.github.jbellis.jvector.vector.DefaultVectorUtilSupport.squareDistanceUnrolled,io.github.jbellis.jvector.vector.DefaultVectorUtilSupport.dotProduct,io.github.jbellis.jvector.vector.DefaultVectorUtilSupport.cosine diff --git a/native/src/main/resources/META-INF/native-image/org.humemai/arcadedb-native-lib/reachability-metadata.json b/native/src/main/resources/META-INF/native-image/org.humemai/arcadedb-native-lib/reachability-metadata.json new file mode 100644 index 0000000..8d9e5e0 --- /dev/null +++ b/native/src/main/resources/META-INF/native-image/org.humemai/arcadedb-native-lib/reachability-metadata.json @@ -0,0 +1,6307 @@ +{ + "reflection": [ + { + "type": "android.app.Application" + }, + { + "type": "android.os.Build$VERSION" + }, + { + "type": "apple.security.AppleProvider", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "boolean[]" + }, + { + "type": "byte[]" + }, + { + "type": "ch.qos.logback.classic.Logger" + }, + { + "type": "char[]" + }, + { + "type": "com.arcadedb.bolt.BoltProtocolPlugin" + }, + { + "type": "com.arcadedb.engine.PaginatedComponentFile" + }, + { + "type": "com.arcadedb.function.sql.coll.SQLFunctionDifference", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.coll.SQLFunctionIntersect", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.coll.SQLFunctionList", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.coll.SQLFunctionMap", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.coll.SQLFunctionSet", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.coll.SQLFunctionSymmetricDifference", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.coll.SQLFunctionUnionAll", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionAstar", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionBellmanFord", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionBoth", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionBothE", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionBothV", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionCCHShortestPath", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionDijkstra", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionDuanSSSP", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionIn", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionInE", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionInV", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionOut", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionOutE", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionOutV", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.graph.SQLFunctionShortestPath", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionAbsoluteValue", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionAverage", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionCount", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionEval", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionMax", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionMedian", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionMin", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionMode", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionPercentile", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionPow", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionRandomInt", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionSquareRoot", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionStandardDeviation", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionStandardDeviationP", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionSum", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionVariance", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.math.SQLFunctionVarianceP", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.misc.SQLFunctionBoolAnd", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.misc.SQLFunctionBoolOr", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.misc.SQLFunctionUUID", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.misc.SQLFunctionVersion", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.text.SQLFunctionConcat", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.text.SQLFunctionStrcmpci", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.time.SQLFunctionCorrelate", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.time.SQLFunctionDelta", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.time.SQLFunctionInterpolate", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.time.SQLFunctionLag", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.time.SQLFunctionLead", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.time.SQLFunctionMovingAvg", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.time.SQLFunctionRank", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.time.SQLFunctionRate", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.time.SQLFunctionRowNumber", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.time.SQLFunctionSysdate", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.time.SQLFunctionTsFirst", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.time.SQLFunctionTsLast", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.time.SQLFunctionTsPercentile", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.vector.SQLFunctionVectorAvg", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.vector.SQLFunctionVectorMax", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.vector.SQLFunctionVectorMin", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.function.sql.vector.SQLFunctionVectorSum", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.graphql.query.GraphQLQueryEngineFactory", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.gremlin.query.GremlinQueryEngineFactory", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.log.LogFormatter", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.mcp.MCPPlugin" + }, + { + "type": "com.arcadedb.metrics.otlp.OtlpMetricsPlugin" + }, + { + "type": "com.arcadedb.metrics.prometheus.PrometheusMetricsPlugin" + }, + { + "type": "com.arcadedb.mongo.MongoDBProtocolPlugin" + }, + { + "type": "com.arcadedb.mongo.query.MongoQueryEngineFactory", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.postgres.PostgresProtocolPlugin" + }, + { + "type": "com.arcadedb.query.opencypher.query.OpenCypherQueryEngineFactory", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.redis.RedisProtocolPlugin" + }, + { + "type": "com.arcadedb.redis.query.RedisQueryEngineFactory", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.server.backup.AutoBackupSchedulerPlugin" + }, + { + "type": "com.arcadedb.server.gremlin.GremlinServerPlugin" + }, + { + "type": "com.arcadedb.server.grpc.GrpcServerPlugin" + }, + { + "type": "com.arcadedb.server.ha.raft.RaftHAPlugin" + }, + { + "type": "com.arcadedb.server.ha.raft.ratis.FixedGrpcRpcType", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.arcadedb.server.security.PinnedInetAddressResolverProvider" + }, + { + "type": "com.arcadedb.tracing.TracingPlugin" + }, + { + "type": "com.arcadedb.utility.AnsiLogFormatter", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.google.common.util.concurrent.AbstractFutureState" + }, + { + "type": "com.google.common.util.concurrent.AbstractFutureState$Waiter" + }, + { + "type": "com.google.gson.internal.bind.JavaTimeTypeAdapters", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.google.protobuf.ExtensionRegistry", + "methods": [ + { + "name": "getEmptyRegistry", + "parameterTypes": [] + } + ] + }, + { + "type": "com.ibm.lang.management.OperatingSystemMXBean" + }, + { + "type": "com.sun.crypto.provider.AESCipher$General", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.sun.crypto.provider.ARCFOURCipher", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.sun.crypto.provider.ChaCha20Cipher$ChaCha20Poly1305", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.sun.crypto.provider.DESCipher", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.sun.crypto.provider.DESedeCipher", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.sun.crypto.provider.DHParameters", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.sun.crypto.provider.GaloisCounterMode$AESGCM", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.sun.crypto.provider.HmacCore$HmacSHA256", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.sun.crypto.provider.PBKDF2Core$HmacSHA256", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.sun.crypto.provider.TlsMasterSecretGenerator", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "com.sun.jndi.dns.DnsContextFactory" + }, + { + "type": "com.sun.management.GarbageCollectionNotificationInfo" + }, + { + "type": "com.sun.management.GarbageCollectorMXBean" + }, + { + "type": "com.sun.management.GcInfo", + "fields": [ + { + "name": "builder" + }, + { + "name": "extAttributes" + } + ] + }, + { + "type": "com.sun.management.HotSpotDiagnosticMXBean" + }, + { + "type": "com.sun.management.OperatingSystemMXBean", + "methods": [ + { + "name": "getCpuLoad", + "parameterTypes": [] + }, + { + "name": "getFreePhysicalMemorySize", + "parameterTypes": [] + }, + { + "name": "getProcessCpuLoad", + "parameterTypes": [] + }, + { + "name": "getProcessCpuTime", + "parameterTypes": [] + }, + { + "name": "getTotalPhysicalMemorySize", + "parameterTypes": [] + } + ] + }, + { + "type": "com.sun.management.ThreadMXBean" + }, + { + "type": "com.sun.management.UnixOperatingSystemMXBean" + }, + { + "type": "com.sun.management.VMOption" + }, + { + "type": "com.sun.management.internal.DiagnosticCommandArgumentInfo", + "jniAccessible": true, + "methods": [ + { + "name": "", + "parameterTypes": [ + "java.lang.String", + "java.lang.String", + "java.lang.String", + "java.lang.String", + "boolean", + "boolean", + "boolean", + "int" + ] + } + ] + }, + { + "type": "com.sun.management.internal.DiagnosticCommandArgumentInfo[]" + }, + { + "type": "com.sun.management.internal.DiagnosticCommandInfo", + "jniAccessible": true, + "methods": [ + { + "name": "", + "parameterTypes": [ + "java.lang.String", + "java.lang.String", + "java.lang.String", + "boolean", + "java.util.List" + ] + } + ] + }, + { + "type": "com.sun.management.internal.DiagnosticCommandInfo[]" + }, + { + "type": "com.sun.management.internal.GarbageCollectorExtImpl" + }, + { + "type": "com.sun.management.internal.HotSpotDiagnostic" + }, + { + "type": "com.sun.management.internal.HotSpotThreadImpl" + }, + { + "type": "com.sun.management.internal.OperatingSystemImpl" + }, + { + "type": "com.sun.management.internal.VirtualThreadSchedulerImpls$VirtualThreadSchedulerImpl" + }, + { + "type": "de.bwaldvogel.mongo.MongoServer$1" + }, + { + "type": "de.bwaldvogel.mongo.wire.MongoDatabaseHandler" + }, + { + "type": "de.bwaldvogel.mongo.wire.MongoExceptionHandler" + }, + { + "type": "de.bwaldvogel.mongo.wire.MongoWireMessageEncoder" + }, + { + "type": "de.bwaldvogel.mongo.wire.MongoWireProtocolHandler" + }, + { + "type": "de.bwaldvogel.mongo.wire.MongoWireReplyEncoder" + }, + { + "type": "double[]" + }, + { + "type": "float[]" + }, + { + "type": "int[]" + }, + { + "type": "io.grpc.census.InternalCensusStatsAccessor" + }, + { + "type": "io.grpc.census.InternalCensusTracingAccessor" + }, + { + "type": "io.grpc.internal.SerializingExecutor" + }, + { + "type": "io.grpc.netty.shaded.io.grpc.netty.NettyServer$1" + }, + { + "type": "io.grpc.netty.shaded.io.grpc.netty.NettyServerHandler" + }, + { + "type": "io.grpc.netty.shaded.io.grpc.netty.ProtocolNegotiators$GrpcNegotiationHandler" + }, + { + "type": "io.grpc.netty.shaded.io.grpc.netty.ProtocolNegotiators$PlaintextHandler" + }, + { + "type": "io.grpc.netty.shaded.io.grpc.netty.ProtocolNegotiators$WaitUntilActiveHandler" + }, + { + "type": "io.grpc.netty.shaded.io.grpc.netty.WriteBufferingAndExceptionHandler" + }, + { + "type": "io.grpc.netty.shaded.io.netty.bootstrap.ServerBootstrap$1" + }, + { + "type": "io.grpc.netty.shaded.io.netty.bootstrap.ServerBootstrap$ServerBootstrapAcceptor" + }, + { + "type": "io.grpc.netty.shaded.io.netty.buffer.AbstractByteBufAllocator" + }, + { + "type": "io.grpc.netty.shaded.io.netty.buffer.AbstractReferenceCountedByteBuf", + "fields": [ + { + "name": "refCnt" + } + ] + }, + { + "type": "io.grpc.netty.shaded.io.netty.buffer.AdaptivePoolingAllocator$Magazine" + }, + { + "type": "io.grpc.netty.shaded.io.netty.channel.AbstractChannelHandlerContext" + }, + { + "type": "io.grpc.netty.shaded.io.netty.channel.ChannelOutboundBuffer" + }, + { + "type": "io.grpc.netty.shaded.io.netty.channel.DefaultChannelConfig" + }, + { + "type": "io.grpc.netty.shaded.io.netty.channel.DefaultChannelPipeline" + }, + { + "type": "io.grpc.netty.shaded.io.netty.channel.DefaultChannelPipeline$HeadContext" + }, + { + "type": "io.grpc.netty.shaded.io.netty.channel.DefaultChannelPipeline$TailContext" + }, + { + "type": "io.grpc.netty.shaded.io.netty.channel.DefaultFileRegion" + }, + { + "type": "io.grpc.netty.shaded.io.netty.channel.epoll.Epoll", + "methods": [ + { + "name": "isAvailable", + "parameterTypes": [] + }, + { + "name": "unavailabilityCause", + "parameterTypes": [] + } + ] + }, + { + "type": "io.grpc.netty.shaded.io.netty.channel.epoll.NativeDatagramPacketArray$NativeDatagramPacket" + }, + { + "type": "io.grpc.netty.shaded.io.netty.channel.unix.PeerCredentials" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.AbstractReferenceCounted", + "fields": [ + { + "name": "refCnt" + } + ] + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.DefaultAttributeMap" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.Recycler$DefaultHandle" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.ReferenceCountUtil" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.ResourceLeakDetector$DefaultResourceLeak" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.concurrent.ConcurrentSkipListIntObjMultimap" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.concurrent.ConcurrentSkipListIntObjMultimap$Index" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.concurrent.ConcurrentSkipListIntObjMultimap$Node" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.concurrent.DefaultPromise" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.concurrent.SingleThreadEventExecutor" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields", + "fields": [ + { + "name": "producerLimit" + } + ] + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields", + "fields": [ + { + "name": "consumerIndex" + } + ] + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields", + "fields": [ + { + "name": "producerIndex" + } + ] + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.atomic.BaseMpscLinkedAtomicArrayQueueColdProducerFields" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.atomic.BaseMpscLinkedAtomicArrayQueueConsumerFields" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.atomic.BaseMpscLinkedAtomicArrayQueueProducerFields" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.atomic.MpmcAtomicArrayQueueConsumerIndexField" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.atomic.MpmcAtomicArrayQueueProducerIndexField" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.atomic.unpadded.MpscAtomicUnpaddedArrayQueueConsumerIndexField" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.atomic.unpadded.MpscAtomicUnpaddedArrayQueueProducerIndexField" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.atomic.unpadded.MpscAtomicUnpaddedArrayQueueProducerLimitField" + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField", + "fields": [ + { + "name": "consumerIndex" + } + ] + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField", + "fields": [ + { + "name": "producerIndex" + } + ] + }, + { + "type": "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField", + "fields": [ + { + "name": "producerLimit" + } + ] + }, + { + "type": "io.grpc.override.ContextStorageOverride" + }, + { + "type": "io.micrometer.core.instrument.distribution.AbstractTimeWindowHistogram" + }, + { + "type": "io.micrometer.core.instrument.distribution.FixedBoundaryHistogram[]" + }, + { + "type": "io.micrometer.core.instrument.distribution.TimeWindowMax" + }, + { + "type": "io.netty.bootstrap.ServerBootstrap$1" + }, + { + "type": "io.netty.bootstrap.ServerBootstrap$ServerBootstrapAcceptor" + }, + { + "type": "io.netty.buffer.AbstractByteBufAllocator" + }, + { + "type": "io.netty.buffer.AdaptivePoolingAllocator$Magazine" + }, + { + "type": "io.netty.buffer.AdaptivePoolingAllocator$SizeClassedChunk" + }, + { + "type": "io.netty.channel.AbstractChannelHandlerContext" + }, + { + "type": "io.netty.channel.ChannelOutboundBuffer" + }, + { + "type": "io.netty.channel.DefaultChannelConfig" + }, + { + "type": "io.netty.channel.DefaultChannelPipeline" + }, + { + "type": "io.netty.channel.DefaultChannelPipeline$HeadContext" + }, + { + "type": "io.netty.channel.DefaultChannelPipeline$TailContext" + }, + { + "type": "io.netty.channel.socket.nio.NioServerSocketChannel", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "io.netty.util.DefaultAttributeMap" + }, + { + "type": "io.netty.util.ReferenceCountUtil" + }, + { + "type": "io.netty.util.ResourceLeakDetector$DefaultResourceLeak" + }, + { + "type": "io.netty.util.concurrent.ConcurrentSkipListIntObjMultimap" + }, + { + "type": "io.netty.util.concurrent.ConcurrentSkipListIntObjMultimap$Index" + }, + { + "type": "io.netty.util.concurrent.ConcurrentSkipListIntObjMultimap$Node" + }, + { + "type": "io.netty.util.concurrent.DefaultPromise" + }, + { + "type": "io.netty.util.concurrent.MpscIntQueue$MpscAtomicIntegerArrayQueue" + }, + { + "type": "io.netty.util.concurrent.SingleThreadEventExecutor" + }, + { + "type": "io.netty.util.internal.shaded.org.jctools.queues.atomic.BaseMpscLinkedAtomicArrayQueueColdProducerFields" + }, + { + "type": "io.netty.util.internal.shaded.org.jctools.queues.atomic.BaseMpscLinkedAtomicArrayQueueConsumerFields" + }, + { + "type": "io.netty.util.internal.shaded.org.jctools.queues.atomic.BaseMpscLinkedAtomicArrayQueueProducerFields" + }, + { + "type": "io.netty.util.internal.shaded.org.jctools.queues.atomic.MpmcAtomicArrayQueueConsumerIndexField" + }, + { + "type": "io.netty.util.internal.shaded.org.jctools.queues.atomic.MpmcAtomicArrayQueueProducerIndexField" + }, + { + "type": "io.netty.util.internal.shaded.org.jctools.queues.varhandle.BaseMpscLinkedVarHandleArrayQueueColdProducerFields", + "fields": [ + { + "name": "producerLimit" + } + ] + }, + { + "type": "io.netty.util.internal.shaded.org.jctools.queues.varhandle.BaseMpscLinkedVarHandleArrayQueueConsumerFields", + "fields": [ + { + "name": "consumerIndex" + } + ] + }, + { + "type": "io.netty.util.internal.shaded.org.jctools.queues.varhandle.BaseMpscLinkedVarHandleArrayQueueProducerFields", + "fields": [ + { + "name": "producerIndex" + } + ] + }, + { + "type": "io.netty.util.internal.shaded.org.jctools.queues.varhandle.MpmcVarHandleArrayQueueConsumerIndexField", + "fields": [ + { + "name": "consumerIndex" + } + ] + }, + { + "type": "io.netty.util.internal.shaded.org.jctools.queues.varhandle.MpmcVarHandleArrayQueueProducerIndexField", + "fields": [ + { + "name": "producerIndex" + } + ] + }, + { + "type": "io.opentelemetry.api.incubator.ExtendedOpenTelemetry" + }, + { + "type": "io.opentelemetry.api.incubator.metrics.ExtendedDefaultMeterProvider" + }, + { + "type": "io.opentelemetry.api.incubator.trace.ExtendedDefaultTracerProvider" + }, + { + "type": "io.opentelemetry.exporter.internal.marshal.VarHandleStringEncoder", + "methods": [ + { + "name": "createIfAvailable", + "parameterTypes": [] + } + ] + }, + { + "type": "io.opentelemetry.exporter.sender.okhttp.internal.OkHttpGrpcSenderProvider" + }, + { + "type": "io.opentelemetry.sdk.trace.internal.shaded.jctools.queues.atomic.MpscAtomicArrayQueueConsumerIndexField" + }, + { + "type": "io.opentelemetry.sdk.trace.internal.shaded.jctools.queues.atomic.MpscAtomicArrayQueueProducerIndexField" + }, + { + "type": "io.opentelemetry.sdk.trace.internal.shaded.jctools.queues.atomic.MpscAtomicArrayQueueProducerLimitField" + }, + { + "type": "io.perfmark.impl.SecretPerfMarkImpl$PerfMarkImpl" + }, + { + "type": "io.prometheus.metrics.expositionformats.internal.PrometheusProtobufWriterImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "io.undertow.UndertowLogger_$logger", + "methods": [ + { + "name": "", + "parameterTypes": [ + "org.jboss.logging.Logger" + ] + } + ] + }, + { + "type": "io.undertow.UndertowLogger_$logger_en" + }, + { + "type": "io.undertow.UndertowLogger_$logger_en_US" + }, + { + "type": "io.undertow.protocols.http2.Http2Channel" + }, + { + "type": "io.undertow.server.ConnectorStatisticsImpl" + }, + { + "type": "io.undertow.server.DefaultByteBufferPool" + }, + { + "type": "io.undertow.server.DefaultByteBufferPool$DefaultPooledBuffer" + }, + { + "type": "io.undertow.server.HttpServerExchange" + }, + { + "type": "io.undertow.server.protocol.framed.AbstractFramedChannel" + }, + { + "type": "io.undertow.server.protocol.framed.AbstractFramedStreamSinkChannel" + }, + { + "type": "io.undertow.server.protocol.http.HttpReadListener" + }, + { + "type": "io.undertow.util.Headers", + "fields": [ + { + "name": "ACCEPT" + }, + { + "name": "ACCEPT_CHARSET" + }, + { + "name": "ACCEPT_ENCODING" + }, + { + "name": "ACCEPT_LANGUAGE" + }, + { + "name": "ACCEPT_RANGES" + }, + { + "name": "AGE" + }, + { + "name": "ALGORITHM" + }, + { + "name": "ALLOW" + }, + { + "name": "AUTHENTICATION_INFO" + }, + { + "name": "AUTHORIZATION" + }, + { + "name": "AUTH_PARAM" + }, + { + "name": "BASIC" + }, + { + "name": "CACHE_CONTROL" + }, + { + "name": "CHUNKED" + }, + { + "name": "CLOSE" + }, + { + "name": "CNONCE" + }, + { + "name": "COMPRESS" + }, + { + "name": "CONNECTION" + }, + { + "name": "CONTENT_DISPOSITION" + }, + { + "name": "CONTENT_ENCODING" + }, + { + "name": "CONTENT_LANGUAGE" + }, + { + "name": "CONTENT_LENGTH" + }, + { + "name": "CONTENT_LOCATION" + }, + { + "name": "CONTENT_MD5" + }, + { + "name": "CONTENT_RANGE" + }, + { + "name": "CONTENT_SECURITY_POLICY" + }, + { + "name": "CONTENT_TRANSFER_ENCODING" + }, + { + "name": "CONTENT_TYPE" + }, + { + "name": "COOKIE" + }, + { + "name": "COOKIE2" + }, + { + "name": "DATE" + }, + { + "name": "DEFLATE" + }, + { + "name": "DIGEST" + }, + { + "name": "DOMAIN" + }, + { + "name": "ETAG" + }, + { + "name": "EXPECT" + }, + { + "name": "EXPIRES" + }, + { + "name": "FORWARDED" + }, + { + "name": "FROM" + }, + { + "name": "GZIP" + }, + { + "name": "HOST" + }, + { + "name": "IDENTITY" + }, + { + "name": "IF_MATCH" + }, + { + "name": "IF_MODIFIED_SINCE" + }, + { + "name": "IF_NONE_MATCH" + }, + { + "name": "IF_RANGE" + }, + { + "name": "IF_UNMODIFIED_SINCE" + }, + { + "name": "KEEP_ALIVE" + }, + { + "name": "LAST_MODIFIED" + }, + { + "name": "LOCATION" + }, + { + "name": "MAX_FORWARDS" + }, + { + "name": "NEGOTIATE" + }, + { + "name": "NEXT_NONCE" + }, + { + "name": "NONCE" + }, + { + "name": "NONCE_COUNT" + }, + { + "name": "OPAQUE" + }, + { + "name": "ORIGIN" + }, + { + "name": "PRAGMA" + }, + { + "name": "PROXY_AUTHENTICATE" + }, + { + "name": "PROXY_AUTHORIZATION" + }, + { + "name": "QOP" + }, + { + "name": "RANGE" + }, + { + "name": "REALM" + }, + { + "name": "REFERER" + }, + { + "name": "REFERRER_POLICY" + }, + { + "name": "REFRESH" + }, + { + "name": "RESPONSE" + }, + { + "name": "RESPONSE_AUTH" + }, + { + "name": "RETRY_AFTER" + }, + { + "name": "SECURE_PROTOCOL" + }, + { + "name": "SEC_WEB_SOCKET_ACCEPT" + }, + { + "name": "SEC_WEB_SOCKET_EXTENSIONS" + }, + { + "name": "SEC_WEB_SOCKET_KEY" + }, + { + "name": "SEC_WEB_SOCKET_KEY1" + }, + { + "name": "SEC_WEB_SOCKET_KEY2" + }, + { + "name": "SEC_WEB_SOCKET_LOCATION" + }, + { + "name": "SEC_WEB_SOCKET_ORIGIN" + }, + { + "name": "SEC_WEB_SOCKET_PROTOCOL" + }, + { + "name": "SEC_WEB_SOCKET_VERSION" + }, + { + "name": "SERVER" + }, + { + "name": "SERVLET_ENGINE" + }, + { + "name": "SET_COOKIE" + }, + { + "name": "SET_COOKIE2" + }, + { + "name": "SSL_CIPHER" + }, + { + "name": "SSL_CIPHER_USEKEYSIZE" + }, + { + "name": "SSL_CLIENT_CERT" + }, + { + "name": "SSL_SESSION_ID" + }, + { + "name": "STALE" + }, + { + "name": "STATUS" + }, + { + "name": "STRICT_TRANSPORT_SECURITY" + }, + { + "name": "TE" + }, + { + "name": "TRAILER" + }, + { + "name": "TRANSFER_ENCODING" + }, + { + "name": "UPGRADE" + }, + { + "name": "URI" + }, + { + "name": "USERNAME" + }, + { + "name": "USER_AGENT" + }, + { + "name": "VARY" + }, + { + "name": "VIA" + }, + { + "name": "WARNING" + }, + { + "name": "WWW_AUTHENTICATE" + }, + { + "name": "X_COMPRESS" + }, + { + "name": "X_CONTENT_LENGTH" + }, + { + "name": "X_CONTENT_TYPE_OPTIONS" + }, + { + "name": "X_DISABLE_PUSH" + }, + { + "name": "X_FORWARDED_FOR" + }, + { + "name": "X_FORWARDED_HOST" + }, + { + "name": "X_FORWARDED_PORT" + }, + { + "name": "X_FORWARDED_PROTO" + }, + { + "name": "X_FORWARDED_SERVER" + }, + { + "name": "X_FRAME_OPTIONS" + }, + { + "name": "X_GZIP" + }, + { + "name": "X_XSS_PROTECTION" + } + ] + }, + { + "type": "io.undertow.util.HttpString" + }, + { + "type": "io.undertow.util.ReferenceCountedPooled" + }, + { + "type": "java.io.FileDescriptor" + }, + { + "type": "java.lang.BaseVirtualThread" + }, + { + "type": "java.lang.Boolean", + "jniAccessible": true, + "fields": [ + { + "name": "TYPE" + } + ], + "methods": [ + { + "name": "getBoolean", + "parameterTypes": [ + "java.lang.String" + ] + } + ] + }, + { + "type": "java.lang.Byte", + "fields": [ + { + "name": "TYPE" + } + ] + }, + { + "type": "java.lang.Character", + "fields": [ + { + "name": "TYPE" + } + ] + }, + { + "type": "java.lang.Class", + "jniAccessible": true, + "methods": [ + { + "name": "getClassLoader", + "parameterTypes": [] + }, + { + "name": "getModule", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.ClassLoader", + "jniAccessible": true, + "fields": [ + { + "name": "classLoaderValueMap" + } + ], + "methods": [ + { + "name": "loadClass", + "parameterTypes": [ + "java.lang.String" + ] + } + ] + }, + { + "type": "java.lang.Deprecated" + }, + { + "type": "java.lang.Double", + "fields": [ + { + "name": "TYPE" + } + ] + }, + { + "type": "java.lang.Float", + "fields": [ + { + "name": "TYPE" + } + ] + }, + { + "type": "java.lang.IllegalArgumentException", + "jniAccessible": true + }, + { + "type": "java.lang.IllegalStateException", + "jniAccessible": true + }, + { + "type": "java.lang.Integer", + "fields": [ + { + "name": "TYPE" + } + ] + }, + { + "type": "java.lang.Long", + "fields": [ + { + "name": "TYPE" + } + ] + }, + { + "type": "java.lang.Math", + "methods": [ + { + "name": "IEEEremainder", + "parameterTypes": [ + "double", + "double" + ] + }, + { + "name": "abs", + "parameterTypes": [ + "double" + ] + }, + { + "name": "abs", + "parameterTypes": [ + "float" + ] + }, + { + "name": "abs", + "parameterTypes": [ + "int" + ] + }, + { + "name": "abs", + "parameterTypes": [ + "long" + ] + }, + { + "name": "absExact", + "parameterTypes": [ + "int" + ] + }, + { + "name": "absExact", + "parameterTypes": [ + "long" + ] + }, + { + "name": "acos", + "parameterTypes": [ + "double" + ] + }, + { + "name": "addExact", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "addExact", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "asin", + "parameterTypes": [ + "double" + ] + }, + { + "name": "atan", + "parameterTypes": [ + "double" + ] + }, + { + "name": "atan2", + "parameterTypes": [ + "double", + "double" + ] + }, + { + "name": "cbrt", + "parameterTypes": [ + "double" + ] + }, + { + "name": "ceil", + "parameterTypes": [ + "double" + ] + }, + { + "name": "ceilDiv", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "ceilDiv", + "parameterTypes": [ + "long", + "int" + ] + }, + { + "name": "ceilDiv", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "ceilDivExact", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "ceilDivExact", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "ceilMod", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "ceilMod", + "parameterTypes": [ + "long", + "int" + ] + }, + { + "name": "ceilMod", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "clamp", + "parameterTypes": [ + "double", + "double", + "double" + ] + }, + { + "name": "clamp", + "parameterTypes": [ + "float", + "float", + "float" + ] + }, + { + "name": "clamp", + "parameterTypes": [ + "long", + "int", + "int" + ] + }, + { + "name": "clamp", + "parameterTypes": [ + "long", + "long", + "long" + ] + }, + { + "name": "copySign", + "parameterTypes": [ + "double", + "double" + ] + }, + { + "name": "copySign", + "parameterTypes": [ + "float", + "float" + ] + }, + { + "name": "cos", + "parameterTypes": [ + "double" + ] + }, + { + "name": "cosh", + "parameterTypes": [ + "double" + ] + }, + { + "name": "decrementExact", + "parameterTypes": [ + "int" + ] + }, + { + "name": "decrementExact", + "parameterTypes": [ + "long" + ] + }, + { + "name": "divideExact", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "divideExact", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "exp", + "parameterTypes": [ + "double" + ] + }, + { + "name": "expm1", + "parameterTypes": [ + "double" + ] + }, + { + "name": "floor", + "parameterTypes": [ + "double" + ] + }, + { + "name": "floorDiv", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "floorDiv", + "parameterTypes": [ + "long", + "int" + ] + }, + { + "name": "floorDiv", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "floorDivExact", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "floorDivExact", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "floorMod", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "floorMod", + "parameterTypes": [ + "long", + "int" + ] + }, + { + "name": "floorMod", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "fma", + "parameterTypes": [ + "double", + "double", + "double" + ] + }, + { + "name": "fma", + "parameterTypes": [ + "float", + "float", + "float" + ] + }, + { + "name": "getExponent", + "parameterTypes": [ + "double" + ] + }, + { + "name": "getExponent", + "parameterTypes": [ + "float" + ] + }, + { + "name": "hypot", + "parameterTypes": [ + "double", + "double" + ] + }, + { + "name": "incrementExact", + "parameterTypes": [ + "int" + ] + }, + { + "name": "incrementExact", + "parameterTypes": [ + "long" + ] + }, + { + "name": "log", + "parameterTypes": [ + "double" + ] + }, + { + "name": "log10", + "parameterTypes": [ + "double" + ] + }, + { + "name": "log1p", + "parameterTypes": [ + "double" + ] + }, + { + "name": "max", + "parameterTypes": [ + "double", + "double" + ] + }, + { + "name": "max", + "parameterTypes": [ + "float", + "float" + ] + }, + { + "name": "max", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "max", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "min", + "parameterTypes": [ + "double", + "double" + ] + }, + { + "name": "min", + "parameterTypes": [ + "float", + "float" + ] + }, + { + "name": "min", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "min", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "multiplyExact", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "multiplyExact", + "parameterTypes": [ + "long", + "int" + ] + }, + { + "name": "multiplyExact", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "multiplyFull", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "multiplyHigh", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "negateExact", + "parameterTypes": [ + "int" + ] + }, + { + "name": "negateExact", + "parameterTypes": [ + "long" + ] + }, + { + "name": "nextAfter", + "parameterTypes": [ + "double", + "double" + ] + }, + { + "name": "nextAfter", + "parameterTypes": [ + "float", + "double" + ] + }, + { + "name": "nextDown", + "parameterTypes": [ + "double" + ] + }, + { + "name": "nextDown", + "parameterTypes": [ + "float" + ] + }, + { + "name": "nextUp", + "parameterTypes": [ + "double" + ] + }, + { + "name": "nextUp", + "parameterTypes": [ + "float" + ] + }, + { + "name": "pow", + "parameterTypes": [ + "double", + "double" + ] + }, + { + "name": "powExact", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "powExact", + "parameterTypes": [ + "long", + "int" + ] + }, + { + "name": "random", + "parameterTypes": [] + }, + { + "name": "rint", + "parameterTypes": [ + "double" + ] + }, + { + "name": "round", + "parameterTypes": [ + "double" + ] + }, + { + "name": "round", + "parameterTypes": [ + "float" + ] + }, + { + "name": "scalb", + "parameterTypes": [ + "double", + "int" + ] + }, + { + "name": "scalb", + "parameterTypes": [ + "float", + "int" + ] + }, + { + "name": "signum", + "parameterTypes": [ + "double" + ] + }, + { + "name": "signum", + "parameterTypes": [ + "float" + ] + }, + { + "name": "sin", + "parameterTypes": [ + "double" + ] + }, + { + "name": "sinh", + "parameterTypes": [ + "double" + ] + }, + { + "name": "sqrt", + "parameterTypes": [ + "double" + ] + }, + { + "name": "subtractExact", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "subtractExact", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "tan", + "parameterTypes": [ + "double" + ] + }, + { + "name": "tanh", + "parameterTypes": [ + "double" + ] + }, + { + "name": "toDegrees", + "parameterTypes": [ + "double" + ] + }, + { + "name": "toIntExact", + "parameterTypes": [ + "long" + ] + }, + { + "name": "toRadians", + "parameterTypes": [ + "double" + ] + }, + { + "name": "ulp", + "parameterTypes": [ + "double" + ] + }, + { + "name": "ulp", + "parameterTypes": [ + "float" + ] + }, + { + "name": "unsignedMultiplyExact", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "unsignedMultiplyExact", + "parameterTypes": [ + "long", + "int" + ] + }, + { + "name": "unsignedMultiplyExact", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "unsignedMultiplyHigh", + "parameterTypes": [ + "long", + "long" + ] + }, + { + "name": "unsignedPowExact", + "parameterTypes": [ + "int", + "int" + ] + }, + { + "name": "unsignedPowExact", + "parameterTypes": [ + "long", + "int" + ] + } + ] + }, + { + "type": "java.lang.Module", + "methods": [ + { + "name": "isNativeAccessEnabled", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.OutOfMemoryError", + "jniAccessible": true + }, + { + "type": "java.lang.ProcessHandle", + "methods": [ + { + "name": "current", + "parameterTypes": [] + }, + { + "name": "pid", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.Short", + "fields": [ + { + "name": "TYPE" + } + ] + }, + { + "type": "java.lang.StackTraceElement" + }, + { + "type": "java.lang.String", + "fields": [ + { + "name": "coder" + }, + { + "name": "value" + } + ] + }, + { + "type": "java.lang.String[]" + }, + { + "type": "java.lang.Thread", + "methods": [ + { + "name": "isVirtual", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.Void", + "fields": [ + { + "name": "TYPE" + } + ] + }, + { + "type": "java.lang.foreign.AddressLayout", + "methods": [ + { + "name": "byteSize", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.foreign.FunctionDescriptor", + "methods": [ + { + "name": "of", + "parameterTypes": [ + "java.lang.foreign.MemoryLayout", + "java.lang.foreign.MemoryLayout[]" + ] + }, + { + "name": "ofVoid", + "parameterTypes": [ + "java.lang.foreign.MemoryLayout[]" + ] + } + ] + }, + { + "type": "java.lang.foreign.Linker", + "methods": [ + { + "name": "defaultLookup", + "parameterTypes": [] + }, + { + "name": "downcallHandle", + "parameterTypes": [ + "java.lang.foreign.MemorySegment", + "java.lang.foreign.FunctionDescriptor", + "java.lang.foreign.Linker$Option[]" + ] + }, + { + "name": "nativeLinker", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.foreign.Linker$Option" + }, + { + "type": "java.lang.foreign.Linker$Option[]" + }, + { + "type": "java.lang.foreign.MemoryLayout" + }, + { + "type": "java.lang.foreign.MemoryLayout[]" + }, + { + "type": "java.lang.foreign.MemorySegment", + "methods": [ + { + "name": "address", + "parameterTypes": [] + }, + { + "name": "asByteBuffer", + "parameterTypes": [] + }, + { + "name": "ofAddress", + "parameterTypes": [ + "long" + ] + }, + { + "name": "ofBuffer", + "parameterTypes": [ + "java.nio.Buffer" + ] + }, + { + "name": "reinterpret", + "parameterTypes": [ + "long" + ] + } + ] + }, + { + "type": "java.lang.foreign.SymbolLookup", + "methods": [ + { + "name": "findOrThrow", + "parameterTypes": [ + "java.lang.String" + ] + } + ] + }, + { + "type": "java.lang.foreign.ValueLayout", + "fields": [ + { + "name": "ADDRESS" + }, + { + "name": "JAVA_LONG" + } + ] + }, + { + "type": "java.lang.foreign.ValueLayout$OfLong" + }, + { + "type": "java.lang.invoke.MethodHandles", + "methods": [ + { + "name": "byteArrayViewVarHandle", + "parameterTypes": [ + "java.lang.Class", + "java.nio.ByteOrder" + ] + }, + { + "name": "byteBufferViewVarHandle", + "parameterTypes": [ + "java.lang.Class", + "java.nio.ByteOrder" + ] + }, + { + "name": "privateLookupIn", + "parameterTypes": [ + "java.lang.Class", + "java.lang.invoke.MethodHandles$Lookup" + ] + } + ] + }, + { + "type": "java.lang.invoke.MethodHandles$Lookup", + "methods": [ + { + "name": "findVarHandle", + "parameterTypes": [ + "java.lang.Class", + "java.lang.String", + "java.lang.Class" + ] + } + ] + }, + { + "type": "java.lang.invoke.VarHandle", + "methods": [ + { + "name": "acquireFence", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.management.BufferPoolMXBean" + }, + { + "type": "java.lang.management.ClassLoadingMXBean" + }, + { + "type": "java.lang.management.CompilationMXBean" + }, + { + "type": "java.lang.management.LockInfo" + }, + { + "type": "java.lang.management.ManagementFactory", + "methods": [ + { + "name": "getRuntimeMXBean", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.management.MemoryMXBean" + }, + { + "type": "java.lang.management.MemoryManagerMXBean" + }, + { + "type": "java.lang.management.MemoryPoolMXBean" + }, + { + "type": "java.lang.management.MemoryUsage", + "methods": [ + { + "name": "from", + "parameterTypes": [ + "javax.management.openmbean.CompositeData" + ] + } + ] + }, + { + "type": "java.lang.management.MonitorInfo" + }, + { + "type": "java.lang.management.PlatformLoggingMXBean" + }, + { + "type": "java.lang.management.RuntimeMXBean", + "methods": [ + { + "name": "getInputArguments", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.management.ThreadInfo" + }, + { + "type": "java.math.BigDecimal" + }, + { + "type": "java.math.BigInteger" + }, + { + "type": "java.nio.Bits", + "fields": [ + { + "name": "UNALIGNED" + } + ] + }, + { + "type": "java.nio.Buffer", + "fields": [ + { + "name": "address" + } + ] + }, + { + "type": "java.nio.ByteBuffer", + "methods": [ + { + "name": "alignedSlice", + "parameterTypes": [ + "int" + ] + }, + { + "name": "put", + "parameterTypes": [ + "int", + "java.nio.ByteBuffer", + "int", + "int" + ] + }, + { + "name": "put", + "parameterTypes": [ + "int", + "byte[]", + "int", + "int" + ] + }, + { + "name": "slice", + "parameterTypes": [ + "int", + "int" + ] + } + ] + }, + { + "type": "java.nio.DirectByteBuffer", + "methods": [ + { + "name": "", + "parameterTypes": [ + "long", + "int" + ] + }, + { + "name": "", + "parameterTypes": [ + "long", + "long" + ] + } + ] + }, + { + "type": "java.nio.channels.FileChannel" + }, + { + "type": "java.nio.channels.spi.AbstractInterruptibleChannel", + "fields": [ + { + "name": "interruptor" + } + ] + }, + { + "type": "java.nio.channels.spi.SelectorProvider" + }, + { + "type": "java.security.AlgorithmParametersSpi" + }, + { + "type": "java.security.KeyStoreSpi" + }, + { + "type": "java.security.SecureRandomParameters" + }, + { + "type": "java.sql.Date" + }, + { + "type": "java.time.Instant" + }, + { + "type": "java.util.Arrays", + "jniAccessible": true, + "methods": [ + { + "name": "asList", + "parameterTypes": [ + "java.lang.Object[]" + ] + } + ] + }, + { + "type": "java.util.Date" + }, + { + "type": "java.util.SplittableRandom", + "methods": [ + { + "name": "nextBytes", + "parameterTypes": [ + "byte[]" + ] + } + ] + }, + { + "type": "java.util.concurrent.atomic.LongAdder", + "methods": [ + { + "name": "", + "parameterTypes": [] + }, + { + "name": "add", + "parameterTypes": [ + "long" + ] + } + ] + }, + { + "type": "java.util.function.Supplier", + "jniAccessible": true, + "methods": [ + { + "name": "get", + "parameterTypes": [] + } + ] + }, + { + "type": "java.util.logging.ConsoleHandler", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "java.util.logging.FileHandler", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "javax.management.MBeanOperationInfo" + }, + { + "type": "javax.management.MBeanServerBuilder", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "javax.management.ObjectName" + }, + { + "type": "javax.management.ObjectName[]" + }, + { + "type": "javax.management.StandardEmitterMBean" + }, + { + "type": "javax.management.openmbean.CompositeData" + }, + { + "type": "javax.management.openmbean.CompositeData[]" + }, + { + "type": "javax.management.openmbean.OpenMBeanOperationInfoSupport" + }, + { + "type": "javax.management.openmbean.TabularData" + }, + { + "type": "javax.naming.directory.InitialDirContext" + }, + { + "type": "jdk.internal.jrtfs.JrtFileSystemProvider" + }, + { + "type": "jdk.internal.loader.ClassLoaders$AppClassLoader" + }, + { + "type": "jdk.internal.misc.Unsafe", + "methods": [ + { + "name": "allocateUninitializedArray", + "parameterTypes": [ + "java.lang.Class", + "int" + ] + }, + { + "name": "getUnsafe", + "parameterTypes": [] + } + ] + }, + { + "type": "jdk.management.VirtualThreadSchedulerMXBean" + }, + { + "type": "jdk.management.jfr.ConfigurationInfo" + }, + { + "type": "jdk.management.jfr.EventTypeInfo" + }, + { + "type": "jdk.management.jfr.FlightRecorderMXBean" + }, + { + "type": "jdk.management.jfr.FlightRecorderMXBeanImpl" + }, + { + "type": "jdk.management.jfr.RecordingInfo" + }, + { + "type": "jdk.management.jfr.SettingDescriptorInfo" + }, + { + "type": "libcore.io.Memory" + }, + { + "type": "long[]" + }, + { + "type": "net.jpountz.lz4.LZ4HCJNICompressor", + "fields": [ + { + "name": "INSTANCE" + } + ], + "methods": [ + { + "name": "", + "parameterTypes": [ + "int" + ] + } + ] + }, + { + "type": "net.jpountz.lz4.LZ4JNICompressor", + "fields": [ + { + "name": "INSTANCE" + } + ] + }, + { + "type": "net.jpountz.lz4.LZ4JNISafeDecompressor", + "fields": [ + { + "name": "INSTANCE" + } + ] + }, + { + "type": "okhttp3.internal.connection.RealCall" + }, + { + "type": "org.apache.log4j.LogManager" + }, + { + "type": "org.apache.logging.log4j.Logger" + }, + { + "type": "org.apache.lucene.analysis.ar.ArabicAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.bg.BulgarianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.bn.BengaliAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.br.BrazilianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.ca.CatalanAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.cjk.CJKAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.ckb.SoraniAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.classic.ClassicAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.core.KeywordAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.core.SimpleAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.core.StopAnalyzer" + }, + { + "type": "org.apache.lucene.analysis.core.UnicodeWhitespaceAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.core.WhitespaceAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.custom.CustomAnalyzer" + }, + { + "type": "org.apache.lucene.analysis.cz.CzechAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.da.DanishAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.de.GermanAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.el.GreekAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.email.UAX29URLEmailAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.en.EnglishAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.es.SpanishAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.et.EstonianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.eu.BasqueAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.fa.PersianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.fi.FinnishAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.fr.FrenchAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.ga.IrishAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.gl.GalicianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.hi.HindiAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.hu.HungarianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.hy.ArmenianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.id.IndonesianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.it.ItalianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.lt.LithuanianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.lv.LatvianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.miscellaneous.ConcatenateGraphFilter$BytesRefBuilderTermAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.miscellaneous.LimitTokenCountAnalyzer" + }, + { + "type": "org.apache.lucene.analysis.ne.NepaliAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.nl.DutchAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.no.NorwegianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.pt.PortugueseAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.query.QueryAutoStopWordAnalyzer" + }, + { + "type": "org.apache.lucene.analysis.ro.RomanianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.ru.RussianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.shingle.ShingleAnalyzerWrapper", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.sr.SerbianAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.standard.StandardAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.sv.SwedishAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.ta.TamilAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.te.TeluguAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.th.ThaiAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.tokenattributes.BytesTermAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.tokenattributes.FlagsAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.tokenattributes.KeywordAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.tokenattributes.OffsetAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.tokenattributes.PayloadAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.tokenattributes.PositionIncrementAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.tokenattributes.PositionLengthAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.tokenattributes.SentenceAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.tokenattributes.TermFrequencyAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.tokenattributes.TypeAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.analysis.tr.TurkishAnalyzer", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.index.Term" + }, + { + "type": "org.apache.lucene.internal.hppc.IntHashSet" + }, + { + "type": "org.apache.lucene.search.BoostAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.search.MaxNonCompetitiveBoostAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.store.DataOutput" + }, + { + "type": "org.apache.lucene.util.BitSet" + }, + { + "type": "org.apache.lucene.util.ByteBlockPool" + }, + { + "type": "org.apache.lucene.util.BytesRef" + }, + { + "type": "org.apache.lucene.util.BytesRefBlockPool" + }, + { + "type": "org.apache.lucene.util.BytesRefHash" + }, + { + "type": "org.apache.lucene.util.SparseFixedBitSet" + }, + { + "type": "org.apache.lucene.util.fst.FST" + }, + { + "type": "org.apache.lucene.util.fst.FSTCompiler$Arc[]" + }, + { + "type": "org.apache.lucene.util.fst.GrowableByteArrayDataOutput" + }, + { + "type": "org.apache.ratis.metrics.impl.MetricRegistriesImpl" + }, + { + "type": "org.apache.ratis.protocol.AdminAsynchronousProtocol", + "methods": [ + { + "name": "getGroupInfoAsync", + "parameterTypes": [ + "org.apache.ratis.protocol.GroupInfoRequest" + ] + } + ] + }, + { + "type": "org.apache.ratis.protocol.RaftClientAsynchronousProtocol", + "methods": [ + { + "name": "submitClientRequestAsync", + "parameterTypes": [ + "org.apache.ratis.protocol.RaftClientRequest" + ] + } + ] + }, + { + "type": "org.apache.ratis.server.DisabledDataStreamServerFactory", + "methods": [ + { + "name": "", + "parameterTypes": [ + "org.apache.ratis.conf.Parameters" + ] + } + ] + }, + { + "type": "org.apache.ratis.server.RaftServer", + "methods": [ + { + "name": "getId", + "parameterTypes": [] + }, + { + "name": "getProperties", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.ratis.server.RaftServerMXBean" + }, + { + "type": "org.apache.ratis.server.impl.FollowerInfoImpl" + }, + { + "type": "org.apache.ratis.server.impl.RaftServerJmxAdapter" + }, + { + "type": "org.apache.ratis.server.impl.ServerImplUtils", + "methods": [ + { + "name": "newRaftServer", + "parameterTypes": [ + "org.apache.ratis.protocol.RaftPeerId", + "org.apache.ratis.protocol.RaftGroup", + "org.apache.ratis.server.storage.RaftStorage$StartupOption", + "org.apache.ratis.statemachine.StateMachine$Registry", + "java.lang.ThreadGroup", + "org.apache.ratis.conf.RaftProperties", + "org.apache.ratis.conf.Parameters" + ] + } + ] + }, + { + "type": "org.apache.ratis.server.leader.LogAppenderDefault" + }, + { + "type": "org.apache.ratis.server.protocol.RaftServerAsynchronousProtocol", + "methods": [ + { + "name": "appendEntriesAsync", + "parameterTypes": [ + "org.apache.ratis.proto.RaftProtos$AppendEntriesRequestProto" + ] + } + ] + }, + { + "type": "org.apache.ratis.server.protocol.RaftServerProtocol", + "methods": [ + { + "name": "requestVote", + "parameterTypes": [ + "org.apache.ratis.proto.RaftProtos$RequestVoteRequestProto" + ] + } + ] + }, + { + "type": "org.apache.ratis.server.raftlog.segmented.SegmentedRaftLog", + "fields": [ + { + "name": "cacheEviction" + } + ] + }, + { + "type": "org.apache.ratis.statemachine.impl.TransactionContextImpl", + "methods": [ + { + "name": "", + "parameterTypes": [ + "org.apache.ratis.proto.RaftProtos$RaftPeerRole", + "org.apache.ratis.statemachine.StateMachine", + "org.apache.ratis.proto.RaftProtos$LogEntryProto" + ] + }, + { + "name": "", + "parameterTypes": [ + "org.apache.ratis.protocol.RaftClientRequest", + "org.apache.ratis.statemachine.StateMachine", + "org.apache.ratis.proto.RaftProtos$StateMachineLogEntryProto", + "org.apache.ratis.thirdparty.com.google.protobuf.ByteString", + "org.apache.ratis.thirdparty.com.google.protobuf.ByteString", + "java.lang.Object" + ] + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.com.codahale.metrics.jmx.JmxReporter$JmxCounter" + }, + { + "type": "org.apache.ratis.thirdparty.com.codahale.metrics.jmx.JmxReporter$JmxCounterMBean" + }, + { + "type": "org.apache.ratis.thirdparty.com.codahale.metrics.jmx.JmxReporter$JmxGauge" + }, + { + "type": "org.apache.ratis.thirdparty.com.codahale.metrics.jmx.JmxReporter$JmxGaugeMBean" + }, + { + "type": "org.apache.ratis.thirdparty.com.codahale.metrics.jmx.JmxReporter$JmxTimer" + }, + { + "type": "org.apache.ratis.thirdparty.com.codahale.metrics.jmx.JmxReporter$JmxTimerMBean" + }, + { + "type": "org.apache.ratis.thirdparty.com.google.common.util.concurrent.AbstractFutureState", + "fields": [ + { + "name": "listenersField" + }, + { + "name": "valueField" + }, + { + "name": "waitersField" + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.com.google.common.util.concurrent.AbstractFutureState$Waiter", + "fields": [ + { + "name": "next" + }, + { + "name": "thread" + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.com.google.protobuf.ExtensionRegistry", + "methods": [ + { + "name": "getEmptyRegistry", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.census.InternalCensusStatsAccessor" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.census.InternalCensusTracingAccessor" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.internal.DnsNameResolverProvider" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.internal.JndiResourceResolverFactory", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.internal.PickFirstLoadBalancerProvider" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.internal.SerializingExecutor" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.netty.NettyClientHandler" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.netty.NettyServer$1" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.netty.NettyServerHandler" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.netty.ProtocolNegotiators$GrpcNegotiationHandler" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.netty.ProtocolNegotiators$PlaintextHandler" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.netty.ProtocolNegotiators$WaitUntilActiveHandler" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.netty.UdsNameResolverProvider" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.netty.WriteBufferingAndExceptionHandler" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.override.ContextStorageOverride" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.util.OutlierDetectionLoadBalancerProvider" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.util.RandomSubsettingLoadBalancerProvider" + }, + { + "type": "org.apache.ratis.thirdparty.io.grpc.util.SecretRoundRobinLoadBalancerProvider$Provider" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.bootstrap.ServerBootstrap$1" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.bootstrap.ServerBootstrap$ServerBootstrapAcceptor" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.buffer.AbstractByteBufAllocator" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.buffer.AbstractReferenceCountedByteBuf", + "fields": [ + { + "name": "refCnt" + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.channel.AbstractChannelHandlerContext" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.channel.ChannelOutboundBuffer" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.channel.DefaultChannelConfig" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.channel.DefaultChannelPipeline" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.channel.DefaultChannelPipeline$TailContext" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.channel.DefaultFileRegion" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.channel.epoll.Epoll", + "methods": [ + { + "name": "isAvailable", + "parameterTypes": [] + }, + { + "name": "unavailabilityCause", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.channel.epoll.NativeDatagramPacketArray$NativeDatagramPacket" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.channel.socket.nio.NioSocketChannel", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.channel.unix.PeerCredentials" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.util.AbstractReferenceCounted", + "fields": [ + { + "name": "refCnt" + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.util.DefaultAttributeMap" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.util.Recycler$DefaultHandle" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.util.ReferenceCountUtil" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.util.ResourceLeakDetector$DefaultResourceLeak" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.util.concurrent.DefaultPromise" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor" + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields", + "fields": [ + { + "name": "producerLimit" + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields", + "fields": [ + { + "name": "consumerIndex" + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields", + "fields": [ + { + "name": "producerIndex" + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField", + "fields": [ + { + "name": "consumerIndex" + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField", + "fields": [ + { + "name": "producerIndex" + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField", + "fields": [ + { + "name": "producerLimit" + } + ] + }, + { + "type": "org.apache.ratis.thirdparty.io.perfmark.impl.SecretPerfMarkImpl$PerfMarkImpl" + }, + { + "type": "org.graalvm.home.impl.DefaultHomeFinder" + }, + { + "type": "org.graalvm.polyglot.io.IOHelper", + "fields": [ + { + "name": "ACCESS" + } + ] + }, + { + "type": "org.graalvm.polyglot.management.Management", + "fields": [ + { + "name": "ACCESS" + } + ] + }, + { + "type": "org.jboss.logmanager.LogManager" + }, + { + "type": "org.jboss.threads.EnhancedQueueExecutor$MXBeanImpl" + }, + { + "type": "org.jboss.threads.management.StandardThreadPoolMXBean" + }, + { + "type": "org.robolectric.Robolectric" + }, + { + "type": "org.slf4j.jul.JULServiceProvider" + }, + { + "type": "org.tartarus.snowball.ext.FinnishStemmer", + "methods": [ + { + "name": "r_LONG", + "parameterTypes": [] + }, + { + "name": "r_VI", + "parameterTypes": [] + } + ] + }, + { + "type": "org.tartarus.snowball.ext.HindiStemmer", + "methods": [ + { + "name": "r_CONSONANT", + "parameterTypes": [] + } + ] + }, + { + "type": "org.tartarus.snowball.ext.IndonesianStemmer", + "methods": [ + { + "name": "r_KER", + "parameterTypes": [] + }, + { + "name": "r_SUFFIX_AN_OK", + "parameterTypes": [] + }, + { + "name": "r_SUFFIX_I_OK", + "parameterTypes": [] + }, + { + "name": "r_SUFFIX_KAN_OK", + "parameterTypes": [] + }, + { + "name": "r_VOWEL", + "parameterTypes": [] + } + ] + }, + { + "type": "org.xnio.Connection" + }, + { + "type": "org.xnio.XnioWorker" + }, + { + "type": "org.xnio._private.Messages_$logger", + "methods": [ + { + "name": "", + "parameterTypes": [ + "org.jboss.logging.Logger" + ] + } + ] + }, + { + "type": "org.xnio._private.Messages_$logger_en" + }, + { + "type": "org.xnio._private.Messages_$logger_en_US" + }, + { + "type": "org.xnio.management.XnioProviderMXBean" + }, + { + "type": "org.xnio.management.XnioServerMXBean" + }, + { + "type": "org.xnio.management.XnioWorkerMXBean" + }, + { + "type": "org.xnio.nio.Log_$logger", + "methods": [ + { + "name": "", + "parameterTypes": [ + "org.jboss.logging.Logger" + ] + } + ] + }, + { + "type": "org.xnio.nio.Log_$logger_en" + }, + { + "type": "org.xnio.nio.Log_$logger_en_US" + }, + { + "type": "org.xnio.nio.NioSocketConduit" + }, + { + "type": "org.xnio.nio.NioTcpServer" + }, + { + "type": "org.xnio.nio.NioTcpServer$1" + }, + { + "type": "org.xnio.nio.NioXnio$3" + }, + { + "type": "org.xnio.nio.NioXnioProvider" + }, + { + "type": "org.xnio.nio.NioXnioWorker" + }, + { + "type": "org.xnio.nio.NioXnioWorker$NioWorkerMetrics" + }, + { + "type": "org.xnio.nio.WorkerThread" + }, + { + "type": "short[]" + }, + { + "type": "sun.management.ClassLoadingImpl" + }, + { + "type": "sun.management.CompilationImpl" + }, + { + "type": "sun.management.ManagementFactoryHelper$1" + }, + { + "type": "sun.management.ManagementFactoryHelper$PlatformLoggingImpl" + }, + { + "type": "sun.management.MemoryImpl" + }, + { + "type": "sun.management.MemoryManagerImpl" + }, + { + "type": "sun.management.MemoryPoolImpl" + }, + { + "type": "sun.management.RuntimeImpl" + }, + { + "type": "sun.management.VMManagementImpl", + "jniAccessible": true, + "fields": [ + { + "name": "compTimeMonitoringSupport" + }, + { + "name": "currentThreadCpuTimeSupport" + }, + { + "name": "objectMonitorUsageSupport" + }, + { + "name": "otherThreadCpuTimeSupport" + }, + { + "name": "remoteDiagnosticCommandsSupport" + }, + { + "name": "synchronizerUsageSupport" + }, + { + "name": "threadAllocatedMemorySupport" + }, + { + "name": "threadContentionMonitoringSupport" + } + ] + }, + { + "type": "sun.misc.Unsafe", + "fields": [ + { + "name": "theUnsafe" + } + ], + "methods": [ + { + "name": "invokeCleaner", + "parameterTypes": [ + "java.nio.ByteBuffer" + ] + } + ] + }, + { + "type": "sun.nio.ch.SelectorImpl", + "fields": [ + { + "name": "publicSelectedKeys" + }, + { + "name": "selectedKeys" + } + ] + }, + { + "type": "sun.reflect.misc.Trampoline", + "methods": [ + { + "name": "invoke", + "parameterTypes": [ + "java.lang.reflect.Method", + "java.lang.Object", + "java.lang.Object[]" + ] + } + ] + }, + { + "type": "sun.security.pkcs12.PKCS12KeyStore", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.pkcs12.PKCS12KeyStore$DualFormatPKCS12", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.provider.DSA$SHA224withDSA", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.provider.DSA$SHA256withDSA", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.provider.MD5", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.provider.NativePRNG", + "methods": [ + { + "name": "", + "parameterTypes": [] + }, + { + "name": "", + "parameterTypes": [ + "java.security.SecureRandomParameters" + ] + } + ] + }, + { + "type": "sun.security.provider.SHA", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.provider.SHA2$SHA224", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.provider.SHA2$SHA256", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.provider.SHA5$SHA384", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.provider.SHA5$SHA512", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.provider.X509Factory", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.rsa.PSSParameters", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.rsa.RSAKeyFactory$Legacy", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.rsa.RSAPSSSignature", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.rsa.RSASignature$SHA224withRSA", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.ssl.KeyManagerFactoryImpl$SunX509", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.ssl.SSLContextImpl$DefaultSSLContext", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.ssl.TrustManagerFactoryImpl$PKIXFactory", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "sun.security.x509.AuthorityInfoAccessExtension", + "methods": [ + { + "name": "", + "parameterTypes": [ + "java.lang.Boolean", + "java.lang.Object" + ] + } + ] + }, + { + "type": "sun.security.x509.AuthorityKeyIdentifierExtension", + "methods": [ + { + "name": "", + "parameterTypes": [ + "java.lang.Boolean", + "java.lang.Object" + ] + } + ] + }, + { + "type": "sun.security.x509.BasicConstraintsExtension", + "methods": [ + { + "name": "", + "parameterTypes": [ + "java.lang.Boolean", + "java.lang.Object" + ] + } + ] + }, + { + "type": "sun.security.x509.CRLDistributionPointsExtension", + "methods": [ + { + "name": "", + "parameterTypes": [ + "java.lang.Boolean", + "java.lang.Object" + ] + } + ] + }, + { + "type": "sun.security.x509.CertificatePoliciesExtension", + "methods": [ + { + "name": "", + "parameterTypes": [ + "java.lang.Boolean", + "java.lang.Object" + ] + } + ] + }, + { + "type": "sun.security.x509.ExtendedKeyUsageExtension", + "methods": [ + { + "name": "", + "parameterTypes": [ + "java.lang.Boolean", + "java.lang.Object" + ] + } + ] + }, + { + "type": "sun.security.x509.KeyUsageExtension", + "methods": [ + { + "name": "", + "parameterTypes": [ + "java.lang.Boolean", + "java.lang.Object" + ] + } + ] + }, + { + "type": "sun.security.x509.NetscapeCertTypeExtension", + "methods": [ + { + "name": "", + "parameterTypes": [ + "java.lang.Boolean", + "java.lang.Object" + ] + } + ] + }, + { + "type": "sun.security.x509.PrivateKeyUsageExtension", + "methods": [ + { + "name": "", + "parameterTypes": [ + "java.lang.Boolean", + "java.lang.Object" + ] + } + ] + }, + { + "type": "sun.security.x509.SubjectKeyIdentifierExtension", + "methods": [ + { + "name": "", + "parameterTypes": [ + "java.lang.Boolean", + "java.lang.Object" + ] + } + ] + }, + { + "type": "sun.text.resources.cldr.FormatData" + }, + { + "type": "sun.text.resources.cldr.FormatData_en" + }, + { + "type": "sun.text.resources.cldr.FormatData_en_US" + }, + { + "type": "sun.util.resources.cldr.CalendarData" + }, + { + "type": "sun.util.resources.cldr.TimeZoneNames" + }, + { + "type": "sun.util.resources.cldr.TimeZoneNames_en" + }, + { + "type": "sun.util.resources.cldr.TimeZoneNames_en_US" + }, + { + "type": { + "proxy": [ + "org.apache.ratis.server.RaftServer" + ] + } + }, + { + "type": { + "proxy": [ + "sun.nio.ch.Interruptible" + ] + } + }, + { + "type": "org.locationtech.spatial4j.context.SpatialContext", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.context.SpatialContextFactory", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.context.jts.JtsSpatialContext", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.context.jts.JtsSpatialContextFactory", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.shape.impl.ShapeFactoryImpl", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.shape.jts.JtsShapeFactory", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.BinaryCodec", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.jts.JtsBinaryCodec", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.WKTReader", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.WKTWriter", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.WktShapeParser", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.jts.JtsWKTReaderShapeParser", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.jts.JtsWKTWriter", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.GeoJSONReader", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.GeoJSONWriter", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.jts.JtsGeoJSONWriter", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.PolyshapeReader", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.PolyshapeWriter", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.jts.JtsPolyshapeWriter", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.LegacyShapeReader", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.io.LegacyShapeWriter", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.distance.CartesianDistCalc", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.distance.GeodesicSphereDistCalc", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.distance.GeodesicSphereDistCalc$Haversine", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.distance.GeodesicSphereDistCalc$LawOfCosines", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "org.locationtech.spatial4j.distance.GeodesicSphereDistCalc$Vincenty", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allPublicMethods": true, + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Alias", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.AlignDatabaseStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.AlterBucketStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.AlterDatabaseStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.AlterGraphAnalyticalViewStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.AlterMaterializedViewStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.AlterPropertyStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.AlterTimeSeriesTypeStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.AlterTypeStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.AlterTypeStatement$Item", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.AndBlock", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ArrayConcatExpression", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ArrayConcatExpressionElement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ArrayLiteralExpression", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ArrayNumberSelector", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ArrayRangeSelector", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ArraySelector", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ArraySingleValuesSelector", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BackupDatabaseStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BaseExpression", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BaseIdentifier", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Batch", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BeginStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BetweenCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BinaryCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BooleanExpression", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BooleanExpression$1", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BooleanExpression$2", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BooleanExpression$3", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BothPathItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BothPathItemOpt", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BreakStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Bucket", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BucketIdentifier", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.BucketList", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CaseAlternative", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CaseExpression", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CheckDatabaseStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CommitStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CompactIndexStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CompactTimeSeriesTypeStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CompareOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ConditionBlock", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ConsoleStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ContainsAllCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ContainsAnyCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ContainsCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ContainsKeyOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ContainsTextCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ContainsValueCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ContainsValueOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateBucketStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateContinuousAggregateStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateDocumentTypeStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateEdgeStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateEdgeTypeStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateGraphAnalyticalViewStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateIndexStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateIndexStatement$Property", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateMaterializedViewStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreatePropertyAttributeStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreatePropertyStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateTimeSeriesTypeStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateTimeSeriesTypeStatement$ColumnDef", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateTriggerStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateTypeAbstractStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateVertexStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateVertexStatementEmpty", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateVertexStatementEmptyNoTarget", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateVertexStatementNoTarget", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.CreateVertexTypeStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.DDLStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.DefineFunctionStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.DeleteFunctionStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.DeleteStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.DropBucketStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.DropContinuousAggregateStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.DropGraphAnalyticalViewStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.DropIndexStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.DropMaterializedViewStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.DropPropertyStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.DropTriggerStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.DropTypeStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.EqualsCompareOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ExplainStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ExportDatabaseStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Expression", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ExpressionStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.FieldMatchPathItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.FindReferencesStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.FirstLevelExpression", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.FloatingPoint", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ForEachBlock", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.FromClause", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.FromItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.FunctionCall", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.GeOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.GroupBy", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.GtOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ILikeOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Identifier", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.IfNotExists", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.IfStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ImportDatabaseStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.InCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.InOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.InPathItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.InPathItemOpt", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.IndexIdentifier", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.IndexMatchCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.InputParameter", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.InsertBody", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.InsertSetExpression", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.InsertStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.InstanceofCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.IsDefinedCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.IsNotDefinedCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.IsNotNullCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.IsNullCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Json", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.JsonArray", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.JsonItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.LeOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.LetClause", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.LetItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.LetStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.LevelZeroIdentifier", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.LikeOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Limit", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.LockStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.LtOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.MatchExpression", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.MatchFilter", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.MatchFilterItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.MatchPathItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.MatchPathItemFirst", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.MatchStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.MatchesCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.MathExpression", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.MethodCall", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Modifier", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.MoveVertexStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.MultiMatchPathItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.MultiMatchPathItemArrows", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.NamedParameter", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.NeOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.NearOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.NeqOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.NestedProjection", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.NestedProjectionItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.NotBlock", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.NullSafeEqualsCompareOperator", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.OrBlock", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.OrderBy", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.OrderByItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.OutPathItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.OutPathItemOpt", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.PCollection", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.PInteger", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.PNumber", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.PString", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ParenthesisBlock", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ParenthesisExpression", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Parse", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ParseCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ParseExpression", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ParseScript", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Pattern", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.PositionalParameter", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ProfileStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Projection", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ProjectionItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.QueryStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.RebuildGraphAnalyticalViewStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.RebuildIndexStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.RebuildTypeStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.RecordAttribute", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.RefreshContinuousAggregateStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.RefreshMaterializedViewStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ResourcePathItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.RestoreDocumentStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.RestoreEdgeStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.RestoreVertexStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Retry", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ReturnStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Rid", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.RightBinaryCondition", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.RollbackStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.SchemaIdentifier", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.SelectStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.SelectWithoutTargetStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.SetGlobalStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.SimpleExecStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.SimpleNode", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Skip", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.SleepStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Statement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.StatementInternal", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.StatementSemicolon", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.SuffixIdentifier", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Timeout", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.TraverseProjectionItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.TraverseStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.TruncateBucketStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.TruncateRecordStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.TruncateTypeStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Unwind", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.UpdateAddItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.UpdateIncrementItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.UpdateItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.UpdateOperations", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.UpdatePutItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.UpdateRemoveItem", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.UpdateStatement", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Url", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.ValueExpression", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.Wait", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.WhereClause", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.WhileBlock", + "allDeclaredFields": true + }, + { + "type": "com.arcadedb.query.sql.parser.WithinOperator", + "allDeclaredFields": true + }, + { + "type": "io.github.jbellis.jvector.vector.NativeVectorizationProvider", + "allPublicConstructors": true, + "allPublicMethods": true + }, + { + "type": "io.github.jbellis.jvector.vector.PanamaVectorizationProvider", + "allPublicConstructors": true + }, + { + "type": "io.github.jbellis.jvector.vector.cnative.NativeSimdOps", + "allPublicMethods": true + }, + { + "type": "io.github.jbellis.jvector.vector.NativeVectorUtilSupport", + "allPublicMethods": true + } + ], + "resources": [ + { + "glob": "META-INF/graalvm/org.graalvm.polyglot/version" + }, + { + "glob": "META-INF/graalvm/org.graalvm.truffle/version" + }, + { + "glob": "META-INF/resources/engine/libtruffleattach/darwin/aarch64/sha256" + }, + { + "glob": "META-INF/services/com.arcadedb.server.ServerPlugin" + }, + { + "glob": "META-INF/services/com.oracle.truffle.api.TruffleRuntimeAccess" + }, + { + "glob": "META-INF/services/com.oracle.truffle.api.impl.TruffleLocator" + }, + { + "glob": "META-INF/services/com.oracle.truffle.api.instrumentation.provider.TruffleInstrumentProvider" + }, + { + "glob": "META-INF/services/com.oracle.truffle.api.library.provider.DefaultExportProvider" + }, + { + "glob": "META-INF/services/com.oracle.truffle.api.provider.InternalResourceProvider" + }, + { + "glob": "META-INF/services/com.oracle.truffle.api.provider.TruffleLanguageProvider" + }, + { + "glob": "META-INF/services/com.oracle.truffle.js.runtime.Evaluator" + }, + { + "glob": "META-INF/services/com.oracle.truffle.runtime.EngineCacheSupport" + }, + { + "glob": "META-INF/services/com.oracle.truffle.runtime.FloodControlHandler" + }, + { + "glob": "META-INF/services/com.oracle.truffle.runtime.LoopNodeFactory" + }, + { + "glob": "META-INF/services/com.oracle.truffle.runtime.TruffleTypes" + }, + { + "glob": "META-INF/services/com.oracle.truffle.runtime.jfr.EventFactory$Provider" + }, + { + "glob": "META-INF/services/io.opentelemetry.context.ContextStorageProvider" + }, + { + "glob": "META-INF/services/io.opentelemetry.sdk.common.export.GrpcSenderProvider" + }, + { + "glob": "META-INF/services/java.net.spi.InetAddressResolverProvider" + }, + { + "glob": "META-INF/services/java.net.spi.URLStreamHandlerProvider" + }, + { + "glob": "META-INF/services/java.nio.channels.spi.SelectorProvider" + }, + { + "glob": "META-INF/services/java.nio.file.spi.FileSystemProvider" + }, + { + "glob": "META-INF/services/java.time.zone.ZoneRulesProvider" + }, + { + "glob": "META-INF/services/org.apache.ratis.metrics.MetricRegistries" + }, + { + "glob": "META-INF/services/org.apache.ratis.thirdparty.io.grpc.LoadBalancerProvider" + }, + { + "glob": "META-INF/services/org.apache.ratis.thirdparty.io.grpc.NameResolverProvider" + }, + { + "glob": "META-INF/services/org.graalvm.home.HomeFinder" + }, + { + "glob": "META-INF/services/org.graalvm.polyglot.impl.AbstractPolyglotImpl" + }, + { + "glob": "META-INF/services/org.jboss.logging.LoggerProvider" + }, + { + "glob": "META-INF/services/org.slf4j.spi.SLF4JServiceProvider" + }, + { + "glob": "META-INF/services/org.xnio.XnioProvider" + }, + { + "glob": "arcadedb-log.properties" + }, + { + "glob": "com/arcadedb/arcadedb.properties" + }, + { + "glob": "io/undertow/version.properties" + }, + { + "glob": "net/jpountz/util/darwin/aarch64/liblz4-java.dylib" + }, + { + "glob": "org/jboss/threads/Version.properties" + }, + { + "glob": "org/slf4j/impl/StaticLoggerBinder.class" + }, + { + "glob": "org/xnio/Version.properties" + }, + { + "glob": "org/xnio/nio/Version.properties" + }, + { + "glob": "prometheus.properties" + }, + { + "glob": "ratis-version.properties" + }, + { + "module": "java.base", + "glob": "jdk/internal/icu/impl/data/icudt76b/nfc.nrm" + }, + { + "module": "jdk.jfr", + "glob": "jdk/jfr/internal/query/view.ini" + }, + { + "module": "jdk.jfr", + "glob": "jdk/jfr/internal/types/metadata.bin" + }, + { + "glob": "META-INF/services/**" + }, + { + "glob": "com/arcadedb/**/*.properties" + }, + { + "glob": "com/arcadedb/**/*.txt" + }, + { + "glob": "com/arcadedb/**/*.json" + }, + { + "glob": "org/apache/lucene/**/*.txt" + }, + { + "glob": "org/apache/lucene/**/*.properties" + }, + { + "glob": "org/apache/lucene/**/*.dict" + }, + { + "glob": "org/apache/lucene/**/*.dat" + }, + { + "glob": "META-INF/maven/**/pom.properties" + }, + { + "glob": "io/undertow/**/*.properties" + }, + { + "glob": "org/xnio/**/*.properties" + }, + { + "glob": "org/jboss/**/*.properties" + }, + { + "glob": "META-INF/versions/22/libjvector-x86_64.so" + }, + { + "glob": "META-INF/versions/22/libjvector-aarch64.so" + } + ], + "foreign": { + "downcalls": [ + { + "returnType": "jlong", + "parameterTypes": [ + "jlong" + ] + }, + { + "returnType": "void", + "parameterTypes": [ + "jlong" + ] + }, + { + "returnType": "jint", + "parameterTypes": [ + "jint" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "jfloat", + "parameterTypes": [ + "void*", + "jlong", + "void*", + "jlong", + "jlong" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "jint", + "parameterTypes": [ + "void*", + "jlong", + "void*", + "jint", + "jlong" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "void", + "parameterTypes": [ + "void*", + "void*", + "jlong" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "void", + "parameterTypes": [ + "void*", + "jfloat", + "jlong" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "void", + "parameterTypes": [ + "void*", + "void*", + "jlong", + "jfloat", + "jlong" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "void", + "parameterTypes": [ + "void*", + "void*", + "void*", + "jlong" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "void", + "parameterTypes": [ + "void*", + "jlong", + "void*", + "jlong", + "void*", + "jlong" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "void", + "parameterTypes": [ + "void*", + "jlong", + "jfloat", + "void*", + "jlong" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "jfloat", + "parameterTypes": [ + "void*", + "jlong" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "jfloat", + "parameterTypes": [ + "void*", + "jint", + "void*", + "jint", + "jlong" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "jfloat", + "parameterTypes": [ + "void*", + "jlong", + "void*", + "jint", + "void*", + "jint", + "jint" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "jfloat", + "parameterTypes": [ + "void*", + "jint", + "jlong", + "jint", + "void*", + "void*", + "jfloat" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "void", + "parameterTypes": [ + "void*", + "jint", + "jlong", + "jint", + "void*", + "jint", + "void*" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "void", + "parameterTypes": [ + "void*", + "jint", + "jlong", + "jint", + "void*" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "void", + "parameterTypes": [ + "void*", + "jlong", + "jfloat", + "jfloat", + "jfloat", + "jfloat", + "void*" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "jfloat", + "parameterTypes": [ + "void*", + "jlong", + "jfloat", + "jfloat", + "jfloat", + "jfloat", + "jint" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "jfloat", + "parameterTypes": [ + "void*", + "jlong", + "jfloat", + "jfloat", + "jint" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "jfloat", + "parameterTypes": [ + "void*", + "void*", + "jlong", + "jfloat", + "jfloat", + "jfloat", + "jfloat" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "jlong", + "parameterTypes": [ + "void*", + "void*", + "jlong", + "jfloat", + "jfloat", + "jfloat", + "jfloat", + "void*" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "void", + "parameterTypes": [ + "void*", + "jlong" + ], + "options": { + "critical": { + "allowHeapAccess": true + } + } + }, + { + "returnType": "void*", + "parameterTypes": [], + "options": { + "critical": { + "allowHeapAccess": true + } + } + } + ] + } +} diff --git a/native/wheel/ARCADEDB_ENGINE_NOTICE b/native/wheel/ARCADEDB_ENGINE_NOTICE new file mode 100644 index 0000000..e86c068 --- /dev/null +++ b/native/wheel/ARCADEDB_ENGINE_NOTICE @@ -0,0 +1,201 @@ +SPDX-License-Identifier: Apache-2.0 + +ArcadeDB +Copyright 2021-present Arcade Data Ltd + +This product includes software developed at +Arcade Data Ltd (https://arcadedata.com/). + +================================================================================ + +This product includes components originally developed for OrientDB, an Open +Source Multi-Model NoSQL Database. +Copyright 2010-2020 OrientDB LTD (https://orientdb.com) + +ArcadeDB was created by Luca Garulli, the same founder of OrientDB, and written +from scratch after SAP's acquisition of OrientDB in 2020. ArcadeDB reuses +OrientDB's SQL engine (now heavily modified) and some utility classes. + +OrientDB source code is available at: https://github.com/orientechnologies/orientdb + +================================================================================ + +This product includes openCypher grammar files and Technology Compatibility Kit +(TCK) test suite, developed by the openCypher community. + + Copyright (c) "Neo4j" + Neo4j Sweden AB [https://neo4j.com] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + + openCypher project website: https://opencypher.org/ + +Cypher® is a registered trademark of Neo4j Inc. + +The included openCypher TCK tests contain the following attribution notice: + + "This work was created by the collective efforts of the openCypher community. + Without limiting the terms of Section 6, any Derivative Work that is not + approved by the public consensus process of the openCypher Implementers Group + should not be described as 'Cypher' (and Cypher® is a registered trademark of + Neo4j Inc.) or as 'openCypher'. Extensions by implementers or prototypes or + proposals for change that have been documented or implemented should only be + described as 'implementation extensions to Cypher' or as 'proposed changes to + Cypher that are not yet approved by the openCypher community'." + +The following files contain openCypher content: + - engine/src/main/antlr4/com/arcadedb/query/opencypher/grammar/Cypher25Parser.g4 + - engine/src/main/antlr4/com/arcadedb/query/opencypher/grammar/Cypher25Lexer.g4 + - engine/src/test/resources/opencypher/tck/features/**/*.feature (220 test files) + + +This product includes software from the OpenCypher-Gremlin project. + + Copyright 2018-2019 Neueda + + The following file is a security-patched override of unmaintained upstream code: + gremlin/src/main/java/org/opencypher/gremlin/translation/groovy/StringTranslationUtils.java + + OpenCypher-Gremlin project: https://github.com/opencypher/cypher-for-gremlin + +================================================================================ + +This product includes software developed by The Apache Software Foundation +(https://www.apache.org/). + +Apache Lucene +Copyright 2001-2024 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (https://www.apache.org/). + +Includes software from other Apache copyright owners, as follows: + - ICU4J, (c) 1995-2014 International Business Machines Corporation and others + - ANTLR4 grammar from C parser (BSD 3-clause license) + - JFlex lexer generator (BSD license) + +Apache ANTLR4 +Copyright 2012-2021 The ANTLR Project + +Apache TinkerPop (Gremlin) +Copyright 2015-2024 The Apache Software Foundation + +Apache Commons Compress +Copyright 2002-2024 The Apache Software Foundation + +Apache Commons Lang +Copyright 2001-2024 The Apache Software Foundation + +Apache Commons Codec +Copyright 2002-2024 The Apache Software Foundation + +Apache HttpComponents +Copyright 1999-2024 The Apache Software Foundation + +================================================================================ + +Apache Ratis +Copyright 2017-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +================================================================================ + +This product includes software developed by GraalVM (Oracle). + + GraalVM SDK, Truffle API, and JavaScript implementation + Copyright (c) 2012, 2024, Oracle and/or its affiliates + + https://www.graalvm.org/ + +================================================================================ + +This product includes software developed by Google Inc. + + Protocol Buffers + Copyright 2008 Google Inc. + + Guava Libraries + Copyright (C) 2010-2024 The Guava Authors + + Gson + Copyright 2008-2024 Google Inc. + + gRPC + Copyright 2015-2024 The gRPC Authors + +================================================================================ + +This product includes software developed by Red Hat, Inc. + + Undertow + Copyright 2013 Red Hat, Inc., and individual contributors + + https://undertow.io/ + +================================================================================ + +This product includes software developed by the Netty project. + + Netty + Copyright 2014 The Netty Project + + https://netty.io/ + +================================================================================ + +This product includes JUnit 5 test framework. + + JUnit 5 + Copyright 2015-2024 the original author or authors + + https://junit.org/junit5/ + +================================================================================ + +This product includes Testcontainers framework. + + Testcontainers + Copyright (c) 2015-2024 Richard North and other authors + + https://testcontainers.com/ + +================================================================================ + +This product includes software developed by FasterXML, LLC. + + Jackson JSON processor + Copyright (c) 2007-2024 Tatu Saloranta and other contributors + + https://github.com/FasterXML/jackson + +================================================================================ + +This product includes Micrometer Tracing (optional tracing module only). + + Micrometer Tracing + Copyright (c) 2017-Present VMware, Inc. All Rights Reserved. + + https://micrometer.io/ + + Micrometer Tracing contains modified portions of: + - 'io.netty.util.internal.logging' from Netty (Copyright 2013 The Netty Project, Apache 2.0, https://netty.io) + - 'StringUtils.isBlank()' from Apache Commons Lang (Copyright 2001-2019 The Apache Software Foundation, Apache 2.0) + - 'JsonUtf8Writer' from Moshi (Copyright 2010 Google Inc., Apache 2.0, https://github.com/square/moshi) + - the 'org.springframework.lang' package from the Spring Framework (Copyright 2002-2019 the original author or authors, Apache 2.0) + +================================================================================ + +This product includes OpenTelemetry Java (optional tracing module only). + + OpenTelemetry Java + Copyright The OpenTelemetry Authors + + https://opentelemetry.io/ + +================================================================================ + +For the complete list of third-party components and their licenses, see the +ATTRIBUTIONS.md file in the root directory of this distribution. diff --git a/native/wheel/NOTICE b/native/wheel/NOTICE new file mode 100644 index 0000000..ea868f8 --- /dev/null +++ b/native/wheel/NOTICE @@ -0,0 +1,25 @@ +arcadedb (native build) +Copyright 2021-present Arcade Data Ltd; Python bindings copyright Taewoon Kim (humem.ai) +Licensed under the Apache License, Version 2.0 (licenses/Apache-2.0.txt, LICENSE). + +This wheel contains the ArcadeDB engine (https://github.com/ArcadeData/arcadedb, release named in +arcadedb_native.ENGINE_VERSION) compiled ahead of time with GraalVM Community Edition 25.4.4.1.1 into libarcadedb.so. +The NOTICE file of ArcadeDB (components developed for OrientDB, the openCypher grammar) and the NOTICE files of the bundled Apache +libraries (Lucene, Commons Lang, Micrometer, Swagger) are reproduced verbatim in THIRD_PARTY_NOTICES.md. + +GraalVM native-image runtime and JDK classes in libarcadedb.so + A native image links the Substrate VM runtime (garbage collector, threads, runtime support) and parts of the OpenJDK class library + into the library. These parts are licensed under the GNU General Public License version 2 WITH the "Classpath" Exception + (licenses/GraalVM-CE-LICENSE.txt, licenses/GraalVM-native-image-LICENSE.txt, and verbatim in THIRD_PARTY_NOTICES.md). The exception lets you link and distribute them with code + under other licences, and it is why this wheel as a whole stays Apache-2.0 for ArcadeDB's own code. The corresponding source: + GraalVM: https://github.com/oracle/graal (release vm-25.4.4.1.1 line; the build used community-25.4.4.1.1+1.1) + OpenJDK: https://github.com/openjdk/jdk (jdk-25 line used by that GraalVM build) + Other GraalVM modules carry the licences in licenses/GraalVM-THIRD_PARTY_LICENSE.txt (graal-sdk and polyglot: UPL-1.0). + Oracle GraalVM (GFTC) is not used; only Community Edition. + +libjvector-x86_64.so + The native vector kernels of JVector 4.0.2 (https://github.com/datastax/jvector), Apache-2.0, unmodified, taken from the jvector jar. + +Java libraries, the Highway/jvector kernels and JDK data inside libarcadedb.so: THIRD_PARTY_NOTICES.md (generated by gen_notices.py). + +System libraries linked dynamically and NOT shipped: libz.so.1, libm.so.6, libc.so.6, libstdc++.so.6, libgcc_s.so.1. diff --git a/native/wheel/THIRD_PARTY_NOTICES.md b/native/wheel/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000..49708d9 --- /dev/null +++ b/native/wheel/THIRD_PARTY_NOTICES.md @@ -0,0 +1,3629 @@ +# Third-party notices (native image, preview) + +Generated by `native-wheel/gen_notices.py` from the classpath that built `libarcadedb.so` (engine 26.10.2, GraalVM CE 25.4.4.1.1). Do not edit by hand: `build_wheel.py` fails when this file differs from the generator output. Licence names come from each artifact's own Maven pom; notice texts are copied verbatim from the jars, the engine tag and the GraalVM home. A text of the Apache License 2.0 is in `licenses/Apache-2.0.txt`. + +## Java libraries compiled into libarcadedb.so + +59 Maven artifacts on the native-image classpath (plus the glue jar of this repository, Apache-2.0). + +| Artifact | Declared in pom | Applied | Verbatim texts | +|---|---|---|---| +| com.arcadedb:arcadedb-engine:26.10.2 | Apache 2 | Apache-2.0 | none in jar | +| org.slf4j:slf4j-api:2.0.20 | MIT | MIT | `LICENSE.txt` [6632f550c207] | +| com.conversantmedia:disruptor:1.2.21 | The Apache License, Version 2.0 | Apache-2.0 | none in jar | +| at.yawk.lz4:lz4-java:1.12.0 | Apache License, Version 2.0 | Apache-2.0 | none in jar | +| org.apache.lucene:lucene-analysis-common:10.5.2 | Apache 2 | Apache-2.0 | `LICENSE.txt` [a2521407b320], `NOTICE.txt` [d3b82734d5e1] | +| org.apache.lucene:lucene-core:10.5.2 | Apache 2 | Apache-2.0 | `LICENSE.txt` [a2521407b320], `NOTICE.txt` [d3b82734d5e1] | +| org.apache.lucene:lucene-spatial-extras:10.5.2 | Apache 2 | Apache-2.0 | `LICENSE.txt` [a2521407b320], `NOTICE.txt` [d3b82734d5e1] | +| org.apache.lucene:lucene-spatial3d:10.5.2 | Apache 2 | Apache-2.0 | `LICENSE.txt` [a2521407b320], `NOTICE.txt` [d3b82734d5e1] | +| io.sgr:s2-geometry-library-java:1.0.0 | The Apache Software License, Version 2.0 | Apache-2.0 | none in jar | +| org.apache.lucene:lucene-queryparser:10.5.2 | Apache 2 | Apache-2.0 | `LICENSE.txt` [a2521407b320], `NOTICE.txt` [d3b82734d5e1] | +| org.apache.lucene:lucene-queries:10.5.2 | Apache 2 | Apache-2.0 | `LICENSE.txt` [a2521407b320], `NOTICE.txt` [d3b82734d5e1] | +| org.apache.lucene:lucene-sandbox:10.5.2 | Apache 2 | Apache-2.0 | `LICENSE.txt` [a2521407b320], `NOTICE.txt` [d3b82734d5e1] | +| org.apache.lucene:lucene-facet:10.5.2 | Apache 2 | Apache-2.0 | `LICENSE.txt` [a2521407b320], `NOTICE.txt` [d3b82734d5e1] | +| org.locationtech.spatial4j:spatial4j:0.8 | The Apache Software License, Version 2.0 | Apache-2.0 | none in jar | +| org.locationtech.jts:jts-core:1.20.0 | Eclipse Public License, Version 2.0 / Eclipse Distribution License - v 1.0 | BSD-3-Clause | `jts-core-1.20.0.LICENSE_EDLv1.txt` [e67a91b5489c] | +| com.google.code.gson:gson:2.14.0 | Apache-2.0 | Apache-2.0 | none in jar | +| com.google.errorprone:error_prone_annotations:2.48.0 | Apache 2.0 | Apache-2.0 | none in jar | +| io.github.jbellis:jvector:4.0.2 | The Apache Software License, Version 2.0 | Apache-2.0 | none in jar | +| org.agrona:agrona:1.20.0 | The Apache License, Version 2.0 | Apache-2.0 | none in jar | +| org.yaml:snakeyaml:2.4 | Apache License, Version 2.0 | Apache-2.0 | none in jar | +| org.antlr:antlr4-runtime:4.13.2 | BSD-3-Clause | BSD-3-Clause | `antlr4-runtime-4.13.2.LICENSE.txt` [3db1fb3ee79a] | +| org.graalvm.sdk:graal-sdk:25.0.2 | Universal Permissive License, Version 1.0 | UPL-1.0 | none in jar | +| org.graalvm.sdk:collections:25.0.2 | Universal Permissive License, Version 1.0 | UPL-1.0 | none in jar | +| org.graalvm.polyglot:polyglot:25.0.2 | Universal Permissive License, Version 1.0 | UPL-1.0 | none in jar | +| com.arcadedb:arcadedb-server:26.10.2 | Apache 2 | Apache-2.0 | none in jar | +| io.undertow:undertow-core:2.4.4.Final | Apache License Version 2.0 | Apache-2.0 | none in jar | +| org.jboss.logging:jboss-logging:3.6.2.Final | Apache License 2.0 | Apache-2.0 | none in jar | +| org.jboss.xnio:xnio-api:3.8.16.Final | Apache License 2.0 | Apache-2.0 | none in jar | +| org.wildfly.common:wildfly-common:1.5.4.Final | Apache License 2.0 | Apache-2.0 | none in jar | +| org.wildfly.client:wildfly-client-config:1.0.1.Final | Apache License 2.0 | Apache-2.0 | none in jar | +| org.jboss.xnio:xnio-nio:3.8.16.Final | Apache License 2.0 | Apache-2.0 | none in jar | +| org.jboss.threads:jboss-threads:3.10.1 | Apache License 2.0 | Apache-2.0 | `LICENSE.txt` [0d542e0c8804] | +| io.smallrye.common:smallrye-common-annotation:2.20.0 | Apache License, Version 2.0 | Apache-2.0 | none in jar | +| io.smallrye.common:smallrye-common-constraint:2.20.0 | Apache License, Version 2.0 | Apache-2.0 | none in jar | +| io.smallrye.common:smallrye-common-cpu:2.20.0 | Apache License, Version 2.0 | Apache-2.0 | none in jar | +| io.smallrye.common:smallrye-common-function:2.20.0 | Apache License, Version 2.0 | Apache-2.0 | none in jar | +| io.micrometer:micrometer-core:1.17.1 | The Apache Software License, Version 2.0 | Apache-2.0 | `LICENSE` [62fb8a3a9621], `NOTICE` [df207ca1a2ea] | +| org.jspecify:jspecify:1.0.1 | The Apache License, Version 2.0 | Apache-2.0 | none in jar | +| io.micrometer:micrometer-commons:1.17.1 | The Apache Software License, Version 2.0 | Apache-2.0 | `LICENSE` [62fb8a3a9621], `NOTICE` [df207ca1a2ea] | +| io.micrometer:micrometer-observation:1.17.1 | The Apache Software License, Version 2.0 | Apache-2.0 | `LICENSE` [62fb8a3a9621], `NOTICE` [df207ca1a2ea] | +| org.hdrhistogram:HdrHistogram:2.2.2 | Public Domain, per Creative Commons CC0 / BSD-2-Clause | CC0-1.0 | `LICENSE.txt` [ba550515c77c] | +| io.swagger.core.v3:swagger-core:2.2.55 | Apache License 2.0 | Apache-2.0 | `LICENSE` [aa4dc81ef2c6], `NOTICE` [921b5b663bdc] | +| jakarta.xml.bind:jakarta.xml.bind-api:2.3.3 | Eclipse Distribution License - v 1.0 | BSD-3-Clause | `LICENSE.md` [dcbbdbb512a2], `NOTICE.md` [190bb88e764b] | +| jakarta.activation:jakarta.activation-api:1.2.2 | EDL 1.0 | BSD-3-Clause | `LICENSE.md` [87eba02f8a41], `NOTICE.md` [6b9afdc018e1] | +| org.apache.commons:commons-lang3:3.20.0 | Apache-2.0 | Apache-2.0 | `LICENSE.txt` [b1d2870f1a00], `NOTICE.txt` [64bc8696af3f] | +| jakarta.validation:jakarta.validation-api:2.0.2 | Apache License 2.0 | Apache-2.0 | none in jar | +| io.swagger.core.v3:swagger-annotations:2.2.55 | Apache License 2.0 | Apache-2.0 | `LICENSE` [aa4dc81ef2c6], `NOTICE` [f1fc76fa8bfb] | +| io.swagger.core.v3:swagger-models:2.2.55 | Apache License 2.0 | Apache-2.0 | `LICENSE` [aa4dc81ef2c6], `NOTICE` [ca0e15ea1bb6] | +| com.arcadedb:arcadedb-network:26.10.2 | Apache 2 | Apache-2.0 | none in jar | +| com.arcadedb:arcadedb-integration:26.10.2 | Apache 2 | Apache-2.0 | none in jar | +| com.sonofab1rd:univocity-parsers:2.10.2 | Apache 2 | Apache-2.0 | none in jar | +| com.arcadedb:arcadedb-console:26.10.2 | Apache 2 | Apache-2.0 | none in jar | +| com.arcadedb:arcadedb-postgresw:26.10.2 | Apache 2 | Apache-2.0 | none in jar | +| com.arcadedb:arcadedb-redisw:26.10.2 | Apache 2 | Apache-2.0 | none in jar | +| com.arcadedb:arcadedb-bolt:26.10.2 | Apache 2 | Apache-2.0 | none in jar | +| com.arcadedb:arcadedb-graphql:26.10.2 | Apache 2 | Apache-2.0 | none in jar | +| com.arcadedb:arcadedb-mcp:26.10.2 | Apache 2 | Apache-2.0 | none in jar | +| org.graalvm.sdk:word:25.4.4.1.1 | Universal Permissive License, Version 1.0 | UPL-1.0 | none in jar | +| org.slf4j:slf4j-jdk14:2.0.20 | MIT | MIT | `LICENSE.txt` [6632f550c207] | + +Where a pom lists several licences they are alternatives (JTS: EPL-2.0 or EDL-1.0; HdrHistogram: CC0-1.0 or BSD-2-Clause); the permissive one is applied. EDL-1.0 is the BSD-3-Clause text. + +## Native code shipped next to the library + +`libjvector-x86_64.so` is taken unmodified from `io.github.jbellis:jvector:4.0.2` (Apache-2.0, pom). Its sources are in the jvector repository at tag `4.0.2` (`jvector-native/src/main/native`): jvector's own C++ kernels (Apache-2.0) and the Google Highway library, pinned there as a git submodule at commit `16f5f5c768064d0bb52952888a14236bec75c104`. Highway's `LICENSE` at that commit says: dual-licensed under the Apache License 2.0 or the BSD 3-Clause License, except `hwy/contrib/random/random-inl.h` (CC0 1.0). Applied here: Apache-2.0. The `LICENSE` of Highway and the `NOTICE.txt` of jvector are reproduced verbatim below. The library links `libstdc++.so.6`, `libm.so.6`, `libgcc_s.so.1` and `libc.so.6` dynamically (not shipped). + +## GraalVM Community Edition runtime and JDK classes in libarcadedb.so + +`libarcadedb.so` contains the Substrate VM runtime and the parts of the OpenJDK class library the engine reaches, built with GraalVM CE 25.4.4.1.1 (Java 25.0.4.1.1, implementor GraalVM Community). Licence: GNU General Public License version 2 with the Classpath Exception (verbatim below, from `LICENSE.txt` of the GraalVM CE home; also in `licenses/`). The exception permits linking and distributing these parts with code under other licences. `graal-sdk`, `polyglot`, `collections` (25.0.2, Maven Central) and `word` (25.4.4.1.1) are UPL-1.0. Oracle GraalVM (GFTC) is not used. Sources: https://github.com/oracle/graal (tag vm-25.4.4.1.1), https://github.com/openjdk/jdk (jdk-25 line). + +Third-party data inside the JDK classes that the image contains (texts from `legal/java.base` of the GraalVM home, verbatim below): ICU normalisation data (`jdk/internal/icu/impl/data/icudt76b/nfc.nrm`, registered resource), Unicode data files, CLDR locale data. Other `legal/java.base` notices (aes.md, c-libutl.md, gcc.md, siphash.md, public_suffix.md) concern files or HotSpot code that are not linked into the image (checked: the library contains no public-suffix data); `jdk.incubator.vector` (sleef.md) is added to the module path but its vector intrinsics are disabled in the build (`-H:-VectorAPISupport`). + +System libraries linked dynamically and not shipped: `libz.so.1`, `libc.so.6`, `ld-linux-x86-64.so.2` (checked with `ldd`). + +## Licence review + +Dependencies whose licence is not in the allowed permissive set or whose provenance is not recorded in the build inputs. The generator lists them; a maintainer decides. + +- **libarcadedb.so JDK module set**: which JDK modules end up linked is not recorded by the build; the notices above cover java.base/java.logging/jdk.unsupported by module inspection of the legal directory only. A native-image `-H:+DashboardDump` or `--emit build-report` run would confirm it + +## Verbatim texts + +### ArcadeDB NOTICE (engine release 26.10.2) + +``` +SPDX-License-Identifier: Apache-2.0 + +ArcadeDB +Copyright 2021-present Arcade Data Ltd + +This product includes software developed at +Arcade Data Ltd (https://arcadedata.com/). + +================================================================================ + +This product includes components originally developed for OrientDB, an Open +Source Multi-Model NoSQL Database. +Copyright 2010-2020 OrientDB LTD (https://orientdb.com) + +ArcadeDB was created by Luca Garulli, the same founder of OrientDB, and written +from scratch after SAP's acquisition of OrientDB in 2020. ArcadeDB reuses +OrientDB's SQL engine (now heavily modified) and some utility classes. + +OrientDB source code is available at: https://github.com/orientechnologies/orientdb + +================================================================================ + +This product includes openCypher grammar files and Technology Compatibility Kit +(TCK) test suite, developed by the openCypher community. + + Copyright (c) "Neo4j" + Neo4j Sweden AB [https://neo4j.com] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + + openCypher project website: https://opencypher.org/ + +Cypher® is a registered trademark of Neo4j Inc. + +The included openCypher TCK tests contain the following attribution notice: + + "This work was created by the collective efforts of the openCypher community. + Without limiting the terms of Section 6, any Derivative Work that is not + approved by the public consensus process of the openCypher Implementers Group + should not be described as 'Cypher' (and Cypher® is a registered trademark of + Neo4j Inc.) or as 'openCypher'. Extensions by implementers or prototypes or + proposals for change that have been documented or implemented should only be + described as 'implementation extensions to Cypher' or as 'proposed changes to + Cypher that are not yet approved by the openCypher community'." + +The following files contain openCypher content: + - engine/src/main/antlr4/com/arcadedb/query/opencypher/grammar/Cypher25Parser.g4 + - engine/src/main/antlr4/com/arcadedb/query/opencypher/grammar/Cypher25Lexer.g4 + - engine/src/test/resources/opencypher/tck/features/**/*.feature (220 test files) + +This product includes software from the OpenCypher-Gremlin project. + + Copyright 2018-2019 Neueda + + The following file is a security-patched override of unmaintained upstream code: + gremlin/src/main/java/org/opencypher/gremlin/translation/groovy/StringTranslationUtils.java + + OpenCypher-Gremlin project: https://github.com/opencypher/cypher-for-gremlin + +================================================================================ + +This product includes software developed by The Apache Software Foundation +(https://www.apache.org/). + +Apache Lucene +Copyright 2001-2024 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (https://www.apache.org/). + +Includes software from other Apache copyright owners, as follows: + - ICU4J, (c) 1995-2014 International Business Machines Corporation and others + - ANTLR4 grammar from C parser (BSD 3-clause license) + - JFlex lexer generator (BSD license) + +Apache ANTLR4 +Copyright 2012-2021 The ANTLR Project + +Apache TinkerPop (Gremlin) +Copyright 2015-2024 The Apache Software Foundation + +Apache Commons Compress +Copyright 2002-2024 The Apache Software Foundation + +Apache Commons Lang +Copyright 2001-2024 The Apache Software Foundation + +Apache Commons Codec +Copyright 2002-2024 The Apache Software Foundation + +Apache HttpComponents +Copyright 1999-2024 The Apache Software Foundation + +================================================================================ + +Apache Ratis +Copyright 2017-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +================================================================================ + +This product includes software developed by GraalVM (Oracle). + + GraalVM SDK, Truffle API, and JavaScript implementation + Copyright (c) 2012, 2024, Oracle and/or its affiliates + + https://www.graalvm.org/ + +================================================================================ + +This product includes software developed by Google Inc. + + Protocol Buffers + Copyright 2008 Google Inc. + + Guava Libraries + Copyright (C) 2010-2024 The Guava Authors + + Gson + Copyright 2008-2024 Google Inc. + + gRPC + Copyright 2015-2024 The gRPC Authors + +================================================================================ + +This product includes software developed by Red Hat, Inc. + + Undertow + Copyright 2013 Red Hat, Inc., and individual contributors + + https://undertow.io/ + +================================================================================ + +This product includes software developed by the Netty project. + + Netty + Copyright 2014 The Netty Project + + https://netty.io/ + +================================================================================ + +This product includes JUnit 5 test framework. + + JUnit 5 + Copyright 2015-2024 the original author or authors + + https://junit.org/junit5/ + +================================================================================ + +This product includes Testcontainers framework. + + Testcontainers + Copyright (c) 2015-2024 Richard North and other authors + + https://testcontainers.com/ + +================================================================================ + +This product includes software developed by FasterXML, LLC. + + Jackson JSON processor + Copyright (c) 2007-2024 Tatu Saloranta and other contributors + + https://github.com/FasterXML/jackson + +================================================================================ + +This product includes Micrometer Tracing (optional tracing module only). + + Micrometer Tracing + Copyright (c) 2017-Present VMware, Inc. All Rights Reserved. + + https://micrometer.io/ + + Micrometer Tracing contains modified portions of: + - 'io.netty.util.internal.logging' from Netty (Copyright 2013 The Netty Project, Apache 2.0, https://netty.io) + - 'StringUtils.isBlank()' from Apache Commons Lang (Copyright 2001-2019 The Apache Software Foundation, Apache 2.0) + - 'JsonUtf8Writer' from Moshi (Copyright 2010 Google Inc., Apache 2.0, https://github.com/square/moshi) + - the 'org.springframework.lang' package from the Spring Framework (Copyright 2002-2019 the original author or authors, Apache 2.0) + +================================================================================ + +This product includes OpenTelemetry Java (optional tracing module only). + + OpenTelemetry Java + Copyright The OpenTelemetry Authors + + https://opentelemetry.io/ + +================================================================================ + +For the complete list of third-party components and their licenses, see the +ATTRIBUTIONS.md file in the root directory of this distribution. +``` + +### GraalVM Community Edition LICENSE.txt (GPLv2, Classpath Exception, UPL-1.0, BSD-3-Clause) + +``` + Product License - GraalVM Community Edition + +GraalVM Community Edition consists of multiple modules. The software as a +whole, is released under version 2 of the GNU General Public License with the +“Classpath” Exception. Certain modules of GraalVM Community Edition are released +under separate and/or additional licenses, as follows: Graal.js is released +under the Universal Permissive License (UPL) Version 1.0; Sulong – LLVM is +released under a 3-clause BSD license. + +The text of the foregoing licenses is reproduced below. + +Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved. + +=========================================================================== + +The GNU General Public License (GPL) + +Version 2, June 1991 + +Copyright (C) 1989, 1991 Free Software Foundation, Inc. +59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Everyone is permitted to copy and distribute verbatim copies of this license +document, but changing it is not allowed. + +Preamble + +The licenses for most software are designed to take away your freedom to share +and change it. By contrast, the GNU General Public License is intended to +guarantee your freedom to share and change free software--to make sure the +software is free for all its users. This General Public License applies to +most of the Free Software Foundation's software and to any other program whose +authors commit to using it. (Some other Free Software Foundation software is +covered by the GNU Library General Public License instead.) You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our +General Public Licenses are designed to make sure that you have the freedom to +distribute copies of free software (and charge for this service if you wish), +that you receive source code or can get it if you want it, that you can change +the software or use pieces of it in new free programs; and that you know you +can do these things. + +To protect your rights, we need to make restrictions that forbid anyone to deny +you these rights or to ask you to surrender the rights. These restrictions +translate to certain responsibilities for you if you distribute copies of the +software, or if you modify it. + +For example, if you distribute copies of such a program, whether gratis or for +a fee, you must give the recipients all the rights that you have. You must +make sure that they, too, receive or can get the source code. And you must +show them these terms so they know their rights. + +We protect your rights with two steps: (1) copyright the software, and (2) +offer you this license which gives you legal permission to copy, distribute +and/or modify the software. + +Also, for each author's protection and ours, we want to make certain that +everyone understands that there is no warranty for this free software. If the +software is modified by someone else and passed on, we want its recipients to +know that what they have is not the original, so that any problems introduced +by others will not reflect on the original authors' reputations. + +Finally, any free program is threatened constantly by software patents. We +wish to avoid the danger that redistributors of a free program will +individually obtain patent licenses, in effect making the program proprietary. +To prevent this, we have made it clear that any patent must be licensed for +everyone's free use or not licensed at all. + +The precise terms and conditions for copying, distribution and modification +follow. + +TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + +0. This License applies to any program or other work which contains a notice +placed by the copyright holder saying it may be distributed under the terms of +this General Public License. The "Program", below, refers to any such program +or work, and a "work based on the Program" means either the Program or any +derivative work under copyright law: that is to say, a work containing the +Program or a portion of it, either verbatim or with modifications and/or +translated into another language. (Hereinafter, translation is included +without limitation in the term "modification".) Each licensee is addressed as +"you". + +Activities other than copying, distribution and modification are not covered by +this License; they are outside its scope. The act of running the Program is +not restricted, and the output from the Program is covered only if its contents +constitute a work based on the Program (independent of having been made by +running the Program). Whether that is true depends on what the Program does. + +1. You may copy and distribute verbatim copies of the Program's source code as +you receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice and +disclaimer of warranty; keep intact all the notices that refer to this License +and to the absence of any warranty; and give any other recipients of the +Program a copy of this License along with the Program. + +You may charge a fee for the physical act of transferring a copy, and you may +at your option offer warranty protection in exchange for a fee. + +2. You may modify your copy or copies of the Program or any portion of it, thus +forming a work based on the Program, and copy and distribute such modifications +or work under the terms of Section 1 above, provided that you also meet all of +these conditions: + + a) You must cause the modified files to carry prominent notices stating + that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in whole or + in part contains or is derived from the Program or any part thereof, to be + licensed as a whole at no charge to all third parties under the terms of + this License. + + c) If the modified program normally reads commands interactively when run, + you must cause it, when started running for such interactive use in the + most ordinary way, to print or display an announcement including an + appropriate copyright notice and a notice that there is no warranty (or + else, saying that you provide a warranty) and that users may redistribute + the program under these conditions, and telling the user how to view a copy + of this License. (Exception: if the Program itself is interactive but does + not normally print such an announcement, your work based on the Program is + not required to print an announcement.) + +These requirements apply to the modified work as a whole. If identifiable +sections of that work are not derived from the Program, and can be reasonably +considered independent and separate works in themselves, then this License, and +its terms, do not apply to those sections when you distribute them as separate +works. But when you distribute the same sections as part of a whole which is a +work based on the Program, the distribution of the whole must be on the terms +of this License, whose permissions for other licensees extend to the entire +whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest your +rights to work written entirely by you; rather, the intent is to exercise the +right to control the distribution of derivative or collective works based on +the Program. + +In addition, mere aggregation of another work not based on the Program with the +Program (or with a work based on the Program) on a volume of a storage or +distribution medium does not bring the other work under the scope of this +License. + +3. You may copy and distribute the Program (or a work based on it, under +Section 2) in object code or executable form under the terms of Sections 1 and +2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable source + code, which must be distributed under the terms of Sections 1 and 2 above + on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three years, to + give any third party, for a charge no more than your cost of physically + performing source distribution, a complete machine-readable copy of the + corresponding source code, to be distributed under the terms of Sections 1 + and 2 above on a medium customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer to + distribute corresponding source code. (This alternative is allowed only + for noncommercial distribution and only if you received the program in + object code or executable form with such an offer, in accord with + Subsection b above.) + +The source code for a work means the preferred form of the work for making +modifications to it. For an executable work, complete source code means all +the source code for all modules it contains, plus any associated interface +definition files, plus the scripts used to control compilation and installation +of the executable. However, as a special exception, the source code +distributed need not include anything that is normally distributed (in either +source or binary form) with the major components (compiler, kernel, and so on) +of the operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the source +code from the same place counts as distribution of the source code, even though +third parties are not compelled to copy the source along with the object code. + +4. You may not copy, modify, sublicense, or distribute the Program except as +expressly provided under this License. Any attempt otherwise to copy, modify, +sublicense or distribute the Program is void, and will automatically terminate +your rights under this License. However, parties who have received copies, or +rights, from you under this License will not have their licenses terminated so +long as such parties remain in full compliance. + +5. You are not required to accept this License, since you have not signed it. +However, nothing else grants you permission to modify or distribute the Program +or its derivative works. These actions are prohibited by law if you do not +accept this License. Therefore, by modifying or distributing the Program (or +any work based on the Program), you indicate your acceptance of this License to +do so, and all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + +6. Each time you redistribute the Program (or any work based on the Program), +the recipient automatically receives a license from the original licensor to +copy, distribute or modify the Program subject to these terms and conditions. +You may not impose any further restrictions on the recipients' exercise of the +rights granted herein. You are not responsible for enforcing compliance by +third parties to this License. + +7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), conditions +are imposed on you (whether by court order, agreement or otherwise) that +contradict the conditions of this License, they do not excuse you from the +conditions of this License. If you cannot distribute so as to satisfy +simultaneously your obligations under this License and any other pertinent +obligations, then as a consequence you may not distribute the Program at all. +For example, if a patent license would not permit royalty-free redistribution +of the Program by all those who receive copies directly or indirectly through +you, then the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply and +the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any patents or +other property right claims or to contest validity of any such claims; this +section has the sole purpose of protecting the integrity of the free software +distribution system, which is implemented by public license practices. Many +people have made generous contributions to the wide range of software +distributed through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing to +distribute software through any other system and a licensee cannot impose that +choice. + +This section is intended to make thoroughly clear what is believed to be a +consequence of the rest of this License. + +8. If the distribution and/or use of the Program is restricted in certain +countries either by patents or by copyrighted interfaces, the original +copyright holder who places the Program under this License may add an explicit +geographical distribution limitation excluding those countries, so that +distribution is permitted only in or among countries not thus excluded. In +such case, this License incorporates the limitation as if written in the body +of this License. + +9. The Free Software Foundation may publish revised and/or new versions of the +General Public License from time to time. Such new versions will be similar in +spirit to the present version, but may differ in detail to address new problems +or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any later +version", you have the option of following the terms and conditions either of +that version or of any later version published by the Free Software Foundation. +If the Program does not specify a version number of this License, you may +choose any version ever published by the Free Software Foundation. + +10. If you wish to incorporate parts of the Program into other free programs +whose distribution conditions are different, write to the author to ask for +permission. For software which is copyrighted by the Free Software Foundation, +write to the Free Software Foundation; we sometimes make exceptions for this. +Our decision will be guided by the two goals of preserving the free status of +all derivatives of our free software and of promoting the sharing and reuse of +software generally. + +NO WARRANTY + +11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR +THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE +STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE +PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND +PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, +YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL +ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE +PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR +INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA +BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER +OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +END OF TERMS AND CONDITIONS + +How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible +use to the public, the best way to achieve this is to make it free software +which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach +them to the start of each source file to most effectively convey the exclusion +of warranty; and each file should have at least the "copyright" line and a +pointer to where the full notice is found. + + One line to give the program's name and a brief idea of what it does. + + Copyright (C) + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., 59 + Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this when it +starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author Gnomovision comes + with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free + software, and you are welcome to redistribute it under certain conditions; + type 'show c' for details. + +The hypothetical commands 'show w' and 'show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may be +called something other than 'show w' and 'show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your school, +if any, to sign a "copyright disclaimer" for the program, if necessary. Here +is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + 'Gnomovision' (which makes passes at compilers) written by James Hacker. + + signature of Ty Coon, 1 April 1989 + + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General Public +License instead of this License. + +"CLASSPATH" EXCEPTION TO THE GPL + +Certain source files distributed by Oracle America and/or its affiliates +are subject to the following clarification and special exception to the GPLv2, +based on the GNU Project exception for its Classpath libraries, known as the +GNU Classpath Exception, but only where Oracle has expressly included in the +particular source file's header the words "Oracle designates this particular +file as subject to the "Classpath" exception as provided by Oracle in the +LICENSE file that accompanied this code." + +You should also note that Oracle includes multiple, independent programs in +this software package. Some of those programs are provided under licenses +deemed incompatible with the GPLv2 by the Free Software Foundation and others. +For example, the package includes programs licensed under the Apache License, +Version 2.0. Such programs are licensed to you under their original licenses. + +Oracle facilitates your further distribution of this package by adding the +Classpath Exception to the necessary parts of its GPLv2 code, which permits you +to use that code in combination with other independent modules not licensed +under the GPLv2. However, note that this would not permit you to commingle code +under an incompatible license with Oracle's GPLv2 licensed code by, for +example, cutting and pasting such code into a file also containing Oracle's +GPLv2 licensed code and then distributing the result. + +Additionally, if you were to remove the Classpath Exception from any of the +files to which it applies and distribute the result, you would likely be +required to license some or all of the other code in that distribution under +the GPLv2 as well, and since the GPLv2 is incompatible with the license terms +of some items included in the distribution by Oracle, removing the Classpath +Exception could therefore effectively compromise your ability to further +distribute the package. + +Proceed with caution and we recommend that you obtain the advice of a lawyer +skilled in open source matters before removing the Classpath Exception or +making modifications to this package which may subsequently be redistributed +and/or involve the use of third party software. + +CLASSPATH EXCEPTION + +Linking this library statically or dynamically with other modules is making a +combined work based on this library. Thus, the terms and conditions of the GNU +General Public License version 2 cover the whole combination. + +As a special exception, the copyright holders of this library give you +permission to link this library with independent modules to produce an +executable, regardless of the license terms of these independent modules, and +to copy and distribute the resulting executable under terms of your choice, +provided that you also meet, for each linked independent module, the terms and +conditions of the license of that module. An independent module is a module +which is not derived from or based on this library. If you modify this library, +you may extend this exception to your version of the library, but you are not +obligated to do so. If you do not wish to do so, delete this exception +statement from your version. + +=========================================================================== + + WRITTEN OFFER FOR SOURCE CODE +For any software that you receive from Oracle in binary form which is licensed +under an open source license that gives you the right to receive the source +code for that binary, you can obtain a copy of the applicable source code by +visiting http://www.oracle.com/goto/opensourcecode. If the source code for the +binary was not provided to you with the binary, you can also receive a copy of +the source code on physical media by submitting a written request to the +address listed below or by sending an email to Oracle using the following link: + http://www.oracle.com/goto/opensourcecode/request. + +Oracle America, Inc. +Attn: Senior Vice President +Development and Engineering Legal +500 Oracle Parkway, 10th Floor +Redwood Shores, CA 94065 + +Your request should include: + • The name of the binary for which you are requesting the source code + • The name and version number of the Oracle product containing the binary + • The date you received the Oracle product + • Your name + • Your company name (if applicable) + • Your return mailing address and email, and + • A telephone number in the event we need to reach you. + +We may charge you a fee to cover the cost of physical media and processing. +Your request must be sent + a. within three (3) years of the date you received the Oracle product that +included the binary that is the subject of your request, or + b. in the case of code licensed under the GPL v3 for as long as Oracle +offers spare parts or customer support for that product model. + +=========================================================================== +``` + +### OpenJDK java.base legal notice: ADDITIONAL_LICENSE_INFO + +``` + ADDITIONAL INFORMATION ABOUT LICENSING + +Certain files distributed by Oracle America, Inc. and/or its affiliates are +subject to the following clarification and special exception to the GPLv2, +based on the GNU Project exception for its Classpath libraries, known as the +GNU Classpath Exception. + +Note that Oracle includes multiple, independent programs in this software +package. Some of those programs are provided under licenses deemed +incompatible with the GPLv2 by the Free Software Foundation and others. +For example, the package includes programs licensed under the Apache +License, Version 2.0 and may include FreeType. Such programs are licensed +to you under their original licenses. + +Oracle facilitates your further distribution of this package by adding the +Classpath Exception to the necessary parts of its GPLv2 code, which permits +you to use that code in combination with other independent modules not +licensed under the GPLv2. However, note that this would not permit you to +commingle code under an incompatible license with Oracle's GPLv2 licensed +code by, for example, cutting and pasting such code into a file also +containing Oracle's GPLv2 licensed code and then distributing the result. + +Additionally, if you were to remove the Classpath Exception from any of the +files to which it applies and distribute the result, you would likely be +required to license some or all of the other code in that distribution under +the GPLv2 as well, and since the GPLv2 is incompatible with the license terms +of some items included in the distribution by Oracle, removing the Classpath +Exception could therefore effectively compromise your ability to further +distribute the package. + +Failing to distribute notices associated with some files may also create +unexpected legal consequences. + +Proceed with caution and we recommend that you obtain the advice of a lawyer +skilled in open source matters before removing the Classpath Exception or +making modifications to this package which may subsequently be redistributed +and/or involve the use of third party software. +``` + +### OpenJDK java.base legal notice: ASSEMBLY_EXCEPTION + +``` + +OPENJDK ASSEMBLY EXCEPTION + +The OpenJDK source code made available by Oracle America, Inc. (Oracle) at +openjdk.org ("OpenJDK Code") is distributed under the terms of the GNU +General Public License version 2 +only ("GPL2"), with the following clarification and special exception. + + Linking this OpenJDK Code statically or dynamically with other code + is making a combined work based on this library. Thus, the terms + and conditions of GPL2 cover the whole combination. + + As a special exception, Oracle gives you permission to link this + OpenJDK Code with certain code licensed by Oracle as indicated at + https://openjdk.org/legal/exception-modules-2007-05-08.html + ("Designated Exception Modules") to produce an executable, + regardless of the license terms of the Designated Exception Modules, + and to copy and distribute the resulting executable under GPL2, + provided that the Designated Exception Modules continue to be + governed by the licenses under which they were offered by Oracle. + +As such, it allows licensees and sublicensees of Oracle's GPL2 OpenJDK Code +to build an executable that includes those portions of necessary code that +Oracle could not provide under GPL2 (or that Oracle has provided under GPL2 +with the Classpath exception). If you modify or add to the OpenJDK code, +that new GPL2 code may still be combined with Designated Exception Modules +if the new code is made subject to this exception by its copyright holder. +``` + +### OpenJDK java.base legal notice: icu.md + +```` +## International Components for Unicode (ICU4J) v76.1 + +### ICU4J License +``` +UNICODE LICENSE V3 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 2016-2024 Unicode, Inc. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR +SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT +DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the +Data Files or Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Data Files or Software, and to permit persons to whom the +Data Files or Software are furnished to do so, provided that either (a) +this copyright and permission notice appear with all copies of the Data +Files or Software, or (b) this copyright and permission notice appear in +associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA +FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. + +SPDX-License-Identifier: Unicode-3.0 + +---------------------------------------------------------------------- + +Third-Party Software Licenses + +This section contains third-party software notices and/or additional +terms for licensed third-party software components included within ICU +libraries. + +---------------------------------------------------------------------- + +ICU License - ICU 1.8.1 to ICU 57.1 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2016 International Business Machines Corporation and others +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, provided that the above +copyright notice(s) and this permission notice appear in all copies of +the Software and that both the above copyright notice(s) and this +permission notice appear in supporting documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY +SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER +RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, use +or other dealings in this Software without prior written authorization +of the copyright holder. + +All trademarks and registered trademarks mentioned herein are the +property of their respective owners. + +``` +```` + +### OpenJDK java.base legal notice: unicode.md + +```` +## The Unicode Standard, Unicode Character Database, Version 16.0.0 + +### Unicode Character Database +``` + +UNICODE LICENSE V3 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2024 Unicode, Inc. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR +SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT +DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the +Data Files or Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Data Files or Software, and to permit persons to whom the +Data Files or Software are furnished to do so, provided that either (a) +this copyright and permission notice appear with all copies of the Data +Files or Software, or (b) this copyright and permission notice appear in +associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA +FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. + +------------------ + +Unicode® Consortium Copyright, Terms of Use, and Licenses + +Welcome to the website of Unicode, Inc. (dba The Unicode Consortium) +(“Unicode”). Except as otherwise noted herein, these terms and conditions +(“Terms of Use”) govern your use of the Unicode website and Unicode Products. +Your use of this website and/or Unicode Products constitutes your agreement +to follow and be bound by these Terms of Use. Unicode provides you with +access to and use of this website and Unicode Products subject to your +compliance with these Terms of Use. If you do not agree to these Terms +of Use, you should not access or use this website or Unicode Products. +Unicode reserves the right to make changes to the website, to Unicode +Products, and to these Terms of Use at any time in its sole d iscretion. + + 1. Unicode Copyright: Copyright © 1991-Present Unicode, Inc. + + 2. Definitions + + a. “Unicode Products” includes the Unicode website and its content, + Data Files, Software, the Unicode Standard, all other Unicode + standards, specifications, technical reports, technical notes, + annexes, code charts, data files, software, publications, + webinars, videos, course materials, online and in-person events, + and all other Unicode products and services made available via + this website or any other channel of distribution, including but + not limited to Github, Maven, YouTube, and other third-party + sites where Unicode maintains and provides products, materials, + and services. + + b. “Unicode Data Files” or "Data Files" include all computer data + files under the following directories: + + i. https://www.unicode.org/Public/ + ii. https://www.unicode.org/reports/ + iii. https://www.unicode.org/ivd/data/ + iv. https://github.com/unicode-org/ + + but exclude any materials present in the above directories that + are not computer data files, such as PDF code charts and + Technical Reports. + + c. “Unicode Software” or "Software" includes any source code or + compiled code in any Unicode Product including but not limited to + the code included in the following directories: + + i. https://www.unicode.org/Public/PROGRAMS/ + ii. https://www.unicode.org/Public/cldr/ + iii. https://github.com/unicode-org/ + + 3. Permissions, Licenses, and Restrictions on Use + + a. You are authorized to freely access and use this website and its + content and all Unicode Products subject to these Terms of Use and + subject to any restriction, permission, or license specifically + associated with any specific material or content. No license is + granted to copy or "mirror" this website. Linking to this website + is permitted. + + b. Except where otherwise more broadly permitted or licensed: + + 1. you may not make copies of or modifications to Unicode + Products for public distribution, or incorporate Unicode + Products in whole or in part into any product or publication, + or otherwise publicly distribute them, without the express + written permission of Unicode, and + + 2. you may not copy or extract fonts or font data from any + Unicode Products, including but not limited to Unicode Code + Charts. + + c. All Unicode Data Files and Unicode Software are subject to the + terms and conditions of the free and open-source Unicode License + v3, unless otherwise indicated by specific restriction, + permission, or license identified at the point of release or in + such software, data file, or other documentation. + + d. You may freely download and make copies of the Unicode® Standard + Core Specification, Unicode Technical Reports, Unicode Technical + Notes, Unicode Code Charts, and other portions of this website + and may annotate and translate such permitted downloads and + copies, provided that such downloads, copies, annotations, and + translations are solely for personal or internal business + purposes and not for public distribution, and further provided + that any permitted copies and modifications fully reproduce all + copyright and other legal notices contained in the original. + Notwithstanding the foregoing, specific versions of the Unicode® + Standard Core Specification, Unicode Technical Reports, Unicode + Technical Notes, and Unicode Code Charts, as well as other Unicode + publications, materials, and portions of this website may be + subject to broader permissions and/or further reservations of + rights and restrictions on use found in the title pages, cover + sheets, front matter, and/or footnotes for each such version, + report, chart, or other publication or material. Consult each + version, report, chart, or other publication or material for any + such permissions and further reservations of rights and + restrictions on use. + + 4. Restricted Rights Legend. Any Unicode Data Files or Software that are + licensed to the United States of America, its agencies and/or + instrumentalities under these Terms of Use is commercial technical + data or commercial computer software developed exclusively at private + expense as defined in FAR 2.101, or DFARS 252.227-7014, as applicable. + For Unicode Data Files, use, duplication, or disclosure by the + Government is subject to restrictions as set forth in DFARS + 202.227-7015 Technical Data, Commercial and Items and these Terms of + Use. For Unicode Software, in accordance with FAR 12-212 or DFARS + 227-7202, as applicable, use, duplication or disclosure by the + Government is subject to the restrictions set forth in these Terms + of Use. + + 5. Disclaimer of Warranties & Limitation of Liability + + a. This website and the Unicode Products are provided “AS-IS” without + charge as a convenience to visitors and users. While Unicode + attempts to provide accurate, error-free, and timely information, + there may be technical or factual inaccuracies and typographical + or other errors in this website and in the Unicode Products. + Unicode reserves the right to make corrections and changes to the + website and the Unicode Products at any time without notice. + + b. YOU ASSUME ALL RESPONSIBILITY AND RISK WITH RESPECT TO YOUR USE + OF THIS WEBSITE AND THE UNICODE PRODUCTS, WHICH ARE PROVIDED "AS + IS" WITHOUT WARRANTY OF ANY KIND EITHER EXPRESS, IMPLIED, OR + STATUTORY, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF + ACCURACY, COMPLETENESS, TITLE, MERCHANTABILITY, FITNESS FOR A + PARTICULAR PURPOSE, OR NON-INFRINGEMENT OF THIRD PARTY RIGHTS. + UNICODE AND ITS LICENSORS AND CONTRIBUTORS ASSUME NO + RESPONSIBILITY FOR ERRORS OR OMISSIONS IN THIS WEBSITE AND/OR THE + UNICODE PRODUCTS. IF YOU ARE DISSATISFIED WITH THIS WEBSITE OR THE + UNICODE PRODUCTS, YOUR SOLE REMEDY IS TO DISCONTINUE USE OF THE + WEBSITE AND THE UNICODE PRODUCTS. + + c. IN NO EVENT SHALL UNICODE, ITS MEMBERS, OR ITS LICENSORS OR + CONTRIBUTORS BE LIABLE FOR ANY CLAIM OR DAMAGES WHATSOEVER OF ANY + KIND, WHETHER DIRECT, INDIRECT, SPECIAL, INCIDENTAL, + CONSEQUENTIAL, EXEMPLARY, OR PUNITIVE DAMAGES, WHETHER OR NOT + UNICODE WAS ADVISED OF THE POSSIBILITY OF SUCH DAMAGE, INCLUDING + BUT NOT LIMITED TO DAMAGE RESULTING FROM LOSS OF USE, DATA, OR + PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE, OR OTHER + TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE, + INABILITY TO USE, PERFORMANCE, FUNCTIONALITY, MODIFICATION, OR + DISTRIBUTION OF THIS WEBSITE, THE UNICODE PRODUCTS, OR ANY + DERIVATIVES THEREOF. + + d. Unicode makes no warranties or representations of any kind + regarding any non-Unicode sites to which you may be directed or + hyperlinked from this website. Hyperlinks are included solely for + your convenience and Unicode makes no warranties or + representations with regard to the accuracy, availability, + suitability, or safety of information, products, or services + provided on such non-Unicode sites. + + 6. Intellectual Property Contributions to Unicode. All contributions or + submissions to Unicode are governed by The Unicode Consortium + Intellectual Property, Licensing & Technical Contribution Policies. + If you wish to make any contribution or submission to Unicode, refer + to the foregoing Policies for further information on how to do so. + In the absence of a signed contributor license or other agreement with + Unicode that expressly governs a particular contribution or + submission, the act of making a contribution or submission of any kind + to Unicode by any communication channel constitutes a binding legal + agreement by the contributor or submitter that they: + + a. represent and agree that the contributed matter is not + proprietary or confidential to the contributor or any third + party, and + + b. grant to Unicode and to recipients of products distributed by + Unicode a perpetual, irrevocable, unrestricted, worldwide, + nonexclusive, no-charge, royalty-free license, without + obligation for accounting, to reproduce, prepare derivative + works of, publicly display, publicly perform, distribute, + make, use, sell, offer to sell, or import that matter for any + purpose, with the unrestricted right to sublicense those + rights. + + 7. Compliance with Unicode Policies. These Terms of Use hereby + incorporate by reference the following Unicode Policies. Your use of + this website and/or Unicode Products and/or your participation in + Unicode activities constitutes your agreement to these Policies + which may be modified at any time in Unicode’s sole discretion: + + a. The Unicode Consortium Code of Conduct + b. The Unicode Consortium Antitrust Policy & Guidelines + c. The Unicode Consortium Intellectual Property, Licensing & + Technical Contribution Policies + d. The Unicode Consortium Name and Trademark Usage Policy + e. The Unicode Consortium Policy on Handling of Confidential + Business Data + f. The Unicode Consortium General Privacy Policy + g. The Unicode Technical Group Procedures + + 8. Trademarks & Logos. The Unicode Word Mark and the Unicode Logo are + trademarks of Unicode, Inc. “The Unicode Consortium'' and “Unicode, + Inc.” are trade names of Unicode, Inc. You hereby acknowledge and + agree to respect Unicode’s exclusive worldwide rights in the Unicode + Word Mark, the Unicode Logo, and the Unicode trade names. The Unicode + Consortium Name and Trademark Usage Policy is incorporated herein by + reference and you agree to abide by its provisions, which may be + changed from time to time in Unicode’s sole discretion. + + 9. Jurisdiction and Venue. This website is operated from, and the + Unicode Products are made available from, locations in the United + States of America. Unicode makes no representation that this website + or Unicode Products are appropriate for use in other locations. If + you access this website or the Unicode Products from other locations, + you are responsible for compliance with local laws. These Terms of + Use, all use of this website and Unicode Products, and any claims and + damages resulting from use of this website or Unicode Products, are + governed by the applicable laws of the United States of America and + the State of California without regard to any principles which would + apply the laws of different jurisdictions. You agree that any disputes + regarding this website and the Unicode Products shall be resolved + solely in the appropriate state and federal courts located in the + counties of San Francisco, San Mateo, or Santa Clara, California. + You agree that these courts have personal jurisdiction over you and + agree to waive any right to transfer the dispute to any other forum. + + 10. Severability. If any provision of these Terms of Use is declared + invalid or unenforceable, the remaining provisions of these Terms of + Use shall remain in effect. +```` + +### OpenJDK java.base legal notice: cldr.md + +```` +## Unicode Common Local Data Repository (CLDR) v47 + +### CLDR License + +``` + +UNICODE LICENSE V3 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2025 Unicode, Inc. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR +SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT +DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the +Data Files or Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Data Files or Software, and to permit persons to whom the +Data Files or Software are furnished to do so, provided that either (a) +this copyright and permission notice appear with all copies of the Data +Files or Software, or (b) this copyright and permission notice appear in +associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA +FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. + +SPDX-License-Identifier: Unicode-3.0 + +———————————— + +Unicode® Copyright and Terms of Use +For the general privacy policy governing access to this site, see the Unicode Privacy Policy. + +Unicode Copyright +Copyright © 1991-2025 Unicode, Inc. All rights reserved. +Definitions +Unicode Data Files ("DATA FILES") include all data files under the directories: +https://www.unicode.org/Public/ +https://www.unicode.org/reports/ +https://www.unicode.org/ivd/data/ + +Unicode Data Files do not include PDF online code charts under the directory: +https://www.unicode.org/Public/ + +Unicode Software ("SOFTWARE") includes any source code published in the Unicode Standard +or any source code or compiled code under the directories: +https://www.unicode.org/Public/PROGRAMS/ +https://www.unicode.org/Public/cldr/ +http://site.icu-project.org/download/ +Terms of Use +Certain documents and files on this website contain a legend indicating that "Modification is permitted." Any person is hereby authorized, without fee, to modify such documents and files to create derivative works conforming to the Unicode® Standard, subject to Terms and Conditions herein. +Any person is hereby authorized, without fee, to view, use, reproduce, and distribute all documents and files, subject to the Terms and Conditions herein. +Further specifications of rights and restrictions pertaining to the use of the Unicode DATA FILES and SOFTWARE can be found in the Unicode Data Files and Software License. +Each version of the Unicode Standard has further specifications of rights and restrictions of use. For the book editions (Unicode 5.0 and earlier), these are found on the back of the title page. +The Unicode PDF online code charts carry specific restrictions. Those restrictions are incorporated as the first page of each PDF code chart. +All other files, including online documentation of the core specification for Unicode 6.0 and later, are covered under these general Terms of Use. +No license is granted to "mirror" the Unicode website where a fee is charged for access to the "mirror" site. +Modification is not permitted with respect to this document. All copies of this document must be verbatim. +Restricted Rights Legend +Any technical data or software which is licensed to the United States of America, its agencies and/or instrumentalities under this Agreement is commercial technical data or commercial computer software developed exclusively at private expense as defined in FAR 2.101, or DFARS 252.227-7014 (June 1995), as applicable. For technical data, use, duplication, or disclosure by the Government is subject to restrictions as set forth in DFARS 202.227-7015 Technical Data, Commercial and Items (Nov 1995) and this Agreement. For Software, in accordance with FAR 12-212 or DFARS 227-7202, as applicable, use, duplication or disclosure by the Government is subject to the restrictions set forth in this Agreement. +Warranties and Disclaimers +This publication and/or website may include technical or typographical errors or other inaccuracies. Changes are periodically added to the information herein; these changes will be incorporated in new editions of the publication and/or website. Unicode, Inc. may make improvements and/or changes in the product(s) and/or program(s) described in this publication and/or website at any time. +If this file has been purchased on magnetic or optical media from Unicode, Inc. the sole and exclusive remedy for any claim will be exchange of the defective media within ninety (90) days of original purchase. +EXCEPT AS PROVIDED IN SECTION E.2, THIS PUBLICATION AND/OR SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND EITHER EXPRESS, IMPLIED, OR STATUTORY, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT. UNICODE, INC. AND ITS LICENSORS ASSUME NO RESPONSIBILITY FOR ERRORS OR OMISSIONS IN THIS PUBLICATION AND/OR SOFTWARE OR OTHER DOCUMENTS WHICH ARE REFERENCED BY OR LINKED TO THIS PUBLICATION OR THE UNICODE WEBSITE. +Waiver of Damages +In no event shall Unicode, Inc. or its licensors be liable for any special, incidental, indirect or consequential damages of any kind, or any damages whatsoever, whether or not Unicode, Inc. was advised of the possibility of the damage, including, without limitation, those resulting from the following: loss of use, data or profits, in connection with the use, modification or distribution of this information or its derivatives. +Trademarks & Logos +The Unicode Word Mark and the Unicode Logo are trademarks of Unicode, Inc. “The Unicode Consortium” and “Unicode, Inc.” are trade names of Unicode, Inc. Use of the information and materials found on this website indicates your acknowledgement of Unicode, Inc.’s exclusive worldwide rights in the Unicode Word Mark, the Unicode Logo, and the Unicode trade names. +The Unicode Consortium Name and Trademark Usage Policy (“Trademark Policy”) are incorporated herein by reference and you agree to abide by the provisions of the Trademark Policy, which may be changed from time to time in the sole discretion of Unicode, Inc. +All third party trademarks referenced herein are the property of their respective owners. +Miscellaneous +Jurisdiction and Venue. This website is operated from a location in the State of California, United States of America. Unicode, Inc. makes no representation that the materials are appropriate for use in other locations. If you access this website from other locations, you are responsible for compliance with local laws. This Agreement, all use of this website and any claims and damages resulting from use of this website are governed solely by the laws of the State of California without regard to any principles which would apply the laws of a different jurisdiction. The user agrees that any disputes regarding this website shall be resolved solely in the courts located in Santa Clara County, California. The user agrees said courts have personal jurisdiction and agree to waive any right to transfer the dispute to any other forum. +Modification by Unicode, Inc. Unicode, Inc. shall have the right to modify this Agreement at any time by posting it to this website. The user may not assign any part of this Agreement without Unicode, Inc.’s prior written consent. +Taxes. The user agrees to pay any taxes arising from access to this website or use of the information herein, except for those based on Unicode’s net income. +Severability. If any provision of this Agreement is declared invalid or unenforceable, the remaining provisions of this Agreement shall remain in effect. +Entire Agreement. This Agreement constitutes the entire agreement between the parties. + +``` +```` + +### Texts found inside the jars (de-duplicated by content; bracket = sha256 prefix) + +#### antlr4-runtime-4.13.2.LICENSE.txt [3db1fb3ee79a] + +Applies to: org.antlr:antlr4-runtime:4.13.2 + +``` +Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +3. Neither name of copyright holders nor the names of its contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` + +#### highway-16f5f5c768064d0bb52952888a14236bec75c104.LICENSE.txt [b8b1fae5894c] + +Applies to: libjvector-x86_64.so (google/highway) + +``` +This project is primarily dual-licensed under your choice of either the Apache +License 2.0 or the BSD 3-Clause License. + +The following files are licensed under different terms: +* hwy/contrib/random/random-inl.h: CC0 1.0 Universal + +The full texts of all applicable licenses are included below, separated by +'---'. + +-------------------------------------------------------------------------------- +Apache License 2.0 +-------------------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- +BSD 3-Clause License +-------------------------------------------------------------------------------- + +Copyright (c) The Highway Project Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- +CC0 1.0 Universal +-------------------------------------------------------------------------------- + +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. +``` + +#### jts-core-1.20.0.LICENSE_EDLv1.txt [e67a91b5489c] + +Applies to: org.locationtech.jts:jts-core:1.20.0 + +``` +Eclipse Distribution License - v 1.0 + +Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + Neither the name of the Eclipse Foundation, Inc. nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` + +#### jvector-4.0.2.NOTICE.txt [3cf7944dd71f] + +Applies to: io.github.jbellis:jvector:4.0.2 + +``` +This product includes software developed for Apache Lucene by +The Apache Software Foundation (http://www.apache.org/). +``` + +#### LICENSE [62fb8a3a9621] + +Applies to: io.micrometer:micrometer-commons:1.17.1, io.micrometer:micrometer-core:1.17.1, io.micrometer:micrometer-observation:1.17.1 + +``` + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` + +#### LICENSE [aa4dc81ef2c6] + +Applies to: io.swagger.core.v3:swagger-annotations:2.2.55, io.swagger.core.v3:swagger-core:2.2.55, io.swagger.core.v3:swagger-models:2.2.55 + +``` + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (c) 2015. SmartBear Software Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` + +#### LICENSE.md [87eba02f8a41] + +Applies to: jakarta.activation:jakarta.activation-api:1.2.2 + +``` + + Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Eclipse Foundation, Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` + +#### LICENSE.md [dcbbdbb512a2] + +Applies to: jakarta.xml.bind:jakarta.xml.bind-api:2.3.3 + +``` + + Copyright (c) 2017, 2018 Oracle and/or its affiliates. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Eclipse Foundation, Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` + +#### LICENSE.txt [b1d2870f1a00] + +Applies to: org.apache.commons:commons-lang3:3.20.0 + +``` + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` + +#### LICENSE.txt [a2521407b320] + +Applies to: org.apache.lucene:lucene-analysis-common:10.5.2, org.apache.lucene:lucene-core:10.5.2, org.apache.lucene:lucene-facet:10.5.2, org.apache.lucene:lucene-queries:10.5.2, org.apache.lucene:lucene-queryparser:10.5.2, org.apache.lucene:lucene-sandbox:10.5.2, org.apache.lucene:lucene-spatial-extras:10.5.2, org.apache.lucene:lucene-spatial3d:10.5.2 + +``` + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from unicode conversion examples available at +http://www.unicode.org/Public/PROGRAMS/CVTUTF. Here is the copyright +from those sources: + +/* + * Copyright 2001-2004 Unicode, Inc. + * + * Disclaimer + * + * This source code is provided as is by Unicode, Inc. No claims are + * made as to fitness for any particular purpose. No warranties of any + * kind are expressed or implied. The recipient agrees to determine + * applicability of information provided. If this file has been + * purchased on magnetic or optical media from Unicode, Inc., the + * sole remedy for any claim will be exchange of defective media + * within 90 days of receipt. + * + * Limitations on Rights to Redistribute This Code + * + * Unicode, Inc. hereby grants the right to freely use the information + * supplied in this file in the creation of products supporting the + * Unicode Standard, and to make copies of this file in any form + * for internal or external distribution as long as this notice + * remains attached. + */ + +Some code in core/src/java/org/apache/lucene/util/ArrayUtil.java was +derived from Python 2.4.2 sources available at +http://www.python.org. Full license is here: + + http://www.python.org/download/releases/2.4.2/license/ + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from Python 3.1.2 sources available at +http://www.python.org. Full license is here: + + http://www.python.org/download/releases/3.1.2/license/ + +Some code in core/src/java/org/apache/lucene/util/automaton was +derived from Brics automaton sources available at +www.brics.dk/automaton/. Here is the copyright from those sources: + +/* + * Copyright (c) 2001-2009 Anders Moeller + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +The levenshtein automata tables in core/src/java/org/apache/lucene/util/automaton +were automatically generated with the moman/finenight FSA package. +Here is the copyright for those sources: + +# Copyright (c) 2010, Jean-Philippe Barrette-LaPierre, +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from ICU (http://www.icu-project.org) +The full license is available here: + https://github.com/unicode-org/icu/blob/main/icu4c/LICENSE + +/* + * Copyright (C) 1999-2010, International Business Machines + * Corporation and others. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * provided that the above copyright notice(s) and this permission notice appear + * in all copies of the Software and that both the above copyright notice(s) and + * this permission notice appear in supporting documentation. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE + * LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR + * ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER + * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT + * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall not + * be used in advertising or otherwise to promote the sale, use or other + * dealings in this Software without prior written authorization of the + * copyright holder. + */ + +The following license applies to the Snowball stemmers: + +Copyright (c) 2001, Dr Martin Porter +Copyright (c) 2002, Richard Boulton +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holders nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The following license applies to the KStemmer: + +Copyright © 2003, +Center for Intelligent Information Retrieval, +University of Massachusetts, Amherst. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. The names "Center for Intelligent Information Retrieval" and +"University of Massachusetts" must not be used to endorse or promote products +derived from this software without prior written permission. To obtain +permission, contact info@ciir.cs.umass.edu. + +THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF MASSACHUSETTS AND OTHER CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +The following license applies to the Morfologik project: + +Copyright (c) 2006 Dawid Weiss +Copyright (c) 2007-2011 Dawid Weiss, Marcin Miłkowski +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of Morfologik nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +The dictionary comes from Morfologik project. Morfologik uses data from +Polish ispell/myspell dictionary hosted at http://www.sjp.pl/slownik/en/ and +is licenced on the terms of (inter alia) LGPL and Creative Commons +ShareAlike. The part-of-speech tags were added in Morfologik project and +are not found in the data from sjp.pl. The tagset is similar to IPI PAN +tagset. + +--- + +The following license applies to the Morfeusz project, +used by org.apache.lucene.analysis.morfologik. + +BSD-licensed dictionary of Polish (SGJP) +http://sgjp.pl/morfeusz/ + +Copyright © 2011 Zygmunt Saloni, Włodzimierz Gruszczyński, +Marcin Woliński, Robert Wołosz + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDERS “AS IS” AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +core/src/java/org/apache/lucene/util/compress/LZ4.java is a Java +implementation of the LZ4 (https://github.com/lz4/lz4/tree/dev/lib) +compression format for Lucene's DataInput/DataOutput abstractions. + +LZ4 Library +Copyright (c) 2011-2016, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` + +#### LICENSE.txt [ba550515c77c] + +Applies to: org.hdrhistogram:HdrHistogram:2.2.2 + +``` +The code in this repository code was Written by Gil Tene, Michael Barker, +and Matt Warren, and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +For users of this code who wish to consume it under the "BSD" license +rather than under the public domain or CC0 contribution text mentioned +above, the code found under this directory is *also* provided under the +following license (commonly referred to as the BSD 2-Clause License). This +license does not detract from the above stated release of the code into +the public domain, and simply represents an additional license granted by +the Author. + +----------------------------------------------------------------------------- +** Beginning of "BSD 2-Clause License" text. ** + + Copyright (c) 2012, 2013, 2014, 2015, 2016 Gil Tene + Copyright (c) 2014 Michael Barker + Copyright (c) 2014 Matt Warren + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. +``` + +#### LICENSE.txt [0d542e0c8804] + +Applies to: org.jboss.threads:jboss-threads:3.10.1 + +``` + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS +``` + +#### LICENSE.txt [6632f550c207] + +Applies to: org.slf4j:slf4j-api:2.0.20, org.slf4j:slf4j-jdk14:2.0.20 + +``` +Copyright (c) 2004-2023 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +``` + +#### NOTICE [df207ca1a2ea] + +Applies to: io.micrometer:micrometer-commons:1.17.1, io.micrometer:micrometer-core:1.17.1, io.micrometer:micrometer-observation:1.17.1 + +``` +Micrometer + +Copyright (c) 2017-Present VMware, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +------------------------------------------------------------------------------- + +This product contains a modified portion of 'io.netty.util.internal.logging', +in the Netty/Common library distributed by The Netty Project: + + * Copyright 2013 The Netty Project + * License: Apache License v2.0 + * Homepage: https://netty.io + +This product contains a modified portion of 'StringUtils.isBlank()', +in the Commons Lang library distributed by The Apache Software Foundation: + + * Copyright 2001-2019 The Apache Software Foundation + * License: Apache License v2.0 + * Homepage: https://commons.apache.org/proper/commons-lang/ + +This product contains a modified portion of 'JsonUtf8Writer', +in the Moshi library distributed by Square, Inc: + + * Copyright 2010 Google Inc. + * License: Apache License v2.0 + * Homepage: https://github.com/square/moshi + +This product contains a modified portion of the 'org.springframework.lang' +package in the Spring Framework library, distributed by VMware, Inc: + + * Copyright 2002-2019 the original author or authors. + * License: Apache License v2.0 + * Homepage: https://spring.io/projects/spring-framework +``` + +#### NOTICE [f1fc76fa8bfb] + +Applies to: io.swagger.core.v3:swagger-annotations:2.2.55 + +``` +Swagger Core - swagger-annotations +Copyright (c) 2015. SmartBear Software Inc. +Swagger Core - swagger-annotations is licensed under Apache 2.0 license. +Copy of the Apache 2.0 license can be found in `LICENSE` file. +``` + +#### NOTICE [921b5b663bdc] + +Applies to: io.swagger.core.v3:swagger-core:2.2.55 + +``` +Swagger Core - swagger-core +Copyright (c) 2015. SmartBear Software Inc. +Swagger Core - swagger-core is licensed under Apache 2.0 license. +Copy of the Apache 2.0 license can be found in `LICENSE` file. +``` + +#### NOTICE [ca0e15ea1bb6] + +Applies to: io.swagger.core.v3:swagger-models:2.2.55 + +``` +Swagger Core - swagger-models +Copyright (c) 2015. SmartBear Software Inc. +Swagger Core - swagger-models is licensed under Apache 2.0 license. +Copy of the Apache 2.0 license can be found in `LICENSE` file. +``` + +#### NOTICE.md [6b9afdc018e1] + +Applies to: jakarta.activation:jakarta.activation-api:1.2.2 + +``` +# Notices for Jakarta Activation + +This content is produced and maintained by Jakarta Activation project. + +* Project home: https://projects.eclipse.org/projects/ee4j.jaf + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Distribution License v. 1.0, +which is available at http://www.eclipse.org/org/documents/edl-v10.php. + +SPDX-License-Identifier: BSD-3-Clause + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/jaf + +## Third-party Content + +This project leverages the following third party content. + +JUnit (4.12) + +* License: Eclipse Public License +``` + +#### NOTICE.md [190bb88e764b] + +Applies to: jakarta.xml.bind:jakarta.xml.bind-api:2.3.3 + +``` +[//]: # " Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. " +[//]: # " " +[//]: # " This program and the accompanying materials are made available under the " +[//]: # " terms of the Eclipse Distribution License v. 1.0, which is available at " +[//]: # " http://www.eclipse.org/org/documents/edl-v10.php. " +[//]: # " " +[//]: # " SPDX-License-Identifier: BSD-3-Clause " + +# Notices for Jakarta XML Binding + +This content is produced and maintained by the Jakarta XML Binding +project. + +* Project home: https://projects.eclipse.org/projects/ee4j.jaxb + +## Trademarks + +Jakarta XML Binding is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Distribution License v. 1.0 which is available at +http://www.eclipse.org/org/documents/edl-v10.php. + +SPDX-License-Identifier: BSD-3-Clause + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/jaxb-api +* https://github.com/eclipse-ee4j/jaxb-tck + +## Third-party Content + +This project leverages the following third party content. + +Apache River (3.0.0) + +* License: Apache-2.0 AND BSD-3-Clause + +ASM 7 (n/a) + +* License: BSD-3-Clause +* Project: https://asm.ow2.io/ +* Source: + https://repository.ow2.org/nexus/#nexus-search;gav~org.ow2.asm~asm-commons~~~~kw,versionexpand + +JTHarness (5.0) + +* License: (GPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0) +* Project: https://wiki.openjdk.java.net/display/CodeTools/JT+Harness +* Source: http://hg.openjdk.java.net/code-tools/jtharness/ + +normalize.css (3.0.2) + +* License: MIT + +SigTest (n/a) + +* License: GPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. +``` + +#### NOTICE.txt [64bc8696af3f] + +Applies to: org.apache.commons:commons-lang3:3.20.0 + +``` +Apache Commons Lang +Copyright 2001-2025 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). +``` + +#### NOTICE.txt [d3b82734d5e1] + +Applies to: org.apache.lucene:lucene-analysis-common:10.5.2, org.apache.lucene:lucene-core:10.5.2, org.apache.lucene:lucene-facet:10.5.2, org.apache.lucene:lucene-queries:10.5.2, org.apache.lucene:lucene-queryparser:10.5.2, org.apache.lucene:lucene-sandbox:10.5.2, org.apache.lucene:lucene-spatial-extras:10.5.2, org.apache.lucene:lucene-spatial3d:10.5.2 + +``` +Apache Lucene +Copyright 2001-2025 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Includes software from other Apache Software Foundation projects, +including, but not limited to: + - Apache Jakarta Regexp + - Apache Commons + - Apache Xerces + +ICU4J, (under analysis/icu) is licensed under an MIT styles license +and Copyright (c) 1995-2008 International Business Machines Corporation and others + +Some data files (under analysis/icu/src/data) are derived from Unicode data such +as the Unicode Character Database. See http://unicode.org/copyright.html for more +details. + +Brics Automaton (under core/src/java/org/apache/lucene/util/automaton) is +BSD-licensed, created by Anders Møller. See http://www.brics.dk/automaton/ + +The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were +automatically generated with the moman/finenight FSA library, created by +Jean-Philippe Barrette-LaPierre. This library is available under an MIT license, +see http://sites.google.com/site/rrettesite/moman and +http://bitbucket.org/jpbarrette/moman/overview/ + +The class org.apache.lucene.util.WeakIdentityMap was derived from +the Apache CXF project and is Apache License 2.0. + +The class org.apache.lucene.util.compress.LZ4 is a Java rewrite of the LZ4 +compression library (https://github.com/lz4/lz4/tree/dev/lib) that is licensed +under the 2-clause BSD license. +(https://opensource.org/licenses/bsd-license.php) + +The Google Code Prettify is Apache License 2.0. +See http://code.google.com/p/google-code-prettify/ + +This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin +g Package (jaspell): http://jaspell.sourceforge.net/ +License: The BSD License (http://www.opensource.org/licenses/bsd-license.php) + +The snowball stemmers in + analysis/common/src/java/net/sf/snowball +were developed by Martin Porter and Richard Boulton. +The snowball stopword lists in + analysis/common/src/resources/org/apache/lucene/analysis/snowball +were developed by Martin Porter and Richard Boulton. +The full snowball package is available from + https://snowballstem.org/ + +The KStem stemmer in + analysis/common/src/org/apache/lucene/analysis/en +was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst) +under the BSD-license. + +The Arabic,Persian,Romanian,Bulgarian, Hindi and Bengali analyzers (common) come with a default +stopword list that is BSD-licensed created by Jacques Savoy. These files reside in: +analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/bn/stopwords.txt +See http://members.unine.ch/jacques.savoy/clef/index.html. + +The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers +(common) are based on BSD-licensed reference implementations created by Jacques Savoy and +Ljiljana Dolamic. These files reside in: +analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java + +The Stempel analyzer (stempel) includes BSD-licensed software developed +by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil, +and Edmond Nolan. + +The Polish analyzer (stempel) comes with a default +stopword list that is BSD-licensed created by the Carrot2 project. The file resides +in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt. +See https://github.com/carrot2/carrot2. + +The SmartChineseAnalyzer source code (smartcn) was +provided by Xiaoping Gao and copyright 2009 by www.imdict.net. + +WordBreakTestUnicode_*.java (under modules/analysis/common/src/test/) +is derived from Unicode data such as the Unicode Character Database. +See http://unicode.org/copyright.html for more details. + +The Morfologik analyzer (morfologik) includes BSD-licensed software +developed by Dawid Weiss and Marcin Miłkowski +(https://github.com/morfologik/morfologik-stemming) and uses +data from the BSD-licensed dictionary of Polish (SGJP, http://sgjp.pl/morfeusz/). + +=========================================================================== +Kuromoji Japanese Morphological Analyzer - Apache Lucene Integration +=========================================================================== + +This software includes a binary and/or source version of data from + + mecab-ipadic-2.7.0-20070801 + +which can be obtained from + + http://atilika.com/releases/mecab-ipadic/mecab-ipadic-2.7.0-20070801.tar.gz + +or + + http://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz + +=========================================================================== +mecab-ipadic-2.7.0-20070801 Notice +=========================================================================== + +Nara Institute of Science and Technology (NAIST), +the copyright holders, disclaims all warranties with regard to this +software, including all implied warranties of merchantability and +fitness, in no event shall NAIST be liable for +any special, indirect or consequential damages or any damages +whatsoever resulting from loss of use, data or profits, whether in an +action of contract, negligence or other tortuous action, arising out +of or in connection with the use or performance of this software. + +A large portion of the dictionary entries +originate from ICOT Free Software. The following conditions for ICOT +Free Software applies to the current dictionary as well. + +Each User may also freely distribute the Program, whether in its +original form or modified, to any third party or parties, PROVIDED +that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear +on, or be attached to, the Program, which is distributed substantially +in the same form as set out herein and that such intended +distribution, if actually made, will neither violate or otherwise +contravene any of the laws and regulations of the countries having +jurisdiction over the User or the intended distribution itself. + +NO WARRANTY + +The program was produced on an experimental basis in the course of the +research and development conducted during the project and is provided +to users as so produced on an experimental basis. Accordingly, the +program is provided without any warranty whatsoever, whether express, +implied, statutory or otherwise. The term "warranty" used herein +includes, but is not limited to, any warranty of the quality, +performance, merchantability and fitness for a particular purpose of +the program and the nonexistence of any infringement or violation of +any right of any third party. + +Each user of the program will agree and understand, and be deemed to +have agreed and understood, that there is no warranty whatsoever for +the program and, accordingly, the entire risk arising from or +otherwise connected with the program is assumed by the user. + +Therefore, neither ICOT, the copyright holder, or any other +organization that participated in or was otherwise related to the +development of the program and their respective officials, directors, +officers and other employees shall be held liable for any and all +damages, including, without limitation, general, special, incidental +and consequential damages, arising out of or otherwise in connection +with the use or inability to use the program or any product, material +or result produced or otherwise obtained by using the program, +regardless of whether they have been advised of, or otherwise had +knowledge of, the possibility of such damages at any time during the +project or thereafter. Each user will be deemed to have agreed to the +foregoing by his or her commencement of use of the program. The term +"use" as used herein includes, but is not limited to, the use, +modification, copying and distribution of the program and the +production of secondary products from the program. + +In the case where the program, whether in its original form or +modified, was distributed or delivered to or received by a user from +any person, organization or entity other than ICOT, unless it makes or +grants independently of ICOT any specific warranty to the user in +writing, such person, organization or entity, will also be exempted +from and not be held liable to the user for any such damages as noted +above as far as the program is concerned. + +=========================================================================== +Nori Korean Morphological Analyzer - Apache Lucene Integration +=========================================================================== + +This software includes a binary and/or source version of data from + + mecab-ko-dic-2.1.1-20180720 + +which can be obtained from + + https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz +``` diff --git a/native/wheel/_jpype_stub.py b/native/wheel/_jpype_stub.py new file mode 100644 index 0000000..6cfd16c --- /dev/null +++ b/native/wheel/_jpype_stub.py @@ -0,0 +1,44 @@ +"""Stand-in for ``jpype`` inside the native `arcadedb` wheel (generated package only). + +The Python sources are shared with ``arcadedb-jvm`` and import ``jpype`` at module level. The native wheel has no JVM and does +not depend on JPype, so the build rewrites those imports to this module. Names resolve to placeholder classes: ``isinstance`` checks +against them are simply False, and any attempt to call, subscript or build one raises :class:`NotOnNativeYet`. ``isJVMStarted()`` is +always False. Code paths that need real JPype (``jvm_kwargs``, ``get_java_database``, Java-object APIs) are the JVM-only APIs listed in the README. +""" + +from arcadedb._native import NotOnNativeYet + + +class JException(Exception): + """Placeholder for ``jpype.JException`` (never raised here).""" + + +class _Placeholder(type): + def __call__(cls, *a, **k): + raise NotOnNativeYet( + f"{cls.__name__} needs a JVM; this is the native build (the arcadedb wheel); the JVM build is arcadedb-jvm" + ) + + def __getitem__(cls, item): + raise NotOnNativeYet( + f"{cls.__name__}[...] needs a JVM; this is the native backend" + ) + + def __getattr__(cls, name): + if name.startswith("__"): + raise AttributeError(name) + raise NotOnNativeYet( + f"{cls.__name__}.{name} needs a JVM; this is the native backend" + ) + + +def isJVMStarted() -> bool: + return False + + +def __getattr__(name: str): + if name.startswith("__"): + raise AttributeError(name) + ph = _Placeholder(name, (), {}) + globals()[name] = ph + return ph diff --git a/native/wheel/build_wheel.py b/native/wheel/build_wheel.py new file mode 100644 index 0000000..26f6f24 --- /dev/null +++ b/native/wheel/build_wheel.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Build the `arcadedb` wheel (the native build) from the shared Python sources and a built native image. + + python native/wheel/build_wheel.py --lib-dir native/target/lib --version 26.11.1.dev0 --engine 26.10.2 --out

+ +Steps: copy src/arcadedb to /stage/src/arcadedb (the shared tree is not duplicated in git; only the pip hints of the extras are renamed from arcadedb-jvm[...] to arcadedb[...]); +rewrite `import jpype` to the bundled stub; flip the default backend to native; add the library of the build platform (libarcadedb.so on Linux, +libarcadedb.dylib on macOS, libarcadedb.dll on Windows; on Linux x86-64 also libjvector-x86_64.so, jvector's Highway library: no other platform ships it); +add LICENSE/NOTICE/GraalVM licence texts; check THIRD_PARTY_NOTICES.md against gen_notices.py (fails on drift); compute the platform tag from the library +(Linux: manylinux__ from the GLIBC symbol versions; macOS: macosx__arm64 from the load commands; Windows: win_amd64); +build with `uv build --wheel`. Run it on the platform the library was built on. +""" + +import argparse +import platform +import re +import shutil +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent.parent # the repository root +PKG = REPO + + +LIB_NAMES = { + "linux": "libarcadedb.so", + "darwin": "libarcadedb.dylib", + "win32": "libarcadedb.dll", +} +JVECTOR = "libjvector-x86_64.so" # Highway, taken from the jvector jar by preview-build.sh on Linux x86-64 only + + +def host_arch() -> str: + m = platform.machine().lower() + return { + "x86_64": "x86_64", + "amd64": "x86_64", + "aarch64": "aarch64", + "arm64": "aarch64", + }.get(m, m) + + +def glibc_floor(lib: Path) -> str: + out = subprocess.run( + ["objdump", "-T", str(lib)], capture_output=True, text=True, check=True + ).stdout + vers = { + tuple(map(int, m.split("."))) + for m in re.findall(r"GLIBC_(\d+\.\d+(?:\.\d+)?)", out) + } + major, minor = max(v[:2] for v in vers) + return f"manylinux_{major}_{minor}_{host_arch()}" + + +def macos_tag(lib: Path) -> str: + """macosx__arm64 from LC_BUILD_VERSION (minos) or LC_VERSION_MIN_MACOSX (version) of the library.""" + out = subprocess.run( + ["otool", "-l", str(lib)], capture_output=True, text=True, check=True + ).stdout + m = re.search(r"minos\s+(\d+)\.(\d+)", out) or re.search( + r"LC_VERSION_MIN_MACOSX.*?\n\s+cmdsize \d+\n\s+version (\d+)\.(\d+)", out, re.S + ) + if not m: + sys.exit(f"no minimum macOS version in the load commands of {lib}:\n{out}") + print(f"load command minimum macOS version: {m.group(1)}.{m.group(2)}") + arch = subprocess.run( + ["lipo", "-archs", str(lib)], capture_output=True, text=True, check=True + ).stdout.split() + if arch != ["arm64"]: + sys.exit(f"{lib} has architectures {arch}, expected only arm64") + return f"macosx_{m.group(1)}_{m.group(2)}_arm64" + + +def platform_tag(lib: Path) -> str: + if sys.platform.startswith("linux"): + return glibc_floor(lib) + if sys.platform == "darwin": + return macos_tag(lib) + if sys.platform == "win32": + if host_arch() != "x86_64": + sys.exit( + f"Windows wheels are win_amd64 only, this machine is {platform.machine()}" + ) + return "win_amd64" + sys.exit(f"unsupported platform {sys.platform}") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( + "--lib-dir", + required=True, + help="directory with the library of this platform (libarcadedb.so / .dylib / .dll) and, on Linux x86-64, libjvector-x86_64.so", + ) + ap.add_argument( + "--version", + default="26.11.1.dev0", + help="PEP 440: A.B.C.devN -> A.B.C -> A.B.C.postN (no aN/bN/rcN)", + ) + ap.add_argument( + "--engine", + required=True, + help="engine release the image was built from, e.g. 26.10.2", + ) + ap.add_argument("--out", required=True) + ap.add_argument( + "--graalvm-home", + default=str( + Path.home() + / "build-scratch/graalvm-dl-25.4.4.1.1/graalvm-community-25.4.4.1.1+1.1" + ), + ) + ap.add_argument( + "--march", + default="compatibility", + help="-march the image was built with (native/scripts/build-image.sh MARCH)", + ) + ap.add_argument( + "--m2-repo", + default="", + help="Maven repository the image classpath jars are read from (default ~/.m2/repository; the CI container build uses another)", + ) + ap.add_argument( + "--notices-out", + default="", + help="also write the generated notices here (CI keeps it in the build record)", + ) + ap.add_argument( + "--skip-notices-check", + action="store_true", + help="do not compare THIRD_PARTY_NOTICES.md with the gen_notices.py output (needs /build.log)", + ) + ap.add_argument("--plat", default="", help="override the computed platform tag") + a = ap.parse_args() + lib_dir, out = Path(a.lib_dir), Path(a.out).resolve() + libname = LIB_NAMES.get(sys.platform) or sys.exit( + f"unsupported platform {sys.platform}" + ) + if not a.skip_notices_check: + # THIRD_PARTY_NOTICES.md is generated from the classpath of the image build (build-image.sh writes /build.log): + # refuse to build a wheel whose notices differ from the library's real contents. Licence-review items are printed, not fatal. + chk = subprocess.run( + [ + sys.executable, + "-I", + str(HERE / "gen_notices.py"), + "--build-log", + str(lib_dir / "build.log"), + "--engine", + a.engine, + "--graalvm-home", + a.graalvm_home, + "--check", + str(HERE / "THIRD_PARTY_NOTICES.md"), + ] + + (["--m2-repo", a.m2_repo] if a.m2_repo else []) + + (["--out", a.notices_out] if a.notices_out else []) + ) + if chk.returncode: + sys.exit( + "THIRD_PARTY_NOTICES.md is out of date: python native/wheel/gen_notices.py --build-log /build.log " + f"--engine {a.engine} --graalvm-home {a.graalvm_home} --out native/wheel/THIRD_PARTY_NOTICES.md" + ) + stage = out / "stage" + shutil.rmtree(stage, ignore_errors=True) + dst = stage / "src" / "arcadedb" + shutil.copytree( + PKG / "src" / "arcadedb", + dst, + ignore=shutil.ignore_patterns( + "__pycache__", "*.pyc", "_version.py", "jars", "jre" + ), + ) + + jp = re.compile( + r"^(\s*)(?:import jpype(?:\.imports)?|import jpype as _jpype|import jpype\.types as jtypes|from jpype import JException)\s*$", + re.M, + ) + left = [] + for f in dst.rglob("*.py"): + t = f.read_text(encoding="utf-8") + t = t.replace( + "'arcadedb-jvm[", "'arcadedb[" + ) # the extras named in install hints + + def sub(m): + line = m.group(0).strip() + ind = m.group(1) + if line == "import jpype.imports": + return f"{ind}pass # jpype.imports: no JVM in this wheel" + tail = { + "import jpype as _jpype": "_jpype", + "import jpype.types as jtypes": "jtypes", + }.get(line, "jpype") + if line == "from jpype import JException": + return f"{ind}from arcadedb._jpype_stub import JException" + return f"{ind}from arcadedb import _jpype_stub as {tail}" + + t = jp.sub(sub, t) + if re.search(r"^\s*(import|from)\s+jpype", t, re.M): + left.append(f.name) + f.write_text(t, encoding="utf-8") + if left: + sys.exit(f"jpype imports left in: {left}") + init = dst / "_native" / "__init__.py" + t = init.read_text(encoding="utf-8") + assert t.count('_DEFAULT_BACKEND = "jpype"') == 1, "default-backend line not found" + init.write_text( + t.replace('_DEFAULT_BACKEND = "jpype"', '_DEFAULT_BACKEND = "native"'), + encoding="utf-8", + ) + shutil.copy(HERE / "_jpype_stub.py", dst / "_jpype_stub.py") + + commit = subprocess.run( + ["git", "-C", str(REPO), "rev-parse", "HEAD"], capture_output=True, text=True + ).stdout.strip() + (dst / "_build_info.py").write_text( + f'"""Written by native-wheel/build_wheel.py."""\n' + f'ENGINE_VERSION = "{a.engine}" # the ArcadeDB engine release compiled into {libname}\n' + f'NATIVE_IMAGE = "GraalVM CE 25.4.4.1.1, -O2, -march={a.march}"\n' + f'NATIVE_MARCH = "{a.march}" # lib.py checks the CPU only for an x86-64-v3 image\n' + f'SOURCE_COMMIT = "{commit}" # ArcadeData/arcadedb-python commit of the bindings and the Java glue\n', + encoding="utf-8", + ) + ip = dst / "__init__.py" + t = ip.read_text(encoding="utf-8") + t = t.replace( + "# Import async execution\n", + "from ._build_info import ENGINE_VERSION, NATIVE_IMAGE, NATIVE_MARCH, SOURCE_COMMIT # noqa: F401\n\n# Import async execution\n", + 1, + ) + t = t.replace( + '__all__ = [\n "__version__",', + '__all__ = [\n "__version__",\n "ENGINE_VERSION",\n "NATIVE_IMAGE",\n "SOURCE_COMMIT",', + 1, + ) + ip.write_text(t, encoding="utf-8") + + ship = [libname] + if sys.platform.startswith("linux") and host_arch() == "x86_64": + ship.append( + JVECTOR + ) # the only platform that ships Highway; lib.py does not need it elsewhere + for n in ship: + shutil.copy( + lib_dir / n, dst / "_native" / n + ) # lib.py looks next to itself (_native/) + lic = out / "licenses-tmp" + shutil.rmtree(lic, ignore_errors=True) + lic.mkdir() + g = Path(a.graalvm_home) + for src, name in ( + (g / "LICENSE.txt", "GraalVM-CE-LICENSE.txt"), + (g / "LICENSE_NATIVEIMAGE.txt", "GraalVM-native-image-LICENSE.txt"), + (g / "THIRD_PARTY_LICENSE.txt", "GraalVM-THIRD_PARTY_LICENSE.txt"), + ): + shutil.copy(src, lic / name) + shutil.copy(REPO / "LICENSE", lic / "Apache-2.0.txt") + shutil.copy(REPO / "LICENSE", stage / "LICENSE") + for n in ("NOTICE", "THIRD_PARTY_NOTICES.md", "README.md"): + shutil.copy(HERE / n, stage / n) + shutil.move( + str(lic), str(stage / "licenses") + ) # dist-info licenses only: one copy in the wheel + (stage / "pyproject.toml").write_text( + (HERE / "pyproject.toml") + .read_text(encoding="utf-8") + .replace("@VERSION@", a.version), + encoding="utf-8", + ) + shutil.copy(HERE / "setup.py", stage / "setup.py") + + plat = a.plat or platform_tag(lib_dir / libname) + print("platform tag:", plat) + import os + + env = dict(os.environ, ARCADEDB_NATIVE_PLAT=plat) + dist = out / "dist" + dist.mkdir(exist_ok=True) + subprocess.run( + ["uv", "build", "--wheel", "--out-dir", str(dist), str(stage)], + check=True, + env=env, + ) + for w in sorted(dist.glob("*.whl")): + print(w, w.stat().st_size) + + +if __name__ == "__main__": + main() diff --git a/native/wheel/gen_notices.py b/native/wheel/gen_notices.py new file mode 100644 index 0000000..a3f8a18 --- /dev/null +++ b/native/wheel/gen_notices.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +"""Generate THIRD_PARTY_NOTICES.md for the native `arcadedb` wheel from what is compiled into libarcadedb.so. + + python gen_notices.py --build-log native/target/lib/build.log --engine 26.10.2 \ + --graalvm-home --out THIRD_PARTY_NOTICES.md + python gen_notices.py ... --check THIRD_PARTY_NOTICES.md # exit 1 on drift (build_wheel.py does this) + +Inputs, all read from disk, nothing typed by hand except the policy tables below: + 1. the `native-image -cp ...` line that native/scripts/build-image.sh logs in build.log (the exact jar set of the image); + 2. each jar: licence names from its Maven pom (parent poms followed), verbatim LICENSE/NOTICE files from the jar itself; + 3. the GraalVM CE home: release file, LICENSE.txt (GPLv2 + Classpath Exception + UPL), legal/java.base notices; + 4. the ArcadeDB NOTICE file: META-INF/NOTICE* of the arcadedb-engine jar on the classpath if it has one (what ships); otherwise the + NOTICE at the engine release tag (`git show`, needs the tag fetched); otherwise the NOTICE of the checked-out repository. + The heading in the output does not name the source (the three are the same text for a release); stderr does. +Paths: the classpath of a container build names jars as /root/.m2/repository/...; they are read from --m2-repo (default +~/.m2/repository) by their path below `.m2/repository/`, and no absolute path reaches the output. +notice-texts/ holds licence texts that are in no jar: the ANTLR and JTS BSD texts (their poms only link them) and, for libjvector, +the Highway LICENSE and jvector NOTICE.txt at jvector tag 4.0.2. + +NO STUDIO: arcadedb-studio is not on the image classpath (its web assets bundle ApexCharts >= 5.1.0, a Community/Commercial licence +that is not compatible with Apache-2.0). This script refuses a classpath that contains it. + +Licence policy: the wheel ships under Apache-2.0. ALLOWED are permissive licences; anything else is listed under "Licence review" +and the script exits 2 with --strict. It never decides such a case; the maintainer does. +""" + +import argparse +import difflib +import hashlib +import os +import re +import subprocess +import sys +import xml.etree.ElementTree as ET +import zipfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent.parent +M2 = Path.home() / ".m2" / "repository" # replaced by --m2-repo in main() +M2_MARK = ".m2/repository/" +TEXTS = HERE / "notice-texts" + +ALLOWED = { + "Apache-2.0", + "MIT", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "0BSD", + "Zlib", + "UPL-1.0", + "CC0-1.0", + "BlueOak-1.0.0", +} +# pom -> SPDX. Several entries in one pom are alternatives (Maven convention), not a conjunction. +POM_NAMES = [ + (r"apache", "Apache-2.0"), + (r"^mit$", "MIT"), + (r"bsd-?2", "BSD-2-Clause"), + (r"bsd-?3", "BSD-3-Clause"), + (r"universal permissive", "UPL-1.0"), + (r"eclipse distribution|^edl", "BSD-3-Clause"), # EDL-1.0 is the BSD-3-Clause text + (r"eclipse public license.*2", "EPL-2.0"), + (r"cc0|public domain", "CC0-1.0"), +] + +HIGHWAY_COMMIT = "16f5f5c768064d0bb52952888a14236bec75c104" # jvector 4.0.2: jvector-native/src/main/native/third_party/highway + + +def sha(b): + return hashlib.sha256(b).hexdigest()[:12] + + +def text(b): + # trailing whitespace is stripped per line (the repository's pre-commit hook does the same to the committed copy) + s = b.decode("utf-8", "replace").replace("\r\n", "\n").replace("\r", "\n") + return "\n".join(l.rstrip() for l in s.split("\n")).rstrip() + "\n" + + +def parse_classpath(build_log): + for line in ( + Path(build_log).read_text(encoding="utf-8", errors="replace").splitlines() + ): + m = re.search(r"native-image -cp (\S+) --shared", line) + if m: + # ':' on Linux/macOS, ';' with C:\... paths on Windows (the repository part is matched below as ".m2/repository/") + cp = m.group(1).strip('"') # an @args file quotes the classpath + return [j.replace("\\", "/") for j in cp.split(";" if ";" in cp else ":")] + sys.exit(f"gen_notices: no 'native-image -cp ... --shared' line in {build_log}") + + +def pom_licenses(pom, depth=0): + if not pom.exists(): + return None + root = ET.parse(pom).getroot() + ns = {"m": root.tag.split("}")[0][1:]} + names = [ + l.findtext("m:name", namespaces=ns) + for l in root.findall("m:licenses/m:license", ns) + ] + if names: + return names + p = root.find("m:parent", ns) + if p is not None and depth < 8: + g, a, v = ( + p.findtext(f"m:{x}", namespaces=ns) + for x in ("groupId", "artifactId", "version") + ) + return pom_licenses( + M2 / g.replace(".", "/") / a / v / f"{a}-{v}.pom", depth + 1 + ) + return None + + +def spdx_from_pom(name): + for pat, spdx in POM_NAMES: + if re.search(pat, name.strip().lower()): + return spdx + return None + + +def choose(spdx_options): + """Alternatives: prefer an ALLOWED one (first in the order the project lists them is not meaningful, so pick by policy order).""" + for o in spdx_options: + if o in ALLOWED: + return o + return None + + +def jar_info(jar): + # The classpath may name the repository as /root/.m2/repository (container build) while it is read from M2 here: map by the part + # below ".m2/repository/", never by the absolute prefix. + i = jar.find(M2_MARK) + if i < 0: + return None # the glue jar of this repository (Apache-2.0, native/) + rel = Path(jar[i + len(M2_MARK) :]) + p = M2 / rel + if not p.exists(): + sys.exit( + f"gen_notices: {jar} is on the classpath but {p} does not exist (--m2-repo?)" + ) + ver = rel.parts[-2] + art = rel.parts[-3] + grp = ".".join(rel.parts[:-3]) + names = pom_licenses(p.with_suffix(".pom")) + opts = [spdx_from_pom(n) or f"UNKNOWN({n})" for n in (names or [])] + chosen = choose(opts) + files = {} + with zipfile.ZipFile(p) as z: + for n in z.namelist(): + if re.search( + r"(^|/)META-INF/(LICEN[CS]E|NOTICE|COPYING)[^/]*$", n, re.I + ) and not n.endswith("/"): + files[n] = z.read(n) + return dict( + gav=f"{grp}:{art}:{ver}", + grp=grp, + art=art, + ver=ver, + pom_names=names, + opts=opts, + chosen=chosen, + files=files, + ) + + +def engine_notice(engine, jars): + """(label, bytes) of the ArcadeDB NOTICE: engine jar first, then the release tag, then this checkout's NOTICE.""" + for j in jars: + if j["grp"] == "com.arcadedb" and j["art"] == "arcadedb-engine": + for n, b in sorted(j["files"].items()): + if re.search(r"(^|/)META-INF/NOTICE[^/]*$", n, re.I): + return f"{j['gav']}!/{n}", b + r = subprocess.run( + ["git", "-C", str(REPO), "show", f"{engine}:NOTICE"], capture_output=True + ) + if r.returncode == 0: + return f"tag {engine}", r.stdout + # This repository has no ArcadeDB tag and no NOTICE of its own: a copy of the engine's NOTICE (from the tag of the release the + # image links) is kept next to this script. Refresh it with `git show :NOTICE` when the engine release changes. + kept = HERE / "ARCADEDB_ENGINE_NOTICE" + if kept.exists(): + return "copy kept in native/wheel", kept.read_bytes() + if (REPO / "NOTICE").exists(): + return "repository checkout (tag not fetched)", (REPO / "NOTICE").read_bytes() + sys.exit( + f"gen_notices: no NOTICE in the engine jar, at tag {engine} or in the checkout" + ) + + +def fence(body): + n = max([3] + [len(m) + 1 for m in re.findall(r"`{3,}", body)]) + return "`" * n + "\n" + body + "`" * n + "\n" + + +def main(): + sys.stderr.reconfigure(encoding="utf-8", errors="replace") + global M2 + ap = argparse.ArgumentParser() + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument( + "--build-log", + help="build.log of build-image.sh (holds the native-image classpath)", + ) + g.add_argument("--lib-dir", help="shortcut: /build.log") + ap.add_argument("--engine", required=True, help="engine release tag, e.g. 26.10.2") + ap.add_argument("--graalvm-home", required=True) + ap.add_argument( + "--m2-repo", + default=str(M2), + help="Maven repository the classpath jars are read from (a container build logs /root/.m2/repository)", + ) + ap.add_argument("--out") + ap.add_argument("--check", help="compare with this file; exit 1 when it differs") + ap.add_argument( + "--strict", + action="store_true", + help="exit 2 when the licence review list is not empty", + ) + a = ap.parse_args() + M2 = Path(a.m2_repo) + log = a.build_log or str(Path(a.lib_dir) / "build.log") + cp = parse_classpath(log) + if any("arcadedb-studio" in c for c in cp): + sys.exit( + "gen_notices: arcadedb-studio is on the classpath; its ApexCharts assets are not Apache-2.0 compatible (see module docstring)" + ) + G = Path(a.graalvm_home) + rel = dict( + re.findall( + r'^(\w+)="?(.*?)"?$', (G / "release").read_text(encoding="utf-8"), re.M + ) + ) + + jars = [j for j in (jar_info(c) for c in cp) if j] + review = [] # (what, why) + out = [] + w = out.append + + w("# Third-party notices (native image, preview)\n") + w( + f"Generated by `native-wheel/gen_notices.py` from the classpath that built `libarcadedb.so` (engine {a.engine}, " + f"GraalVM CE {rel.get('GRAALVM_VERSION')}). Do not edit by hand: `build_wheel.py` fails when this file differs from the " + "generator output. Licence names come from each artifact's own Maven pom; notice texts are copied " + "verbatim from the jars, the engine tag and the GraalVM home. A text of the Apache License 2.0 is in " + "`licenses/Apache-2.0.txt`.\n" + ) + + # ---- Java ---- + w("## Java libraries compiled into libarcadedb.so\n") + w( + f"{len(jars)} Maven artifacts on the native-image classpath (plus the glue jar of this repository, Apache-2.0).\n" + ) + w("| Artifact | Declared in pom | Applied | Verbatim texts |") + w("|---|---|---|---|") + appendix_java = {} # sha -> (name, text, [gavs]) + for j in jars: + decl = " / ".join(j["pom_names"] or ["none"]) + if j["chosen"] is None: + review.append((j["gav"], f"licence not in the allowed set: {decl}")) + refs = [] + for n, b in sorted(j["files"].items()): + k = sha(b) + appendix_java.setdefault(k, [n.split("/")[-1], text(b), []])[2].append( + j["gav"] + ) + refs.append(f"`{n.split('/')[-1]}` [{k}]") + extra = { + "org.antlr:antlr4-runtime": "antlr4-runtime-4.13.2.LICENSE.txt", + "org.locationtech.jts:jts-core": "jts-core-1.20.0.LICENSE_EDLv1.txt", + }.get(f"{j['grp']}:{j['art']}") + if extra: + b = (TEXTS / "java" / extra).read_bytes() + k = sha(b) + appendix_java.setdefault(k, [extra, text(b), []])[2].append(j["gav"]) + refs.append(f"`{extra}` [{k}]") + w( + f"| {j['gav']} | {decl} | {j['chosen'] or 'REVIEW'} | {', '.join(refs) or 'none in jar'} |" + ) + w("") + w( + "Where a pom lists several licences they are alternatives (JTS: EPL-2.0 or EDL-1.0; HdrHistogram: CC0-1.0 or BSD-2-Clause); " + "the permissive one is applied. EDL-1.0 is the BSD-3-Clause text.\n" + ) + + # ---- native code ---- + w("## Native code shipped next to the library\n") + w( + "`libjvector-x86_64.so` is taken unmodified from `io.github.jbellis:jvector:4.0.2` (Apache-2.0, pom). Its sources are in the " + "jvector repository at tag `4.0.2` (`jvector-native/src/main/native`): jvector's own C++ kernels (Apache-2.0) and the Google Highway " + f"library, pinned there as a git submodule at commit `{HIGHWAY_COMMIT}`. Highway's `LICENSE` at that commit says: " + "dual-licensed under the Apache License 2.0 or the BSD 3-Clause License, except `hwy/contrib/random/random-inl.h` (CC0 1.0). " + "Applied here: Apache-2.0. The `LICENSE` of Highway and the `NOTICE.txt` of jvector are reproduced verbatim below. " + "The library links `libstdc++.so.6`, `libm.so.6`, `libgcc_s.so.1` and `libc.so.6` dynamically (not shipped).\n" + ) + for f, who in ( + ( + f"highway-{HIGHWAY_COMMIT}.LICENSE.txt", + "libjvector-x86_64.so (google/highway)", + ), + ("jvector-4.0.2.NOTICE.txt", "io.github.jbellis:jvector:4.0.2"), + ): + b_ = (TEXTS / "java" / f).read_bytes() + appendix_java.setdefault(sha(b_), [f, text(b_), []])[2].append(who) + + # ---- GraalVM / JDK ---- + w("## GraalVM Community Edition runtime and JDK classes in libarcadedb.so\n") + w( + f"`libarcadedb.so` contains the Substrate VM runtime and the parts of the OpenJDK class library the engine reaches, built with " + f"GraalVM CE {rel.get('GRAALVM_VERSION')} (Java {rel.get('JAVA_VERSION')}, implementor {rel.get('IMPLEMENTOR')}). Licence: GNU General Public License " + "version 2 with the Classpath Exception (verbatim below, from `LICENSE.txt` of the GraalVM CE home; also in `licenses/`). The exception " + "permits linking and distributing these parts with code under other licences. `graal-sdk`, `polyglot`, `collections` (25.0.2, Maven Central) " + f"and `word` ({rel.get('GRAALVM_VERSION')}) are UPL-1.0. Oracle GraalVM (GFTC) is not used. Sources: https://github.com/oracle/graal (tag vm-{rel.get('GRAALVM_VERSION')}), " + "https://github.com/openjdk/jdk (jdk-25 line).\n" + ) + w( + "Third-party data inside the JDK classes that the image contains (texts from `legal/java.base` of the GraalVM home, verbatim below): " + "ICU normalisation data (`jdk/internal/icu/impl/data/icudt76b/nfc.nrm`, registered resource), Unicode data files, CLDR locale data. " + "Other `legal/java.base` notices (aes.md, c-libutl.md, gcc.md, siphash.md, public_suffix.md) concern files or HotSpot code that are not " + "linked into the image (checked: the library contains no public-suffix data); `jdk.incubator.vector` (sleef.md) is added to the module " + "path but its vector intrinsics are disabled in the build (`-H:-VectorAPISupport`).\n" + ) + review.append( + ( + "libarcadedb.so JDK module set", + "which JDK modules end up linked is not recorded by the build; the notices above cover java.base/java.logging/jdk.unsupported by module inspection of the legal directory only. A native-image `-H:+DashboardDump` or `--emit build-report` run would confirm it", + ) + ) + w( + "System libraries linked dynamically and not shipped: `libz.so.1`, `libc.so.6`, `ld-linux-x86-64.so.2` (checked with `ldd`).\n" + ) + + # ---- review ---- + w("## Licence review\n") + w( + "Dependencies whose licence is not in the allowed permissive set or whose provenance is not recorded in the build inputs. " + "The generator lists them; a maintainer decides.\n" + ) + for what, why in review: + w(f"- **{what}**: {why}") + w("") + + # ---- appendix ---- + w("## Verbatim texts\n") + src, nb = engine_notice(a.engine, jars) + print(f"gen_notices: ArcadeDB NOTICE read from {src}", file=sys.stderr) + w("### ArcadeDB NOTICE (engine release %s)\n" % a.engine) + w(fence(text(nb))) + w( + "### GraalVM Community Edition LICENSE.txt (GPLv2, Classpath Exception, UPL-1.0, BSD-3-Clause)\n" + ) + w(fence(text((G / "LICENSE.txt").read_bytes()))) + for f in ( + "ADDITIONAL_LICENSE_INFO", + "ASSEMBLY_EXCEPTION", + "icu.md", + "unicode.md", + "cldr.md", + ): + w(f"### OpenJDK java.base legal notice: {f}\n") + w(fence(text((G / "legal" / "java.base" / f).read_bytes()))) + w( + "### Texts found inside the jars (de-duplicated by content; bracket = sha256 prefix)\n" + ) + for k, (n, t, who) in sorted( + appendix_java.items(), key=lambda kv: (kv[1][0].lower(), kv[1][2]) + ): + w(f"#### {n} [{k}]\n") + w("Applies to: " + ", ".join(sorted(who)) + "\n") + w(fence(t)) + body = "\n".join(out) + body = re.sub(r"\n{3,}", "\n\n", body).rstrip() + "\n" + if a.out: + Path(a.out).write_text(body, encoding="utf-8", newline="\n") + status = 0 + if a.check: + have = Path(a.check).read_text(encoding="utf-8") + if have != body: + print( + f"gen_notices: {a.check} differs from the generated notices; run gen_notices.py --out {a.check}", + file=sys.stderr, + ) + diff = list( + difflib.unified_diff( + have.splitlines(), + body.splitlines(), + "committed", + "generated", + lineterm="", + ) + ) + for l in diff[:80]: + print(l[:300], file=sys.stderr) + if len(diff) > 80: + print(f"... {len(diff) - 80} more diff lines", file=sys.stderr) + status = 1 + for what, why in review: + print(f"licence review: {what}: {why}", file=sys.stderr) + if a.strict and review and status == 0: + status = 2 + return status + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/native/wheel/notice-texts/java/antlr4-runtime-4.13.2.LICENSE.txt b/native/wheel/notice-texts/java/antlr4-runtime-4.13.2.LICENSE.txt new file mode 100644 index 0000000..5d27694 --- /dev/null +++ b/native/wheel/notice-texts/java/antlr4-runtime-4.13.2.LICENSE.txt @@ -0,0 +1,28 @@ +Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +3. Neither name of copyright holders nor the names of its contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/native/wheel/notice-texts/java/highway-16f5f5c768064d0bb52952888a14236bec75c104.LICENSE.txt b/native/wheel/notice-texts/java/highway-16f5f5c768064d0bb52952888a14236bec75c104.LICENSE.txt new file mode 100644 index 0000000..d5cff9c --- /dev/null +++ b/native/wheel/notice-texts/java/highway-16f5f5c768064d0bb52952888a14236bec75c104.LICENSE.txt @@ -0,0 +1,371 @@ +This project is primarily dual-licensed under your choice of either the Apache +License 2.0 or the BSD 3-Clause License. + +The following files are licensed under different terms: +* hwy/contrib/random/random-inl.h: CC0 1.0 Universal + +The full texts of all applicable licenses are included below, separated by +'---'. + +-------------------------------------------------------------------------------- +Apache License 2.0 +-------------------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- +BSD 3-Clause License +-------------------------------------------------------------------------------- + +Copyright (c) The Highway Project Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- +CC0 1.0 Universal +-------------------------------------------------------------------------------- + +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/native/wheel/notice-texts/java/jts-core-1.20.0.LICENSE_EDLv1.txt b/native/wheel/notice-texts/java/jts-core-1.20.0.LICENSE_EDLv1.txt new file mode 100644 index 0000000..52646c2 --- /dev/null +++ b/native/wheel/notice-texts/java/jts-core-1.20.0.LICENSE_EDLv1.txt @@ -0,0 +1,30 @@ +Eclipse Distribution License - v 1.0 + +Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + Neither the name of the Eclipse Foundation, Inc. nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/native/wheel/notice-texts/java/jvector-4.0.2.NOTICE.txt b/native/wheel/notice-texts/java/jvector-4.0.2.NOTICE.txt new file mode 100644 index 0000000..8050191 --- /dev/null +++ b/native/wheel/notice-texts/java/jvector-4.0.2.NOTICE.txt @@ -0,0 +1,2 @@ +This product includes software developed for Apache Lucene by +The Apache Software Foundation (http://www.apache.org/). diff --git a/native/wheel/parity_wheel.py b/native/wheel/parity_wheel.py new file mode 100644 index 0000000..6a85347 --- /dev/null +++ b/native/wheel/parity_wheel.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""parity_wheel.py: run the test suite of this repository against the INSTALLED `arcadedb` (native) wheel, file by file, and count outcomes per file. + + python native/wheel/parity_wheel.py --repo --python --out parity-native.tsv + +A run per test file in its own process (a hang or crash costs one file), junit XML per file, then one table: +passed / failed / error / NOTYET (the test needs a Java-object API, reported by tests/conftest.py as a skip whose reason starts +with NOTYET) / other skips / TIMEOUT-or-crash. Files in tests/conftest.py's NATIVE_NOT_APPLICABLE are not collected (JVM-only). +""" + +import argparse +import glob +import os +import shutil +import subprocess +import sys +import xml.etree.ElementTree as ET + +p = argparse.ArgumentParser() +p.add_argument("--repo", required=True) +p.add_argument("--python", default=sys.executable) +p.add_argument("--out", required=True) +p.add_argument("--timeout", type=int, default=900) +p.add_argument( + "--cores", + default="0-5,8-11", + help="taskset core list (Linux only; ignored where taskset does not exist)", +) +p.add_argument("--files", default="") +p.add_argument( + "--max-notyet", + type=int, + default=None, + help="fail (exit 1) when more tests than this are NOTYET skips; a final release uses 0", +) +a = p.parse_args() + +pkg = os.path.abspath(a.repo) +files = sorted(glob.glob(os.path.join(pkg, "tests", "test_*.py"))) +import ast + +na_files, win_files = {}, {} +for node in ast.parse(open(os.path.join(pkg, "tests", "conftest.py")).read()).body: + if isinstance(node, ast.Assign): + name = getattr(node.targets[0], "id", "") + if name == "NATIVE_NOT_APPLICABLE": + na_files = ast.literal_eval(node.value) + elif name == "WINDOWS_NOT_APPLICABLE": + win_files = ast.literal_eval(node.value) +files = [f for f in files if os.path.basename(f) not in na_files] +if ( + sys.platform == "win32" +): # conftest.py's collect_ignore does not apply to a file named on the command line + files = [f for f in files if os.path.basename(f) not in win_files] +if a.files: + want = set(a.files.split(",")) + files = [f for f in files if os.path.basename(f) in want] +env = dict(os.environ, PYTHONUNBUFFERED="1") +env.pop("PYTHONPATH", None) # the wheel, not a source tree, must be what is imported +env.pop("ARCADEDB_BACKEND", None) # the backend comes from the installed distribution +env.pop("ARCADEDB_NATIVE_LIB", None) # the library comes from the installed wheel +chk = subprocess.run( + [ + a.python, + "-I", + "-c", + "import arcadedb as n; print(n.__file__, n.__version__, n.__backend__, n.ENGINE_VERSION, n.SOURCE_COMMIT); " + 'assert n.__backend__ == "native"; assert "site-packages" in n.__file__', + ], + env=env, + cwd="/", + capture_output=True, + text=True, +) +print("WHEEL CHECK:", chk.stdout.strip(), chk.stderr.strip()[-300:], flush=True) +assert chk.returncode == 0, "tests would not run against the installed wheel" +a.out = os.path.abspath(a.out) +xmldir = os.path.splitext(a.out)[0] + "-xml" +os.makedirs(xmldir, exist_ok=True) +rows, tot = [], dict.fromkeys( + ["passed", "failed", "error", "notyet", "na_jvm", "skipped", "timeout"], 0 +) +for f in files: + name = os.path.basename(f) + xml = os.path.join(xmldir, name + ".xml") + cmd = [ + *( + ["taskset", "-c", a.cores] + if sys.platform == "linux" and shutil.which("taskset") + else [] + ), + *( + ["nice", "-n", "10"] + if sys.platform != "win32" and shutil.which("nice") + else [] + ), + a.python, + "-m", + "pytest", + f"tests/{name}", + "-q", + "-p", + "no:cacheprovider", + f"--junitxml={xml}", + "-o", + "junit_family=xunit1", + ] + try: + r = subprocess.run( + cmd, cwd=pkg, env=env, capture_output=True, text=True, timeout=a.timeout + ) + status = "" + open(os.path.join(xmldir, name + ".log"), "w").write( + r.stdout[-20000:] + "\n" + r.stderr[-5000:] + ) + except subprocess.TimeoutExpired as e: + status = "TIMEOUT" + open(os.path.join(xmldir, name + ".log"), "w").write( + ((e.stdout or b"")[-8000:]).decode() + if isinstance(e.stdout, bytes) + else str(e.stdout)[-8000:] + ) + c = dict.fromkeys(["passed", "failed", "error", "notyet", "na_jvm", "skipped"], 0) + fails = [] + if os.path.exists(xml): + try: + for tc in ET.parse(xml).getroot().iter("testcase"): + if tc.find("failure") is not None: + c["failed"] += 1 + fails.append(tc.get("name")) + elif tc.find("error") is not None: + c["error"] += 1 + fails.append(tc.get("name")) + elif tc.find("skipped") is not None: + msg = tc.find("skipped").get("message") or "" + c[ + ( + "notyet" + if msg.startswith("NOTYET") + else "na_jvm" if msg.startswith("NA-JVM") else "skipped" + ) + ] += 1 + else: + c["passed"] += 1 + except ET.ParseError: + status = status or "BADXML" + else: + status = status or "CRASH" + for k in c: + tot[k] += c[k] + if status == "TIMEOUT": + tot["timeout"] += 1 + rows.append((name, c, status, fails)) + print( + f"{name:45s} pass={c['passed']:4d} fail={c['failed']:3d} err={c['error']:3d} notyet={c['notyet']:3d} na_jvm={c['na_jvm']:3d} skip={c['skipped']:3d} {status}", + flush=True, + ) +with open(a.out, "w") as fh: + fh.write( + "file\tpassed\tfailed\terror\tnotyet\tna_jvm\tskipped\tstatus\tfailing_tests\n" + ) + for name, c, status, fails in rows: + fh.write( + f"{name}\t{c['passed']}\t{c['failed']}\t{c['error']}\t{c['notyet']}\t{c['na_jvm']}\t{c['skipped']}\t{status}\t{';'.join(fails[:40])}\n" + ) + fh.write( + f"TOTAL\t{tot['passed']}\t{tot['failed']}\t{tot['error']}\t{tot['notyet']}\t{tot['na_jvm']}\t{tot['skipped']}\ttimeouts={tot['timeout']}\t\n" + ) +print("TOTAL", tot) +if a.max_notyet is not None and tot["notyet"] > a.max_notyet: + sys.exit(f"{tot['notyet']} tests are NOTYET skips, above the limit {a.max_notyet}") diff --git a/native/wheel/pyproject.toml b/native/wheel/pyproject.toml new file mode 100644 index 0000000..d63ebd1 --- /dev/null +++ b/native/wheel/pyproject.toml @@ -0,0 +1,47 @@ +# Template: build_wheel.py copies this into a staging directory next to the copied src/arcadedb tree and replaces @VERSION@. +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "arcadedb" +version = "@VERSION@" +description = "ArcadeDB embedded multi-model database, native build: the engine as a GraalVM native-image library, no JVM, no JPype. For heavy long-running loads use the JVM build, arcadedb-jvm; both import as arcadedb" +readme = "README.md" +requires-python = ">=3.11" +license = "Apache-2.0" +license-files = ["LICENSE", "NOTICE", "THIRD_PARTY_NOTICES.md", "licenses/**"] +authors = [{ name = "Taewoon Kim", email = "taewoon@humem.ai" }] +keywords = ["database", "graph", "document", "multi-model", "arcadedb", "embedded", "native-image", "graalvm", "native"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Database", +] +dependencies = [] + +[project.optional-dependencies] +numpy = ["numpy>=1.26.4"] +arrow = ["pyarrow>=23.0.1", "numpy>=1.26.4"] +pandas = ["pandas>=2.2.0", "numpy>=1.26.4"] +polars = ["polars>=1.0", "pyarrow>=23.0.1", "numpy>=1.26.4"] + +[project.urls] +Homepage = "https://github.com/ArcadeData/arcadedb-python" +Repository = "https://github.com/ArcadeData/arcadedb-python" +"Bug Tracker" = "https://github.com/ArcadeData/arcadedb-python/issues" +"ArcadeDB Upstream" = "https://github.com/ArcadeData/arcadedb" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +arcadedb = ["_native/*.so", "_native/*.dylib", "_native/*.dll"] diff --git a/native/wheel/setup.py b/native/wheel/setup.py new file mode 100644 index 0000000..0dab00b --- /dev/null +++ b/native/wheel/setup.py @@ -0,0 +1,35 @@ +"""arcadedb (the native build): a pure-Python tree plus one prebuilt shared library, tagged py3-none-. + +The platform tag comes from ARCADEDB_NATIVE_PLAT (build_wheel.py computes it from the library: glibc floor, macOS minimum version, win_amd64). +""" + +import os + +from setuptools import setup +from setuptools.dist import Distribution + +try: + from wheel.bdist_wheel import bdist_wheel as _bdist_wheel +except ImportError: # pragma: no cover + _bdist_wheel = None + + +class BinaryDistribution(Distribution): + def has_ext_modules(self): + return True + + +cmdclass = {} +if _bdist_wheel is not None: + + class bdist_wheel(_bdist_wheel): + def finalize_options(self): + super().finalize_options() + self.root_is_pure = False # platlib, not purelib + + def get_tag(self): + return "py3", "none", os.environ["ARCADEDB_NATIVE_PLAT"] + + cmdclass["bdist_wheel"] = bdist_wheel + +setup(distclass=BinaryDistribution, cmdclass=cmdclass) diff --git a/pyproject.toml b/pyproject.toml index f6b000c..67892ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,11 @@ test = [ "duckdb>=1.4.2", "ladybug>=0.21.2", "networkx>=3.4", + # tests/test_wheel_packages.py runs `setup.py build_py` on a copy of the sources to prove the JVM wheel ships every + # subpackage; Python 3.12+ venvs have no setuptools, so without it here the test fails on those legs instead of running. + # Floor 84.1.0: the dependency-floor audit resolves the lowest version; pip-audit (2026-10-24) finds no known + # vulnerability in 84.1.0, while 78.1.1 still has PYSEC-2026-3447 and GHSA-xmj3-9gf7-h52w. + "setuptools>=84.1.0", ] dev = [ "black", @@ -129,7 +134,7 @@ Repository = "https://github.com/ArcadeData/arcadedb-python" "ArcadeDB Upstream" = "https://github.com/ArcadeData/arcadedb" [tool.setuptools] -packages = ["arcadedb"] +packages = ["arcadedb", "arcadedb._native"] package-dir = { "" = "src" } include-package-data = true diff --git a/scripts/build.sh b/scripts/build.sh index 2e4efb1..2b11470 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -261,7 +261,7 @@ else fi echo -e "${CYAN}📋 Build Configuration:${NC}" -echo -e " Package: ${YELLOW}arcadedb${NC}" +echo -e " Package: ${YELLOW}arcadedb-jvm${NC}" echo -e " Platform: ${YELLOW}${PLATFORM}${NC}" echo -e " Python Version: ${YELLOW}${PYTHON_VERSION}${NC}" echo -e " JAR Source: ${YELLOW}${JAR_SOURCE_DESC}${NC}" @@ -271,7 +271,7 @@ echo "" # Package configuration PACKAGE_NAME="arcadedb-jvm" -DESCRIPTION="ArcadeDB embedded multi-model database with bundled JRE - no Java installation required" +DESCRIPTION="ArcadeDB embedded multi-model database, JVM build with a bundled JRE (no Java installation needed)" echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${BLUE}Building: ${YELLOW}${PACKAGE_NAME}${NC}" diff --git a/src/arcadedb/__init__.py b/src/arcadedb/__init__.py index c5a98d7..348fffd 100644 --- a/src/arcadedb/__init__.py +++ b/src/arcadedb/__init__.py @@ -7,14 +7,19 @@ # Which build this is: 'jvm' (the arcadedb-jvm distribution, the Java engine through JPype) or 'native' (the # arcadedb distribution, the engine as a GraalVM native-image library). The import name is arcadedb for both. -__backend__ = "jvm" - -# Both distributions install a package named arcadedb into the same directory; pip overwrites one with the other -# silently. Refuse to import when the other one is installed too, before anything else runs. +# `_native.distribution_backend()` is fixed when the wheel is built (the native wheel build flips one line in +# `_native/__init__.py`). `__backend__` is the engine that runs now: ARCADEDB_BACKEND=native can switch it on in a +# source checkout, where a developer loads a library they built themselves. +from . import _native as _native_backend from ._dist_conflict import DISTRIBUTIONS as _DISTRIBUTIONS from ._dist_conflict import check as _check_distributions -_check_distributions(__backend__) +_DISTRIBUTION_BACKEND = _native_backend.distribution_backend() +__backend__ = "native" if _native_backend.enabled() else "jvm" + +# Both distributions install a package named arcadedb into the same directory; pip overwrites one with the other +# silently. Refuse to import when the other one is installed too, before anything else runs. +_check_distributions(_DISTRIBUTION_BACKEND) # The installed distribution's version is the single source of truth: the # wheel is versioned from the release tag, while the generated _version.py @@ -26,7 +31,7 @@ try: from importlib.metadata import version as _dist_version - __version__ = _dist_version(_DISTRIBUTIONS[__backend__]) + __version__ = _dist_version(_DISTRIBUTIONS[_DISTRIBUTION_BACKEND]) except Exception: # not installed: fall back to the build-time file try: from ._version import __version__ diff --git a/src/arcadedb/_native/__init__.py b/src/arcadedb/_native/__init__.py new file mode 100644 index 0000000..27820cb --- /dev/null +++ b/src/arcadedb/_native/__init__.py @@ -0,0 +1,41 @@ +"""The native backend: the engine as a GraalVM native-image library, no JVM and no JPype. + +It is the only backend of the `arcadedb` wheel. In a source checkout or in the `arcadedb-jvm` wheel, ``ARCADEDB_BACKEND=native`` +selects it (with ``ARCADEDB_NATIVE_LIB`` pointing at a library you built); the default there is JPype. The engine runs as a GraalVM native-image shared +library (``libarcadedb``) that Python loads with ``ctypes``: no JVM, no JPype. The surface is the DSL-first core: +databases, ``query``/``command`` for SQL and openCypher, transactions, results and the in-process server. + +What is not on native yet (each raises :class:`NotOnNativeYet`): Python UDFs and record callbacks, ``GraphBatch``, the +async executor, the Java-object APIs (``Document``/``Vertex``/``Edge`` wrappers, ``new_vertex``, ``new_document``, +``schema``, ``get_java_database``), the columnar/Arrow readers other than a simple row conversion, importers and +exporters that go through Java objects, and every JVM setting (``jvm_kwargs``, ``start_jvm``). +""" + +import os + + +class NotOnNativeYet(NotImplementedError): + """The feature needs the Java object API, which the native backend does not expose (yet).""" + + +# The backend used when ARCADEDB_BACKEND is unset. "jpype" here, which is the arcadedb-jvm wheel; the build of the native +# `arcadedb` wheel (native/wheel/build_wheel.py) rewrites this one line to "native". +_DEFAULT_BACKEND = "jpype" + + +def distribution_backend() -> str: + """The build this code was packaged as: "native" (the `arcadedb` distribution) or "jvm" (the `arcadedb-jvm` distribution). + + Fixed when the wheel is built. ``ARCADEDB_BACKEND`` does not change it: the distribution check in ``arcadedb/__init__.py`` + asks which package this file belongs to, not which engine a developer switched on. + """ + return "native" if _DEFAULT_BACKEND == "native" else "jvm" + + +def enabled() -> bool: + """True when the native backend is selected: ``ARCADEDB_BACKEND=native``, or unset in the `arcadedb` (native) wheel (read at call time, so a test can switch it).""" + return os.environ.get("ARCADEDB_BACKEND", _DEFAULT_BACKEND).lower() == "native" + + +def not_yet(what: str): + raise NotOnNativeYet(f"{what} is not on the native backend yet") diff --git a/src/arcadedb/_native/bulk.py b/src/arcadedb/_native/bulk.py new file mode 100644 index 0000000..e8510a3 --- /dev/null +++ b/src/arcadedb/_native/bulk.py @@ -0,0 +1,96 @@ +"""Columnar bulk insert through one crossing (``arcadedb_n_insert_cols``): numpy int/float/bool columns and float32 vector +columns (a 2-D array or a list of equal-length 1-D float arrays) go over as one buffer; anything else returns None and the +caller falls back to the per-row path. The documents are built Java-side as ``DocumentBatcher.insertColumns`` builds them on JPype. +""" + +import json +import os + +from . import lib + +try: + import numpy as _np +except ImportError: # pragma: no cover + _np = None + +_ENABLED = os.environ.get("ARCADEDB_NATIVE_BULK", "1") != "0" + + +def insert_columns_fast(db, type_name, columns, commit_every): + if ( + not _ENABLED + or _np is None + or lib._lib is None + or not hasattr(lib._lib, "arcadedb_n_insert_cols") + ): + return None + cols, parts, n = [], [], None + for name, values in columns.items(): + if hasattr(values, "to_numpy") and not isinstance(values, (list, tuple)): + return None + kind, dim = None, 0 + if isinstance(values, _np.ndarray): + k = values.dtype.kind + if ( + values.ndim == 1 + and k in "iu" + and ( + k == "i" + or not values.size + or int(values.max()) <= _np.iinfo(_np.int64).max + ) + ): + kind, arr = "q", _np.ascontiguousarray(values, dtype=_np.int64) + elif values.ndim == 1 and k == "f": + kind, arr = "d", _np.ascontiguousarray(values, dtype=_np.float64) + elif values.ndim == 1 and k == "b": + kind, arr = "b", _np.ascontiguousarray(values, dtype=_np.uint8) + elif values.ndim == 2 and k == "f" and values.shape[1] > 0: + kind, arr, dim = ( + "f", + _np.ascontiguousarray(values, dtype=_np.float32), + int(values.shape[1]), + ) + elif ( + isinstance(values, (list, tuple)) + and values + and all( + isinstance(v, _np.ndarray) + and v.ndim == 1 + and v.dtype.kind == "f" + and v.shape == values[0].shape + for v in values + ) + and values[0].shape[0] > 0 + ): + kind, arr, dim = ( + "f", + _np.ascontiguousarray(_np.stack(values), dtype=_np.float32), + int(values[0].shape[0]), + ) + if kind is None: + return None + rows = int(arr.shape[0]) + if n is not None and rows != n: + return None # the per-row path reports the ragged columns + n = rows + cols.append([name, kind, dim]) + parts.append(arr.tobytes()) + if not n: + return None + header = json.dumps( + {"n": n, "commit": min(int(commit_every), 2**31 - 1), "cols": cols} + ) + buf = b"".join(parts) + from .database import _call + + reply = _call( + "Failed to bulk-insert", + "arcadedb_n_insert_cols", + db._h, + lib.b(type_name), + lib.b(header), + buf, + len(buf), + ) + return int(reply["n"]) diff --git a/src/arcadedb/_native/codec.py b/src/arcadedb/_native/codec.py new file mode 100644 index 0000000..c57504f --- /dev/null +++ b/src/arcadedb/_native/codec.py @@ -0,0 +1,229 @@ +"""Python <-> native JSON: query parameters out, typed rows in.""" + +import json +import os +from collections.abc import Mapping +from datetime import date, datetime, timezone +from decimal import Decimal + +try: # numpy is optional + import numpy as _np +except ImportError: # pragma: no cover + _np = None + +TAG = "\u0001" +_EPOCH_NAIVE = datetime(1970, 1, 1) + + +def _default(value, _bin=None): + """json.dumps hook: the values JSON cannot carry, as tagged objects (see Json.java). + + ``_bin`` (binary mode): ``[arrays, floats_so_far]``; a float ndarray then goes to the side buffer as float32 and the JSON carries + ``["B", offset, count]`` instead of the number list (no float formatting in Python, no text parsing in Java). + """ + if isinstance(value, RID): + return {TAG: ["R", value.text]} + if isinstance(value, Decimal): + return {TAG: ["n", str(value)]} + if isinstance(value, datetime): + utc = ( + value.astimezone(timezone.utc).replace(tzinfo=None) + if value.utcoffset() is not None + else value + ) + return {TAG: ["T", utc.isoformat()]} + if isinstance(value, date): + return {TAG: ["D", value.isoformat()]} + if isinstance(value, (set, frozenset)): + return {TAG: ["s", list(value)]} + if isinstance(value, (bytes, bytearray, memoryview)): + import base64 + + return {TAG: ["b", base64.b64encode(bytes(value)).decode("ascii")]} + if _np is not None: + if isinstance(value, _np.ndarray): + if value.dtype.kind == "f": + if _bin is not None: + arr = _np.ascontiguousarray(value.ravel(), dtype=_np.float32) + off = _bin[1] + _bin[0].append(arr) + _bin[1] = off + arr.size + return {TAG: ["B", off, arr.size]} + return {TAG: ["F", value.tolist()]} + return value.tolist() + if isinstance(value, _np.bool_): + return bool(value) + if isinstance(value, _np.integer): + return int(value) + if isinstance(value, _np.floating): + return float(value) + if isinstance(value, Mapping): + return dict(value) + raise TypeError( + f"cannot pass {type(value).__name__} as a parameter on the native backend" + ) + + +_PLAIN = frozenset((str, int, float, bool, type(None))) +_dumps = json.dumps +_fast = os.environ.get("ARCADEDB_NATIVE_FASTENC", "1") != "0" + + +def _prep(value): + """A value ready for json.dumps: a dict with a key that is not a str becomes a tagged ``[[k, v], ..]`` map (JSON would turn + the key into text; the JPype path keeps it), and containers are walked for such dicts. Plain scalars pass untouched. + """ + t = type(value) + if t in _PLAIN: + return value + if isinstance(value, Mapping): + if all(type(k) is str for k in value): + return {k: _prep(v) for k, v in value.items()} + return {TAG: ["m", [[_prep(k), _prep(v)] for k, v in value.items()]]} + if t is list or t is tuple: + return [_prep(v) for v in value] + return value + + +def encode_params(args, binary=False): + """The ``params`` text for ``arcadedb_n_exec``: '' for none, else ``{"named":{..}}`` or ``{"pos":[..]}``. + + Same selection as the JPype path: a lone list/tuple is the positional list itself, a lone mapping (also alone in + a list) is the named map, anything else is positional. + """ + if not args: + return ("", None, 0) if binary else "" + values = args[0] if len(args) == 1 and isinstance(args[0], (list, tuple)) else args + plain = _PLAIN + if _fast and set(map(type, values)) <= plain: + # all plain scalars (the per-row INSERT of a load): the cached default encoder, no default hook, no per-call encoder object + text = _dumps({"pos": values if type(values) is list else list(values)}) + return (text, None, 0) if binary else text + if len(values) == 1 and isinstance(values[0], Mapping): + named = values[0] + if not all(type(v) in plain for v in named.values()): + named = {k: _prep(v) for k, v in named.items()} + return _dump({"named": dict(named)}, binary) + if not all(type(v) in plain for v in values): + values = [_prep(v) for v in values] + return _dump({"pos": list(values)}, binary) + + +def _dump(obj, binary): + """JSON text; binary mode returns ``(text, float32 side buffer bytes or None, float count)``.""" + if not binary: + return json.dumps(obj, default=_default) + state = [[], 0] + text = json.dumps(obj, default=lambda v: _default(v, state)) + arrays = state[0] + if not arrays: + return text, None, 0 + buf = arrays[0].tobytes() if len(arrays) == 1 else _np.concatenate(arrays).tobytes() + return text, buf, state[1] + + +class RID: + """A record id as the engine returns it in a row; passed back as a parameter it is a link, not a string.""" + + __slots__ = ("text",) + + def __init__(self, text): + self.text = text + + def __str__(self): + return self.text + + def __repr__(self): + return f"RID({self.text!r})" + + def __eq__(self, other): + return self.text == (other.text if isinstance(other, RID) else other) + + def __hash__(self): + return hash(self.text) + + +def _restore_float(text): + return float(text) + + +def _restore_map(pairs): + return {k: v for k, v in pairs} + + +_SIMPLE = { + "n": Decimal, + "i": int, + "D": date.fromisoformat, + "T": datetime.fromisoformat, + "Z": datetime.fromisoformat, + "s": set, + "f": _restore_float, + "m": _restore_map, + "R": RID, +} + + +def _hook(obj): + tagged = obj.get(TAG) + # a column may be called like the tag: only a lone [kind, payload] pair of a known kind is a tagged value + if ( + tagged is None + or len(obj) != 1 + or type(tagged) is not list + or len(tagged) != 2 + or tagged[0] not in _SIMPLE + ): + return obj + kind, payload = tagged + return _SIMPLE[kind](payload) + + +def loads(text: str, floats=None): + """Decode a reply, restoring tagged values. ``floats``: the float32 tail of a binary reply (``["B", offset, count]`` values).""" + if floats is None: + return json.loads(text, object_hook=_hook) + + def hook(obj): + tagged = obj.get(TAG) + if ( + tagged is not None + and len(obj) == 1 + and type(tagged) is list + and len(tagged) == 3 + and tagged[0] == "B" + ): + return floats[ + tagged[1] : tagged[1] + tagged[2] + ].tolist() # float32 widens to the same double as the JPype path + return _hook(obj) + + return json.loads(text, object_hook=hook) + + +# --- JSON-native form of a row (ResultSet.to_json_list): DATE/DATETIME as epoch millis, DECIMAL as float. + + +def json_native(value): + if isinstance(value, datetime): + return ( + int((value.replace(tzinfo=None) - _EPOCH_NAIVE).total_seconds() * 1000) + if value.tzinfo is None + else int(value.timestamp() * 1000) + ) + if isinstance(value, date): + return int( + ( + datetime.combine(value, datetime.min.time()) - _EPOCH_NAIVE + ).total_seconds() + * 1000 + ) + if isinstance(value, Decimal): + return float(value) + if isinstance(value, (set, frozenset)): + return [json_native(v) for v in value] + if isinstance(value, list): + return [json_native(v) for v in value] + if isinstance(value, dict): + return {k: json_native(v) for k, v in value.items()} + return value diff --git a/src/arcadedb/_native/columns.py b/src/arcadedb/_native/columns.py new file mode 100644 index 0000000..90f59df --- /dev/null +++ b/src/arcadedb/_native/columns.py @@ -0,0 +1,167 @@ +"""Rows to columns for the native backend (the columnar path is Python over the rows; the JPype path decodes a Java buffer).""" + +from datetime import date, datetime, timezone +from decimal import Decimal + + +def _names(rows, columns): + if columns is not None: + return list(columns) + seen = {} + for row in rows: + for k in row: + seen.setdefault(k, None) + return list(seen) + + +def _kind(values): + """'b', 'i', 'f', 'dt', 'dec', 'obj' for the non-null values of a column.""" + kinds = set() + for v in values: + if v is None: + continue + t = type(v) + if t is bool: + kinds.add("b") + elif t is int: + kinds.add("i") + elif t is float: + kinds.add("f") + elif t is datetime or t is date: + kinds.add("dt") + elif t is Decimal: + kinds.add("dec") + else: + kinds.add("obj") + if not kinds: + return "obj" + if len(kinds) == 1: + return kinds.pop() + if kinds == {"i", "f"}: + return "f" + return "obj" + + +def _ms(v): + if isinstance(v, datetime): + if v.tzinfo is not None: + v = v.astimezone(timezone.utc).replace(tzinfo=None) + return v + return datetime(v.year, v.month, v.day) + + +def _float_vectors(values): + """A column of equal-length float lists (None allowed) that float32 holds exactly, as a (count, dim) float32 array with + NaN rows for the nulls; None for any other column. Rows cannot say float[] from List, so only a column whose every + value survives float32 unchanged is read as a vector column (the JPype path decides on the Java element type). + """ + import numpy as np + + dim = 0 + for v in values: + if v is None: + continue + if type(v) is not list or not v or not all(type(x) is float for x in v): + return None + if dim == 0: + dim = len(v) + elif len(v) != dim: + return None + if dim == 0: + return None + arr = np.full((len(values), dim), np.nan, dtype=np.float32) + for i, v in enumerate(values): + if v is not None: + arr[i] = v + if not np.array_equal(arr[i].astype(np.float64), np.array(v)): + return None + return arr + + +def to_columns(rows, columns=None): + import numpy as np + + out = {} + for name in _names(rows, columns): + values = [row.get(name) for row in rows] + if _kind(values) == "obj": + vectors = _float_vectors(values) + if vectors is not None: + out[name] = vectors + continue + has_null = any(v is None for v in values) + kind = _kind(values) + if kind == "i": + out[name] = ( + np.array([np.nan if v is None else v for v in values], dtype=np.float64) + if has_null + else np.array(values, dtype=np.int64) + ) + elif kind == "f": + out[name] = np.array( + [np.nan if v is None else v for v in values], dtype=np.float64 + ) + elif kind == "b": + out[name] = values if has_null else np.array(values, dtype=bool) + elif kind == "dt": + out[name] = np.array( + [ + ( + np.datetime64("NaT", "ms") + if v is None + else np.datetime64(_ms(v), "ms") + ) + for v in values + ], + dtype="datetime64[ms]", + ) + elif kind == "dec": + arr = np.empty(len(values), dtype=object) + arr[:] = values + out[name] = arr + else: + out[name] = values + return out + + +def to_arrow(rows, columns=None): + import pyarrow as pa + + arrays = {} + for name in _names(rows, columns): + values = [row.get(name) for row in rows] + kind = _kind(values) + if kind == "i": + arrays[name] = pa.array(values, type=pa.int64()) + elif kind == "f": + try: + arrays[name] = pa.array(values, type=pa.float64()) + except ( + pa.ArrowInvalid + ): # an int beyond +-2**53 next to floats: the column degrades to exact text + arrays[name] = pa.array( + [None if v is None else str(v) for v in values], type=pa.string() + ) + elif kind == "b": + arrays[name] = pa.array(values, type=pa.bool_()) + elif kind == "dt": + arrays[name] = pa.array( + [None if v is None else _ms(v) for v in values], type=pa.timestamp("ms") + ) + elif kind == "dec": + try: + arrays[name] = pa.array(values) + except ( + pa.ArrowInvalid + ): # wider than decimal256 (76 digits): an exact string + arrays[name] = pa.array( + [None if v is None else str(v) for v in values], type=pa.string() + ) + else: + try: + arrays[name] = pa.array([None if v is None else v for v in values]) + except (pa.ArrowInvalid, pa.ArrowTypeError): + arrays[name] = pa.array( + [None if v is None else str(v) for v in values], type=pa.string() + ) + return pa.table(arrays) diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py new file mode 100644 index 0000000..a487f2a --- /dev/null +++ b/src/arcadedb/_native/database.py @@ -0,0 +1,1361 @@ +"""NativeDatabase, NativeDatabaseFactory, NativeResultSet and NativeResult: the DSL-first core over libarcadedb.""" + +import itertools +import json +import os +import re +import time as _time +from collections import deque +from datetime import date, datetime +from typing import Any, Dict, Iterator, List, Optional + +from ..exceptions import ArcadeDBError +from ..transactions import TransactionContext +from ..type_conversion import json_bulk_dumps +from . import lib, not_yet +from .codec import _default as codec_default +from .codec import _prep as _prep_row +from .codec import encode_params, json_native, loads + +_LAZY_COMMAND_ROWS = os.environ.get("ARCADEDB_NATIVE_LAZY_CMD", "1") != "0" +_BINARY_REPLY = os.environ.get("ARCADEDB_NATIVE_BIN_REPLY", "1") != "0" +_INSERT_MANY_CHUNK = 10_000 +_RID_RE = re.compile(r"#?-?\d+:-?\d+") +_FIRST_BATCH = 256 +_EPOCH_DAY_ORDINAL = date(1970, 1, 1).toordinal() +_NEXT_BATCH = 2048 +_KIND = {"v": "v", "e": "e", "d": "d"} + + +def _error(prefix: str, reply: dict) -> ArcadeDBError: + """The ArcadeDBError for a failed reply, worded like the JPype path (``prefix: java.class: message``).""" + chain = reply.get("chain") or [] + head = f"{reply.get('cls')}: {reply.get('error')}" + exc = ArcadeDBError(f"{prefix}: {head}" if prefix else head) + exc.native = reply + exc.java_chain = chain + return exc + + +def _read_error(reply: dict) -> ArcadeDBError: + exc = ArcadeDBError( + f"Reading the result set failed: {reply.get('cls')}: {reply.get('error')}" + ) + exc.native = reply + exc.java_chain = reply.get("chain") or [] + return exc + + +def _call(prefix: str, name: str, *args) -> dict: + reply = json.loads(lib.call(name, *args)) + if not reply.get("ok"): + raise _error(prefix, reply) + return reply + + +class NativeResult: + """One row. Values are already Python types (datetime, Decimal, lists, dicts, RID text).""" + + __slots__ = ("_props", "_meta", "_database") + + def __init__(self, props: Dict[str, Any], meta, database=None): + self._props = props + self._meta = meta or None + self._database = database + + def _check_open(self) -> None: + database = self._database + if database is not None and database._closed and self._meta is not None: + raise ArcadeDBError( + "Database is closed: a record row cannot be read after its database was closed. " + "Read what you need before closing it." + ) + + def has_property(self, name: str) -> bool: + self._check_open() + return name in self._props + + def get(self, name: str, convert_types: bool = True) -> Any: + self._check_open() + return self._props.get(name) + + def get_raw(self, name: str) -> Any: + return self.get(name) + + def get_rid(self) -> Optional[str]: + return self._meta[0] if self._meta else None + + def _record(self, kinds): + from .records import _wrap + + self._check_open() + if not self._meta or self._meta[2] not in kinds: + return None + return _wrap(self._database, self) + + def get_vertex(self): + return self._record("v") + + def get_edge(self): + return self._record("e") + + def get_element(self): + return self._record("vde") + + def get_property_names(self) -> List[str]: + self._check_open() + return list(self._props) + + @property + def property_names(self) -> List[str]: + return self.get_property_names() + + def to_dict(self, convert_types: bool = True) -> Dict[str, Any]: + self._check_open() + return dict(self._props) + + def to_json(self) -> str: + self._check_open() + out = json_native(self._props) + for key, value in self._props.items(): + if isinstance(value, date) and not isinstance(value, datetime): + out[key] = ( + value.toordinal() - _EPOCH_DAY_ORDINAL + ) # as the engine writes a DATE property: days since the epoch + if self._meta: + rid, type_name, kind = self._meta + out = dict(out) + out["@cat"] = kind + out["@type"] = type_name + out["@rid"] = rid + return json.dumps(out, default=str, separators=(",", ":")) + + def __repr__(self) -> str: + names = list(self._props) + preview = ", ".join(repr(n) for n in names[:3]) + ( + ", ..." if len(names) > 3 else "" + ) + rid = self.get_rid() + return ( + f"Result(rid={rid!r}, properties=[{preview}])" + if rid + else f"Result(properties=[{preview}])" + ) + + +class NativeResultSet: + """Cursor over a statement's rows, fetched in batches through ``arcadedb_n_next``.""" + + def __init__(self, database, reply: dict): + self._database = database # strong reference: a result keeps its database alive + self._rs = reply["rs"] + self._buf = deque(zip(reply["rows"], reply["meta"])) + self._more = not reply["drained"] + self._closed = False + self._exhausted = False + # the engine computes lazily: an error met while the first rows were read is raised when the rows before it + # have been consumed, as it is on the JPype path (the cursor is already finished on the Java side) + self._pending_error = reply.get("rerr") + + @classmethod + def _lazy(cls, database, rs_id: int) -> "NativeResultSet": + """The result of arcadedb_n_cmd: a cursor id and nothing read yet (id 0: no cursor).""" + self = cls.__new__(cls) + self._database = database + self._rs = rs_id + self._buf = deque() + self._more = rs_id > 0 + self._closed = False + self._exhausted = False + self._pending_error = None + return self + + # -- state + + def _readable(self) -> bool: + database = self._database + if database is not None and database._closed and not self._exhausted: + raise ArcadeDBError( + "Database is closed: the rows of this result set cannot be read after their database was closed. " + "Read them before closing the database." + ) + if not self._closed: + return True + if self._exhausted: + return False + raise ArcadeDBError( + "This result set was closed before all its rows were read (by first(), one(), close(), or leaving its " + "with block), so the rows it had not returned are gone. Run the query again to read them." + ) + + def _fill(self) -> bool: + """Fetch the next batch into the buffer; False once the cursor is drained.""" + if self._pending_error is not None: + err, self._pending_error = self._pending_error, None + self._more = False + self._closed = True + self._exhausted = True + raise _read_error(err) + if not self._more: + return False + try: + reply = loads(lib.call("arcadedb_n_next", self._rs, _NEXT_BATCH)) + except ArcadeDBError: + raise + if not reply.get("ok"): + self._more = False + self._closed = True + self._exhausted = True + self._rs = 0 + raise ArcadeDBError( + f"Reading the result set failed: {reply.get('cls')}: {reply.get('error')}" + ) + self._buf.extend(zip(reply["rows"], reply["meta"])) + self._more = not reply["drained"] + self._pending_error = reply.get("rerr") + return bool(self._buf) or self._pending_error is not None + + def _finish(self) -> None: + self._exhausted = True + self.close() + + def close(self) -> None: + if self._closed: + return + self._closed = True + if self._more and self._rs: + try: + lib.close_rs(self._rs) + except Exception: # nosec B110 - best effort + pass + self._more = False + self._buf.clear() + + # -- reading + + def __iter__(self) -> Iterator[NativeResult]: + return self + + def __next__(self) -> NativeResult: + if self._readable(): + while True: + if self._buf: + props, meta = self._buf.popleft() + return NativeResult(props, meta, self._database) + if not self._fill(): + break + if not self._closed: + self._finish() + raise StopIteration + + def _drain_dicts(self) -> List[Dict[str, Any]]: + out: List[Dict[str, Any]] = [] + if not self._readable(): + return out + while True: + out.extend(p for p, _ in self._buf) + self._buf.clear() + if not self._fill(): + break + self._finish() + return out + + def to_list(self, convert_types: bool = True) -> List[Dict[str, Any]]: + return self._drain_dicts() + + def iter_dicts(self, convert_types: bool = True) -> Iterator[Dict[str, Any]]: + for row in self: + yield row.to_dict() + + def to_json_list(self, batch_size: int = 10_000) -> List[Dict[str, Any]]: + if int(batch_size) < 1: + raise ValueError("batch_size must be at least 1") + return [json_native(row) for row in self._drain_dicts()] + + def iter_json_batches(self, batch_size: int = 10_000, **kwargs): + if int(batch_size) < 1: + raise ValueError("batch_size must be at least 1") + rows = self.to_json_list() + for i in range(0, len(rows), batch_size): + yield rows[i : i + batch_size] + + def to_dataframe(self, convert_types: bool = True): + import pandas as pd + + return pd.DataFrame(self._drain_dicts()) + + def to_columns(self, batch_size: int = 25_000, columns=None): + """Columns as the JPype path returns them (numpy int64/float64/bool/datetime64[ms] arrays, lists for text and + JSON-typed values, an object array of Decimal), built in Python from the rows: a columnar JSON path, no + binary buffer. None when numpy is missing.""" + try: + import numpy # noqa: F401 + except ImportError: + return None + from .columns import to_columns + + return to_columns(self._drain_dicts(), columns) + + def to_arrow(self, batch_size: int = 25_000, columns=None): + """A ``pyarrow.Table`` with the types of :meth:`to_columns` but nulls kept (None when pyarrow is missing).""" + try: + import numpy # noqa: F401 + import pyarrow # noqa: F401 + except ImportError: + return None + from .columns import to_arrow + + return to_arrow(self._drain_dicts(), columns) + + def to_polars(self, batch_size: int = 25_000, columns=None): + try: + import polars as pl + except ImportError as exc: + raise ImportError( + "polars is required for to_polars(). " + "Install with: pip install 'arcadedb[polars]'" + ) from exc + table = self.to_arrow(batch_size, columns) + if table is None: + raise ImportError( + "to_polars() reads through Arrow, which needs pyarrow and numpy. " + "Install with: pip install 'arcadedb[polars]'" + ) + return pl.from_arrow(table) + + def iter_chunks(self, size: int = 1000, convert_types: bool = True): + chunk = [] + for row in self.iter_dicts(convert_types=convert_types): + chunk.append(row) + if len(chunk) >= size: + yield chunk + chunk = [] + if chunk: + yield chunk + + def count(self) -> int: + n = 0 + for _ in self: + n += 1 + return n + + def first(self) -> Optional[NativeResult]: + try: + return next(iter(self)) + except StopIteration: + return None + finally: + self.close() + + def one(self) -> NativeResult: + iterator = iter(self) + try: + try: + result = next(iterator) + except StopIteration as exc: + raise ValueError("Query returned no results") from exc + try: + next(iterator) + raise ValueError("Query returned multiple results") + except StopIteration: + return result + finally: + self.close() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.close() + + def __del__(self): + try: + self.close() + except Exception: # nosec B110 + pass + + +class NativeDatabase: + """The DSL-first database handle: statements, transactions, lifecycle. See the package docstring for the rest.""" + + def __init__(self, handle: int, owned_by_server: bool = False): + self._h = handle + self._closed = False + self._server_owned = owned_by_server + self._async_executors: list = [] + self._vector_meta: dict = {} + self._vector_dimensions: dict = {} + self._vector_id_properties: dict = {} + + # -- lifecycle + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def _check_not_closed(self): + if self._closed: + raise ArcadeDBError("Database is closed") + + def close(self): + if self._closed: + return + self._close_async_executors() + if self._server_owned: + # owned by the server: the engine refuses the close, and this handle is done either way + self._closed = True + return + try: + lib.flush_closes() + _call("Failed to close database", "arcadedb_n_close", self._h, 0) + finally: + self._closed = True + + def _discard_async_executor(self, executor) -> None: + try: + self._async_executors.remove(executor) + except ValueError: + pass + + def _close_async_executors(self): + first_error = None + while self._async_executors: + executor = self._async_executors.pop() + try: + executor.close() + except Exception as e: + first_error = first_error or ArcadeDBError( + f"Failed to close async executor: {e}" + ) + if first_error is not None: + raise first_error + + def drop(self): + self._check_not_closed() + lib.flush_closes() + _call("Failed to drop database", "arcadedb_n_close", self._h, 1) + self._closed = True + + def __del__(self): + try: + self.close() + except Exception: # nosec B110 + return + + def _info(self) -> dict: + return _call("Failed to read the database", "arcadedb_n_info", self._h) + + def is_open(self) -> bool: + return not self._closed and self._info()["open"] + + def get_name(self) -> str: + self._check_not_closed() + return self._info()["name"] + + def get_database_path(self) -> str: + self._check_not_closed() + return self._info()["path"] + + def is_read_only(self) -> bool: + self._check_not_closed() + return self._info()["mode"] == "READ_ONLY" + + # -- statements + + def _exec( + self, flags: int, prefix: str, language: str, command: str, args + ) -> NativeResultSet: + self._check_not_closed() + if flags & 1 and _LAZY_COMMAND_ROWS and lib.CMD_PATH: + # phase 8: the cursor id comes back as a long (no reply text to free and decode); the rows are read late or never + text, buf, count = encode_params(args, binary=True) + rid = lib.cmd( + self._h, + 1, + lib.bc(language), + lib.bc(command), + lib.bc(text), + buf, + count, + _FIRST_BATCH, + ) + if rid < 0: + raise _error(prefix, json.loads(lib.last_error())) + return NativeResultSet._lazy(self, rid) + # phase 7 (arcadedb_n_exec flags): 2 = a command's rows are pulled but encoded only when read, 4 = float arrays come back as a float32 tail + if flags & 1 and _LAZY_COMMAND_ROWS: + flags |= 2 + if _BINARY_REPLY: + flags |= 4 + if lib.BINARY_PARAMS: + text, buf, count = encode_params(args, binary=True) + if buf is not None: + raw, floats = lib.call_bin( + "arcadedb_n_exec_bin", + self._h, + flags, + lib.b(language), + lib.b(command), + lib.b(text), + _FIRST_BATCH, + buf, + count, + ) + else: + raw, floats = lib.call_bin( + "arcadedb_n_exec", + self._h, + flags, + lib.b(language), + lib.b(command), + lib.b(text), + _FIRST_BATCH, + ) + else: + raw, floats = lib.call_bin( + "arcadedb_n_exec", + self._h, + flags, + lib.b(language), + lib.b(command), + lib.b(encode_params(args)), + _FIRST_BATCH, + ) + reply = loads(raw, floats) + if not reply.get("ok"): + raise _error(prefix, reply) + return NativeResultSet(self, reply) + + def query(self, language: str, command: str, *args) -> NativeResultSet: + """Run a read statement. Parameters bind as on the JPype path (``?`` positional, ``:name`` from one dict).""" + return self._exec(0, "Query failed", language, command, args) + + def command(self, language: str, command: str, *args) -> NativeResultSet: + """Run a statement that may write.""" + return self._exec(1, "Command failed", language, command, args) + + # -- transactions + + def _tx(self, op: int, prefix: str) -> dict: + self._check_not_closed() + if (op == 1 or op == 2) and lib.CLOSE_BATCH: + lib.flush_closes() + return _call(prefix, "arcadedb_n_tx", self._h, op) + + def begin(self): + self._tx(0, "Failed to begin transaction") + + def commit(self): + self._tx(1, "Failed to commit transaction") + + def rollback(self): + self._tx(2, "Failed to rollback transaction") + + def transaction(self) -> TransactionContext: + return TransactionContext(self) + + def is_transaction_active(self) -> bool: + return self._tx(3, "Failed to check transaction status")["active"] + + def _transaction_stack_mark(self): + state = self._tx(3, "Failed to read the transaction state") + return state["active"], state["depth"] + + def _transaction_commit_mark(self): + """The commit count of the current transaction (the witness that a BATCH boundary published part of its work).""" + return self._tx(3, "Failed to read the transaction state")["commits"] + + def _end_transactions_opened_since(self, mark): + was_active, depth_before = mark + first_error = None + state = self._tx(3, "Failed to read the transaction state") + for _ in range(state["depth"] - depth_before): + if not self.is_transaction_active(): + break + try: + self.rollback() + except Exception as e: + first_error = first_error or e + if not was_active and self.is_transaction_active(): + try: + self.rollback() + except Exception as e: + first_error = first_error or e + if first_error is not None: + raise first_error + + def run_in_transaction(self, fn, retries: int = 12, backoff_s: float = 0.005): + """``fn`` inside a transaction, retried on NeedRetry conflicts (see the JPype ``Database.run_in_transaction``).""" + duplicated_key_retried = False + for attempt in range(retries + 1): + mark = self._transaction_stack_mark() + commits_at_start = None + try: + self.begin() + commits_at_start = self._transaction_commit_mark() + result = fn() + self.commit() + return result + except BaseException as e: + try: + self._end_transactions_opened_since(mark) + except Exception: # nosec B110 + pass + if attempt >= retries or not isinstance(e, Exception): + raise + reply = getattr(e, "native", None) or {} + kind = reply.get("retry") + if kind is None: + raise + if kind == "duplicated_key": + if duplicated_key_retried: + raise + duplicated_key_retried = True + if commits_at_start is not None: + try: + if self._transaction_commit_mark() != commits_at_start: + raise + except ArcadeDBError: + pass + _time.sleep( + max(backoff_s * (attempt + 1), reply.get("retry_ms", 0) / 1000.0) + ) + + # -- small helpers over statements (DSL-first: no Java record objects) + + def count_type(self, type_name: str) -> int: + self._check_not_closed() + try: + count_sql = f"SELECT count(*) AS c FROM `{type_name}`" # nosec B608 - backtick-quoted schema identifier, no values + row = self.query("sql", count_sql).first() + except ArcadeDBError as e: + if "was not found" in str(e): + return 0 + raise ArcadeDBError(f"Failed to count type '{type_name}': {e}") from e + return int(row.get("c")) if row is not None else 0 + + def lookup_by_rid(self, rid: str): + """The record at ``rid`` as a Document, Vertex or Edge wrapper.""" + self._check_not_closed() + if not _RID_RE.fullmatch(str(rid)): + raise ArcadeDBError(f"Failed to lookup RID '{rid}': not a record id") + rid_sql = f"SELECT FROM {rid}" # nosec B608 - rid matched _RID_RE above + row = self.query("sql", rid_sql).first() + if row is None: + raise ArcadeDBError(f"Failed to lookup RID '{rid}': record not found") + return row.get_element() + + def lookup_by_key(self, type_name: str, keys: List[str], values: List[Any]): + self._check_not_closed() + where = " AND ".join(f"`{k}` = ?" for k in keys) + try: + lookup_sql = f"SELECT FROM `{type_name}` WHERE {where} LIMIT 1" # nosec B608 - backtick-quoted identifiers; values go through ? parameters + row = self.query("sql", lookup_sql, list(values)).first() + return row.get_element() if row is not None else None + except ArcadeDBError as e: + raise ArcadeDBError(f"Failed to lookup by key in '{type_name}': {e}") from e + + def insert_many( + self, type_name: str, rows, commit_every: int = 10_000, parallel: bool = False + ) -> int: + """Bulk insert. Rows go in chunks of 10,000, one JSON text and one crossing per chunk (``arcadedb_n_insert_rows``: a + newDocument/set/save loop inside the image, as ``DocumentBatcher.insertManyJson`` on the JPype path); a chunk with a value the + JSON text cannot carry unchanged goes row by row through ``INSERT INTO t CONTENT :row``. Inside the caller's transaction nothing is + committed; otherwise every ``commit_every`` rows are one transaction (0: all rows in one). + """ + self._check_not_closed() + if parallel: + return self._insert_many_parallel(type_name, rows) + sql = f"INSERT INTO `{type_name}` CONTENT :row" + rows = iter(rows) + was_active = self.is_transaction_active() + every = int(commit_every) + n = 0 + try: + if not was_active: + self.begin() + while True: + size = _INSERT_MANY_CHUNK + if not was_active and every > 0: + size = min(size, every - n % every) + chunk = list(itertools.islice(rows, size)) + if not chunk: + break + self._insert_chunk(type_name, sql, chunk) + n += len(chunk) + if not was_active and every > 0 and n % every == 0: + self.commit() + self.begin() + if not was_active: + self.commit() + return n + except BaseException: + if not was_active: + try: + if self.is_transaction_active(): + self.rollback() + except Exception: # nosec B110 - best-effort rollback + pass + raise + + def _insert_chunk(self, type_name: str, sql: str, chunk) -> None: + if lib.INSERT_ROWS: + try: + payload = json_bulk_dumps(chunk) + except (TypeError, ValueError): + payload = None + if payload is not None: + reply = json.loads( + lib.call( + "arcadedb_n_insert_rows", + self._h, + lib.b(type_name), + lib.b(payload), + ) + ) + if not reply.get("ok"): + raise _error(f"Failed to bulk-insert into '{type_name}'", reply) + return + if lib.EXEC_MANY: + # values JSON text cannot carry unchanged (a date-time, a decimal, NaN, a big integer): the same statement per + # row, but all rows of the chunk in one crossing (arcadedb_n_exec_many), not one call per row + self._command_many("sql", sql, [{"row": row} for row in chunk]) + return + for row in chunk: + self.command("sql", sql, {"row": row}).close() + + def _command_many(self, language: str, command: str, rows) -> int: + """Experimental (phase 8): the statement once per row of ``rows`` (each a list of positional values or a dict of named values), + all in one crossing and in the open transaction; the cursors are drained and closed. Not part of the public API. + """ + self._check_not_closed() + payload = json.dumps([_prep_row(r) for r in rows], default=codec_default) + reply = json.loads( + lib.call( + "arcadedb_n_exec_many", + self._h, + lib.b(language), + lib.b(command), + lib.b(payload), + ) + ) + if not reply.get("ok"): + raise _error("Command failed", reply) + return reply["n"] + + def _parallel_load_failed(self, type_name, error) -> ArcadeDBError: + """The error for a parallel load that failed while handing rows to the writers: waits for the queue to drain first, so + the caller sees a final state, and says that the rows handed over before the failure may have been stored. + """ + try: + self._async().wait_completion() + except Exception: # nosec B110 - the original error is the one to report + pass + text = str(error) + prefix = "Failed to bulk-insert: " + if text.startswith(prefix): + text = text[len(prefix) :] + return ArcadeDBError( + f"Failed to bulk-insert into '{type_name}': {text}; the rows handed to the " + f"parallel writers before the failure may have been stored (the call waited " + f"for them before raising)" + ) + + def _insert_many_parallel(self, type_name, rows) -> int: + """insert_many(parallel=True): chunks go to the async writers; the failure count is read after the final wait.""" + import itertools + + from . import extras + + iterator = iter(rows) + tokens, n = [], 0 + try: + while True: + chunk = list(itertools.islice(iterator, 10_000)) + if not chunk: + break + try: + tokens.append(extras.insert_parallel(self, type_name, chunk)["fid"]) + except ArcadeDBError as e: + # as the JPype path: the rows handed to the writers before the failure are written whatever this call does + raise self._parallel_load_failed(type_name, e) from e + n += len(chunk) + except ArcadeDBError: + raise + except BaseException: + try: + self._async().wait_completion() + except Exception: # nosec B110 - the original error is the one to report + pass + raise + try: + self._async().wait_completion() + except Exception as e: + raise ArcadeDBError(f"Failed to bulk-insert into '{type_name}': {e}") from e + n_failed, first = 0, None + for token in tokens: + result = extras.failures(self, token) + if result["count"] and first is None: + first = result["first"] + n_failed += result["count"] + if n_failed: + raise ArcadeDBError( + f"Failed to bulk-insert into '{type_name}': the parallel writers reported {n_failed} failed record(s) " + f"of {n}; the rest may have been stored (first failure: {first})" + ) + return n + + def _kind_of(self, type_name): + """'v', 'e' or 'd' for a type (cached): which wrapper a record of that type gets.""" + kinds = self.__dict__.setdefault("_type_kinds", {}) + if type_name not in kinds: + row = self.query( + "sql", "SELECT type FROM schema:types WHERE name = ?", type_name + ).first() + kinds[type_name] = {"vertex": "v", "edge": "e"}.get( + row.get("type") if row else None, "d" + ) + return kinds[type_name] + + def _async(self): + from .extras import NativeAsyncExecutor + + return NativeAsyncExecutor(self) + + # -- not on native yet + + def get_java_database(self): + not_yet("get_java_database") + + def new_vertex(self, type_name: str): + """A vertex that is created by ``save()`` (``CREATE VERTEX .. CONTENT``).""" + from .records import Vertex + + self._check_not_closed() + return Vertex(self, type_name) + + def new_document(self, type_name: str): + """A document that is created by ``save()`` (``INSERT INTO .. CONTENT``).""" + from .records import Document + + self._check_not_closed() + return Document(self, type_name) + + def graph_batch(self, **options): + """A GraphBatch for bulk graph loads (keywords as :meth:`Database.graph_batch`).""" + from .extras import NativeGraphBatch + + self._check_not_closed() + return NativeGraphBatch.create(self, **options) + + def async_executor(self): + """The database's async executor.""" + from .extras import NativeAsyncExecutor + + self._check_not_closed() + executor = NativeAsyncExecutor(self) + self._async_executors.append(executor) + return executor + + def register_function(self, name, fn, min_args=0, max_args=-1): + """Make a Python callable available to SQL (process-wide, see ``extras.register_function``).""" + from .extras import register_function + + register_function(name, fn, min_args, max_args) + + @property + def schema(self): + """DDL helpers over SQL statements (the DSL-first subset); everything else is not on native yet.""" + self._check_not_closed() + return NativeSchema(self) + + def create_vector_index( + self, + vertex_type, + vector_property, + dimensions, + id_property=None, + distance_function="cosine", + max_connections=32, + beam_width=100, + quantization="INT8", + encoding=None, + location_cache_size=None, + graph_build_cache_size=None, + mutations_before_rebuild=None, + store_vectors_in_graph=False, + add_hierarchy=True, + pq_subspaces=None, + pq_clusters=None, + pq_center_globally=None, + pq_training_limit=None, + build_graph_now=True, + ): + """A vector index (``CREATE INDEX .. LSM_VECTOR``) and its :class:`VectorIndex` wrapper; arguments as the JPype path.""" + from .schema_objects import create_vector_index + + self._check_not_closed() + if location_cache_size is not None: + raise ValueError( + "location_cache_size is no longer supported (ArcadeDB issues #5559 and #5568): a vector location is the only " + "mapping from a vector id to its record, so capping the location index drops vectors from searches and from " + "countEntries() instead of spilling them to disk. Remove the argument and size the heap for the live vector " + "set instead." + ) + return create_vector_index( + self, + vertex_type, + vector_property, + dimensions, + id_property, + distance_function, + max_connections, + beam_width, + quantization, + encoding, + graph_build_cache_size, + mutations_before_rebuild, + store_vectors_in_graph, + add_hierarchy, + pq_subspaces, + pq_clusters, + pq_center_globally, + pq_training_limit, + build_graph_now, + ) + + def import_documents( + self, + source, + document_type: str = "Document", + *, + file_type=None, + delimiter=None, + header=None, + skip_entries=None, + properties_include=None, + commit_every=None, + parallel=None, + wal=None, + verbose_level=None, + probe_only=None, + force_database_create=None, + trim_text=None, + on_row_error=None, + extra_settings=None, + ): + """Import document-shaped files (CSV, JSON, ...) through the engine's importer; same arguments as the JPype path.""" + from ..importer import ( + ImportResult, + _build_import_settings, + normalize_import_source, + ) + from . import extras + + self._check_not_closed() + source_url = normalize_import_source(source) + settings = _build_import_settings( + source_url, + document_type=document_type, + file_type=file_type, + delimiter=delimiter, + header=header, + skip_entries=skip_entries, + properties_include=properties_include, + commit_every=commit_every, + parallel=parallel, + wal=wal, + verbose_level=verbose_level, + probe_only=probe_only, + force_database_create=force_database_create, + trim_text=trim_text, + on_row_error=on_row_error, + extra_settings=extra_settings, + ) + try: + stats = extras.run_import(self, settings, parallel, commit_every, wal) + return ImportResult( + result="PROBE_ONLY" if probe_only and stats is None else "OK", + operation="import documents", + source_url=source_url, + statistics=stats or {}, + ) + except Exception as e: + raise ArcadeDBError(f"Document import failed: {e}") from e + + def export_database( + self, + file_path, + format: str = "jsonl", + overwrite: bool = False, + include_types=None, + exclude_types=None, + verbose: int = 1, + ) -> dict: + self._check_not_closed() + from ..exporter import export_database + + return export_database( + self, file_path, format, overwrite, include_types, exclude_types, verbose + ) + + def export_to_csv( + self, query: str, file_path: str, language: str = "sql", fieldnames=None + ): + self._check_not_closed() + from ..exporter import export_to_csv + + export_to_csv(self.query(language, query), file_path, fieldnames) + + def insert_columns( + self, type_name, columns, commit_every: int = 10_000, parallel: bool = False + ) -> int: + """Bulk insert from whole columns (dict of name -> numpy array or list), as rows through :meth:`insert_many`.""" + self._check_not_closed() + items = list(columns.items()) if hasattr(columns, "items") else None + if not items: + raise ValueError("insert_columns needs at least one column") + for name, values in items: + if not isinstance(name, str): + raise ValueError(f"column names must be strings, got {name!r}") + if isinstance(values, (str, bytes, bytearray)): + raise TypeError( + f"column {name!r} is a {type(values).__name__}, not a sequence of values; " + f"wrap a single value in a list" + ) + known = {n: len(v) for n, v in items if hasattr(v, "__len__")} + if len(set(known.values())) > 1: + raise ValueError(f"columns differ in length: {known}") + if not parallel: + from .bulk import insert_columns_fast + + done = insert_columns_fast(self, type_name, columns, commit_every) + if done is not None: + return done + names = [n for n, _ in items] + data = [_column_values(n, v) for n, v in items] + lengths = {n: len(c) for n, c in zip(names, data)} + if len(set(lengths.values())) != 1: + raise ValueError(f"columns differ in length: {lengths}") + if not data[0]: + return 0 + rows = (dict(zip(names, vals)) for vals in zip(*data)) + return self.insert_many(type_name, rows, commit_every, parallel) + + def insert_dataframe( + self, type_name, df, *, batch_size: int = 10_000, parallel: bool = False + ) -> int: + """A pandas or Polars DataFrame into a type, in batches: the strict Arrow loading of ``Database.insert_dataframe`` + (dtype refusals, declared types, NaN rules, the committed-rows message), batches through :meth:`insert_many` for a + document type and ``GraphBatch.create_vertices`` for a vertex type (``arcadedb/dataframe.py``). + """ + from ..dataframe import insert_table, to_arrow_table + + self._check_not_closed() + table = to_arrow_table(df, what="insert_dataframe()") + return insert_table( + self, + type_name, + table, + batch_size=batch_size, + parallel=parallel, + what="insert_dataframe()", + ) + + def insert_arrow( + self, type_name, table, *, batch_size: int = 10_000, parallel: bool = False + ) -> int: + """An Arrow table (or ``RecordBatch``, ``RecordBatchReader``, Arrow PyCapsule object) into a type, in batches; as + :meth:`insert_dataframe`. + """ + from ..dataframe import insert_table, to_arrow_table + + self._check_not_closed() + arrow_table = to_arrow_table(table, what="insert_arrow()") + return insert_table( + self, + type_name, + arrow_table, + batch_size=batch_size, + parallel=parallel, + what="insert_arrow()", + ) + + def set_wal_flush(self, mode: str): + self._check_not_closed() + valid = {"no": "NO", "yes_nometadata": "YES_NOMETADATA", "yes_full": "YES_FULL"} + if mode not in valid: + raise ValueError( + f"Invalid WAL flush mode: {mode}. Must be one of: {list(valid)}" + ) + from .extras import db_set + + try: + db_set(self, "wal_flush", valid[mode]) + except ArcadeDBError as e: + raise ArcadeDBError(f"Failed to set WAL flush mode: {e}") from e + + def set_read_your_writes(self, enabled: bool): + self._check_not_closed() + from .extras import db_set + + db_set(self, "read_your_writes", "true" if enabled else "false") + + def is_read_your_writes(self) -> bool: + self._check_not_closed() + from .extras import db_set + + return bool(db_set(self, "read_your_writes")["v"]) + + def set_auto_transaction(self, enabled: bool): + self._check_not_closed() + from .extras import db_set + + db_set(self, "auto_transaction", "true" if enabled else "false") + + +def _column_values(name, values): + """One ``insert_columns`` column as a list of Python values, with the JPype path's refusals (a dtype that does not + cross, a value beyond the 64-bit signed range, a column that is not one-dimensional). + """ + try: + import numpy as np + except ImportError: # pragma: no cover + np = None + if hasattr(values, "to_numpy") and not isinstance(values, (list, tuple)): + dtype = getattr(values, "dtype", None) + if np is not None and isinstance(dtype, np.dtype): + values = values.to_numpy() + else: # a pandas extension dtype (Int64, string, boolean, category): its is a null + values = values.to_numpy(dtype=object, na_value=None) + if np is not None and isinstance(values, np.ndarray): + if values.ndim != 1: + raise ValueError( + f"column {name!r} is {values.ndim}-dimensional; a column is one-dimensional" + ) + kind = values.dtype.kind + if kind == "u" and values.size and int(values.max()) > np.iinfo(np.int64).max: + raise ValueError( + f"column {name!r} holds a value beyond the 64-bit signed range" + ) + if kind in "MmcV": + raise TypeError( + f"column {name!r} has dtype {values.dtype}, which does not cross natively; " + "convert it to Python values or use insert_many" + ) + return values.tolist() + return list(values) + + +class _JavaObject: + """What the JPype Schema returns where the native one has no Java object: any use says so.""" + + def __bool__(self): + return True + + def __getattr__(self, name): + not_yet(f"the Java object a Schema method returns ({name})") + + +class NativeSchema: + """Type, property and index creation as SQL statements. Returns None where the JPype Schema returns a Java object.""" + + def __init__(self, db): + self._db = db + + def _run(self, sql): + self._db.command("sql", sql).close() + + def _type(self, kind, name, buckets, if_not_exists=False): + ine = " IF NOT EXISTS" if if_not_exists else "" + suffix = f" BUCKETS {int(buckets)}" if buckets else "" + self._run(f"CREATE {kind} TYPE `{name}`{ine}{suffix}") + return _JavaObject() + + def create_document_type(self, name, buckets=None): + return self._type("DOCUMENT", name, buckets) + + def create_vertex_type(self, name, buckets=None): + return self._type("VERTEX", name, buckets) + + def create_edge_type(self, name, buckets=None): + return self._type("EDGE", name, buckets) + + def get_or_create_document_type(self, name, buckets=None): + return self._type("DOCUMENT", name, buckets, True) + + def get_or_create_vertex_type(self, name, buckets=None): + return self._type("VERTEX", name, buckets, True) + + def get_or_create_edge_type(self, name, buckets=None): + return self._type("EDGE", name, buckets, True) + + def exists_type(self, name): + rows = self._db.query( + "sql", "SELECT FROM schema:types WHERE name = ?", name + ).to_list() + return bool(rows) + + def drop_type(self, name): + self._run(f"DROP TYPE `{name}`") + + def create_property(self, type_name, property_name, property_type, of_type=None): + t = getattr(property_type, "value", property_type) + o = f" OF {getattr(of_type, 'value', of_type)}" if of_type else "" + self._run(f"CREATE PROPERTY `{type_name}`.`{property_name}` {t}{o}") + return _JavaObject() + + def get_or_create_property( + self, type_name, property_name, property_type, of_type=None + ): + t = getattr(property_type, "value", property_type) + o = f" OF {getattr(of_type, 'value', of_type)}" if of_type else "" + self._run( + f"CREATE PROPERTY `{type_name}`.`{property_name}` IF NOT EXISTS {t}{o}" + ) + return _JavaObject() + + def drop_property(self, type_name, property_name): + self._run(f"DROP PROPERTY `{type_name}`.`{property_name}`") + + _INDEX_KIND = { + ("LSM_TREE", True): "UNIQUE", + ("LSM_TREE", False): "NOTUNIQUE", + ("HASH", True): "UNIQUE_HASH", + ("HASH", False): "NOTUNIQUE_HASH", + } + + def create_index( + self, type_name, property_names, unique=False, index_type="LSM_TREE" + ): + kind = getattr(index_type, "value", index_type) + kind = self._INDEX_KIND.get((kind, bool(unique)), kind) + cols = ", ".join(f"`{p}`" for p in property_names) + self._run(f"CREATE INDEX ON `{type_name}` ({cols}) {kind}") + return _JavaObject() + + def exists_index(self, index_name): + return bool( + self._db.query( + "sql", "SELECT FROM schema:indexes WHERE name = ?", index_name + ).to_list() + ) + + def drop_index(self, index_name, force=False): + self._run(f"DROP INDEX `{index_name}`") + + # -- reads, as small objects with the engine's accessors (schema_objects.py) + + def get_type(self, name): + from .schema_objects import NativeType + + rows = self._db.query( + "sql", "SELECT FROM schema:types WHERE name = ?", name + ).to_list() + return NativeType(rows[0]) if rows else None + + def get_types(self): + from .schema_objects import NativeType + + return [ + NativeType(r) + for r in self._db.query("sql", "SELECT FROM schema:types").to_list() + ] + + def get_indexes(self): + from .schema_objects import NativeIndex + + return [ + NativeIndex(r) + for r in self._db.query("sql", "SELECT FROM schema:indexes").to_list() + ] + + def get_index_by_name(self, index_name): + from .schema_objects import NativeIndex + + rows = self._db.query( + "sql", "SELECT FROM schema:indexes WHERE name = ?", index_name + ).to_list() + return NativeIndex(rows[0]) if rows else None + + def get_or_create_index( + self, type_name, property_names, unique=False, index_type="LSM_TREE" + ): + kind = getattr(index_type, "value", index_type) + kind = self._INDEX_KIND.get((kind, bool(unique)), kind) + cols = ", ".join(f"`{p}`" for p in property_names) + try: + self._run(f"CREATE INDEX IF NOT EXISTS ON `{type_name}` ({cols}) {kind}") + except ArcadeDBError as e: + raise ArcadeDBError( + f"Failed to get/create index on '{type_name}[{','.join(property_names)}]': {e}" + ) from e + name = f"{type_name}[{','.join(property_names)}]" + return self.get_index_by_name(name) or _JavaObject() + + def _vector_rows(self): + # the type-level rows (named Type[prop]); the per-bucket rows carry a file id + return [ + r + for r in self._db.query( + "sql", "SELECT FROM schema:indexes WHERE indexType = 'LSM_VECTOR'" + ).to_list() + if "[" in r["name"] and "fileId" not in r + ] + + def get_vector_index(self, vertex_type, vector_property): + from .schema_objects import NativeVectorIndex + + for r in self._vector_rows(): + if r["typeName"] == vertex_type and r["properties"] == [[vector_property]]: + return NativeVectorIndex( + self._db, vertex_type, vector_property, r["name"] + ) + return None + + def list_vector_indexes(self): + return [r["name"] for r in self._vector_rows()] + + def __getattr__(self, name): + if name.startswith("_"): + raise AttributeError(name) + not_yet(f"Schema.{name} (returns Java objects)") + + +class NativeDatabaseFactory: + """Create/open databases by path.""" + + def __init__(self, path: str, jvm_kwargs: Optional[dict] = None): + if jvm_kwargs: + import logging + + logging.getLogger("arcadedb").warning( + "jvm_kwargs %s ignored: the native backend has no JVM", + sorted(jvm_kwargs), + ) + self._path = os.fspath(path) + + def create(self) -> NativeDatabase: + return NativeDatabase( + _call("Failed to create database", "arcadedb_n_open", lib.b(self._path), 0)[ + "h" + ] + ) + + def open(self, read_only: bool = False) -> NativeDatabase: + mode = 2 if read_only else 1 + return NativeDatabase( + _call( + "Failed to open database", "arcadedb_n_open", lib.b(self._path), mode + )["h"] + ) + + def exists(self) -> bool: + return _call( + "Failed to check if database exists", + "arcadedb_n_open", + lib.b(self._path), + 3, + )["exists"] diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py new file mode 100644 index 0000000..69c6789 --- /dev/null +++ b/src/arcadedb/_native/extras.py @@ -0,0 +1,819 @@ +"""Native backend coverage beyond the DSL core: GraphBatch, the async executor, database settings, import/export and Python +functions in SQL, over the ``arcadedb_n_gb*``, ``arcadedb_n_async``, ``arcadedb_n_dbset``, ``arcadedb_n_import``, +``arcadedb_n_export`` and ``arcadedb_n_udf_register`` entry points (``NativeExtras.java``). + +Callbacks (async completions, error handlers, Python functions called from SQL) go through ONE ctypes function pointer that +``_ensure_upcall`` registers with the library: the engine thread (or an async writer thread) calls ``_dispatch(id, json)``, +ctypes takes the GIL for the call, and the reply is JSON (see the header of ``NativeExtras.java`` for GIL and exception rules). +""" + +import ctypes +import itertools +import json +import threading +from collections.abc import Mapping +from typing import Any, Callable, Iterable, Optional + +from ..exceptions import ArcadeDBError +from ..graph_batch import GraphBatch as _BaseGraphBatch +from . import lib, not_yet +from .codec import _PLAIN, _default, _prep, loads + +# -------------------------------------------------------------------------------------------------------- upcall + +_UPCALL_TYPE = ctypes.CFUNCTYPE( + ctypes.c_void_p, ctypes.c_longlong, ctypes.c_char_p +) # (Java long id, char *json) +_callbacks: dict = {} +_callback_ids = itertools.count(1) +_callbacks_lock = threading.Lock() +_reply = threading.local() +_upcall_ref = None + + +def _enc(value) -> str: + """JSON text of a value with the tagged encoding of query parameters.""" + return json.dumps(_prep(value), default=_default) + + +def _dispatch(cid, raw): + try: + fn = _callbacks.get(cid) + message = loads(raw.decode("utf-8")) if raw else {} + reply = fn(message) if fn is not None else None + data = None if reply is None else _enc(reply).encode("utf-8") + except BaseException as exc: # nosec B110 - an exception must not cross the C frame + data = json.dumps({"error": f"{type(exc).__name__}: {exc}"}).encode("utf-8") + if data is None: + return None + # kept alive until this thread's next callback: Java copies the text before it returns + buf = ctypes.create_string_buffer(data) + _reply.buf = buf + return ctypes.addressof(buf) + + +def _ensure_upcall() -> None: + global _upcall_ref + if _upcall_ref is not None: + return + lib.call_void( + "arcadedb_n_set_upcall", ctypes.cast(_make_pointer(), ctypes.c_void_p) + ) + + +def _make_pointer(): + global _upcall_ref + _upcall_ref = _UPCALL_TYPE(_dispatch) + return _upcall_ref + + +def _register(fn: Callable) -> int: + _ensure_upcall() + with _callbacks_lock: + cid = next(_callback_ids) + _callbacks[cid] = fn + return cid + + +def _error_from(message: dict) -> ArcadeDBError: + reply = message.get("err") or {} + exc = ArcadeDBError(f"{reply.get('cls')}: {reply.get('error')}") + exc.native = reply + exc.java_chain = reply.get("chain") or [] + return exc + + +def _rid_of(value) -> str: + """The RID text of a record wrapper, a RID or its text.""" + get_rid = getattr(value, "get_rid", None) + return get_rid() if get_rid is not None else str(value) + + +def _call(prefix: str, name: str, *args) -> dict: + from .database import _call as call + + return call(prefix, name, *args) + + +# -------------------------------------------------------------------------------------------------------- GraphBatch + + +class NativeGraphBatch(_BaseGraphBatch): + """``GraphBatch`` over the engine's batch builder in the image (same names and behaviour as the JPype wrapper).""" + + _BULK_CHUNK = 10_000 # rows per crossing: small enough that a chunk's text, rows and reply die in the young generation + + def __init__(self, database, handle: int): + self._db = database + self._gb = handle + self._closed = False + + @classmethod + def create(cls, database, **options) -> "NativeGraphBatch": + wal_flush = options.get("wal_flush") + if wal_flush is not None: + options["wal_flush"] = cls._to_java_wal_flush_name(wal_flush) + given = {k: v for k, v in options.items() if v is not None} + try: + reply = _call( + "Failed to create GraphBatch", + "arcadedb_n_gb_create", + database._h, + lib.b(_enc(given)), + ) + except ArcadeDBError: + raise + return cls(database, reply["h"]) + + @classmethod + def _to_java_wal_flush_name(cls, mode: str) -> str: + normalized = mode.lower() + if normalized not in cls._VALID_WAL_FLUSH_MODES: + raise ValueError( + f"Invalid wal_flush mode: {mode}. Must be one of: {list(cls._VALID_WAL_FLUSH_MODES.keys())}" + ) + return cls._VALID_WAL_FLUSH_MODES[normalized] + + def _op(self, prefix: str, op: str, **args) -> dict: + return _call(prefix, "arcadedb_n_gb", self._gb, lib.b(op), lib.b(_enc(args))) + + def new_vertex(self, type_name: str): + from .records import Vertex + + self._check_not_closed() + return Vertex(self._db, type_name) + + def create_vertex(self, type_name: str, **properties): + from .records import Vertex + + self._check_not_closed() + db = self._db + started = False + try: + if not db.is_transaction_active(): + started = True + db.begin() + reply = self._op( + f"Failed to create batch vertex of type '{type_name}'", + "create_vertex", + type=type_name, + props=self._to_java_varargs(properties), + ) + if started: + db.commit() + return Vertex(db, type_name, reply["rid"], properties) + except BaseException as e: + if started: + try: + if db.is_transaction_active(): + db.rollback() + except ( + Exception + ): # nosec B110 - the original error is the one to report + pass + if isinstance(e, ArcadeDBError) or not isinstance(e, Exception): + raise + raise ArcadeDBError( + f"Failed to create batch vertex of type '{type_name}': {e}" + ) from e + + def _to_java_varargs(self, properties: dict) -> dict: + """The properties of one vertex as the library takes them (the step the JPype path converts to Java values in).""" + if not isinstance(properties, dict): + raise TypeError("vertex properties must be a dict") + for key in properties: + if type(key) is not str: + raise TypeError(f"property names must be str, got {key!r}") + return properties + + def _create_vertices_json_bulk(self, type_name: str, rows): + """The bulk path (JSON rows in, RIDs out): the RIDs, or None for rows JSON would change (the caller falls back).""" + from ..type_conversion import json_bulk_scalar_ok + + for row in rows: + for key, value in (row or {}).items(): + if type(key) is not str or not json_bulk_scalar_ok(value): + return None + return self._op( + f"Failed to create batch vertices for type '{type_name}'", + "create_vertices", + type=type_name, + rows=[r or {} for r in rows], + )["rids"] + + def create_vertices(self, type_name: str, count_or_properties) -> list: + self._check_not_closed() + db = self._db + owns_transaction = not db.is_transaction_active() + prefix = f"Failed to create batch vertices for type '{type_name}'" + try: + chunk = self._BULK_CHUNK + if isinstance(count_or_properties, int): + rids = [] + for start in range(0, count_or_properties, chunk): + rids.extend( + self._op( + prefix, + "create_vertices", + type=type_name, + count=min(chunk, count_or_properties - start), + )["rids"] + ) + if not count_or_properties: + rids = self._op(prefix, "create_vertices", type=type_name, count=0)[ + "rids" + ] + return rids + rows = list(count_or_properties) + for row in rows: + if row is not None and not isinstance(row, dict): + raise TypeError( + "create_vertices() property rows must be dicts or None" + ) + # chunked, as the JPype wrapper does: one crossing, one JSON text and one reply per chunk, so a 1M-vertex load + # never holds the text, the parsed rows and the reply of all of it at once (the Serial GC copies what lives) + rids = [] + for start in range(0, len(rows), chunk): + rids.extend( + self._op( + prefix, + "create_vertices", + type=type_name, + rows=[r or {} for r in rows[start : start + chunk]], + )["rids"] + ) + if not rows: + rids = self._op(prefix, "create_vertices", type=type_name, rows=[])[ + "rids" + ] + return rids + except BaseException as e: + if owns_transaction: + try: + if db.is_transaction_active(): + db.rollback() + except Exception: # nosec B110 + pass + if isinstance(e, (TypeError, ArcadeDBError)) or not isinstance( + e, Exception + ): + raise + raise ArcadeDBError(f"{prefix}: {e}") from e + + @staticmethod + def _rid_text(value) -> str: + rid = getattr(value, "get_rid", None) + if rid is not None: + return rid() + return str(value) + + def new_edge( + self, + source_vertex_or_rid, + edge_type: str, + destination_vertex_or_rid, + **properties, + ): + self._check_not_closed() + self._op( + f"Failed to buffer batch edge '{edge_type}'", + "new_edge", + src=self._rid_text(source_vertex_or_rid), + type=edge_type, + dst=self._rid_text(destination_vertex_or_rid), + props=properties, + ) + return self + + def new_edges(self, source_rids, edge_type: str, destination_rids, properties=None): + self._check_not_closed() + source_rids = [self._rid_text(r) for r in source_rids] + destination_rids = [self._rid_text(r) for r in destination_rids] + if properties is not None: + properties = list(properties) + lengths = [len(source_rids), len(destination_rids)] + if properties is not None: + lengths.append(len(properties)) + if len(set(lengths)) != 1: + names = ["source_rids", "destination_rids", "properties"] + got = ", ".join(f"{n} {k}" for n, k in zip(names, lengths)) + raise ArcadeDBError( + f"Failed to buffer batch edges '{edge_type}': source_rids, destination_rids and properties must have " + f"the same length ({got})" + ) + for p in properties or (): + for key in p or (): + if not isinstance(key, str): + raise ArcadeDBError( + f"Failed to buffer batch edges '{edge_type}': property names must be str, got {key!r}" + ) + chunk = self._BULK_CHUNK + if properties is None: + # no properties: the RIDs of a chunk go over as two ';'-joined texts (as EdgeBatcher.newEdgesJoined takes them on the + # JPype path) and are read straight into RIDs, not as two JSON lists of Strings + for start in range(0, len(source_rids), chunk): + self._op( + f"Failed to buffer batch edges '{edge_type}'", + "new_edges", + src=";".join(source_rids[start : start + chunk]), + type=edge_type, + dst=";".join(destination_rids[start : start + chunk]), + props=None, + ) + return self + for start in range(0, len(source_rids), chunk): + self._op( + f"Failed to buffer batch edges '{edge_type}'", + "new_edges", + src=source_rids[start : start + chunk], + type=edge_type, + dst=destination_rids[start : start + chunk], + props=( + None + if properties is None + else [p or {} for p in properties[start : start + chunk]] + ), + ) + return self + + def flush(self): + self._check_not_closed() + self._op("Failed to flush GraphBatch", "flush") + return self + + def close(self): + if self._closed: + return + refused = False + try: + self._op("Failed to close GraphBatch", "close") + except ArcadeDBError as e: + # an IllegalStateException while the caller has a transaction open: the batch is still open with its work pending + refused = (getattr(e, "native", None) or {}).get( + "cls" + ) == "java.lang.IllegalStateException" and bool( + self._db.is_transaction_active() + ) + raise + finally: + if not refused: + self._closed = True + + def get_total_edges_created(self) -> int: + self._check_not_closed() + return int(self._op("Failed to read GraphBatch", "stats")["total_edges"]) + + def get_buffered_edge_count(self) -> int: + self._check_not_closed() + return int(self._op("Failed to read GraphBatch", "stats")["buffered"]) + + def get_deferred_incoming_edge_count(self) -> int: + self._check_not_closed() + return int(self._op("Failed to read GraphBatch", "stats")["deferred"]) + + +# -------------------------------------------------------------------------------------------------------- async + + +class NativeAsyncExecutor: + """The database's async executor (one per database) with the JPype wrapper's names; callbacks run on the engine's writer + threads and take the GIL, as the JPype proxies do.""" + + def __init__(self, database): + self._db = database + self._closed = False + self._owner = database + self._cb_ids: list = [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def is_closed(self) -> bool: + return self._closed + + def _op(self, op: str, **args) -> dict: + return _call( + "Async executor failed", + "arcadedb_n_async", + self._db._h, + lib.b(op), + lib.b(_enc(args)), + ) + + def _settings(self) -> dict: + return self._op("get") + + def _set(self, key, value): + self._op("set", key=key, value=value) + return self + + # configuration + def set_parallel_level(self, level: int): + if level < 1: + raise ValueError("parallel_level must be at least 1") + return self._set("parallel", level) + + def set_commit_every(self, count: int): + if count < 1: + raise ValueError( + f"commit_every must be >= 1 (got {count}); the async executor always commits in batches" + ) + return self._set("commit_every", count) + + def set_transaction_use_wal(self, use_wal: bool): + return self._set("wal", bool(use_wal)) + + def set_transaction_sync(self, sync_mode: str): + valid = {"no": "NO", "yes_nometadata": "YES_NOMETADATA", "yes_full": "YES_FULL"} + if sync_mode not in valid: + raise ValueError(f"sync_mode must be one of {set(valid)}, got {sync_mode}") + return self._set("sync", valid[sync_mode]) + + def set_back_pressure(self, percentage: int): + if percentage < 0 or percentage > 100: + raise ValueError("back_pressure must be between 0 and 100") + return self._set("back_pressure", percentage) + + def get_parallel_level(self) -> int: + return int(self._settings()["parallel"]) + + def get_back_pressure(self) -> int: + return int(self._settings()["back_pressure"]) + + def get_commit_every(self) -> int: + return int(self._settings()["commit_every"]) + + def is_transaction_use_wal(self) -> bool: + return bool(self._settings()["wal"]) + + def get_transaction_sync(self) -> str: + return self._settings()["sync"].lower() + + def get_thread_count(self) -> int: + return int(self._settings()["threads"]) + + def is_processing(self) -> bool: + return bool(self._settings()["processing"]) + + def is_pending(self) -> bool: + return self.is_processing() + + def kill(self): + self._op("kill") + + # work + def _record_callback(self, callback, kind="d"): + from .records import _wrap_dict + + def handler(message): + callback(_wrap_dict(self._db, message["rec"], kind)) + + return _register(handler) + + def _error_callback(self, callback): + return _register(lambda message: callback(_error_from(message))) + + def create_record(self, document, callback=None, error_callback=None): + kind = "v" if hasattr(document, "get_out_edges") else "d" + cb = self._record_callback(callback, kind) if callback is not None else 0 + err = self._error_callback(error_callback) if error_callback is not None else 0 + self._op( + "create_record", + type=document._type, + kind=kind, + props=document._props, + cb=cb, + err=err, + ) + + def _statement(self, op, language, text, callback, args, params, error_callback): + from .database import NativeResultSet + + args = tuple(args or ()) + if args and params: + raise ValueError("Use either positional args or named params, not both") + cb = 0 + if callback is not None: + + def on_result(message): + # the JPype wrapper calls the callback once per row of the completed statement + for row in NativeResultSet(self._db, message["reply"]): + callback(row) + + cb = _register(on_result) + err = self._error_callback(error_callback) if error_callback is not None else 0 + if params: + values = dict(params) + elif args: + values = ( + dict(args[0]) + if len(args) == 1 and isinstance(args[0], Mapping) + else list(args) + ) + else: + values = None + self._op(op, lang=language, text=text, params=values, cb=cb, err=err) + + def query( + self, language, query_text, callback, args=None, error_callback=None, **params + ): + self._statement( + "query", language, query_text, callback, args, params, error_callback + ) + + def command( + self, + language, + command_text, + callback=None, + args=None, + error_callback=None, + **params, + ): + self._statement( + "command", language, command_text, callback, args, params, error_callback + ) + + def append_samples( + self, type_name, timestamps, *column_values, primitive: bool = False + ): + def plain(values): + kind = getattr(getattr(values, "dtype", None), "kind", None) + if ( + kind == "b" + ): # a numpy bool array is a 0/1 numeric column (a list of bools is not: the engine refuses it) + values = values.astype("int64") + return values.tolist() if hasattr(values, "tolist") else list(values) + + stamps = [int(t) for t in plain(timestamps)] + if primitive: + # the engine's primitive TimeSeriesBatch is a JVM-boundary optimisation (no boxing per value); here the rows cross + # as one JSON text either way, so the columns take the same path, with the primitive path's length check + for index, values in enumerate(column_values): + if len(values) != len(stamps): + raise ValueError( + f"column {index} has {len(values)} values for {len(stamps)} timestamps" + ) + self._op( + "append_samples", + type=type_name, + ts=stamps, + cols=[plain(c) for c in column_values], + ) + + def _edge_callback(self, callback): + """The handler of an edge-created event: ``callback(edge, created_source, created_destination)``.""" + from .records import _wrap_dict + + def handler(message): + callback( + _wrap_dict(self._db, message["edge"], "e"), + bool(message["src"]), + bool(message["dst"]), + ) + + return _register(handler) + + def new_edge( + self, + source_vertex, + edge_type: str, + destination_vertex_or_rid, + light: bool = False, + callback=None, + **properties, + ): + cb = self._edge_callback(callback) if callback is not None else 0 + self._op( + "new_edge", + src=_rid_of(source_vertex), + type=edge_type, + dst=_rid_of(destination_vertex_or_rid), + light=bool(light), + props=properties, + cb=cb, + ) + + def new_edge_by_keys( + self, + source_vertex_type: str, + source_key_names, + source_key_values, + destination_vertex_type: str, + destination_key_names, + destination_key_values, + create_vertex_if_not_exist: bool, + edge_type: str, + bidirectional: bool, + light: bool, + callback=None, + **properties, + ): + def listed(names, values, what): + if isinstance(names, str): + names = [names] + values = list(values) if isinstance(values, (list, tuple)) else [values] + names, values = list(names), list(values) + if len(names) != len(values): + raise ValueError( + f"{what}_key_names and {what}_key_values must have same size" + ) + return names, values + + src_names, src_values = listed(source_key_names, source_key_values, "source") + dst_names, dst_values = listed( + destination_key_names, destination_key_values, "destination" + ) + cb = self._edge_callback(callback) if callback is not None else 0 + self._op( + "new_edge_by_keys", + src_type=source_vertex_type, + src_names=src_names, + src_values=src_values, + dst_type=destination_vertex_type, + dst_names=dst_names, + dst_values=dst_values, + create=bool(create_vertex_if_not_exist), + type=edge_type, + bidirectional=bool(bidirectional), + light=bool(light), + props=properties, + cb=cb, + ) + + def scan_type( + self, + type_name: str, + callback, + polymorphic: bool = True, + error_callback=None, + ): + """Scan every record of a type on the writer threads: ``callback(record) -> bool`` (False stops the scan; None goes on).""" + from .records import _wrap_dict + + def on_record(message): + result = callback(_wrap_dict(self._db, message["rec"], "d")) + return {"v": True if result is None else bool(result)} + + cb = _register(on_record) + err = 0 + if error_callback is not None: + + def on_error(message): + result = error_callback( + message.get("rid"), _error_from({"err": message.get("err")}) + ) + return {"v": True if result is None else bool(result)} + + err = _register(on_error) + self._op( + "scan_type", type=type_name, polymorphic=bool(polymorphic), cb=cb, err=err + ) + + def transaction(self, *args, **kwargs): + not_yet("AsyncExecutor.transaction") + + def wait_completion(self, timeout_ms: Optional[int] = None): + if timeout_ms is None: + self._op("wait") + elif timeout_ms < 0: + raise ValueError(f"timeout_ms must be None or >= 0, got {timeout_ms}") + elif timeout_ms == 0: + if self.is_processing(): + raise TimeoutError("Async operations have not completed (0ms poll)") + elif not self._op("wait", timeout_ms=timeout_ms)["done"]: + raise TimeoutError( + f"Async operations did not complete within {timeout_ms}ms" + ) + + def on_ok(self, callback: Callable[[], None]): + self._op("on_ok", cb=_register(lambda message: callback())) + return self + + def on_error(self, callback: Callable[[Exception], None]): + self._op("on_error", cb=self._error_callback(callback)) + return self + + def close(self): + if self._closed: + return + try: + self._op("close") + finally: + self._closed = True + owner, self._owner = self._owner, None + if owner is not None: + owner._discard_async_executor(self) + + +def insert_parallel(database, type_name: str, rows) -> dict: + """Hand ``rows`` (a list of dicts) to the async writers; returns the failure token for :func:`failures`.""" + return _call( + "Failed to bulk-insert", + "arcadedb_n_async", + database._h, + b"insert_parallel", + lib.b(_enc({"type": type_name, "rows": rows})), + ) + + +def failures(database, token: int) -> dict: + return _call( + "Failed to read the parallel writers", + "arcadedb_n_async", + database._h, + b"failures", + lib.b(_enc({"fid": token})), + ) + + +# -------------------------------------------------------------------------------------------------------- settings + + +def db_set(database, op: str, value: str = "") -> dict: + return _call( + f"Failed to set {op}", "arcadedb_n_dbset", database._h, lib.b(op), lib.b(value) + ) + + +# -------------------------------------------------------------------------------------------------------- import / export + + +def import_documents(database, settings: dict) -> Optional[dict]: + from ..importer import _stringify_import_setting + + text = json.dumps({k: _stringify_import_setting(v) for k, v in settings.items()}) + return _call("Failed to import", "arcadedb_n_import", database._h, lib.b(text))[ + "stats" + ] + + +def run_import(database, settings: dict, parallel, commit_every, wal) -> Optional[dict]: + """The importer with the async executor's settings overridden for the run and restored after it (as the JPype path does).""" + executor = database._async() + saved = executor._settings() + saved_ryw = database.is_read_your_writes() + try: + database.set_read_your_writes(False) + if parallel is not None: + executor.set_parallel_level(parallel) + if commit_every is not None: + executor.set_commit_every(commit_every) + if wal is not None: + executor.set_transaction_use_wal(wal) + return import_documents(database, settings) + finally: + try: + executor.wait_completion() + finally: + database.set_read_your_writes(saved_ryw) + executor._set("parallel", saved["parallel"]) + executor._set("commit_every", saved["commit_every"]) + executor._set("wal", saved["wal"]) + + +def export_database( + database, path: str, fmt: str, overwrite: bool, settings: dict +) -> dict: + return _call( + "Database export failed", + "arcadedb_n_export", + database._h, + lib.b(path), + lib.b(fmt), + 1 if overwrite else 0, + lib.b(json.dumps(settings)), + )["stats"] + + +# -------------------------------------------------------------------------------------------------------- Python functions in SQL + + +def register_function( + name: str, fn: Callable, min_args: int = 0, max_args: int = -1 +) -> None: + """Make ``fn`` callable from SQL as ``name(..)``: ``SELECT name(a, b) FROM T``. + + ``fn`` receives the argument values as Python values and returns a value of a type a parameter can carry (str, int, float, + bool, None, list, dict, Decimal, datetime). An exception raised by ``fn`` fails the statement with its text. Registration is + process-wide (the engine's function factory), like any engine function; ``max_args`` < 0 means any number. + """ + + def handler(message): + try: + return {"v": fn(*message.get("args", []))} + except Exception as exc: + return {"error": f"{type(exc).__name__}: {exc}"} + + cid = _register(handler) + _call( + "Failed to register the function", + "arcadedb_n_udf_register", + lib.b(name), + cid, + int(min_args), + int(max_args if max_args >= 0 else 2**31 - 1), + ) diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py new file mode 100644 index 0000000..b2f16ac --- /dev/null +++ b/src/arcadedb/_native/lib.py @@ -0,0 +1,338 @@ +"""ctypes loader and call layer for ``libarcadedb`` (native-image shared library).""" + +import ctypes +import json +import os +import sys +import threading +from pathlib import Path + +from ..exceptions import ArcadeDBError + +_LIB_NAMES = { + "linux": "libarcadedb.so", + "darwin": "libarcadedb.dylib", + "win32": "libarcadedb.dll", +} + +# ARCADEDB_NATIVE_BINARY=0 turns off the float32 side buffer for numpy float parameters (phase 6: on by default when the library has it) +BINARY_PARAMS = os.environ.get("ARCADEDB_NATIVE_BINARY", "1") != "0" + +# phase 8 switches (each defaults on when the library has the entry point; 0 turns one off, for A/B runs in one image): +# ARCADEDB_NATIVE_CMD: commands go through arcadedb_n_cmd (cursor id back as a long: no reply text, no free, no JSON decode) +# ARCADEDB_NATIVE_CLOSE_BATCH: result sets closed by Python are closed 64 at a time in one crossing (0: one crossing each) +# ARCADEDB_NATIVE_FASTENC: positional parameters of plain scalars are encoded by the cached json encoder, no default hook +CMD_PATH = os.environ.get("ARCADEDB_NATIVE_CMD", "1") != "0" +CLOSE_BATCH = int(os.environ.get("ARCADEDB_NATIVE_CLOSE_BATCH", "64")) +FAST_ENCODE = os.environ.get("ARCADEDB_NATIVE_FASTENC", "1") != "0" +INSERT_ROWS = os.environ.get("ARCADEDB_NATIVE_INSERT_ROWS", "1") != "0" +# ARCADEDB_NATIVE_EXEC_MANY: the rows of an insert_many chunk that JSON text cannot carry unchanged go in one crossing (arcadedb_n_exec_many), not one call each +EXEC_MANY = os.environ.get("ARCADEDB_NATIVE_EXEC_MANY", "1") != "0" + +_lock = threading.Lock() +_lib = None +_isolate = None +_main_thread = None +_creator = None +_local = threading.local() + + +def library_path() -> Path: + """``ARCADEDB_NATIVE_LIB`` if set, else the library next to this module.""" + env = os.environ.get("ARCADEDB_NATIVE_LIB") + if env: + return Path(env) + name = _LIB_NAMES.get(sys.platform, "libarcadedb.so") + return Path(__file__).with_name(name) + + +class _Thread: + """The isolate thread of one Python thread; detached when the Python thread ends.""" + + def __init__(self, handle, owned): + self.handle = handle + self.owned = owned + + def __del__(self): + if self.owned and _lib is not None: + try: + _lib.graal_detach_thread(self.handle) + except Exception: # nosec B110 - interpreter shutdown + pass + + +def _check_cpu() -> None: + """An image built for x86-64-v3 (AVX2, BMI2, FMA) needs them: fail with a clear error instead of SIGILL on an older CPU.""" + if sys.platform != "linux" or os.environ.get("ARCADEDB_NATIVE_SKIP_CPU_CHECK"): + return + try: # the arcadedb wheel (native build) records its image's -march; a source checkout uses a locally built (default v3) image + from .._build_info import NATIVE_MARCH + except ImportError: + NATIVE_MARCH = "x86-64-v3" + if NATIVE_MARCH == "compatibility": + return # baseline x86-64: runs on every x86-64 CPU + import platform + + if platform.machine() not in ("x86_64", "AMD64"): + return + try: + with open("/proc/cpuinfo") as f: + flags = next((ln for ln in f if ln.startswith("flags")), "").split() + except OSError: + return + missing = [x for x in ("avx2", "bmi2", "fma") if flags and x not in flags] + if missing: + raise ArcadeDBError( + f"The native library needs an x86-64-v3 CPU (AVX2, BMI2, FMA); this CPU lacks {', '.join(missing)}. " + "Use the arcadedb-jvm package instead, or set ARCADEDB_NATIVE_SKIP_CPU_CHECK=1 at your own risk." + ) + + +def _load(): + global _lib, _isolate, _main_thread, _creator + with _lock: + if _lib is not None: + return _lib + path = library_path() + if not path.exists(): + raise ArcadeDBError( + f"Native library not found at {path}. Build it from the native-spike workflow or set " + "ARCADEDB_NATIVE_LIB." + ) + _check_cpu() + # jvector's Highway library (libjvector-.so) is looked for next to the engine library + os.environ.setdefault("ARCADEDB_NATIVE_LIBDIR", str(path.resolve().parent)) + lib = ctypes.CDLL(str(path)) + isolate = ctypes.c_void_p() + thread = ctypes.c_void_p() + lib.graal_create_isolate.argtypes = [ + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(ctypes.c_void_p), + ] + lib.graal_create_isolate.restype = ctypes.c_int + params = None + iso_args = os.environ.get("ARCADEDB_NATIVE_ISOLATE_ARGS", "").split() + if iso_args: + # run-time image options for the isolate (e.g. "-XX:InitialCollectionPolicy=Adaptive -XX:MaximumYoungGenerationSizePercent=20") + class _Params(ctypes.Structure): + _fields_ = [ + ("version", ctypes.c_int), + ("reserved_address_space_size", ctypes.c_size_t), + ("auxiliary_image_path", ctypes.c_char_p), + ("auxiliary_image_reserved_space_size", ctypes.c_size_t), + ("argc", ctypes.c_int), + ("argv", ctypes.POINTER(ctypes.c_char_p)), + ("pkey", ctypes.c_int), + ("ignore_unrecognized_args", ctypes.c_char), + ("_reserved_4", ctypes.c_char), + ("_reserved_5", ctypes.c_char), + ] + + argv = (ctypes.c_char_p * (len(iso_args) + 1))( + b"arcadedb", *[a.encode() for a in iso_args] + ) + params = _Params(version=5, argc=len(argv), argv=argv) + lib.graal_create_isolate.argtypes[0] = ctypes.POINTER(_Params) + rc = lib.graal_create_isolate( + ctypes.byref(params) if params is not None else None, + ctypes.byref(isolate), + ctypes.byref(thread), + ) + if rc != 0: + raise ArcadeDBError(f"graal_create_isolate failed (rc={rc})") + lib.graal_attach_thread.argtypes = [ + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_void_p), + ] + lib.graal_attach_thread.restype = ctypes.c_int + lib.graal_detach_thread.argtypes = [ctypes.c_void_p] + lib.graal_detach_thread.restype = ctypes.c_int + # a Java long is 64 bits on every platform; C long is 32 bits on Windows, so L is long long (the same size as c_long elsewhere) + P, L, I, S = ctypes.c_void_p, ctypes.c_longlong, ctypes.c_int, ctypes.c_char_p + sigs = { + "arcadedb_n_open": [P, S, I], + "arcadedb_n_close": [P, L, I], + "arcadedb_n_info": [P, L], + "arcadedb_n_tx": [P, L, I], + "arcadedb_n_exec": [P, L, I, S, S, S, I], + "arcadedb_n_exec_bin": [P, L, I, S, S, S, I, S, L], + "arcadedb_n_insert_cols": [P, L, S, S, S, L], + "arcadedb_n_next": [P, L, I], + "arcadedb_n_rs_close": [P, L], + "arcadedb_n_cmd": [P, L, I, S, S, S, S, L, I], + "arcadedb_n_last_error": [P], + "arcadedb_n_rs_close_many": [P, P, I], + "arcadedb_n_exec_many": [P, L, S, S, S], + "arcadedb_n_insert_rows": [P, L, S, S], + "arcadedb_n_server_start": [P, S], + "arcadedb_n_server_stop": [P, L], + "arcadedb_n_server_db": [P, L, S, I], + "arcadedb_n_diag": [P], + "arcadedb_n_sample_dump": [P], + "arcadedb_n_dbset": [P, L, S, S], + "arcadedb_n_gb_create": [P, L, S], + "arcadedb_n_gb": [P, L, S, S], + "arcadedb_n_async": [P, L, S, S], + "arcadedb_n_import": [P, L, S], + "arcadedb_n_export": [P, L, S, S, I, S], + "arcadedb_n_udf_register": [P, S, L, I, I], + "arcadedb_n_set_upcall": [P, P], + "arcadedb_free": [P, P], + } + for name, argtypes in sigs.items(): + fn = getattr(lib, name, None) + if fn is None: # a library built before this entry point existed + continue + fn.argtypes = argtypes + fn.restype = ( + None + if name + in ( + "arcadedb_free", + "arcadedb_n_rs_close_many", + "arcadedb_n_set_upcall", + ) + else ctypes.c_longlong if name == "arcadedb_n_cmd" else ctypes.c_void_p + ) + global BINARY_PARAMS, CMD_PATH, CLOSE_BATCH, INSERT_ROWS, EXEC_MANY + if not hasattr( + lib, "arcadedb_n_exec_bin" + ): # a library built before the binary path + BINARY_PARAMS = False + if not hasattr(lib, "arcadedb_n_cmd"): # a library built before phase 8 + CMD_PATH = False + if not hasattr(lib, "arcadedb_n_rs_close_many"): + CLOSE_BATCH = 0 + if not hasattr(lib, "arcadedb_n_insert_rows"): + INSERT_ROWS = False + if not hasattr(lib, "arcadedb_n_exec_many"): + EXEC_MANY = False + _isolate = isolate + _creator = threading.get_ident() + _local.thread = _Thread(thread, owned=False) + _lib = lib + return lib + + +def loaded() -> bool: + return _lib is not None + + +def _thread_handle(): + t = getattr(_local, "thread", None) + if t is None: + thread = ctypes.c_void_p() + rc = _lib.graal_attach_thread(_isolate, ctypes.byref(thread)) + if rc != 0: + raise ArcadeDBError(f"graal_attach_thread failed (rc={rc})") + t = _local.thread = _Thread(thread, owned=True) + return t.handle + + +def call(name: str, *args) -> dict: + """Run one entry point; returns the decoded reply, raises ArcadeDBError-free (callers map errors).""" + lib = _lib or _load() + th = _thread_handle() + ptr = getattr(lib, name)(th, *args) + try: + text = ctypes.string_at(ptr).decode("utf-8") + finally: + lib.arcadedb_free(th, ptr) + return text + + +def call_bin(name: str, *args): + """Like :func:`call` for a reply that may carry a float32 tail (``arcadedb_n_exec*`` with flag 4): returns ``(text, floats)``, + floats a float32 numpy array or None. The JSON ends with ``,"bin":N}`` and the N floats follow its NUL, 4-byte aligned. + """ + lib = _lib or _load() + th = _thread_handle() + ptr = getattr(lib, name)(th, *args) + try: + raw = ctypes.string_at(ptr) + floats = None + if raw.endswith(b"}"): + i = raw.rfind(b',"bin":', max(0, len(raw) - 40)) + if i >= 0: + n = int(raw[i + 7 : -1]) + import numpy as np + + off = (len(raw) + 1 + 3) & ~3 + floats = np.frombuffer( + ctypes.string_at(ptr + off, 4 * n), dtype=np.float32 + ) + raw = raw[:i] + b"}" + finally: + lib.arcadedb_free(th, ptr) + return raw.decode("utf-8"), floats + + +def call_void(name: str, *args) -> None: + """Run an entry point that returns nothing (``arcadedb_n_set_upcall``).""" + lib = _lib or _load() + getattr(lib, name)(_thread_handle(), *args) + + +def b(value) -> bytes: + return value.encode("utf-8") if isinstance(value, str) else value + + +_bytes_cache = {} + + +def bc(value: str) -> bytes: + """UTF-8 of a statement text or language name, remembered (the same few texts cross a million times in a load).""" + r = _bytes_cache.get(value) + if r is None: + r = value.encode("utf-8") + if len(_bytes_cache) < 4096: + _bytes_cache[value] = r + return r + + +def cmd(handle, flags, language, command, params, buf, count, first) -> int: + """``arcadedb_n_cmd``: the cursor id (> 0), 0 for no cursor, -1 for an error (read it with :func:`last_error`).""" + lib = _lib or _load() + t = _local.__dict__.get("thread") + th = t.handle if t is not None else _thread_handle() + return lib.arcadedb_n_cmd( + th, handle, flags, language, command, params, buf, count, first + ) + + +def last_error() -> str: + return call("arcadedb_n_last_error") + + +# --- result sets closed in batches (one crossing for CLOSE_BATCH cursors) +_pending = [] +_pending_lock = threading.Lock() + + +def close_rs(rs_id: int) -> None: + if CLOSE_BATCH <= 0: + call("arcadedb_n_rs_close", rs_id) + return + _pending.append(rs_id) + if len(_pending) >= CLOSE_BATCH: + flush_closes() + + +def flush_closes() -> None: + """Close every cursor whose close is waiting; called before a commit, a rollback and a database close, and when the batch is full.""" + if not _pending or _lib is None: + return + with _pending_lock: + ids = _pending[:] + del _pending[: len(ids)] + if ids: + arr = (ctypes.c_long * len(ids))( + *ids + ) # arcadedb_n_rs_close_many takes a C `long *` (CLongPointer): c_long on every platform + _lib.arcadedb_n_rs_close_many(_thread_handle(), arr, len(ids)) + + +def diag() -> dict: + """Runtime facts of the image: Java version, jdk.incubator.vector present, CPUs, max heap, jvector provider classes.""" + return json.loads(call("arcadedb_n_diag")) diff --git a/src/arcadedb/_native/records.py b/src/arcadedb/_native/records.py new file mode 100644 index 0000000..7f03d2f --- /dev/null +++ b/src/arcadedb/_native/records.py @@ -0,0 +1,206 @@ +"""Record wrappers for the native backend: ``Document``, ``Vertex`` and ``Edge`` as Python objects over SQL statements. + +The JPype wrappers hold a Java record; here a record is its type, its RID (``None`` until saved) and its properties as a +dict, and every write is one DSL statement (``CREATE VERTEX .. CONTENT``, ``INSERT .. CONTENT``, ``UPDATE .. MERGE``, +``DELETE``, ``CREATE EDGE``). They subclass the JPype wrapper classes so ``isinstance`` checks keep their meaning. +""" + +from typing import Any, Dict, List, Optional + +from ..exceptions import ArcadeDBError +from ..graph import Document as _BaseDocument +from ..graph import Edge as _BaseEdge +from ..graph import Vertex as _BaseVertex +from . import not_yet +from .codec import RID + + +def _java_object(): + from .database import _JavaObject + + return _JavaObject() + + +_UNSAVED = "#-1:-1" + + +def _q(name: str) -> str: + return "`" + name.replace("`", "``") + "`" + + +class _Mixin: + """What the three native wrappers share (state in ``_db``, ``_type``, ``_rid``, ``_props``, ``_mutable``).""" + + _CREATE = "INSERT INTO {t} CONTENT :row" + _DELETE = "DELETE FROM {rid}" + + def __init__(self, database, type_name, rid=None, props=None, mutable=False): + self._java_document = _java_object() + self._database = database + self._db = database + self._type = type_name + self._rid = rid or None + self._props: Dict[str, Any] = dict(props or {}) + self._mutable = mutable or not rid + self._property_names_cache = None + + # -- reading + + def _check_open(self) -> None: + database = self._database + if database is not None and database._closed and self._rid: + raise ArcadeDBError( + "Database is closed: a record cannot be read after its database was closed. " + "Read what you need before closing it." + ) + + def get(self, name: str, convert_types: bool = True) -> Any: + self._check_open() + return self._props.get(name) + + def get_raw(self, name: str) -> Any: + return self.get(name) + + def has_property(self, name: str) -> bool: + self._check_open() + return name in self._props + + def get_property_names(self) -> List[str]: + self._check_open() + return list(self._props) + + def to_dict(self, convert_types: bool = True) -> Dict[str, Any]: + self._check_open() + return dict(self._props) + + def get_rid(self) -> str: + return self._rid or _UNSAVED + + def get_identity(self): + return RID(self.get_rid()) + + def get_type_name(self) -> str: + self._check_open() + return self._type + + def get_java_document(self): + not_yet("get_java_document") + + # -- writing + + def set(self, name: str, value: Any): + if not self._mutable: + raise AttributeError( + f"{type(self).__name__} is immutable. Call .modify() first to get a mutable version." + ) + self._props[name] = value + return self + + def save(self): + db = self._db + if self._rid is None: + row = db.command( + "sql", self._CREATE.format(t=_q(self._type)), {"row": self._props} + ).first() + self._adopt(row) + else: + db.command( + "sql", f"UPDATE {self._rid} MERGE :row", {"row": self._props} + ).close() + return self + + def _adopt(self, row) -> None: + if row is None or not row.get_rid(): + raise ArcadeDBError( + f"Failed to save a {self._type} record: the statement returned no record" + ) + self._rid = row.get_rid() + self._props = row.to_dict() + + def delete(self): + if self._rid is None: + raise ArcadeDBError("Cannot delete a record that was never saved") + self._db.command("sql", self._DELETE.format(rid=self._rid)).close() + + def modify(self): + self._check_open() + return type(self)(self._db, self._type, self._rid, self._props, mutable=True) + + def __repr__(self) -> str: + return f"<{type(self).__name__} rid={self.get_rid()} type={self._type}>" + + +class Document(_Mixin, _BaseDocument): + pass + + +class Vertex(_Mixin, _BaseVertex): + _CREATE = "CREATE VERTEX {t} CONTENT :row" + _DELETE = "DELETE FROM {rid}" + + def new_edge(self, label: str, target: "Vertex", **kwargs) -> "Edge": + self._check_open() + if self._rid is None or getattr(target, "_rid", None) is None: + raise ArcadeDBError( + "Both vertices of an edge must be saved before new_edge()" + ) + sql = f"CREATE EDGE {_q(label)} FROM {self._rid} TO {target._rid}" + if kwargs: + sql += " CONTENT :row" + row = self._db.command( + "sql", sql, *(({"row": kwargs},) if kwargs else ()) + ).first() + edge = Edge(self._db, label, mutable=False) + edge._adopt(row) + return edge + + def _edges(self, fn: str, labels) -> List["Edge"]: + self._check_open() + args = ", ".join("'" + label.replace("'", "\\'") + "'" for label in labels) + edges_sql = f"SELECT expand({fn}({args})) FROM {self._rid}" # nosec B608 - fn is a fixed function name, labels are escaped literals, _rid is the record's own id + rows = self._db.query("sql", edges_sql) + return [_wrap(self._db, row) for row in rows] + + def get_out_edges(self, *labels: str) -> List["Edge"]: + return self._edges("outE", labels) + + def get_in_edges(self, *labels: str) -> List["Edge"]: + return self._edges("inE", labels) + + def get_both_edges(self, *labels: str) -> List["Edge"]: + return self._edges("bothE", labels) + + +class Edge(_Mixin, _BaseEdge): + + def _end(self, fn: str) -> "Vertex": + self._check_open() + end_sql = f"SELECT expand({fn}()) FROM {self._rid}" # nosec B608 - fn is a fixed function name, _rid is the record's own id + row = self._db.query("sql", end_sql).first() + return _wrap(self._db, row) + + def get_in(self) -> Vertex: + return self._end("inV") + + def get_out(self) -> Vertex: + return self._end("outV") + + +def _wrap(database, row): + """The wrapper for a result row that is a record (vertex, edge or document); None for a projection.""" + meta = row._meta + if not meta: + return None + rid, type_name, kind = meta + cls = {"v": Vertex, "e": Edge}.get(kind, Document) + return cls(database, type_name, rid, row._props) + + +def _wrap_dict(database, record: dict, kind: str = "d"): + """The wrapper for a nested record the engine sent as a dict with ``@rid``, ``@type`` and ``@cat`` members.""" + props = dict(record) + rid = props.pop("@rid", None) + type_name = props.pop("@type", None) + kind = props.pop("@cat", kind) + cls = {"v": Vertex, "e": Edge}.get(kind, Document) + return cls(database, type_name, rid, props) diff --git a/src/arcadedb/_native/schema_objects.py b/src/arcadedb/_native/schema_objects.py new file mode 100644 index 0000000..eb14755 --- /dev/null +++ b/src/arcadedb/_native/schema_objects.py @@ -0,0 +1,374 @@ +"""What the JPype ``Schema`` returns as Java objects, rebuilt from ``schema:types`` and ``schema:indexes`` rows for the native +backend: types and indexes with the engine's camelCase accessors (``getName``, ``existsProperty``, ``getTypeName``, ...) and the +vector index wrapper over ``vectorNeighbors``. Read-only views: DDL stays SQL statements. +""" + +import json +import re + +from ..exceptions import ArcadeDBError +from ..vector import VectorIndex as _BaseVectorIndex +from . import not_yet + +_PROPERTY = re.compile(r"name: (.+?), type: ([A-Z_]+)") + + +class NativeType: + """A type as a ``schema:types`` row.""" + + def __init__(self, row: dict): + self._row = row + + def getName(self) -> str: + return self._row["name"] + + def getType(self) -> str: + return self._row.get("type") + + def getPropertyNames(self): + return [ + m.group(1) + for m in map(_PROPERTY.search, self._row.get("properties") or []) + if m + ] + + def existsProperty(self, name: str) -> bool: + return name in self.getPropertyNames() + + def getParentTypes(self): + return list(self._row.get("parentTypes") or []) + + getSuperTypes = getParentTypes + + def getBuckets(self): + return list(self._row.get("buckets") or []) + + def __getattr__(self, name): + if name.startswith("_"): + raise AttributeError(name) + not_yet(f"the Java object a Schema method returns ({name})") + + def __repr__(self): + return f"" + + +class NativeIndex: + """An index as a ``schema:indexes`` row.""" + + def __init__(self, row: dict): + self._row = row + + def getName(self) -> str: + return self._row["name"] + + def getTypeName(self) -> str: + return self._row.get("typeName") + + def getPropertyNames(self): + props = self._row.get("properties") or [] + return list(props[0]) if props and isinstance(props[0], list) else list(props) + + def isUnique(self) -> bool: + return bool(self._row.get("unique")) + + def getType(self) -> str: + return self._row.get("indexType") + + def isAutomatic(self) -> bool: + return bool(self._row.get("automatic")) + + def __getattr__(self, name): + if name.startswith("_"): + raise AttributeError(name) + not_yet(f"the Java object a Schema method returns ({name})") + + def __repr__(self): + return f"" + + +class _IndexShim: + """The few LSMVectorIndex methods the shared ``VectorIndex`` code calls, over statements.""" + + def __init__(self, database, type_name, prop, name): + self._db, self._type, self._prop, self._name = database, type_name, prop, name + + def getName(self): + return self._name + + def getTypeName(self): + return self._type + + def getPropertyNames(self): + return [self._prop] + + def getMetadata(self): + """Just what the PRODUCT-readiness check reads: the session's quantization and pq_clusters (engine default 256).""" + from types import SimpleNamespace + + meta = self._db._vector_meta.get((self._type, self._prop), {}) + return SimpleNamespace( + quantizationType=str(meta.get("quantization") or "NONE"), + pqClusters=int(meta.get("pqClusters", 256)), + ) + + def getIdPropertyName(self): + return self._db._vector_id_properties.get((self._type, self._prop), "id") + + def countEntries(self): + count_sql = f"SELECT count(*) AS c FROM `{self._type}` WHERE `{self._prop}` IS NOT NULL" # nosec B608 - backtick-quoted schema identifiers, no values + row = self._db.query("sql", count_sql).first() + return int(row.get("c")) + + def warmUp(self): + return None + + def buildVectorGraphNow(self): + # the engine builds the graph on the first search: one search now, on a vector of the index's dimension + dims = self._db._vector_dimensions.get((self._type, self._prop)) + if dims: + self._db.query( + "sql", "SELECT vectorNeighbors(?, ?, 1)", self._name, [0.0] * dims + ).close() + + +def _rid_text(value) -> str: + get_rid = getattr(value, "get_rid", None) + return get_rid() if get_rid is not None else str(value) + + +class NativeVectorIndex(_BaseVectorIndex): + """``VectorIndex`` over ``vectorNeighbors``: ``find_nearest`` returns ``[(record, distance)]`` like the JPype wrapper.""" + + def __init__(self, database, type_name, prop, name=None): + shim = _IndexShim(database, type_name, prop, name or f"{type_name}[{prop}]") + super().__init__(shim, database) + self._shim = shim + + def _resolve_lsm_indexes(self): + return [self._shim] + + def _run_search( + self, *, query_vector, k, allowed_rids, approximate, ef_search=None + ): + if approximate: + not_yet("VectorIndex.find_nearest_approximate over PRODUCT quantization") + from .records import _wrap_dict + + vector = ( + query_vector.tolist() + if hasattr(query_vector, "tolist") + else list(query_vector) + ) + # the statement's own options: efSearch, and filter (the RIDs the search may return), as the JPype path passes them + options = {} + if ef_search is not None: + options["efSearch"] = int(ef_search) + if allowed_rids: + options["filter"] = [_rid_text(r) for r in allowed_rids] + if options: + rows = self._database.query( + "sql", + "SELECT expand(vectorNeighbors(?, ?, ?, ?))", + self._shim._name, + vector, + int(k), + options, + ) + else: + rows = self._database.query( + "sql", + "SELECT expand(vectorNeighbors(?, ?, ?))", + self._shim._name, + vector, + int(k), + ) + out = [] + for row in rows: + nested = row.get("record") + record = ( + None + if nested is None + else _wrap_dict( + self._database, nested, self._database._kind_of(nested.get("@type")) + ) + ) + out.append((record, float(row.get("distance")))) + return self._sort_results(out)[:k] + + def find_nearest_approximate(self, query_vector, k=10, allowed_rids=None): + """PQ approximate search needs PRODUCT quantization; the search itself has no statement form (not on native yet).""" + if self.get_quantization() != "PRODUCT": + raise ArcadeDBError("Approximate search requires quantization=PRODUCT (PQ)") + not_yet("VectorIndex.find_nearest_approximate over PRODUCT quantization") + + def _lookup_query_vector_by_key(self, key): + type_name, prop = self._shim._type, self._shim._prop + id_prop = self._shim.getIdPropertyName() + lookup_sql = f"SELECT `{prop}` AS `query_vector` FROM `{type_name}` WHERE `{id_prop}` = ? LIMIT 1" # nosec B608 - backtick-quoted schema identifiers; the key goes through the ? parameter + row = self._database.query("sql", lookup_sql, key).first() + if row is None: + raise ArcadeDBError( + f"No record found in type '{type_name}' where {id_prop} = {key!r}" + ) + if row.get("query_vector") is None: + raise ArcadeDBError( + f"Record found for {id_prop} = {key!r} but property '{prop}' is null" + ) + return row.get("query_vector") + + def get_stats(self): + not_yet("VectorIndex.get_stats") + + def get_metadata(self): + """The configuration this session created the index with (no engine-chosen values: the image has no accessor for them).""" + meta = self._database._vector_meta.get((self._shim._type, self._shim._prop)) + if meta is None: + not_yet("VectorIndex.get_metadata of an index this session did not create") + names = { + "dimensions": "dimensions", + "similarity": "similarity_function", + "idPropertyName": "id_property", + "quantization": "quantization", + "maxConnections": "max_connections", + "beamWidth": "beam_width", + "graphBuildCacheSize": "graph_build_cache_size", + "mutationsBeforeRebuild": "mutations_before_rebuild", + "storeVectorsInGraph": "store_vectors_in_graph", + "addHierarchy": "add_hierarchy", + "pqSubspaces": "pq_subspaces", + "pqClusters": "pq_clusters", + "pqCenterGlobally": "pq_center_globally", + "pqTrainingLimit": "pq_training_limit", + } + buckets = [ + r["name"] + for r in self._database.query( + "sql", "SELECT FROM schema:indexes WHERE indexType = 'LSM_VECTOR'" + ).to_list() + if r["typeName"] == self._shim._type + and r["properties"] == [[self._shim._prop]] + and "fileId" in r + ] + out = { + "index_name": self._shim._name, + "bucket_index_name": buckets[0] if buckets else None, + "type_name": self._shim._type, + "vector_property": self._shim._prop, + } + out.update({names[k]: v for k, v in meta.items() if k in names}) + if "similarity_function" in out: + out["similarity_function"] = str(out["similarity_function"]).upper() + if "quantization" in out: + out["quantization"] = str(out["quantization"]).upper() + return out + + def get_quantization(self): + meta = self._database._vector_meta.get((self._shim._type, self._shim._prop)) + if meta is None: + not_yet( + "VectorIndex.get_quantization of an index this session did not create" + ) + return str(meta.get("quantization") or "NONE") + + +def _check_vector_arguments( + quantization, + encoding, + pq_subspaces, + pq_clusters, + pq_center_globally, + pq_training_limit, +): + if any( + v is not None + for v in (pq_subspaces, pq_clusters, pq_center_globally, pq_training_limit) + ): + if not quantization or quantization.upper() != "PRODUCT": + raise ValueError("PQ parameters require quantization='PRODUCT'") + if ( + encoding + and encoding.upper() == "INT8" + and quantization + and quantization.upper() == "INT8" + ): + raise ValueError( + "encoding='INT8' cannot be combined with quantization='INT8'; use quantization='NONE' for native INT8 storage" + ) + + +def create_vector_index( + db, + vertex_type, + vector_property, + dimensions, + id_property, + distance_function, + max_connections, + beam_width, + quantization, + encoding, + graph_build_cache_size, + mutations_before_rebuild, + store_vectors_in_graph, + add_hierarchy, + pq_subspaces, + pq_clusters, + pq_center_globally, + pq_training_limit, + build_graph_now, +): + """``CREATE INDEX .. LSM_VECTOR METADATA {..}`` with the JPype builder's defaults and checks; returns the wrapper.""" + try: + _check_vector_arguments( + quantization, + encoding, + pq_subspaces, + pq_clusters, + pq_center_globally, + pq_training_limit, + ) + except ValueError as e: + raise ArcadeDBError(f"Failed to create vector index: {e}") from e + meta = { + "dimensions": int(dimensions), + "similarity": distance_function, + "maxConnections": int(max_connections), + "beamWidth": int(beam_width), + } + if id_property: + meta["idPropertyName"] = id_property + if quantization: + meta["quantization"] = quantization + if encoding: + meta["encoding"] = encoding + for key, value in ( + ("pqSubspaces", pq_subspaces), + ("pqClusters", pq_clusters), + ("pqCenterGlobally", pq_center_globally), + ("pqTrainingLimit", pq_training_limit), + ): + if value is not None: + meta[key] = bool(value) if key == "pqCenterGlobally" else int(value) + if store_vectors_in_graph: + meta["storeVectorsInGraph"] = True + if add_hierarchy is not None: + meta["addHierarchy"] = bool(add_hierarchy) + if graph_build_cache_size is not None: + meta["graphBuildCacheSize"] = int(graph_build_cache_size) + if mutations_before_rebuild is not None: + meta["mutationsBeforeRebuild"] = int(mutations_before_rebuild) + try: + db.command( + "sql", + f"CREATE INDEX ON `{vertex_type}` (`{vector_property}`) LSM_VECTOR METADATA {json.dumps(meta)}", + ).close() + key = (vertex_type, vector_property) + db._vector_meta[key] = meta + db._vector_dimensions[key] = int(dimensions) + db._vector_id_properties[key] = id_property or "id" + index = NativeVectorIndex(db, vertex_type, vector_property) + if build_graph_now: + index.build_graph_now() + return index + except Exception as e: + raise ArcadeDBError(f"Failed to create vector index: {e}") from e diff --git a/src/arcadedb/_native/server.py b/src/arcadedb/_native/server.py new file mode 100644 index 0000000..090239c --- /dev/null +++ b/src/arcadedb/_native/server.py @@ -0,0 +1,118 @@ +"""In-process server on the native backend (HTTP API, wire-protocol plugins as configured; no Studio in the native wheel).""" + +import os +from typing import Any, Dict, Optional + +from ..exceptions import ArcadeDBError +from . import lib +from .database import NativeDatabase, _call + + +def _config_key(key: str) -> str: + return f"arcadedb.{key.replace('_', '.')}" + + +class NativeServer: + def __init__( + self, + root_path: str = "./databases", + root_password: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, + ): + self._config = config or {} + self._root_path = root_path + self._root_password = root_password + self._h = 0 + self._started = False + + def _settings(self) -> dict: + cfg = self._config + settings: Dict[str, Any] = {"arcadedb.server.rootPath": self._root_path} + if self._root_password: + settings["arcadedb.server.rootPassword"] = self._root_password + settings["arcadedb.server.mode"] = cfg.get("mode", "development") + settings["arcadedb.server.httpIncomingHost"] = cfg.get("host", "localhost") + settings["arcadedb.server.httpIncomingPort"] = cfg.get("http_port", 2480) + for key, value in cfg.items(): + if key not in ("mode", "host", "http_port"): + settings[_config_key(key)] = value + return settings + + def start(self): + import json + + if self._started: + raise ArcadeDBError("Server is already started") + body = json.dumps( + { + "settings": self._settings(), + "host": self._config.get("host", "localhost"), + } + ) + try: + self._h = _call( + "Failed to start server", "arcadedb_n_server_start", lib.b(body) + )["h"] + except ArcadeDBError: + raise + self._started = True + + def stop(self): + if not self._started: + return + _call("Failed to stop server", "arcadedb_n_server_stop", self._h) + self._started = False + + def __del__(self): + try: + if self._started: + self.stop() + except Exception: # nosec B110 + return + + def get_database(self, name: str) -> NativeDatabase: + if not self._started: + raise ArcadeDBError("Server is not started") + return NativeDatabase( + _call( + f"Failed to get database '{name}'", + "arcadedb_n_server_db", + self._h, + lib.b(name), + 0, + )["h"], + owned_by_server=True, + ) + + def create_database(self, name: str) -> NativeDatabase: + if not self._started: + raise ArcadeDBError("Server is not started") + return NativeDatabase( + _call( + f"Failed to create database '{name}'", + "arcadedb_n_server_db", + self._h, + lib.b(name), + 1, + )["h"], + owned_by_server=True, + ) + + def is_started(self) -> bool: + return self._started + + def get_http_port(self) -> int: + return self._config.get("http_port", 2480) + + def get_studio_url(self) -> str: + host = self._config.get("host", "localhost") + if host in ("0.0.0.0", "::"): # nosec B104 + host = "localhost" + return f"http://{host}:{self.get_http_port()}/" + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop() diff --git a/src/arcadedb/core.py b/src/arcadedb/core.py index cc4794c..3a36c54 100644 --- a/src/arcadedb/core.py +++ b/src/arcadedb/core.py @@ -2316,6 +2316,16 @@ def __del__(self): class DatabaseFactory: """Factory for creating/opening ArcadeDB databases.""" + def __new__(cls, path: str, jvm_kwargs: Optional[dict] = None): + # ARCADEDB_BACKEND=native (experimental, native-spike branch): the factory over libarcadedb + from . import _native + + if _native.enabled(): + from ._native.database import NativeDatabaseFactory + + return NativeDatabaseFactory(path, jvm_kwargs) + return super().__new__(cls) + def __init__( self, path: str, diff --git a/src/arcadedb/dataframe.py b/src/arcadedb/dataframe.py index a38a170..634ff9c 100644 --- a/src/arcadedb/dataframe.py +++ b/src/arcadedb/dataframe.py @@ -9,8 +9,16 @@ ``GraphBatch.create_vertices`` takes (a vertex type). No Java was added: both paths use the existing glue (``DocumentBatcher.insertColumns`` and the ``GraphBatch`` vertex batcher). + +The native backend (the ``arcadedb`` wheel) shares everything but the +last step: the checks, the type mapping, the batching and the error wording are +the same code. A document batch goes over as one JSON text through +``Database.insert_many`` (one crossing per 10,000 rows), a vertex batch through +``GraphBatch.create_vertices``, and a value JSON cannot carry (a date-time, a +decimal, bytes) travels as a tagged value the image decodes. """ +import re from typing import Any, Dict, List, Optional, Tuple import jpype @@ -140,6 +148,46 @@ def to_arrow_table(data, *, what: str): ) +def _is_native(db) -> bool: + """True for the native backend's database (no JVM: schema reads are SQL, values cross as tagged JSON).""" + return not hasattr(db, "_java_db") + + +_PROPERTY = re.compile(r"name: (.+?), type: ([A-Z_]+)") + + +def _declared_types_native( + db, type_name: str, names: List[str] +) -> Tuple[bool, Dict[str, str]]: + """:func:`_declared_types` for the native backend: the ``schema:types`` rows, walking the parent types.""" + rows = {r["name"]: r for r in db.query("sql", "SELECT FROM schema:types").to_list()} + row = rows.get(type_name) + if row is None: + raise ArcadeDBError( + f"Cannot load into '{type_name}': the type does not exist " + f"(create it first, with or without declared properties)" + ) + kind = row.get("type") + if kind == "edge": + raise ArcadeDBError( + f"Cannot load into '{type_name}': it is an edge type. Load the vertices, " + f"then buffer the edges with graph_batch().new_edges()" + ) + declared: Dict[str, str] = {} + seen, queue = set(), [type_name] + while queue: # the type first, then its parents: the nearest declaration wins + current = queue.pop(0) + if current in seen or current not in rows: + continue + seen.add(current) + for text in rows[current].get("properties") or []: + m = _PROPERTY.search(text) + if m and m.group(1) in names: + declared.setdefault(m.group(1), m.group(2)) + queue.extend(rows[current].get("parentTypes") or []) + return kind == "vertex", declared + + def _declared_types( db, type_name: str, names: List[str] ) -> Tuple[bool, Dict[str, str]]: @@ -147,6 +195,8 @@ def _declared_types( Raises ArcadeDBError when the type does not exist or is an edge type. """ + if _is_native(db): + return _declared_types_native(db, type_name, names) java_schema = db._java_db.getSchema() if not java_schema.existsType(type_name): raise ArcadeDBError( @@ -218,7 +268,50 @@ def _temporal_to_java(chunk, pa): ] -def _column_values(name: str, chunk, pa): +def _temporal_to_tagged(chunk, pa): + """A timestamp or date chunk as tagged values (``T`` date-time, ``D`` date) for the native backend, with the same + reading rules as :func:`_temporal_to_java`: ISO text of the integer epoch value, so a nanosecond value keeps its + nanoseconds (a Python ``datetime`` would cut them to microseconds). + """ + import numpy as np + + from ._native.codec import TAG + + types = pa.types + mask = chunk.is_null().to_numpy(zero_copy_only=False) + if types.is_date(chunk.type): + raw = chunk.cast(pa.int64() if types.is_date64(chunk.type) else pa.int32()) + days = raw.fill_null(0).to_numpy(zero_copy_only=False).astype(np.int64) + if types.is_date64(chunk.type): + days = days // 86_400_000 + text = days.astype("datetime64[D]").astype(str).tolist() + tag = "D" + else: + unit = chunk.type.unit + values = chunk.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) + text = ( + values.astype(np.int64).astype(f"datetime64[{unit}]").astype(str).tolist() + ) + tag = "T" + return [None if m else {TAG: [tag, s]} for s, m in zip(text, mask)] + + +def _scalars_to_tagged(kind: str, chunk): + """A decimal or binary chunk as tagged values (``n`` decimal text, ``b`` base64) for the native backend.""" + from ._native.codec import TAG + + values = chunk.to_pylist() + if kind == "decimal": + return [None if v is None else {TAG: ["n", str(v)]} for v in values] + import base64 + + return [ + None if v is None else {TAG: ["b", base64.b64encode(v).decode("ascii")]} + for v in values + ] + + +def _column_values(name: str, chunk, pa, native: bool = False): """One Arrow array as what ``Database.insert_columns`` takes: a numpy array when the column crosses as one buffer (integers, floats and booleans without nulls), else a list of values (``None`` is a null). """ @@ -227,6 +320,15 @@ def _column_values(name: str, chunk, pa): chunk = chunk.cast(chunk.type.value_type) kind = _kind(pa, chunk.type) if kind in ("integer", "floating", "bool"): + if native and types.is_uint64(chunk.type) and len(chunk) > chunk.null_count: + import pyarrow.compute as pc + + if ( + pc.max(chunk).as_py() > 2**63 - 1 + ): # insert_columns refuses it on the JPype path, with this text + raise ValueError( + f"column {name!r} holds a value beyond the 64-bit signed range" + ) if chunk.null_count == 0: values = chunk.to_numpy(zero_copy_only=False) if values.dtype.kind == "f" and values.dtype.itemsize < 8: @@ -234,7 +336,11 @@ def _column_values(name: str, chunk, pa): return values return chunk.to_pylist() if kind in ("timestamp", "date"): - return _temporal_to_java(chunk, pa) + return ( + _temporal_to_tagged(chunk, pa) if native else _temporal_to_java(chunk, pa) + ) + if native and kind in ("decimal", "binary"): + return _scalars_to_tagged(kind, chunk) if kind == "null": return [None] * len(chunk) # strings, binary, decimal, list, struct: Python values, which insert_columns converts element by element @@ -286,17 +392,18 @@ def insert_table( if table.num_rows == 0: return 0 + native = _is_native(db) written = 0 # a document load inside the caller's own transaction joins it (insert_columns commits # nothing then); a vertex load refuses to run inside one - in_caller_transaction = (not is_vertex) and bool(db._java_db.isTransactionActive()) + in_caller_transaction = (not is_vertex) and bool(db.is_transaction_active()) graph_batch = db.graph_batch() if is_vertex else None try: for batch in table.to_batches(max_chunksize=batch_size): if batch.num_rows == 0: continue columns = { - name: _column_values(name, batch.column(i), pa) + name: _column_values(name, batch.column(i), pa, native) for i, name in enumerate(names) } try: @@ -304,6 +411,10 @@ def insert_table( _insert_vertices( graph_batch, type_name, names, columns, batch.num_rows ) + elif native: + _insert_documents_native( + db, type_name, names, columns, batch_size, parallel + ) else: db.insert_columns( type_name, columns, commit_every=batch_size, parallel=parallel @@ -340,3 +451,27 @@ def _insert_vertices( graph_batch.create_vertices( type_name, [dict(zip(names, row)) for row in zip(*lists)] ) + + +def _insert_documents_native( + db, + type_name: str, + names: List[str], + columns: Dict[str, Any], + batch_size: int, + parallel: bool, +): + """One batch of documents on the native backend: the columns as one dict per row, handed to ``insert_many`` + (one crossing per 10,000 rows; the batch is one transaction, or joins the caller's). + """ + lists = [ + # jvm-payload: not-a-crossing -- no JVM here: insert_many serialises the rows to JSON text + columns[name].tolist() if hasattr(columns[name], "tolist") else columns[name] + for name in names + ] + db.insert_many( + type_name, + [dict(zip(names, row)) for row in zip(*lists)], + commit_every=batch_size, + parallel=parallel, + ) diff --git a/src/arcadedb/exceptions.py b/src/arcadedb/exceptions.py index f33d800..6133ec4 100644 --- a/src/arcadedb/exceptions.py +++ b/src/arcadedb/exceptions.py @@ -126,6 +126,14 @@ class ArcadeDBError(Exception): def __str__(self): text = super().__str__() + chain = getattr(self, "java_chain", None) + if chain and len(chain) > 1: + # native backend: the cause chain came over as text, innermost last + name, message = chain[-1] + if not message or message not in text: + detail = f"{name}: {message}" if message and message != "None" else name + text = f"{text} (caused by {detail})" + return text try: root = _java_root_cause(self.__cause__) if root is not None: diff --git a/src/arcadedb/exporter.py b/src/arcadedb/exporter.py index 63ec6ef..20cdf47 100644 --- a/src/arcadedb/exporter.py +++ b/src/arcadedb/exporter.py @@ -73,8 +73,6 @@ def export_database( - JSONL format is recommended for full backup/restore - Exported files are compressed (.tgz format) """ - start_jvm() - # Validate format supported_formats = ["jsonl", "graphml", "graphson"] if export_format.lower() not in supported_formats: @@ -92,6 +90,26 @@ def export_database( if export_dir and not os.path.exists(export_dir): os.makedirs(export_dir, exist_ok=True) + from . import _native + + if _native.enabled(): + settings = {} + if include_types: + settings["includeTypes"] = ",".join(include_types) + if exclude_types: + settings["excludeTypes"] = ",".join(exclude_types) + if verbose is not None: + settings["verboseLevel"] = str(verbose) + try: + from ._native.extras import export_database as native_export + + return native_export( + db, file_path, export_format.lower(), overwrite, settings + ) + except Exception as e: + raise _export_error(e, export_format, file_path) from e + + start_jvm() try: import jpype @@ -136,31 +154,36 @@ def export_database( return python_result except Exception as e: - # Check for specific error messages - error_msg = str(e) - # Only GraphML and GraphSON come from arcadedb-gremlin; "not found" also matches a missing type or file. - if export_format.lower() in ("graphml", "graphson") and ( - "arcadedb-gremlin" in error_msg - or "Format not supported" in error_msg - or "not found" in error_msg - ): - raise ArcadeDBError( - f"Export format '{export_format}' requires additional modules: " - f"GraphML and GraphSON come from the engine's optional " - f"arcadedb-gremlin module, which this package does not bundle. " - f"Use export_format='jsonl'. Error: {error_msg}" - ) from e - elif ( - "already exists" in error_msg - or "already exist" in error_msg - or "cannot be overwritten" in error_msg - ): - raise ArcadeDBError( - f"Export file '{file_path}' already exists. " - f"Use overwrite=True to replace it. Error: {error_msg}" - ) from e - else: - raise ArcadeDBError(f"Database export failed: {error_msg}") from e + raise _export_error(e, export_format, file_path) from e + + +def _export_error(e, export_format, file_path): + """The ArcadeDBError for a failed export (shared by the JPype and native paths).""" + # Check for specific error messages + error_msg = str(e) + # Only GraphML and GraphSON come from arcadedb-gremlin; "not found" also matches a missing type or file. + if export_format.lower() in ("graphml", "graphson") and ( + "arcadedb-gremlin" in error_msg + or "Format not supported" in error_msg + or "not found" in error_msg + ): + return ArcadeDBError( + f"Export format '{export_format}' requires additional modules: " + f"GraphML and GraphSON come from the engine's optional " + f"arcadedb-gremlin module, which this package does not bundle. " + f"Use export_format='jsonl'. Error: {error_msg}" + ) + elif ( + "already exists" in error_msg + or "already exist" in error_msg + or "cannot be overwritten" in error_msg + ): + return ArcadeDBError( + f"Export file '{file_path}' already exists. " + f"Use overwrite=True to replace it. Error: {error_msg}" + ) + else: + return ArcadeDBError(f"Database export failed: {error_msg}") def _column_union(rows) -> List[str]: @@ -218,7 +241,7 @@ def export_to_csv( if file_dir and not os.path.exists(file_dir): os.makedirs(file_dir, exist_ok=True) - if isinstance(results, ResultSet): + if isinstance(results, ResultSet) or hasattr(results, "iter_json_batches"): # Stream rows via batched Java-side JSON serialization (one JPype # crossing per batch instead of several per row — measured ~5x on # 100k-row exports). Values carry JSON-native types, so DATE and diff --git a/src/arcadedb/server.py b/src/arcadedb/server.py index e92c857..f62dec5 100644 --- a/src/arcadedb/server.py +++ b/src/arcadedb/server.py @@ -50,6 +50,22 @@ def _all_interfaces_host_settings(): class ArcadeDBServer: """ArcadeDB Server wrapper for enabling HTTP API and Studio access.""" + def __new__( + cls, + root_path: str = "./databases", + root_password: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, + jvm_kwargs: Optional[dict] = None, + ): + # ARCADEDB_BACKEND=native (experimental, native-spike branch): the in-process server over libarcadedb + from . import _native + + if _native.enabled(): + from ._native.server import NativeServer + + return NativeServer(root_path, root_password, config) + return super().__new__(cls) + def __init__( self, root_path: str = "./databases", diff --git a/src/arcadedb/vector.py b/src/arcadedb/vector.py index 3b5bbfd..c1ed3cb 100644 --- a/src/arcadedb/vector.py +++ b/src/arcadedb/vector.py @@ -27,6 +27,12 @@ def _quote_identifier(identifier: str) -> str: _FLOAT_ARRAY = None +def _native_enabled(): + from . import _native + + return _native.enabled() + + def to_java_float_array(vector): """ Convert a Python array-like object to a Java float array. @@ -42,6 +48,10 @@ def to_java_float_array(vector): Returns: Java float array compatible with ArcadeDB vector indexes """ + if _native_enabled(): + return _np.asarray( + vector if hasattr(vector, "__array__") else list(vector), dtype=_np.float32 + ) global _FLOAT_ARRAY float_array = _FLOAT_ARRAY if float_array is None: diff --git a/tests/conftest.py b/tests/conftest.py index 732373d..21549b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,18 +13,34 @@ # A test file that cannot run here is not collected, so nothing is reported as skipped: a skip means # a test that should have run, and scripts/check_test_skips.py fails the CI job on any skip. collect_ignore = [] -if sys.platform == "win32": + +import arcadedb # noqa: E402 (the build under test decides which tests apply) + +# "jvm" for the arcadedb-jvm distribution, "native" for the arcadedb distribution (the GraalVM native-image library). +BACKEND = arcadedb.__backend__ + +# Files that cannot run on Windows (native/wheel/parity_wheel.py reads this table too: it passes every file to pytest by name, +# and a file named on the command line is collected even when it is in collect_ignore). +WINDOWS_NOT_APPLICABLE = { # test_sigint.py sends SIGINT to a child process, which Windows cannot deliver - collect_ignore.append("test_sigint.py") + "test_sigint.py": "SIGINT to a child process", # test_python_function_interrupt.py sends SIGINT to its own process, which Windows cannot do either - collect_ignore.append("test_python_function_interrupt.py") + "test_python_function_interrupt.py": "SIGINT to its own process", # LadybugDB's compiled module does not load in this process on Windows (it falls back to a C-API library its # wheel does not ship), so the Cypher half of the equivalence checks runs on Linux and macOS # (humemai/arcadedb-embedded-python#347) - collect_ignore.append("test_equivalence_ladybug.py") + "test_equivalence_ladybug.py": "LadybugDB does not load on Windows", # runs scripts/stage_jars.sh under bash with a fake docker; the bash a Windows # subprocess finds may be WSL's rather than Git Bash - collect_ignore.append("test_stage_jars.py") + "test_stage_jars.py": "bash with a fake docker", +} +if sys.platform == "win32": + collect_ignore.extend(WINDOWS_NOT_APPLICABLE) +if BACKEND != "native": + # Java -> Python upcalls (Python functions in SQL) exist on the native build only + collect_ignore.append("test_native_udf.py") + # the native library's start-up output (#477) can only be checked with the native library loaded + collect_ignore.append("test_native_startup_runtime.py") if not os.path.isfile( os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md") ): @@ -33,6 +49,145 @@ collect_ignore.append("test_docs_examples.py") +# The native build (the arcadedb distribution): the same suite runs against libarcadedb. A test that needs a Java object API +# raises NotOnNativeYet and is reported as a skip whose reason starts with "NOTYET"; a test about the JVM itself is reported as +# a skip that starts with "NA-JVM"; a file about the JVM, the JPype wheel or the JVM wheel's build tooling is not collected and +# listed in NATIVE_NOT_APPLICABLE. native/wheel/parity_wheel.py counts the three kinds apart, and the native workflow fails on +# any other skip (this repository accepts no skips, scripts/check_test_skips.py). +NATIVE_NOT_APPLICABLE = { + "test_jvm.py": "JVM start-up and settings", + "test_jvm_args.py": "JVM arguments", + "test_jar_provenance.py": "jar fingerprint of the JVM wheel", + "test_jar_closure.py": "jar closure of the JVM wheel", + "test_java_package_shadowing.py": "JPype package import hooks", + "test_wheel_platform_tag.py": "JVM wheel tag", + "test_wheel_size.py": "JVM wheel size", + "test_wheel_licences.py": "JVM wheel licences", + "test_wheel_packages.py": "JVM wheel contents", + "test_server_packaging.py": "JVM wheel contents", + "test_runtime_cache.py": "JRE runtime cache of the JVM wheel", + "test_aot.py": "JVM class-data cache", + "test_stage_jars.py": "staging of the JVM wheel's jars", + "test_extract_version.py": "version of the JVM wheel's engine jar", + "test_extract_version_classifiers.py": "version of the JVM wheel's engine jar", + "test_build_scripts.py": "build scripts of the JVM wheel", + "test_build_no_uv_refresh.py": "build scripts of the JVM wheel", + "test_build_wheel_check.py": "build scripts of the JVM wheel", +} +if BACKEND == "native" and not os.environ.get("NATIVE_PARITY_ALL"): + collect_ignore.extend(NATIVE_NOT_APPLICABLE) + + +# Failure texts that mean the test is about the JPype bridge, not about database behaviour. +NATIVE_JVM_MARKERS = ( + "JVMNotRunning", + "without JVM", + "Java Virtual Machine is not running", + "'_java_db'", + "'_java_result'", + "bridge jar", + "'nextRows'", + "getClass", + "DID NOT RAISE KeyboardInterrupt", + "Cannot drop the database in transaction", # teardown after an injected KeyboardInterrupt + "assert (None is not None)", # fixtures that need a bridge class + "KeyError: 'flush'", + "KeyError: 'max_connections'", + "cause is ", # the Java exception as __cause__ + "findNeighborsFromVector", # the test calls the Java vector index directly + "to_java_rid", +) + + +# Failure texts that mean the test needs a Java-object wrapper (Document, Vertex, Edge) or the exporter's Java path. +NATIVE_NOTYET_MARKERS = ( + "'NativeResult' object is not iterable", + "assert 'NativeResult' ==", +) + + +# Tests that count the JPype parameter conversions (an internal of the JPype path). +NATIVE_NA_TESTS = { + "test_non_scalars_keep_the_general_path", + "test_a_subclass_of_a_scalar_is_not_assumed_to_be_one", + "test_lsm_vector_search_looks_up_all_hits_in_one_bridge_call", # counts GraphCalls bridge crossings +} + + +# Tests that exercise the JVM itself, so the native backend (no JVM, no jars) has nothing to run. Skipped before they start, +# with the reason; a test whose behaviour native shares (Ctrl-C in a Python loop, the zone of to_json_list) is not listed. +NATIVE_NA_JVM_TESTS = { + "test_ctrl_c_raises_keyboard_interrupt_and_runs_cleanup[java]": "a Java call blocked in JPype's SIGINT handler (#179)", + "test_interrupt_true_keeps_the_jvm_handling_sigint": "start_jvm(interrupt=True): the JVM handles SIGINT", + "test_docs_installation_jvm_examples": "documents start_jvm() and jvm_kwargs", + "test_to_java_int_array_keeps_values_that_fit": "to_java_int_array builds a Java int[] (JPype)", + "test_to_java_int_array_refuses_values_beyond_32_bits_in_an_array": "to_java_int_array builds a Java int[] (JPype)", + "test_to_java_byte_array_accepts_bytes_like_fast_paths": "to_java_byte_array builds a Java byte[] (JPype)", + "test_to_java_int_array": "to_java_int_array builds a Java int[] (JPype)", + "test_to_java_int_array_accepts_numpy_directly": "to_java_int_array builds a Java int[] (JPype)", + "test_to_java_int_array_round_trips_through_a_sparse_vector": "to_java_int_array builds a Java int[] (JPype)", + "test_the_host_reaches_exactly_the_listener_bind_settings": "reads the engine's GlobalConfiguration through the JVM", +} + + +def pytest_collection_modifyitems(config, items): + if BACKEND != "native": + return + for item in items: + reason = NATIVE_NA_JVM_TESTS.get(item.name) + if reason: + item.add_marker(pytest.mark.skip(reason=f"NA-JVM: {reason}")) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + if BACKEND != "native": + return + report = outcome.get_result() + if call.excinfo is None or report.passed: + return + from arcadedb._native import NotOnNativeYet + + if item.name in NATIVE_NA_TESTS: + report.outcome = "skipped" + report.longrepr = ( + str(item.fspath), + item.location[1] or 0, + "NA-JVM: counts JPype conversions", + ) + return + exc, seen = call.excinfo.value, 0 + while exc is not None and seen < 8: + if isinstance(exc, NotOnNativeYet): + report.outcome = "skipped" + report.longrepr = ( + str(item.fspath), + item.location[1] or 0, + f"NOTYET: {exc}", + ) + return + text = f"{type(exc).__name__}: {exc}" + if any(marker in text for marker in NATIVE_NOTYET_MARKERS): + report.outcome = "skipped" + report.longrepr = ( + str(item.fspath), + item.location[1] or 0, + "NOTYET: Java-object wrapper or exporter path", + ) + return + if any(marker in text for marker in NATIVE_JVM_MARKERS): + # the test reaches into the JVM directly (jpype.JClass, Java arrays, bridge classes): about the JPype path + report.outcome = "skipped" + report.longrepr = ( + str(item.fspath), + item.location[1] or 0, + "NA-JVM: the test uses the JVM directly", + ) + return + exc, seen = exc.__cause__ or exc.__context__, seen + 1 + + @pytest.hookimpl(trylast=True) def pytest_configure(config): # HotSpot routinely raises access violations it handles itself diff --git a/tests/test_bridge_fixes.py b/tests/test_bridge_fixes.py index e0f4941..419774e 100644 --- a/tests/test_bridge_fixes.py +++ b/tests/test_bridge_fixes.py @@ -5,6 +5,7 @@ """ import json +import os import subprocess # nosec B404 - test-controlled child process import sys @@ -48,9 +49,12 @@ def test_insert_many_nested_values_read_as_python_in_the_same_transaction( from datetime import date import arcadedb -from arcadedb.jvm import start_jvm -start_jvm(heap_size="1g", jvm_args=["-Duser.timezone=" + sys.argv[2]]) +native = arcadedb.__backend__ == "native" # the native build has no JVM: its zone is the TZ the process was started with +if not native: + from arcadedb.jvm import start_jvm + + start_jvm(heap_size="1g", jvm_args=["-Duser.timezone=" + sys.argv[2]]) db = arcadedb.create_database(sys.argv[1]) db.command("sql", "CREATE DOCUMENT TYPE Event") with db.transaction(): @@ -65,9 +69,12 @@ def test_insert_many_nested_values_read_as_python_in_the_same_transaction( ) batch = db.query("sql", "SELECT FROM Event").to_json_list()[0] single = json.loads(db.query("sql", "SELECT FROM Event").first().to_json()) -from jpype import JClass # noqa: E402 (after start_jvm) -mapped = JClass("com.arcadedb.schema.Type").getTypeByValue(JClass("java.time.LocalDate").of(2024, 1, 2)) -print(json.dumps({"batch": batch, "single": single, "local_date_type": str(mapped)}), flush=True) +if native: + mapped = "native" # no Java class to ask: the check below accepts either form +else: + from jpype import JClass # noqa: E402 (after start_jvm) + mapped = str(JClass("com.arcadedb.schema.Type").getTypeByValue(JClass("java.time.LocalDate").of(2024, 1, 2))) +print(json.dumps({"batch": batch, "single": single, "local_date_type": mapped}), flush=True) db.close() """ @@ -94,6 +101,9 @@ def test_to_json_list_writes_a_date_as_midnight_utc_in_any_jvm_zone(tmp_path, zo capture_output=True, text=True, timeout=180, + env=dict( + os.environ, TZ=zone + ), # the native image takes its zone from here; the JVM gets -Duser.timezone ) assert proc.returncode == 0, proc.stderr[-2000:] out = json.loads(proc.stdout.strip().splitlines()[-1]) @@ -102,6 +112,8 @@ def test_to_json_list_writes_a_date_as_midnight_utc_in_any_jvm_zone(tmp_path, zo assert batch["day"] == MIDNIGHT_UTC_2024_01_02 if out["local_date_type"] == "DATE": assert single["day"] == EPOCH_DAY_2024_01_02 + elif out["local_date_type"] == "native": + assert single["day"] in (EPOCH_DAY_2024_01_02, MIDNIGHT_UTC_2024_01_02) elif out["local_date_type"] == "None": assert single["day"] == MIDNIGHT_UTC_2024_01_02 else: diff --git a/tests/test_dataframe_ingest.py b/tests/test_dataframe_ingest.py index 3404cd0..5f14ff1 100644 --- a/tests/test_dataframe_ingest.py +++ b/tests/test_dataframe_ingest.py @@ -717,7 +717,7 @@ def test_missing_polars_is_an_import_error_with_the_fix(self, temp_db, monkeypat monkeypatch.setitem(sys.modules, "polars", None) with pytest.raises( ImportError, - match=r"polars is required for to_polars\(\).*arcadedb\[polars\]", + match=r"polars is required for to_polars\(\).*arcadedb(-jvm)?\[polars\]", ): temp_db.query("sql", "SELECT FROM NoPolars").to_polars() diff --git a/tests/test_dist_conflict.py b/tests/test_dist_conflict.py index de7b870..b0db7d9 100644 --- a/tests/test_dist_conflict.py +++ b/tests/test_dist_conflict.py @@ -99,11 +99,14 @@ def import_arcadedb(extra_site): ) +def own_backend(): + """The build the installed package is: fixed at build time, not switched by ARCADEDB_BACKEND.""" + return arcadedb._native_backend.distribution_backend() + + def sibling(): """The distribution of the other build than the one under test.""" - return _dist_conflict.DISTRIBUTIONS[ - _dist_conflict.other_backend(arcadedb.__backend__) - ] + return _dist_conflict.DISTRIBUTIONS[_dist_conflict.other_backend(own_backend())] def test_import_fails_when_the_other_distribution_is_installed_too(tmp_path): @@ -118,7 +121,7 @@ def test_import_fails_when_the_other_distribution_is_installed_too(tmp_path): def test_import_is_silent_when_only_this_distribution_is_installed(tmp_path): site = tmp_path / "site" - fake_dist_info(site, _dist_conflict.DISTRIBUTIONS[arcadedb.__backend__], "26.11.1") + fake_dist_info(site, _dist_conflict.DISTRIBUTIONS[own_backend()], "26.11.1") result = import_arcadedb(site) assert result.returncode == 0, result.stderr assert "pip uninstall" not in result.stderr diff --git a/tests/test_docs_examples.py b/tests/test_docs_examples.py index ab23f4e..8c2b360 100644 --- a/tests/test_docs_examples.py +++ b/tests/test_docs_examples.py @@ -266,6 +266,11 @@ def test_docs_installation_examples(temp_dir_factory): "ArcadeDB Python bindings version:", base_dir / "installation_verify", ) + + +def test_docs_installation_jvm_examples(temp_dir_factory): + base_dir = Path(temp_dir_factory("docs_installation_jvm_")) + _run_doc_block( "getting-started/installation.md", "from arcadedb.jvm import start_jvm", diff --git a/tests/test_free_threading.py b/tests/test_free_threading.py index cb31514..7a110f6 100644 --- a/tests/test_free_threading.py +++ b/tests/test_free_threading.py @@ -29,7 +29,7 @@ ) # documents (and vertices with a vector) per thread DIM = 8 BASE = 64 # vertices in the indexed type the searches run against -NATIVE = os.environ.get("ARCADEDB_BACKEND", "jpype").lower() == "native" +NATIVE = arcadedb.__backend__ == "native" NN = "SELECT bid FROM (SELECT expand(vectorNeighbors('Base[embedding]', ?, ?)))" diff --git a/tests/test_interrupt_hygiene.py b/tests/test_interrupt_hygiene.py index dcd9922..10e0bae 100644 --- a/tests/test_interrupt_hygiene.py +++ b/tests/test_interrupt_hygiene.py @@ -15,6 +15,8 @@ @author Luca Garulli (l.garulli@arcadedata.com) """ +import importlib + import pytest import arcadedb @@ -22,6 +24,10 @@ from arcadedb.core import Database from arcadedb.server import ArcadeDBServer +# the wrapper-level checks run against the class the active build's databases are instances of +if arcadedb.__backend__ == "native": + Database = importlib.import_module("arcadedb._native.database").NativeDatabase + @pytest.fixture def db(temp_db_path): @@ -30,7 +36,11 @@ def db(temp_db_path): yield database # a failed assertion can leave levels open, and drop() refuses inside a transaction try: - database._java_db.rollbackAllNested() + if hasattr(database, "_java_db"): + database._java_db.rollbackAllNested() + else: + while database.is_transaction_active(): + database.rollback() except Exception: # nosec B110 - teardown must not hide the test failure pass database.drop() @@ -42,7 +52,9 @@ def _count(database): def _depth(database): """How many transaction levels are on this thread's stack (nesting depth).""" - return int(database._java_db.getNestedTransactions()) + if hasattr(database, "_java_db"): + return int(database._java_db.getNestedTransactions()) + return database._transaction_stack_mark()[1] class TestRunInTransactionBeginHygiene: diff --git a/tests/test_native_startup.py b/tests/test_native_startup.py new file mode 100644 index 0000000..15b107f --- /dev/null +++ b/tests/test_native_startup.py @@ -0,0 +1,29 @@ +"""The native wheel starts quietly: no polyglot-engine warning (#477).""" + +import re +from pathlib import Path + +import pytest + +ARCADE_NATIVE = ( + Path(__file__).resolve().parents[3] + / "native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java" +) + + +def test_the_image_glue_switches_the_polyglot_engine_off_before_any_entry_point(): + """The image has no polyglot languages; leaving the engine on logs 'no languages found' at WARNING in every process. + + Static check (no native build needed): the glue class that holds the entry points must set the option in a static initializer. + """ + if not ARCADE_NATIVE.is_file(): + pytest.skip("the repository checkout is not available") + source = ARCADE_NATIVE.read_text() + assert re.search( + r"static\s*\{[^{}]*GlobalConfiguration\.POLYGLOT_ENGINE_ENABLED\.setValue\(false\);[^{}]*\}", + source, + ), "ArcadeNative must run GlobalConfiguration.POLYGLOT_ENGINE_ENABLED.setValue(false) in a static block" + + +# The run-time half (the first open logs nothing at WARNING) needs the native library and lives in +# test_native_startup_runtime.py, which conftest.py collects on the native backend only. diff --git a/tests/test_native_startup_runtime.py b/tests/test_native_startup_runtime.py new file mode 100644 index 0000000..f816df1 --- /dev/null +++ b/tests/test_native_startup_runtime.py @@ -0,0 +1,25 @@ +"""The native wheel starts quietly at run time: the first open logs nothing at WARNING or above (#477). + +Needs the native library, so conftest.py does not collect this file on the JPype backend (a skip there would fail +scripts/check_test_skips.py); it runs in the native parity run. +""" + +import re +import subprocess # nosec B404 - test-controlled child process +import sys + + +def test_the_first_open_writes_nothing_at_warning(tmp_path): + code = ( + "import arcadedb as a, sys\n" + f"db = a.create_database({str(tmp_path / 'db')!r})\n" + "db.query('sql', 'SELECT 1 AS v').first()\n" + "db.close()\n" + ) + done = subprocess.run( # nosec B603 - fixed argv, no shell, test-owned code string + [sys.executable, "-c", code], capture_output=True, text=True, timeout=120 + ) + assert done.returncode == 0, done.stderr + output = done.stdout + done.stderr + assert "no languages found" not in output + assert not re.search(r"\b(WARNI|WARNING|SEVERE|ERROR)\b", output), output diff --git a/tests/test_native_udf.py b/tests/test_native_udf.py new file mode 100644 index 0000000..e5dc83b --- /dev/null +++ b/tests/test_native_udf.py @@ -0,0 +1,125 @@ +"""Python functions called from SQL and async callbacks, on the native backend (Java -> Python upcalls); not collected on JPype.""" + +import threading + +import pytest + +import arcadedb +from arcadedb import ArcadeDBError + + +def test_a_python_function_in_a_select(temp_db): + temp_db.register_function("py_double", lambda x: x * 2, 1, 1) + row = temp_db.query("sql", "SELECT py_double(21) AS v").first() + assert row.get("v") == 42 + + +def test_a_python_function_reads_columns_and_returns_text(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE T") + with temp_db.transaction(): + for i in range(3): + temp_db.command("sql", "INSERT INTO T SET n = ?, s = ?", i, f"x{i}") + temp_db.register_function("py_tag", lambda n, s: f"{s}:{n + 1}", 2, 2) + rows = temp_db.query("sql", "SELECT py_tag(n, s) AS v FROM T ORDER BY n").to_list() + assert [r["v"] for r in rows] == ["x0:1", "x1:2", "x2:3"] + + +def test_an_exception_in_the_function_fails_the_statement_with_its_text(temp_db): + def boom(x): + raise ValueError("no way") + + temp_db.register_function("py_boom", boom, 1, 1) + with pytest.raises(ArcadeDBError, match="no way"): + temp_db.query("sql", "SELECT py_boom(1) AS v").first() + + +def test_a_function_may_call_back_into_the_database(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE C") + with temp_db.transaction(): + temp_db.command("sql", "INSERT INTO C SET k = 7") + + def count_c(): + return temp_db.query("sql", "SELECT count(*) AS c FROM C").first().get("c") + + temp_db.register_function("py_count_c", count_c, 0, 0) + assert temp_db.query("sql", "SELECT py_count_c() AS v").first().get("v") == 1 + + +def test_async_callbacks_run_on_a_writer_thread_and_see_the_record(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE A") + seen, threads = [], set() + + def on_record(record): + seen.append(record.get("n")) + threads.add(threading.get_ident()) + + executor = temp_db.async_executor() + for n in range(20): + doc = temp_db.new_document("A") + doc.set("n", n) + executor.create_record(doc, callback=on_record) + executor.wait_completion() + assert sorted(seen) == list(range(20)) + assert threading.get_ident() not in threads + + +def _two_vertices(db): + db.command("sql", "CREATE VERTEX TYPE P") + db.command("sql", "CREATE PROPERTY P.n INTEGER") + db.command("sql", "CREATE INDEX ON P (n) UNIQUE") + db.command("sql", "CREATE EDGE TYPE K") + with db.transaction(): + a = db.new_vertex("P").set("n", 1).save() + b = db.new_vertex("P").set("n", 2).save() + return a, b + + +def test_async_new_edge_runs_the_callback_with_the_edge(temp_db): + a, b = _two_vertices(temp_db) + seen = [] + executor = temp_db.async_executor() + executor.new_edge( + a, + "K", + b, + callback=lambda edge, src, dst: seen.append((edge.get("w"), src, dst)), + w=5, + ) + executor.wait_completion() + assert seen == [(5, False, False)] + assert temp_db.query("sql", "SELECT count(*) AS c FROM K").first().get("c") == 1 + + +def test_async_new_edge_by_keys_creates_the_missing_vertex(temp_db): + _two_vertices(temp_db) + seen = [] + executor = temp_db.async_executor() + executor.new_edge_by_keys( + "P", + "n", + 1, + "P", + "n", + 3, + True, + "K", + True, + False, + callback=lambda edge, src, dst: seen.append((src, dst)), + ) + executor.wait_completion() + assert seen == [(False, True)] + assert temp_db.query("sql", "SELECT count(*) AS c FROM P").first().get("c") == 3 + + +def test_async_scan_type_calls_back_for_every_record_and_can_stop(temp_db): + _two_vertices(temp_db) + ns = [] + executor = temp_db.async_executor() + executor.scan_type("P", lambda record: ns.append(record.get("n"))) + executor.wait_completion() + assert sorted(ns) == [1, 2] + stopped = [] + executor.scan_type("P", lambda record: stopped.append(record.get("n")) or False) + executor.wait_completion() + assert len(stopped) == 1 diff --git a/tests/test_native_wheel_template.py b/tests/test_native_wheel_template.py new file mode 100644 index 0000000..0808993 --- /dev/null +++ b/tests/test_native_wheel_template.py @@ -0,0 +1,49 @@ +"""The template of the native `arcadedb` wheel (native/wheel/) names the right distribution and says which package is the sibling. + +Repository tooling: no engine, no JVM and no native library needed. +""" + +import re +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +WHEEL = ROOT / "native" / "wheel" + + +def test_the_template_is_the_arcadedb_distribution(): + meta = tomllib.loads((WHEEL / "pyproject.toml").read_text())["project"] + assert meta["name"] == "arcadedb" + assert meta["version"] == "@VERSION@" # build_wheel.py replaces it + assert meta["dependencies"] == [] # no JVM and no JPype + assert "arcadedb-jvm" in meta["description"] + + +def test_python_classifiers_match_the_jvm_wheel(): + def pythons(path): + classifiers = tomllib.loads(path.read_text())["project"]["classifiers"] + return [ + c + for c in classifiers + if c.startswith("Programming Language :: Python :: 3.") + ] + + assert pythons(WHEEL / "pyproject.toml") == pythons(ROOT / "pyproject.toml") + + +def test_the_pypi_description_names_the_sibling_in_its_first_paragraph(): + text = (WHEEL / "README.md").read_text() + paragraphs = [ + p for p in re.split(r"\n\s*\n", text) if p.strip() and not p.startswith("#") + ] + first = " ".join(p for p in paragraphs[:1]) + assert "`arcadedb-jvm`" in first, first + jvm = (ROOT / "README.md").read_text() + head = jvm.split("##", 1)[0] + assert "`arcadedb`" in head and "arcadedb-jvm" in head + + +def test_the_builder_has_no_other_package_name(): + text = (WHEEL / "build_wheel.py").read_text() + assert 'dst = stage / "src" / "arcadedb"' in text + assert "arcadedb_native" not in text diff --git a/tests/test_select_native_matrix.py b/tests/test_select_native_matrix.py new file mode 100644 index 0000000..16e575b --- /dev/null +++ b/tests/test_select_native_matrix.py @@ -0,0 +1,75 @@ +"""The legs of the native workflow per event: a pull request must stay small (the org shares a runner cap with upstream).""" + +import importlib.util +import json +import re +import subprocess # nosec B404 - runs this repository's script with sys.executable +import sys +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "select_native_matrix.py" +spec = importlib.util.spec_from_file_location("select_native_matrix", SCRIPT) +sel = importlib.util.module_from_spec(spec) +spec.loader.exec_module(sel) + + +def test_pull_request_builds_one_platform_and_tests_one_python(): + chosen = sel.select("pull_request") + assert [b["key"] for b in chosen["build"]] == ["linux-x86_64"] + assert [(t["key"], t["python"]) for t in chosen["test"]] == [ + ("linux-x86_64", "3.12") + ] + assert chosen["publish"] is False + + +def test_main_nightly_and_dispatch_build_all_four_platforms(): + for event in ("push", "schedule", "workflow_dispatch"): + chosen = sel.select(event) + assert [b["key"] for b in chosen["build"]] == [ + "linux-x86_64", + "linux-aarch64", + "macos-arm64", + "windows-x64", + ] + assert len(chosen["test"]) == 4 * len(sel.MAIN_PYTHONS) + assert chosen["publish"] is False + + +def test_a_release_tag_tests_every_supported_python_on_every_platform(): + chosen = sel.select("tag") + assert len(chosen["build"]) == 4 + assert len(chosen["test"]) == 4 * 4 + assert {t["python"] for t in chosen["test"]} == {"3.11", "3.12", "3.13", "3.14"} + assert chosen["publish"] is True + + +def test_pythons_are_the_classifiers_of_the_native_wheel(): + text = tomllib.loads((ROOT / "native" / "wheel" / "pyproject.toml").read_text()) + prefix = "Programming Language :: Python :: 3." + pythons = [ + "3." + c.rsplit(".", 1)[1] + for c in text["project"]["classifiers"] + if c.startswith(prefix) + ] + assert pythons == sel.ALL_PYTHONS + + +def test_graalvm_is_community_edition_with_a_sha256_per_platform(): + for key, leg in sel.PLATFORMS.items(): + assert "graalvm-community" in leg["graalvm_asset"], key + assert re.fullmatch(r"[0-9a-f]{64}", leg["graalvm_sha256"]), key + assert "oracle" not in leg["graalvm_asset"].lower(), key + + +def test_the_script_prints_github_output_lines(): + out = subprocess.run( # nosec B603 - fixed argv, no shell + [sys.executable, str(SCRIPT), "pull_request"], + capture_output=True, + text=True, + check=True, + ).stdout.splitlines() + assert [line.split("=", 1)[0] for line in out] == ["build", "test", "publish"] + assert json.loads(out[0].split("=", 1)[1])[0]["key"] == "linux-x86_64" + assert out[2] == "publish=false" diff --git a/tests/test_sigint.py b/tests/test_sigint.py index 143ded2..902bd3d 100644 --- a/tests/test_sigint.py +++ b/tests/test_sigint.py @@ -32,9 +32,11 @@ _CHILD = """ import atexit, sys, time import arcadedb -from arcadedb.jvm import start_jvm -start_jvm(heap_size="512m", interrupt=(sys.argv[3] == "1")) +if arcadedb.__backend__ != "native": # the native build has no JVM to start + from arcadedb.jvm import start_jvm + + start_jvm(heap_size="512m", interrupt=(sys.argv[3] == "1")) db = arcadedb.create_database(sys.argv[2]) db.command("sql", "CREATE DOCUMENT TYPE T") atexit.register(lambda: print("ATEXIT", flush=True)) diff --git a/tests/test_vector_sql.py b/tests/test_vector_sql.py index 03932b2..1fbdb66 100644 --- a/tests/test_vector_sql.py +++ b/tests/test_vector_sql.py @@ -470,6 +470,17 @@ def test_create_index_with_native_int8_encoding_sql(self, test_db): def test_vector_neighbors_on_native_int8_storage_sql(self, test_db): """Verify that vectorNeighbors works against native INT8-encoded storage.""" + from arcadedb._native import enabled + + # a BINARY value and a float[] parameter: Java arrays on the JPype path, bytes and a float32 array on the native one + if enabled(): + import numpy as np + + to_bytes = bytes + to_floats = lambda v: np.array(v, dtype=np.float32) # noqa: E731 + else: + to_bytes = arcadedb.to_java_byte_array + to_floats = arcadedb.to_java_float_array test_db.command("sql", "CREATE VERTEX TYPE SqlNativeInt8SearchDoc") test_db.command("sql", "CREATE PROPERTY SqlNativeInt8SearchDoc.id STRING") test_db.command("sql", "CREATE PROPERTY SqlNativeInt8SearchDoc.vec BINARY") @@ -492,19 +503,19 @@ def test_vector_neighbors_on_native_int8_storage_sql(self, test_db): "sql", "INSERT INTO SqlNativeInt8SearchDoc SET id = ?, vec = ?", "doc_a", - arcadedb.to_java_byte_array([127, 0, 0, 0]), + to_bytes([127, 0, 0, 0]), ) test_db.command( "sql", "INSERT INTO SqlNativeInt8SearchDoc SET id = ?, vec = ?", "doc_b", - arcadedb.to_java_byte_array([120, 10, 0, 0]), + to_bytes([120, 10, 0, 0]), ) test_db.command( "sql", "INSERT INTO SqlNativeInt8SearchDoc SET id = ?, vec = ?", "doc_c", - arcadedb.to_java_byte_array([0, 127, 0, 0]), + to_bytes([0, 127, 0, 0]), ) rows = test_db.query( @@ -514,7 +525,7 @@ def test_vector_neighbors_on_native_int8_storage_sql(self, test_db): "(SELECT expand(vectorNeighbors('SqlNativeInt8SearchDoc[vec]', ?, 2))) " "ORDER BY distance" ), - arcadedb.to_java_float_array([1.0, 0.0, 0.0, 0.0]), + to_floats([1.0, 0.0, 0.0, 0.0]), ).to_list() assert [row.get("id") for row in rows] == ["doc_a", "doc_b"] diff --git a/tests/test_wheel_packages.py b/tests/test_wheel_packages.py new file mode 100644 index 0000000..af217db --- /dev/null +++ b/tests/test_wheel_packages.py @@ -0,0 +1,69 @@ +"""The JVM wheel must ship every subpackage the shared Python code imports. + +``core.py``, ``server.py``, ``exporter.py`` and ``vector.py`` do ``from . import _native`` to ask which backend is +selected, in the JVM wheel too (the native wheel is built from the same sources). ``pyproject.toml`` lists the packages +by hand, so a subpackage left out of that list gives an ``ImportError`` on the first ``create_database`` in the JVM wheel. + +In a source checkout the test builds the package with setuptools (no JARs, no JRE, no native library needed) and checks the +build. In the wheel's test container there is no ``setup.py``, so it checks the installed package instead. +""" + +import os +import shutil +import subprocess # nosec B404 - test-controlled child process +import sys +from pathlib import Path + +PYTHON_DIR = Path(__file__).resolve().parents[1] +SOURCE = PYTHON_DIR / "src" / "arcadedb" + + +def _built_root(tmp_path: Path): + """The directory that holds the built ``arcadedb``, or None to use the installed one.""" + if not (PYTHON_DIR / "setup.py").exists(): + return None + # Build from a copy: setuptools writes an egg-info next to the sources, and a stale one lists files + # the current pyproject.toml would not ship. + project = tmp_path / "project" + project.mkdir() + shutil.copy(PYTHON_DIR / "pyproject.toml", project) + shutil.copy(PYTHON_DIR / "setup.py", project) + shutil.copytree( + SOURCE, + project / "src" / "arcadedb", + ignore=shutil.ignore_patterns("__pycache__"), + ) + build_lib = tmp_path / "lib" + subprocess.run( # nosec B603 - fixed argv (sys.executable setup.py build_py), no shell + [sys.executable, "setup.py", "-q", "build_py", "--build-lib", str(build_lib)], + cwd=project, + check=True, + ) + return build_lib + + +def test_build_ships_every_subpackage(tmp_path): + root = _built_root(tmp_path) + if root is None: + import arcadedb + + package = Path(arcadedb.__file__).parent + assert (package / "_native" / "__init__.py").exists() + return + expected = {p.parent.name for p in SOURCE.glob("*/__init__.py")} + built = {p.parent.name for p in (root / "arcadedb").glob("*/__init__.py")} + assert ( + expected == built + ), f"subpackages missing from the build: {sorted(expected - built)}" + + +def test_backend_selection_imports_without_native_library(tmp_path): + """A fresh interpreter, as the wheel's ``test_install.py`` runs: the JVM backend is the default.""" + root = _built_root(tmp_path) + env = {"PATH": os.environ["PATH"]} + if root is not None: + env["PYTHONPATH"] = str(root) + code = "import arcadedb.core, arcadedb._native as n;" "assert n.enabled() is False" + subprocess.run( # nosec B603 - fixed argv, no shell, test-owned code string + [sys.executable, "-c", code], check=True, cwd=tmp_path, env=env + ) From dd0eea8ca0e4d6eeae168743aedc2007b8fe8db0 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 20:35:13 +0900 Subject: [PATCH 04/33] CI: a native workflow with a selective matrix, and a gated release of arcadedb .github/workflows/native.yml builds the native image per platform, builds the wheel, runs the gates (glibc floor, auditwheel, twine, name and version) and the test suite against the INSTALLED wheel. scripts/select_native_matrix.py holds the legs (pinned by tests/test_select_native_matrix.py), because a native image build takes 1 to 2.5 hours per platform and the organisation shares a runner cap with upstream: - pull request: only when a native path changed (paths filter) and not for a draft; linux-x86_64, Python 3.12. - push to main (native paths only), nightly (only when something besides docs changed in the last 25 hours, a full run on Mondays) and dispatch: linux-x86_64, linux-aarch64, macos-arm64, windows-x64; Python 3.12 and 3.14. - release tag (A.B.C.devN, A.B.C, A.B.C.postN): all four platforms on Python 3.11 to 3.14. Publishing: a pre-release (A.B.C.devN) goes to the environment pypi-prerelease when linux-x86_64, linux-aarch64 and macos-arm64 passed on every Python. A final release (A.B.C, A.B.C.postN) goes to the environment pypi only when the repository variable ARCADEDB_NATIVE_READINESS_BAR_MET is true, all four platforms passed on all four Pythons, and no test is a NOTYET skip. GraalVM Community Edition only, pinned by sha256 per platform. The JVM workflows skip native/** and native.yml in their paths-ignore. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/native.yml | 676 +++++++++++++++++++++ .github/workflows/test-python-bindings.yml | 6 + .github/workflows/test-python-examples.yml | 6 + scripts/select_native_matrix.py | 132 ++++ 4 files changed, 820 insertions(+) create mode 100644 .github/workflows/native.yml create mode 100644 scripts/select_native_matrix.py diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml new file mode 100644 index 0000000..2a4c5df --- /dev/null +++ b/.github/workflows/native.yml @@ -0,0 +1,676 @@ +# The native build: the `arcadedb` distribution (a GraalVM native-image shared library loaded with ctypes). The JVM build, +# `arcadedb-jvm`, is release.yml / test-python-bindings.yml. +# +# A native image build takes about 1 to 2.5 hours per platform and the ArcadeData organisation shares a runner cap with +# upstream ArcadeDB, so the load is selective (scripts/select_native_matrix.py holds the legs, tests/test_select_native_matrix.py pins them): +# pull request only when a native path changed (the `paths` filter below); linux-x86_64, Python 3.12; not for a draft +# push to main only when a native path changed; all four platforms, Python 3.12 and 3.14 +# nightly all four platforms, Python 3.12 and 3.14, but only when something besides docs changed in the last 25 hours (a full run on Mondays) +# dispatch all four platforms, Python 3.12 and 3.14; never publishes +# tag A.B.C.devN | A.B.C | A.B.C.postN all four platforms, Python 3.11 to 3.14; then publishes (see below) +# +# One wheel per platform, py3-none-: +# linux-x86_64 manylinux_2_34 container manylinux_2_34_x86_64 libarcadedb.so + libjvector-x86_64.so +# linux-aarch64 manylinux_2_34 aarch64 container manylinux_2_34_aarch64 libarcadedb.so +# macos-arm64 native, heap and threads capped macosx__arm64 libarcadedb.dylib +# windows-x64 native, MSVC environment loaded win_amd64 libarcadedb.dll +# Only free GitHub-hosted standard runners. GraalVM Community Edition only (pinned by sha256 per platform), never Oracle GraalVM. +# +# Gates per platform: the glibc floor, `auditwheel show`, `twine check`, wheel name `arcadedb` and version == requested version, and the +# test suite of this repository run against the INSTALLED wheel (native/wheel/parity_wheel.py): no failure, error, crash, timeout, or skip +# besides the counted NOTYET and NA-JVM kinds (tests/conftest.py). +# +# Publishing (tags only; PyPI trusted publishing, no token; a trusted publisher on the PyPI project `arcadedb` for this workflow file): +# A.B.C.devN environment `pypi-prerelease`. Needs the parity run of linux-x86_64, linux-aarch64 and macos-arm64 on every Python. +# Windows is published too when it passed, and is informational otherwise. +# A.B.C, A.B.C.postN environment `pypi`, ONLY when the readiness bar is met: the repository variable ARCADEDB_NATIVE_READINESS_BAR_MET +# is `true` (a maintainer sets it after checking the correctness fixes and the fault and concurrency probes), the parity run +# of ALL four platforms on ALL four Pythons passed, and no test is a NOTYET skip. Speed parity is not part of the bar. +# The bar is written down in docs/modules/ROOT/pages/development/release.adoc. +name: Native wheel (arcadedb) + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + paths: &native_paths + - "native/**" + - "src/arcadedb/_native/**" + - "tests/test_native_*.py" + - "tests/conftest.py" + - "scripts/select_native_matrix.py" + - "tests/test_select_native_matrix.py" + - ".github/workflows/native.yml" + push: + branches: [main] + paths: *native_paths + tags: + - "[0-9]+.[0-9]+.[0-9]+*" # 26.11.1.dev0, 26.11.1, 26.11.1.post1; the plan job rejects the rest + schedule: + - cron: "40 3 * * *" + workflow_dispatch: + inputs: + version: + description: "Wheel version to build (PEP 440: A.B.C.devN, A.B.C or A.B.C.postN); a tag push takes it from the tag" + required: false + default: "26.11.1.dev0" + type: string + engine: + description: "Engine release (Maven Central) the image links; empty = the release segment of a final version, else the newest release" + required: false + default: "" + type: string + +concurrency: + group: native-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} + # A dry run is superseded by a newer one; a tag run is never cancelled (a half-done release is worse than a queued one). + cancel-in-progress: ${{ github.ref_type != 'tag' }} + +permissions: + contents: read + +env: + GRAALVM_VERSION: "25.4.4.1.1" + GRAALVM_BASE: https://github.com/graalvm/graalvm-ce-builds/releases/download/graal-25.4.4.1.1 + UV_VERSION: "0.12.0" + PYTHONUTF8: "1" # Windows defaults to cp1252; the notices contain non-ASCII author names + # The version and engine of a run that is not a tag. The newest engine release on Maven Central is what an A.B.C.devN wheel links + # (the A.B.C release does not exist yet); raise it when a newer release is out. + DEFAULT_VERSION: "26.11.1.dev0" + DEV_ENGINE: "26.10.2" + +jobs: + plan: + name: Plan the legs + # a draft pull request starts the platform legs when it is marked ready, as in test-python-bindings.yml + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + run: ${{ steps.plan.outputs.run }} + version: ${{ steps.ver.outputs.version }} + engine: ${{ steps.ver.outputs.engine }} + final: ${{ steps.ver.outputs.final }} + build: ${{ steps.plan.outputs.build }} + test: ${{ steps.plan.outputs.test }} + publish: ${{ steps.plan.outputs.publish }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 50 + + - name: Resolve the version and the engine + id: ver + env: + REF_TYPE: ${{ github.ref_type }} + REF_NAME: ${{ github.ref_name }} + INPUT_VERSION: ${{ inputs.version }} + INPUT_ENGINE: ${{ inputs.engine }} + run: | + set -euo pipefail + if [ "$REF_TYPE" = "tag" ]; then + version="$REF_NAME" + [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]] || { echo "::error::Tag '$version' is not A.B.C, A.B.C.devN or A.B.C.postN (bare version, no v, no aN/bN/rcN)"; exit 1; } + else + version="${INPUT_VERSION:-$DEFAULT_VERSION}" + fi + base="${version%%.dev*}" + base="${base%%.post*}" + if [[ "$version" == *.dev* ]]; then + final=false + engine="${INPUT_ENGINE:-$DEV_ENGINE}" + else + final=true + engine="${INPUT_ENGINE:-$base}" + # a final or post wheel links exactly the engine release it is named after + [ "$engine" = "$base" ] || { echo "::error::Version $version links engine $base, not $engine" >&2; exit 1; } + fi + { echo "version=$version"; echo "engine=$engine"; echo "final=$final"; } >> "$GITHUB_OUTPUT" + echo "wheel version $version, engine $engine, final=$final" + + - name: Tag must be the next version + if: ${{ github.ref_type == 'tag' }} + env: + TAG: ${{ github.ref_name }} + run: python3 -I scripts/check_version.py "$TAG" --git-remote origin --pypi arcadedb + + - name: Choose the legs + id: plan + env: + EVENT: ${{ github.event_name }} + REF_TYPE: ${{ github.ref_type }} + run: | + set -euo pipefail + run=true + event="$EVENT" + if [ "$REF_TYPE" = "tag" ]; then event=tag; fi + if [ "$EVENT" = "schedule" ]; then + changed="$(git log --since='25 hours ago' --format=%h -- src native tests scripts pyproject.toml .github/workflows/native.yml | wc -l)" + monday="$(date -u +%u)" + if [ "$changed" = 0 ] && [ "$monday" != 1 ]; then + run=false + echo "nothing changed in the last 25 hours and it is not Monday: no native run tonight" + fi + fi + echo "run=$run" >> "$GITHUB_OUTPUT" + python3 -I scripts/select_native_matrix.py "$event" >> "$GITHUB_OUTPUT" + + build: + name: Image and wheel (${{ matrix.key }}) + needs: plan + if: ${{ needs.plan.outputs.run == 'true' }} + runs-on: ${{ matrix.runner }} + timeout-minutes: ${{ matrix.build_minutes }} + defaults: + run: + shell: bash + strategy: + fail-fast: false # a failing platform must not stop the others from building + matrix: + include: ${{ fromJSON(needs.plan.outputs.build) }} + env: + KEY: ${{ matrix.key }} + KIND: ${{ matrix.kind }} + steps: + # Windows checks scripts out with CRLF by default, which breaks bash and mvnw: keep LF everywhere. + - name: Keep LF line endings + run: | + git config --global core.autocrlf false + git config --global core.eol lf + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # T: the runner temp directory in a form bash and the native tools both accept (D:/a/_temp on Windows) + - name: Paths + run: | + if [ "$RUNNER_OS" = Windows ]; then T="$(cygpath -m "$RUNNER_TEMP")"; else T="$RUNNER_TEMP"; fi + echo "T=$T" >> "$GITHUB_ENV" + + - name: Install uv + if: ${{ matrix.kind != 'windows' }} + run: | + installer="$(mktemp)" + curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" "https://astral.sh/uv/${UV_VERSION}/install.sh" + sh "$installer" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install uv (Windows) + if: ${{ matrix.kind == 'windows' }} + shell: pwsh + run: | + $env:UV_NO_MODIFY_PATH = "1" + Invoke-RestMethod "https://astral.sh/uv/$env:UV_VERSION/install.ps1" | Invoke-Expression + Add-Content -Path $env:GITHUB_PATH -Value "$env:USERPROFILE\.local\bin" + + # The builder heap is 12g (8g ran out of memory on the laptop, 2026-10-25): swap keeps a 16 GB runner from being killed. + - name: Add swap + if: ${{ matrix.kind == 'linux' }} + run: | + set -euo pipefail + sudo fallocate -l 8G /swapfile.native + sudo chmod 600 /swapfile.native + sudo mkswap /swapfile.native + sudo swapon /swapfile.native + free -m + + - name: Download GraalVM CE and verify its sha256 + env: + GRAALVM_ASSET: ${{ matrix.graalvm_asset }} + GRAALVM_SHA256: ${{ matrix.graalvm_sha256 }} + run: | + set -euo pipefail + curl -fsSL --retry 5 --retry-all-errors --retry-delay 5 -o "$T/graalvm.archive" "$GRAALVM_BASE/$GRAALVM_ASSET" + if command -v sha256sum > /dev/null; then + echo "$GRAALVM_SHA256 $T/graalvm.archive" | sha256sum -c - + else + echo "$GRAALVM_SHA256 $T/graalvm.archive" | shasum -a 256 -c - + fi + mkdir -p "$T/graalvm" + case "$GRAALVM_ASSET" in + *.zip) + 7z x -y -o"$T/graalvm-x" "$T/graalvm.archive" > /dev/null + mv "$T"/graalvm-x/*/* "$T/graalvm/" + ;; + *) tar -xzf "$T/graalvm.archive" -C "$T/graalvm" --strip-components=1 ;; + esac + rm -rf "$T/graalvm.archive" "$T/graalvm-x" + home="$T/graalvm" + # the macOS archive is a bundle: the JDK home is below Contents/Home + if [ -f "$home/Contents/Home/release" ]; then home="$home/Contents/Home"; fi + echo "GRAALVM_HOME_P5=$home" >> "$GITHUB_ENV" + grep -F "GRAALVM_VERSION=\"$GRAALVM_VERSION\"" "$home/release" + ni="$home/bin/native-image" + if [ "$KIND" = windows ]; then ni="$ni.cmd"; fi + "$ni" --version + + # native-image on Windows needs the MSVC toolchain on PATH and the INCLUDE/LIB variables (what the x64 Native Tools prompt sets). + - name: Load the MSVC environment + if: ${{ matrix.kind == 'windows' }} + shell: pwsh + run: | + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" + $vs = & $vswhere -latest -products '*' -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + if (-not $vs) { throw "no Visual Studio with the C++ tools" } + $bat = Join-Path $vs 'VC\Auxiliary\Build\vcvars64.bat' + Write-Host "vcvars: $bat" + $keep = '^(PATH|INCLUDE|LIB|LIBPATH|VC.*|VS.*|WindowsSdk.*|WindowsSDK.*|UCRT.*|UniversalCRT.*|DevEnv.*|Framework.*|NETFXSDK.*|Platform|CommandPromptType)$' + cmd /c "`"$bat`" >nul && set" | ForEach-Object { + # the name and value are taken before the second -match, which would overwrite $Matches + $i = $_.IndexOf('=') + if ($i -gt 0) { + $name = $_.Substring(0, $i) + if ($name -match $keep) { Add-Content -Path $env:GITHUB_ENV -Value $_ } + } + } + + - name: Hide Git's GNU link.exe (it shadows the MSVC linker) + if: ${{ matrix.kind == 'windows' }} + run: | + mv /usr/bin/link.exe /usr/bin/link-gnu.exe || true + echo "link is now: $(command -v link || echo none)" + + # -march=compatibility, -J-Xmx12g, -H:-VectorAPISupport, fastops.mode 1, inside the manylinux_2_34 container (glibc 2.34) so the + # glibc floor of libarcadedb.so is a guarantee, not a property of the runner. + - name: Build the native image in manylinux_2_34 + if: ${{ matrix.kind == 'linux' }} + env: + ENGINE: ${{ needs.plan.outputs.engine }} + MANYLINUX_IMAGE: ${{ matrix.container }} + XMX: ${{ matrix.xmx }} + NI_EXTRA: ${{ matrix.ni_extra }} + run: | + set -euo pipefail + mkdir -p "$T/lib" "$T/m2" + docker run --rm \ + -v "$GITHUB_WORKSPACE:/work" -v "$GRAALVM_HOME_P5:/graalvm:ro" -v "$T/lib:/out" -v "$T/m2:/root/.m2" \ + -e GRAALVM_HOME_P5=/graalvm -e ARCADEDB_ENGINE_VERSION="$ENGINE" -e MARCH=compatibility -e XMX="$XMX" -e NI_EXTRA="$NI_EXTRA" \ + "$MANYLINUX_IMAGE" bash -euo pipefail -c ' + ldd --version | sed -n 1p # not `head -1`: under pipefail its early exit can SIGPIPE ldd + uname -m + # zlib-devel, unzip and which are in the base repositories; EPEL is not needed and its mirrors fail now and then + for i in 1 2 3 4 5; do + dnf install -y --disablerepo="epel*" --setopt=install_weak_deps=False zlib-devel unzip which >/dev/null && break + [ "$i" -lt 5 ] && { echo "dnf failed, retry $i in $((i * 20))s" >&2; sleep $((i * 20)); } || exit 1 + done + cd /work + bash native/scripts/build-image.sh /out + ' || { tail -n 80 "$T/lib/build.log" || true; exit 1; } + # the container ran as root: give the files back to the runner user + sudo chown -R "$(id -u):$(id -g)" "$T/lib" "$GITHUB_WORKSPACE" + cat "$T/lib/build.done" + cat "$T/lib/sha256.txt" + ls -l "$T/lib" + + # macOS and Windows build on the runner itself (no container): same script, capped heap/threads where the runner is small. + - name: Build the native image on the runner + if: ${{ matrix.kind != 'linux' }} + env: + ARCADEDB_ENGINE_VERSION: ${{ needs.plan.outputs.engine }} + MARCH: compatibility + XMX: ${{ matrix.xmx }} + NI_EXTRA: ${{ matrix.ni_extra }} + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_target }} + run: | + set -euo pipefail + mkdir -p "$T/lib" + bash native/scripts/build-image.sh "$T/lib" || { tail -n 120 "$T/lib/build.log" || true; exit 1; } + cat "$T/lib/build.done" + cat "$T/lib/sha256.txt" + ls -l "$T/lib" + + - name: Glibc floor of the libraries + if: ${{ matrix.kind == 'linux' }} + run: | + set -euo pipefail + for so in "$T"/lib/*.so; do + top="$(objdump -T "$so" | grep -Eo 'GLIBC_[0-9]+\.[0-9]+(\.[0-9]+)?' | sort -uV | tail -n 1)" + echo "$(basename "$so"): highest $top" + [ "$(printf '%s\nGLIBC_2.34\n' "$top" | sort -V | tail -n 1)" = "GLIBC_2.34" ] || { echo "$so needs $top, above the manylinux_2_34 floor" >&2; exit 1; } + done + + - name: Build the wheel + env: + VERSION: ${{ needs.plan.outputs.version }} + ENGINE: ${{ needs.plan.outputs.engine }} + run: | + set -euo pipefail + # the Linux container logged /root/.m2/repository in the native-image classpath; on the runner that is $T/m2/repository + case "$KIND" in + linux) m2="$T/m2/repository" ;; + windows) m2="$(cygpath -m "$HOME")/.m2/repository" ;; + *) m2="$HOME/.m2/repository" ;; + esac + mkdir -p "$T/record" + uv run --no-project --python 3.12 python native/wheel/build_wheel.py --lib-dir "$T/lib" --version "$VERSION" --engine "$ENGINE" \ + --graalvm-home "$GRAALVM_HOME_P5" --m2-repo "$m2" \ + --notices-out "$T/record/THIRD_PARTY_NOTICES.generated.md" --out "$T/wheel" + ls -l "$T/wheel/dist" + shopt -s nullglob + wheels=("$T"/wheel/dist/*.whl) + [ "${#wheels[@]}" = 1 ] + + - name: "Gate: auditwheel grants the manylinux tag, no ISA warning" + if: ${{ matrix.kind == 'linux' }} + env: + PLAT: ${{ matrix.plat_re }} + run: | + set -euo pipefail + w="$(ls "$T"/wheel/dist/*.whl)" + uvx auditwheel show "$w" | tee "$T/auditwheel.txt" + grep -F 'consistent with the following platform tag' "$T/auditwheel.txt" + grep -F "\"$PLAT\"" "$T/auditwheel.txt" + if grep -Eiq 'instruction set|x86_64_v[0-9]' "$T/auditwheel.txt"; then + echo "auditwheel reports an ISA warning: the image is not baseline x86-64" >&2 + exit 1 + fi + + - name: "Gate: the macOS library links system libraries only" + if: ${{ matrix.kind == 'macos' }} + run: | + set -euo pipefail + w="$(ls "$T"/wheel/dist/*.whl)" + mkdir -p "$T/unz" + unzip -q -o "$w" -d "$T/unz" + dylib="$(ls "$T"/unz/arcadedb/_native/libarcadedb.dylib)" + otool -L "$dylib" + otool -l "$dylib" | grep -A4 -E 'LC_BUILD_VERSION|LC_VERSION_MIN' || true + lipo -info "$dylib" + # delocate lists the libraries the wheel would have to bundle: none may remain (system libraries are not listed) + deps="$(uvx --from delocate delocate-listdeps "$w")" + printf 'delocate-listdeps: [%s]\n' "$deps" + [ -z "$deps" ] + + # Informational: the imported DLLs, so a dependency on the Visual C++ redistributable would be visible. + - name: List the DLLs the Windows library imports + if: ${{ matrix.kind == 'windows' }} + run: | + set -euo pipefail + dll="$(ls "$T"/lib/libarcadedb.dll)" + dumpbin //dependents "$dll" | tee "$T/dumpbin.txt" + + - name: "Gate: twine check" + run: uvx twine check --strict "$T"/wheel/dist/*.whl + + - name: "Gate: the wheel is arcadedb, its version equals the requested (tag) version, the tag fits the platform" + env: + VERSION: ${{ needs.plan.outputs.version }} + PLAT_RE: ${{ matrix.plat_re }} + run: | + set -euo pipefail + w="$(ls "$T"/wheel/dist/*.whl)" + uv run --no-project --with packaging python - "$w" "$VERSION" "$PLAT_RE" <<'PY' + import re + import sys + from packaging.utils import parse_wheel_filename + from packaging.version import Version + name, version, _build, tags = parse_wheel_filename(sys.argv[1].replace("\\", "/").rsplit("/", 1)[-1]) + want = Version(sys.argv[2]) + print(f"wheel {name} {version} {sorted(map(str, tags))}; requested {want}") + assert str(name) == "arcadedb", name + assert str(want) == sys.argv[2], f"{sys.argv[2]} is not the canonical PEP 440 form {want}: fix the tag" + assert version == want, f"wheel version {version} != requested {want}" + assert len(tags) == 1, tags + (tag,) = tags + assert (tag.interpreter, tag.abi) == ("py3", "none"), tag + assert re.fullmatch(sys.argv[3], tag.platform), f"platform tag {tag.platform} does not match {sys.argv[3]}" + PY + + # also on failure: the generated notices and the full build log are what a drift needs to be diagnosed + - name: Collect the build record + if: ${{ always() }} + run: | + mkdir -p "$T/dist-out" "$T/record" + cp "$T/lib/build.log" "$T/record/build.log" || true + cp "$T"/wheel/dist/*.whl "$T/dist-out/" || true + for f in auditwheel.txt dumpbin.txt; do + if [ -f "$T/$f" ]; then cp "$T/$f" "$T/record/"; fi + done + cp "$T/lib/sha256.txt" "$T/lib/build.done" "$T/record/" || true + tail -n 400 "$T/lib/build.log" > "$T/record/build.log.tail" || true + (cd "$T/dist-out" && if command -v sha256sum > /dev/null; then sha256sum ./*.whl; else shasum -a 256 ./*.whl; fi) | tee "$T/record/wheel.sha256" || true + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: native-wheel-${{ matrix.key }} + path: ${{ runner.temp }}/dist-out/*.whl + if-no-files-found: error + retention-days: 30 + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: ${{ always() }} + with: + name: native-build-record-${{ matrix.key }} + path: ${{ runner.temp }}/record/ + if-no-files-found: ignore + retention-days: 30 + + test: + name: Parity against the installed wheel (${{ matrix.key }}, Python ${{ matrix.python }}) + needs: [plan, build] + # A platform whose build failed has no wheel to download and shows red here; the other platforms still run. + if: ${{ !cancelled() && needs.plan.outputs.run == 'true' }} + runs-on: ${{ matrix.runner }} + timeout-minutes: ${{ matrix.test_minutes }} + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.test) }} + env: + KEY: ${{ matrix.key }} + PY: ${{ matrix.python }} + steps: + - name: Keep LF line endings + run: | + git config --global core.autocrlf false + git config --global core.eol lf + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Paths + run: | + if [ "$RUNNER_OS" = Windows ]; then T="$(cygpath -m "$RUNNER_TEMP")"; else T="$RUNNER_TEMP"; fi + echo "T=$T" >> "$GITHUB_ENV" + + - name: Install uv + if: ${{ runner.os != 'Windows' }} + run: | + installer="$(mktemp)" + curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" "https://astral.sh/uv/${UV_VERSION}/install.sh" + sh "$installer" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install uv (Windows) + if: ${{ runner.os == 'Windows' }} + shell: pwsh + run: | + $env:UV_NO_MODIFY_PATH = "1" + Invoke-RestMethod "https://astral.sh/uv/$env:UV_VERSION/install.ps1" | Invoke-Expression + Add-Content -Path $env:GITHUB_PATH -Value "$env:USERPROFILE\.local\bin" + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: native-wheel-${{ matrix.key }} + path: ${{ runner.temp }}/wheel + + # The tests of this checkout run in a clean environment that contains the wheel and the test dependencies, not the source tree. + # The repository tooling tests that are about the wheel template and the matrix run here too (they need the checkout). + - name: Run the parity suite + env: + FINAL: ${{ needs.plan.outputs.final }} + run: | + set -euo pipefail + w="$(ls "$T"/wheel/*.whl)" + mkdir -p "$T/parity" + cores="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)" + limit=() + if [ "$FINAL" = true ]; then limit=(--max-notyet 0); fi + uv run --no-project --python "$PY" --with "$w" --with pytest --with numpy --with requests --with jpype1 --with pyarrow \ + --with polars --with pandas --with duckdb --with networkx --with "psycopg[binary]" --with adbc-driver-postgresql \ + --with redis --with neo4j --with ladybug \ + python native/wheel/parity_wheel.py --repo "$GITHUB_WORKSPACE" \ + --out "$T/parity/parity-wheel.tsv" --cores "0-$((cores - 1))" "${limit[@]}" | tee "$T/parity/parity-wheel.run.log" + + - name: "Gate: no failure, error, crash, timeout or uncounted skip" + run: | + set -euo pipefail + tsv="$T/parity/parity-wheel.tsv" + awk -F'\t' ' + NR == 1 { next } + $1 == "TOTAL" { passed = $2; failed = $3; err = $4; notyet = $5; na = $6; skipped = $7; tot = 1; next } + $8 != "" { bad = bad " " $1 "(" $8 ")" } + END { + if (!tot) { print "no TOTAL row" > "/dev/stderr"; exit 1 } + printf "parity %s py%s: passed=%s failed=%s error=%s notyet=%s na_jvm=%s skipped=%s\n", ENVIRON["KEY"], ENVIRON["PY"], passed, failed, err, notyet, na, skipped + if (bad != "") { print "files with a status:" bad > "/dev/stderr"; exit 1 } + if (passed + 0 == 0 || failed + 0 != 0 || err + 0 != 0) exit 1 + # this repository accepts no skips; NOTYET and NA-JVM are the two counted kinds of the native build + if (skipped + 0 != 0) { print "skipped tests that are neither NOTYET nor NA-JVM: " skipped > "/dev/stderr"; exit 1 } + }' "$tsv" | tee -a "$GITHUB_STEP_SUMMARY" + + # Reached only when the gate passed: the publish jobs read these markers (one per platform and Python, holding the wheel file name). + - name: Mark this leg as passed + run: | + set -euo pipefail + mkdir -p "$T/passed" + basename "$(ls "$T"/wheel/*.whl)" > "$T/passed/$KEY-py$PY.ok" + cat "$T/passed/$KEY-py$PY.ok" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: native-passed-${{ matrix.key }}-py${{ matrix.python }} + path: ${{ runner.temp }}/passed/${{ matrix.key }}-py${{ matrix.python }}.ok + if-no-files-found: error + retention-days: 30 + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: ${{ !cancelled() }} + with: + name: native-parity-${{ matrix.key }}-py${{ matrix.python }} + path: | + ${{ runner.temp }}/parity/parity-wheel.tsv + ${{ runner.temp }}/parity/parity-wheel.run.log + ${{ runner.temp }}/parity/parity-wheel-xml/ + if-no-files-found: warn + retention-days: 30 + + # What may be published, decided once for both publish jobs: a list of the wheels whose platform passed on every required Python. + verify: + name: Verify what may be published + needs: [plan, build, test] + if: ${{ !cancelled() && needs.plan.outputs.publish == 'true' }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + outputs: + ok: ${{ steps.verify.outputs.ok }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: native-passed-* + path: passed + merge-multiple: true + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: native-wheel-* + path: wheels + merge-multiple: true + + - name: Every required platform passed on every Python, and the wheels are the ones that were tested + id: verify + env: + VERSION: ${{ needs.plan.outputs.version }} + FINAL: ${{ needs.plan.outputs.final }} + BAR: ${{ vars.ARCADEDB_NATIVE_READINESS_BAR_MET }} + run: | + set -euo pipefail + mkdir -p passed dist + ls -l passed wheels + pythons="3.11 3.12 3.13 3.14" + if [ "$FINAL" = true ]; then + required="linux-x86_64 linux-aarch64 macos-arm64 windows-x64" + if [ "$BAR" != true ]; then + echo "::error::$VERSION is a final release of arcadedb (native), and the readiness bar is not marked as met (repository variable ARCADEDB_NATIVE_READINESS_BAR_MET): nothing is published. See docs/modules/ROOT/pages/development/release.adoc" >&2 + exit 1 + fi + else + required="linux-x86_64 linux-aarch64 macos-arm64" + fi + for key in $required; do + for py in $pythons; do + [ -f "passed/$key-py$py.ok" ] || { echo "::error::required leg $key on Python $py did not pass its parity run: nothing is published" >&2; exit 1; } + done + done + # every platform that passed on every Python is published (Windows is informational for a pre-release) + for key in linux-x86_64 linux-aarch64 macos-arm64 windows-x64; do + all=true + for py in $pythons; do [ -f "passed/$key-py$py.ok" ] || all=false; done + [ "$all" = true ] || continue + w="$(cat "passed/$key-py3.12.ok")" + case "$w" in + "arcadedb-${VERSION}-py3-none-"*.whl) ;; + *) echo "::error::unexpected wheel $w for version $VERSION" >&2; exit 1 ;; + esac + [ -f "wheels/$w" ] || { echo "::error::wheel $w is missing" >&2; exit 1; } + cp "wheels/$w" dist/ + done + ls -l dist + echo "ok=true" >> "$GITHUB_OUTPUT" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: native-release-dist + path: dist/*.whl + if-no-files-found: error + retention-days: 7 + + publish-prerelease: + name: Publish a pre-release to PyPI + needs: [plan, verify] + if: ${{ !cancelled() && needs.verify.result == 'success' && needs.plan.outputs.final == 'false' }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + environment: + name: pypi-prerelease + url: https://pypi.org/project/arcadedb/ + permissions: + id-token: write # PyPI trusted publishing (OIDC); no API token anywhere + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: native-release-dist + path: dist + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + packages-dir: dist + + # The first FINAL arcadedb (native) release. `verify` already refused it unless ARCADEDB_NATIVE_READINESS_BAR_MET is true. + publish: + name: Publish to PyPI + needs: [plan, verify] + if: ${{ !cancelled() && needs.verify.result == 'success' && needs.plan.outputs.final == 'true' }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + environment: + name: pypi + url: https://pypi.org/project/arcadedb/ + permissions: + id-token: write + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: native-release-dist + path: dist + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + packages-dir: dist diff --git a/.github/workflows/test-python-bindings.yml b/.github/workflows/test-python-bindings.yml index b32da7e..3a160f7 100644 --- a/.github/workflows/test-python-bindings.yml +++ b/.github/workflows/test-python-bindings.yml @@ -16,6 +16,9 @@ on: - "CLAUDE.md" - "examples/README.md" - "LICENSE*" + # the native build has its own workflow (native.yml); nothing under native/ is in the JVM wheel + - "native/**" + - ".github/workflows/native.yml" pull_request: # ready_for_review: draft PRs skip the platform matrices (see the job conditions), # so marking one ready is what starts them. @@ -27,6 +30,9 @@ on: - "CLAUDE.md" - "examples/README.md" - "LICENSE*" + # the native build has its own workflow (native.yml); nothing under native/ is in the JVM wheel + - "native/**" + - ".github/workflows/native.yml" # A new upstream snapshot is published as arcadedata/arcadedb:latest without # any change here, so build and test against it every night as well. The tag diff --git a/.github/workflows/test-python-examples.yml b/.github/workflows/test-python-examples.yml index 65b08a5..5a5a77c 100644 --- a/.github/workflows/test-python-examples.yml +++ b/.github/workflows/test-python-examples.yml @@ -16,6 +16,9 @@ on: - "CLAUDE.md" - "examples/README.md" - "LICENSE*" + # the native build has its own workflow (native.yml); nothing under native/ is in the JVM wheel + - "native/**" + - ".github/workflows/native.yml" pull_request: # ready_for_review: draft PRs skip the platform matrices (see the job conditions), # so marking one ready is what starts them. @@ -27,6 +30,9 @@ on: - "CLAUDE.md" - "examples/README.md" - "LICENSE*" + # the native build has its own workflow (native.yml); nothing under native/ is in the JVM wheel + - "native/**" + - ".github/workflows/native.yml" # A new upstream snapshot is published as arcadedata/arcadedb:latest without # any change here, so build and test against it every night as well. The tag diff --git a/scripts/select_native_matrix.py b/scripts/select_native_matrix.py new file mode 100644 index 0000000..03ae360 --- /dev/null +++ b/scripts/select_native_matrix.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Choose the legs of the native workflow (.github/workflows/native.yml) for one event. + +A native image build takes about 1 to 2.5 hours per platform and the ArcadeData organisation shares a runner cap with +upstream ArcadeDB, so a pull request builds one platform (linux-x86_64) and tests one Python (3.12), and only when +native paths changed (the workflow's `paths` filter). Push to main, the nightly run and a dispatch build all four +platforms and test Python 3.12 and 3.14 on each. A release tag tests every supported Python (3.11 to 3.14) on all four. + +usage: select_native_matrix.py EVENT [--tag-kind final|dev|none] EVENT: pull_request | push | schedule | workflow_dispatch | tag + +Prints `build=`, `test=` and `publish=` lines (GITHUB_OUTPUT format). `publish` is true for a +tag only. The GraalVM CE archives are pinned by sha256 per platform (GraalVM Community Edition only, never Oracle GraalVM). +""" + +from __future__ import annotations + +import argparse +import json +import sys + +GRAALVM_VERSION = "25.4.4.1.1" + +# key -> the build leg. `graalvm_sha256` is the release asset's digest (GitHub API), equal to the release's own .sha256. +PLATFORMS: dict[str, dict] = { + "linux-x86_64": { + "kind": "linux", + "runner": "ubuntu-latest", + "container": "quay.io/pypa/manylinux_2_34_x86_64:latest", + "graalvm_asset": "graalvm-community-jdk-25i4-25.0.4.1.1_linux-x64_bin.tar.gz", + "graalvm_sha256": "05ccbbe783210b6886ff7b08fcd0b061c5dce4852b05db87284fc0e24abb08e2", + "plat_re": "manylinux_2_34_x86_64", + "xmx": "12g", + "ni_extra": "", + "macos_target": "", + "build_minutes": 150, + "test_minutes": 120, + }, + "linux-aarch64": { + "kind": "linux", + "runner": "ubuntu-24.04-arm", + "container": "quay.io/pypa/manylinux_2_34_aarch64:latest", + "graalvm_asset": "graalvm-community-jdk-25i4-25.0.4.1.1_linux-aarch64_bin.tar.gz", + "graalvm_sha256": "e5f5e2f59643cf96765c741dc00b206f86c69c8c1bf843fe26050c871e0e2dbc", + "plat_re": "manylinux_2_34_aarch64", + "xmx": "12g", + "ni_extra": "", + "macos_target": "", + "build_minutes": 150, + "test_minutes": 120, + }, + "macos-arm64": { + "kind": "macos", + "runner": "macos-15", + "container": "", + "graalvm_asset": "graalvm-community-jdk-25i4-25.0.4.1.1_macos-aarch64_bin.tar.gz", + "graalvm_sha256": "4ec9932aa4aab1afca0ea31982d32d6f956615c61fe21a227b49756b25adda4f", + "plat_re": "macosx_[0-9]+_[0-9]+_arm64", + # 7 GB runner: a capped builder heap and two threads; -mmacosx-version-min because the link ignores + # MACOSX_DEPLOYMENT_TARGET alone + "xmx": "5g", + "ni_extra": "-H:NumberOfThreads=2 -H:NativeLinkerOption=-mmacosx-version-min=11.0", + "macos_target": "11.0", + "build_minutes": 180, + "test_minutes": 150, + }, + "windows-x64": { + "kind": "windows", + "runner": "windows-latest", + "container": "", + "graalvm_asset": "graalvm-community-jdk-25i4-25.0.4.1.1_windows-x64_bin.zip", + "graalvm_sha256": "2733ea1331f1a98b05dd55a768b07347051dad48f030dbbfa191b11a2ccf4144", + "plat_re": "win_amd64", + "xmx": "12g", + "ni_extra": "", + "macos_target": "", + "build_minutes": 150, + "test_minutes": 150, + }, +} + +# Python versions the wheel supports (the classifiers of native/wheel/pyproject.toml). +ALL_PYTHONS = ["3.11", "3.12", "3.13", "3.14"] + +# Platforms whose parity run must pass before a pre-release (A.B.C.devN) is published. Windows is the newest wheel and is +# informational for a pre-release; a FINAL release needs all four (see ALL_PLATFORMS and native.yml). +PRERELEASE_REQUIRED = ["linux-x86_64", "linux-aarch64", "macos-arm64"] +ALL_PLATFORMS = list(PLATFORMS) + +PULL_REQUEST_PLATFORMS = ["linux-x86_64"] +PULL_REQUEST_PYTHONS = ["3.12"] +MAIN_PYTHONS = ["3.12", "3.14"] + + +def select(event: str) -> dict: + if event == "pull_request": + platforms, pythons = PULL_REQUEST_PLATFORMS, PULL_REQUEST_PYTHONS + elif event in ("push", "schedule", "workflow_dispatch"): + platforms, pythons = ALL_PLATFORMS, MAIN_PYTHONS + elif event == "tag": + platforms, pythons = ALL_PLATFORMS, ALL_PYTHONS + else: + raise ValueError(f"unknown event {event!r}") + build = [{"key": k, **PLATFORMS[k]} for k in platforms] + test = [ + { + "key": k, + "runner": PLATFORMS[k]["runner"], + "python": py, + "test_minutes": PLATFORMS[k]["test_minutes"], + } + for k in platforms + for py in pythons + ] + return {"build": build, "test": test, "publish": event == "tag"} + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser() + ap.add_argument( + "event", + choices=["pull_request", "push", "schedule", "workflow_dispatch", "tag"], + ) + a = ap.parse_args(argv) + chosen = select(a.event) + print(f"build={json.dumps(chosen['build'], separators=(',', ':'))}") + print(f"test={json.dumps(chosen['test'], separators=(',', ':'))}") + print(f"publish={'true' if chosen['publish'] else 'false'}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 9d28df95cf8e254a8b483344ce0f9e960d62f129 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 20:35:57 +0900 Subject: [PATCH 05/33] Docs: which package? The choice on every page a user lands on README, the installation page, the quickstart and both PyPI descriptions (README.md for arcadedb-jvm, native/wheel/README.md for arcadedb) name both packages up front: arcadedb is the native build and the default for scripts, notebooks, tests, CLIs, serverless functions and agents; arcadedb-jvm is the JVM build for heavy, long-running loads; both import as arcadedb. The install lines are shown side by side with a one-line rule, never a bare pip install arcadedb. New permanent page docs/modules/ROOT/pages/which-package.adoc (replaces packages.adoc): the table, how to check (arcadedb.__backend__), what changes between the builds, switching and the import guard, server use (the official Docker images arcadedata/arcadedb), drivers, and a marked transition section for the old names. Transition-only callouts carry a REMOVE WHEN comment. The CI and release pages describe native.yml and the readiness bar for a final arcadedb release. tests/test_native_wheel_template.py pins the template of the native wheel (name, classifiers, sibling named in the first paragraph). Co-Authored-By: Claude Sonnet 5.5 --- README.md | 53 +++++-- docs/antora.yml | 16 +- docs/modules/ROOT/nav.adoc | 2 +- docs/modules/ROOT/pages/development/ci.adoc | 10 +- .../ROOT/pages/development/release.adoc | 30 +++- .../ROOT/pages/how-to/concurrency.adoc | 9 +- .../pages/how-to/databases-and-queries.adoc | 2 +- docs/modules/ROOT/pages/index.adoc | 61 ++++---- docs/modules/ROOT/pages/installation.adoc | 70 +++++---- docs/modules/ROOT/pages/packages.adoc | 56 ------- docs/modules/ROOT/pages/quickstart.adoc | 22 ++- docs/modules/ROOT/pages/which-package.adoc | 145 ++++++++++++++++++ native/wheel/README.md | 69 +++++++++ 13 files changed, 394 insertions(+), 151 deletions(-) delete mode 100644 docs/modules/ROOT/pages/packages.adoc create mode 100644 docs/modules/ROOT/pages/which-package.adoc create mode 100644 native/wheel/README.md diff --git a/README.md b/README.md index 4f274cf..3ca0caa 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,38 @@ # ArcadeDB Python -[![PyPI](https://img.shields.io/pypi/v/arcadedb-jvm)](https://pypi.org/project/arcadedb-jvm/) +[![PyPI arcadedb](https://img.shields.io/pypi/v/arcadedb?label=arcadedb)](https://pypi.org/project/arcadedb/) +[![PyPI arcadedb-jvm](https://img.shields.io/pypi/v/arcadedb-jvm?label=arcadedb-jvm)](https://pypi.org/project/arcadedb-jvm/) [![Python versions](https://img.shields.io/pypi/pyversions/arcadedb-jvm)](https://pypi.org/project/arcadedb-jvm/) [![Tests](https://github.com/ArcadeData/arcadedb-python/actions/workflows/test-python-bindings.yml/badge.svg)](https://github.com/ArcadeData/arcadedb-python/actions/workflows/test-python-bindings.yml) [![License](https://img.shields.io/badge/license-Apache--2.0-blue)](https://github.com/ArcadeData/arcadedb-python/blob/main/LICENSE) -Python bindings for [ArcadeDB](https://arcadedb.com), the multi-model database (graph, document, key/value, vector and time series, queried with SQL or openCypher). The engine runs inside your Python process through [JPype](https://jpype.readthedocs.io/), with no server and no network hop. The wheel bundles its own Java 25 runtime, so you install no Java. Status: beta. +Python bindings for [ArcadeDB](https://arcadedb.com), the multi-model database (graph, document, key/value, vector and time series, queried with SQL or openCypher). The engine runs inside your Python process, with no server and no network hop. This repository builds two packages, and both import as `arcadedb`. `arcadedb-jvm` has a sibling, `arcadedb`, the native build: pick `arcadedb` for scripts, notebooks, tests, CLIs, serverless functions and agents, and `arcadedb-jvm` for heavy, long-running loads. Status: beta. + +| Package | Engine | Pick it for | +|---|---|---| +| [`arcadedb`](https://pypi.org/project/arcadedb/) (native, the default) | The engine compiled by GraalVM Community Edition into a shared library, loaded with `ctypes`. No JVM, no JPype | Scripts, notebooks, tests, CLIs, serverless functions, agents | +| [`arcadedb-jvm`](https://pypi.org/project/arcadedb-jvm/) (JVM) | The Java engine in your process through [JPype](https://jpype.readthedocs.io/), with a bundled Java 25 runtime | Heavy, long-running loads, where the JIT wins over time. Also the Java-object APIs and JVM options | + +Both packages `import arcadedb`, and they have the same public API. Install one, not both. Not sure which? Read [Which package?](https://github.com/ArcadeData/arcadedb-python/blob/main/docs/modules/ROOT/pages/which-package.adoc). ## Install ```bash -pip install arcadedb-jvm # or: uv add arcadedb-jvm +pip install arcadedb # native, the default (or: uv add arcadedb) +pip install arcadedb-jvm # JVM, for heavy long-running loads (or: uv add arcadedb-jvm) ``` -This package was published as arcadedb-embedded until 26.10.x; install arcadedb now. + +> Transition note: the first release of this layout is 26.11.1. Until `arcadedb` passes its readiness bar, it may appear only as pre-releases, `26.11.1.devN`. Install it with `pip install --pre arcadedb` (or `uv add --prerelease allow arcadedb`). + + +> Old names: `arcadedb-jvm` was formerly `arcadedb-embedded`, and `arcadedb` was formerly `arcadedb-embedded-native`. The old packages will get a deprecation shim. - Python 3.11 to 3.14. CI builds and tests all four versions on every platform below for `main`, every night and each release. A pull request runs a smaller selection. -- Linux x86_64 and ARM64, macOS on Apple Silicon, Windows x86_64. One wheel per platform and Python version. -- Import name: `arcadedb`. Optional extras: `arrow`, `pandas`, `vector`. +- Linux x86_64 and ARM64, macOS on Apple Silicon, Windows x86_64, for both packages. The Windows wheel of `arcadedb` is the newest. +- Import name: `arcadedb`. Check which package you have with `python -c "import arcadedb; print(arcadedb.__backend__)"`. It prints `native` or `jvm`. +- Optional extras: `arrow`, `pandas`, `vector` for `arcadedb-jvm`; `numpy`, `arrow`, `pandas`, `polars` for `arcadedb`. +- `arcadedb-jvm` bundles its own Java 25 runtime, so you install no Java. `arcadedb` needs no Java at all. ## Quick start @@ -45,23 +60,29 @@ This prints `Name: Alice` (the engine also logs to standard error). `create_data - Graph, document, key/value, vector (JVector HNSW) and time series models; ACID transactions. - Import from CSV, XML and ArcadeDB JSONL; export to JSONL. - Columnar results with `to_columns()`, `to_arrow()` and `to_dataframe()`. -- An optional in-process HTTP server (`create_server()`), off until you call it, for other processes that need the same data. From 26.11.1 the wheels also carry the Studio web UI, which the server then serves. Studio used to bundle ApexCharts, which is no longer open source ([ArcadeData/arcadedb#10579](https://github.com/ArcadeData/arcadedb/issues/10579)), and [ArcadeData/arcadedb#10627](https://github.com/ArcadeData/arcadedb/pull/10627) replaced it with Apache ECharts. The earlier 26.10.x wheels on PyPI have no Studio. +- An optional in-process HTTP server (`create_server()`), off until you call it, for other processes that need the same data. -For several processes, other languages, HA or replication, run the regular [ArcadeDB server](https://docs.arcadedb.com) and connect with [`arcadedb-driver`](https://pypi.org/project/arcadedb-driver/) (HTTP client, developed in [ArcadeData/arcadedb-drivers](https://github.com/ArcadeData/arcadedb-drivers)). + +> New in 26.11.1: the `arcadedb-jvm` wheels carry the Studio web UI, which the server then serves. Studio used to bundle ApexCharts, which is no longer open source ([ArcadeData/arcadedb#10579](https://github.com/ArcadeData/arcadedb/issues/10579)), and [ArcadeData/arcadedb#10627](https://github.com/ArcadeData/arcadedb/pull/10627) replaced it with Apache ECharts. The earlier 26.10.x wheels on PyPI have no Studio. + +For several processes, other languages, HA or replication, run the official Docker image `arcadedata/arcadedb` ([ArcadeDB server](https://docs.arcadedb.com)) and connect with [`arcadedb-driver`](https://pypi.org/project/arcadedb-driver/) (HTTP client, developed in [ArcadeData/arcadedb-drivers](https://github.com/ArcadeData/arcadedb-drivers)). ## Packages on PyPI | Package | What it is | Status | |---|---|---| -| [`arcadedb`](https://pypi.org/project/arcadedb/) | These bindings: the engine on a bundled JVM. The package this README describes | First release from this repository: 26.11.1 | -| [`arcadedb-native`](https://pypi.org/project/arcadedb-native/) | The same API with the engine compiled ahead of time by GraalVM Community Edition: no JVM, no JPype. Python 3.11 or later. It imports as `arcadedb`, like the JVM wheel, so install only one of the two | Not in this repository yet (porting is tracked in [#19](https://github.com/ArcadeData/arcadedb-python/issues/19)). Until then a placeholder on PyPI | -| Formerly `arcadedb-embedded` and `arcadedb-embedded-native` | The names of these packages until 26.10.x. They import as `arcadedb_embedded` and `arcadedb_native` | Deprecated: install `arcadedb` | +| [`arcadedb`](https://pypi.org/project/arcadedb/) | The native build: the engine compiled by GraalVM Community Edition, no JVM, no JPype. The default for embedded use | First release of this layout: 26.11.1. It may first appear as `26.11.1.devN` pre-releases | +| [`arcadedb-jvm`](https://pypi.org/project/arcadedb-jvm/) | The JVM build: the Java engine on a bundled JVM, through JPype. The package this README described before the two-package layout | First release from this repository: 26.11.1 | +| `arcadedb-native` | An alias of `arcadedb`: same version, it only installs `arcadedb` | Planned | + + +Formerly `arcadedb-embedded` (now `arcadedb-jvm`, the same build) and formerly `arcadedb-embedded-native` (now `arcadedb`). They imported as `arcadedb_embedded` and `arcadedb_native`, and they are deprecated. Releases from this repository start with 26.11.1. The 26.10.x wheels were built and published from [humemai/arcadedb-embedded-python](https://github.com/humemai/arcadedb-embedded-python). ## Documentation -- Here: [`docs/`](https://github.com/ArcadeData/arcadedb-python/tree/main/docs/modules/ROOT/pages) is an Antora component (AsciiDoc) with Installation, Which package for which mode, Quickstart, Databases and queries, and Concurrency, processes and crashes. It covers only what is specific to the Python bindings. +- Here: [`docs/`](https://github.com/ArcadeData/arcadedb-python/tree/main/docs/modules/ROOT/pages) is an Antora component (AsciiDoc) with Installation, [Which package?](https://github.com/ArcadeData/arcadedb-python/blob/main/docs/modules/ROOT/pages/which-package.adoc), Quickstart, Databases and queries, and Concurrency, processes and crashes. It covers only what is specific to the Python bindings. - The component is not published on docs.arcadedb.com yet; adding it as a content source is open in [ArcadeData/arcadedb-docs#505](https://github.com/ArcadeData/arcadedb-docs/pull/505). - The guide, API reference and example walkthroughs for the published package are still at [docs.humem.ai/arcadedb](https://docs.humem.ai/arcadedb/latest/) until those pages are ported here ([#19](https://github.com/ArcadeData/arcadedb-python/issues/19)). - ArcadeDB itself (SQL, openCypher, schema, server): [docs.arcadedb.com](https://docs.arcadedb.com). It is not repeated here. @@ -73,10 +94,12 @@ Releases from this repository start with 26.11.1. The 26.10.x wheels were built ## Build and test from source ```bash -./scripts/build.sh # a wheel for your platform in dist/ +./scripts/build.sh # an arcadedb-jvm wheel for your platform in dist/ ``` -Linux builds run in Docker. macOS and Windows need a JDK 25 or later. The jars come from the `arcadedata/arcadedb` Docker image: `latest` by default, or the tag in `ARCADEDB_TAG` (for example `ARCADEDB_TAG=26.10.2 ./scripts/build.sh`). The package version is read from the engine jar in that image, so a `26.11.1-SNAPSHOT` image builds `26.11.1.devN`. +The `arcadedb` (native) wheel is built from `native/` with GraalVM Community Edition; [`CLAUDE.md`](https://github.com/ArcadeData/arcadedb-python/blob/main/CLAUDE.md) has the commands. A native image build takes about 1 to 2.5 hours per platform. + +The JVM build runs in Docker on Linux. macOS and Windows need a JDK 25 or later. The jars come from the `arcadedata/arcadedb` Docker image: `latest` by default, or the tag in `ARCADEDB_TAG` (for example `ARCADEDB_TAG=26.10.2 ./scripts/build.sh`). The package version is read from the engine jar in that image, so a `26.11.1-SNAPSHOT` image builds `26.11.1.devN`. Tests run against the built wheel, not the source tree: install the wheel with its `test` extra into a virtual environment, then run `pytest`. [`CLAUDE.md`](https://github.com/ArcadeData/arcadedb-python/blob/main/CLAUDE.md) has the exact commands, the build options and the lint setup. @@ -86,7 +109,7 @@ Open an [issue](https://github.com/ArcadeData/arcadedb-python/issues) or a pull ## Where this code comes from -It moved here from `bindings/python` in [ArcadeData/arcadedb](https://github.com/ArcadeData/arcadedb) at `ee04bd202abce0183c6263a3c4d8a61a4d526960`, with its git history. Issue numbers in older commit messages refer to that repository. [humemai/arcadedb-embedded-python](https://github.com/humemai/arcadedb-embedded-python) is the fork where the bindings kept developing (runtime fixes, JRE trim, native build, benchmarks). Its benchmark tooling stays there. +It moved here from `bindings/python` in [ArcadeData/arcadedb](https://github.com/ArcadeData/arcadedb) at `ee04bd202abce0183c6263a3c4d8a61a4d526960`, with its git history. Issue numbers in older commit messages refer to that repository. [humemai/arcadedb-embedded-python](https://github.com/humemai/arcadedb-embedded-python) is the fork where the bindings kept developing (runtime fixes, JRE trim, the native build, benchmarks). Its benchmark tooling stays there. ## Citation diff --git a/docs/antora.yml b/docs/antora.yml index 1c06e68..810da11 100644 --- a/docs/antora.yml +++ b/docs/antora.yml @@ -6,17 +6,21 @@ nav: - modules/ROOT/nav.adoc asciidoc: attributes: - # Package names. Every page takes them from here. - pkg-name: arcadedb-jvm + # Package names. Every page takes them from here. There is no bare pkg-name: + # a page always says which of the two packages it means. + # native-pkg-name: the native build (default for embedded use). + # jvm-pkg-name: the JVM build (heavy, long-running loads, Java-object APIs). + # Both import as arcadedb. + native-pkg-name: arcadedb + jvm-pkg-name: arcadedb-jvm import-name: arcadedb import-stmt: import arcadedb - native-pkg-name: arcadedb - native-import-name: arcadedb # Other names a reader meets on PyPI (the pages that compare packages use these). - jvm-pkg-name: arcadedb-jvm driver-pkg-name: arcadedb-driver grpc-driver-pkg-name: arcadedb-driver-grpc community-driver-pkg-name: arcadedb-python community-repo-url: https://github.com/stevereiner/arcadedb-python repo-url: https://github.com/ArcadeData/arcadedb-python - pypi-url: https://pypi.org/project/arcadedb-jvm/ + native-pypi-url: https://pypi.org/project/arcadedb/ + jvm-pypi-url: https://pypi.org/project/arcadedb-jvm/ + which-package-url: https://github.com/ArcadeData/arcadedb-python/blob/main/docs/modules/ROOT/pages/which-package.adoc diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index a4014ee..ce55e5c 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -1,6 +1,6 @@ * xref:index.adoc[ArcadeDB Python] * xref:installation.adoc[Installation] -* xref:packages.adoc[Which package for which mode] +* xref:which-package.adoc[Which package?] * xref:quickstart.adoc[Quickstart] * How-to ** xref:how-to/databases-and-queries.adoc[Databases and queries] diff --git a/docs/modules/ROOT/pages/development/ci.adoc b/docs/modules/ROOT/pages/development/ci.adoc index e867358..d6c467e 100644 --- a/docs/modules/ROOT/pages/development/ci.adoc +++ b/docs/modules/ROOT/pages/development/ci.adoc @@ -1,7 +1,7 @@ = Continuous integration :description: Which platforms and Python versions the CI runs for a pull request, and which for main, the nightly run and a release. -Two workflows test the package: `test-python-bindings.yml` builds a wheel and runs the unit tests, and `test-python-examples.yml` runs the scripts in `examples/`. Both build the wheels on a native runner of each platform. +Two workflows test the `arcadedb-jvm` package: `test-python-bindings.yml` builds a wheel and runs the unit tests, and `test-python-examples.yml` runs the scripts in `examples/`. Both build the wheels on a native runner of each platform. A third workflow, `native.yml`, builds and tests the `arcadedb` (native) package; see <<_native_build,Native build>>. The package supports four platforms and four Python versions (3.11 to 3.14). That is 16 combinations for each workflow. A pull request does not run all of them, because the organisation shares one runner limit with ArcadeData/arcadedb. Each macOS or Windows job on a pull request delays the upstream jobs too. @@ -34,6 +34,14 @@ A release always runs the full matrix, because `release.yml` expects one wheel f * A draft pull request skips the platform jobs. Marking it ready for review starts them. * A new push to a pull request cancels the run it replaces. A run on `main` is never cancelled. +== Native build + +The workflow `native.yml` builds the native image of the `arcadedb` package. It does not follow the table above. + +* A pull request runs it only when native paths change: `native/**`, `src/arcadedb/_native/**`, the native tests, or `native.yml` itself. It then runs on `linux-x86_64` with Python 3.12 only. +* A push to `main`, the nightly run, a manual run (dispatch) and a release tag run the full platform matrix: `linux-x86_64`, `linux-aarch64`, `macos-arm64` and `windows-x64`. +* A native image build takes about 1 to 2.5 hours for each platform. The organisation shares one runner limit with upstream ArcadeDB, so do not add runs lightly. Change the paths or the matrix of `native.yml` only for a reason that you can write down. + == Change the selection Edit `PULL_REQUEST_LEGS` in `scripts/select_matrix.py`. The test `tests/test_select_matrix.py` fixes the full matrix at 16 combinations, so a change that shrinks it fails. When you add a Python version, add it to `PYTHONS` in the script and to the classifiers in `pyproject.toml`. A test checks that both lists are the same, and that `requires-python` is the oldest of them. When you drop one, remove it from the same places and update the counts in the test. diff --git a/docs/modules/ROOT/pages/development/release.adoc b/docs/modules/ROOT/pages/development/release.adoc index 5067337..a487bbe 100644 --- a/docs/modules/ROOT/pages/development/release.adoc +++ b/docs/modules/ROOT/pages/development/release.adoc @@ -1,7 +1,7 @@ = Release -:description: How a release of the arcadedb-jvm package is numbered, tagged, checked and published to PyPI. +:description: How a release of the arcadedb-jvm and arcadedb (native) packages is numbered, tagged, checked and published to PyPI. -A release is a Git tag. Pushing the tag starts the `release.yml` workflow. It builds the wheels, tests them, checks them and publishes them to PyPI with trusted publishing. The workflow stores no token. +A release is a Git tag. Pushing the tag starts the release workflows. This repository releases two packages: `arcadedb-jvm` (the JVM build) with `release.yml`, and `arcadedb` (the native build) with `native.yml`. They build the wheels, test them, check them and publish them to PyPI with trusted publishing. The workflows store no token. The sections below describe `release.yml`; the native release is in <<_native_release,Native release>>. == Version numbering @@ -63,3 +63,29 @@ The second argument is the previous release. Use `-` for the first release of th * A `pip download` for each of the four platforms, and a fresh install on Python 3.12 and 3.14 that opens a database and runs one SQL and one openCypher query. It prints `PASS` or `FAIL` for each step, and its exit code is the number of failures. + +[[_native_release]] +== Native release + +The two packages release from two workflows with two PyPI environments: + +[cols="2,2,2"] +|=== +| Package | Workflow | GitHub environment + +| `arcadedb-jvm` +| `release.yml` +| `pypi-jvm` + +| `arcadedb` (native) +| `native.yml` +| `pypi` +|=== + +The version rule above holds for both packages. + +* A native pre-release (`26.11.1.devN`) publishes when the builds on `linux-x86_64`, `linux-aarch64` and `macos-arm64` pass. +* A FINAL native release (`A.B.C` or `A.B.C.postN`) needs two more things: +** The repository variable `ARCADEDB_NATIVE_READINESS_BAR_MET` is `true`. A maintainer sets it after checking the readiness bar: the correctness fixes are merged; the full suite is green on every platform, Windows included, and on Python 3.11 to 3.14; the fault and concurrency probes pass; and the documented API raises no `NotOnNativeYet`. +** The parity run of every platform and every Python version passes with zero `NotOnNativeYet` skips. +* Speed parity with the JVM build is not part of the bar. diff --git a/docs/modules/ROOT/pages/how-to/concurrency.adoc b/docs/modules/ROOT/pages/how-to/concurrency.adoc index 96198fe..f322624 100644 --- a/docs/modules/ROOT/pages/how-to/concurrency.adoc +++ b/docs/modules/ROOT/pages/how-to/concurrency.adoc @@ -12,11 +12,12 @@ Every number on this page comes from probe scripts that were run on 2026-10-24 o | Label | Package | Python | JVM -| `{pkg-name}` 26.10.2.post1, OpenJDK 25.0.4.1 +| `{jvm-pkg-name}` 26.10.2.post1, OpenJDK 25.0.4.1 | 3.12.13, GIL on +// REMOVE WHEN: the old arcadedb-embedded* shims are retired and two releases have shipped with the two-package layout allow-old-name | Native -| `{native-pkg-name}` 26.10.2.dev1, GraalVM CE 25.4.4.1.1 native image. A preview of alpha quality +| The native wheel (formerly `arcadedb-embedded-native`) 26.10.2.dev1, GraalVM CE 25.4.4.1.1 native image. A preview of alpha quality, measured before the rename | 3.12.13, GIL on | Native, Python 3.14t @@ -157,7 +158,7 @@ In all 15 phases (5 call styles, 3 builds) every operation landed (2,000 of 2,00 == Processes -One process owns an embedded database at a time. Start worker processes with `spawn` and give each its own database. If several processes must share data, run a server and use a driver: see xref:packages.adoc#drivers[Packages that talk to a server]. +One process owns an embedded database at a time. Start worker processes with `spawn` and give each its own database. If several processes must share data, run a server and use a driver: see xref:which-package.adoc#drivers[Packages that talk to a server]. The probe has these cases. Each ran once per build. @@ -251,4 +252,4 @@ If a process must stop and it does not react to a signal, `SIGKILL` cost no ackn The native wheel is a `py3-none` wheel, and it installed and ran on free-threaded Python 3.14.5 with the GIL off. The Python 3.14t rows in the tables above are those runs: threads, several writers, asyncio and the signal and kill probes. The probe on several processes was not run there. The runs are shorter than those on Python 3.12, as listed at the top of this page. -The JVM wheel does not install on Python 3.14t. `uv` could not resolve `{pkg-name}` 26.10.2.post1 there: it reported no wheel with a matching Python version tag. +The JVM wheel does not install on Python 3.14t. `uv` could not resolve `{jvm-pkg-name}` 26.10.2.post1 there: it reported no wheel with a matching Python version tag. diff --git a/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc b/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc index e46ec5d..9d41722 100644 --- a/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc +++ b/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc @@ -5,7 +5,7 @@ This guide covers the life of a database directory and the two calls that talk to it. Prefer SQL or openCypher through `db.command(...)` and `db.query(...)` for schema, inserts and graph work. The bindings also expose wrapper objects for the rest of the Java API; they are not covered here. -Every script on this page ran against the published `{pkg-name}` wheel. Run each one in an empty directory. +Every script on this page ran against the published `{jvm-pkg-name}` wheel. Run each one in an empty directory. == The life of a database diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index df982dd..07ddc28 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -1,10 +1,25 @@ = ArcadeDB Python -:description: Run the ArcadeDB engine inside your Python process: install, first steps and how-to guides for the {pkg-name} package. +:description: Run the ArcadeDB engine inside your Python process: install, first steps and how-to guides for the arcadedb (native) and arcadedb-jvm (JVM) packages. // Sources: bindings docs index.md, api-access-methods.md and getting-started/installation.md, -// and the PyPI description of arcadedb-native. +// and the PyPI descriptions of arcadedb and arcadedb-jvm. -`{pkg-name}` runs the ArcadeDB engine inside your Python process. Calls cross into the engine through https://jpype.readthedocs.io/[JPype], so there is no network hop and no server to start. The wheel bundles its own Java runtime (Java 25), so you do not install Java. +These packages run the ArcadeDB engine inside your Python process, with no network hop and no server to start. Two packages exist, and both use the import name `{import-name}`: + +[cols="1,3,3"] +|=== +| Package | Engine | Pick it for + +| `{native-pkg-name}` (the default) +| Native: the engine compiled by GraalVM Community Edition, loaded with `ctypes`. No JVM, no JPype. +| Scripts, notebooks, tests, CLIs, serverless functions and agents. + +| `{jvm-pkg-name}` +| JVM: the Java engine in your process through https://jpype.readthedocs.io/[JPype], with a bundled Java 25 runtime. +| Heavy, long-running loads, and the Java-object APIs. +|=== + +Not sure which one? Read xref:which-package.adoc[Which package?]. Install one of them, not both. ArcadeDB is a multi-model database: graph, document, key/value, vector and time series data live in one database, and you query it with SQL or openCypher. The bindings expose the two calls you need for that, `db.command(...)` and `db.query(...)`. @@ -49,54 +64,34 @@ ArcadeDB reaches Python in two ways. Pick by where the engine should run. | One per call | Setup -| `pip install {pkg-name}` -| A running server, plus `pip install arcadedb-driver` +| `pip install {native-pkg-name}` or `pip install {jvm-pkg-name}` +| A running server (the official Docker image `arcadedata/arcadedb`), plus `pip install arcadedb-driver` |=== Choose the embedded bindings for a single script, a notebook, or an application with one Python process that owns its data. Choose the xref:arcadedb::how-to/connectivity/drivers/python-http.adoc[HTTP driver] when more than one process, another language, or a remote machine must reach the same database, or when the server must outlive your Python process. -The two meet in one case. The bindings bundle an optional HTTP server, with the Studio web UI from 26.11.1. It is off until you call `create_server()`. The process that owns the database keeps direct access, and other processes connect over HTTP. That server lives and dies with your Python process. For a database that must outlive any one client, or for HA, replication and TLS, run the regular xref:arcadedb::index.adoc[ArcadeDB server] instead. +The two meet in one case. The bindings bundle an optional HTTP server. It is off until you call `create_server()`. The process that owns the database keeps direct access, and other processes connect over HTTP. That server lives and dies with your Python process. For a database that must outlive any one client, or for HA, replication and TLS, run the regular xref:arcadedb::index.adoc[ArcadeDB server] instead. [[jvm-vs-native]] -== JVM build or native build - -Two packages ship for now. Both give you the same engine and the same API. +== Native build or JVM build -[cols="1,3,3"] -|=== -| | `{pkg-name}` (default) | `{native-pkg-name}` (preview) - -| Runtime -| The engine on a bundled JVM, reached through JPype -| The engine compiled ahead of time with GraalVM Community Edition to a shared library, loaded with `ctypes`: no JVM and no JPype - -| Import name -| `{import-name}` -| `{native-import-name}` - -| Status -| The package these docs describe -| A preview of alpha quality. It is an experiment, not a release: do not use it for data you cannot lose - -| Install -| `pip install {pkg-name}` -| `pip install --pre {native-pkg-name}` -|=== +Both packages give you the same engine and the same API. The table in xref:which-package.adoc[Which package?] lists what differs: start-up time, memory, platforms, and the APIs that exist only on the JVM build (`get_java_database`, `jvm_kwargs`, `start_jvm`). The pages in this component apply to both packages. Parts that apply to one only say so. -Both packages import as `{import-name}`, so an environment holds one of them. The preview's public API follows the one of the default package, so the pages in this component apply to both; the JVM-specific parts (JVM options, `jvm_kwargs`) do not apply to the native build. Check the preview's https://pypi.org/project/{native-pkg-name}/[PyPI page] for its current status before you rely on it. +Check which one you have with `python -c "import arcadedb; print(arcadedb.__backend__)"`. It prints `native` or `jvm`. == Requirements * Python 3.11 to 3.14. The wheels' CI runs all four versions on every supported platform for `main`, the nightly run and each release. A pull request runs a smaller selection (xref:development/ci.adoc[Continuous integration]). -* Linux (x86_64 and ARM64), macOS on Apple Silicon, or Windows (x86_64). -* No Java installation. +* Linux (x86_64 and ARM64), macOS on Apple Silicon, or Windows (x86_64). The Windows wheel of the native build is the newest. +* No Java installation, for either package. == Links +* xref:which-package.adoc[Which package?] * xref:installation.adoc[Installation] * xref:quickstart.adoc[Quickstart] * xref:how-to/databases-and-queries.adoc[Databases and queries] -* {pypi-url}[`{pkg-name}` on PyPI] +* {native-pypi-url}[`{native-pkg-name}` on PyPI] and {jvm-pypi-url}[`{jvm-pkg-name}` on PyPI] * {repo-url}[Source repository] and {repo-url}/issues[issue tracker] The bindings are licensed under Apache 2.0, like the ArcadeDB engine. diff --git a/docs/modules/ROOT/pages/installation.adoc b/docs/modules/ROOT/pages/installation.adoc index b720055..b972406 100644 --- a/docs/modules/ROOT/pages/installation.adoc +++ b/docs/modules/ROOT/pages/installation.adoc @@ -1,37 +1,48 @@ = Installation -:description: Install the {pkg-name} wheel, check that it works, and set JVM options. +:description: Install the arcadedb (native) or arcadedb-jvm (JVM) wheel, check that it works, and set JVM options. // Sources: bindings docs getting-started/installation.md, the PyPI metadata of -// arcadedb and arcadedb-native. +// arcadedb and arcadedb-jvm. == Install -[source,shell,subs="attributes+"] ----- -pip install {pkg-name} ----- +Pick one of the two packages. Both import as `{import-name}`. -This package was published as arcadedb-embedded until 26.10.x; install arcadedb now. +[cols="1,2,2"] +|=== +| Package | Install | Pick this if -Or with https://docs.astral.sh/uv/[uv]: +| `{native-pkg-name}` (native, the default) +| `pip install {native-pkg-name}` + +`uv add {native-pkg-name}` +| You write scripts, notebooks, tests, CLIs, serverless functions or agents. You want a fast start and a small footprint, and you need no JVM. -[source,shell,subs="attributes+"] ----- -uv add {pkg-name} ----- +| `{jvm-pkg-name}` (JVM) +| `pip install {jvm-pkg-name}` + +`uv add {jvm-pkg-name}` +| Your load is heavy and long-running, so the JIT compiler pays off. Or you need the Java-object APIs, JVM options, `jvm_kwargs` or `start_jvm`. +|=== + +Install one of them, not both. If both are installed, `import arcadedb` raises an `ImportError` that names both packages. See xref:which-package.adoc[Which package?] for the comparison, and for how to switch. -The package is self-contained. It carries a Java 25 runtime for its platform, so you do not install Java. Its only Python dependency is `JPype1` 1.5.0 or later (the Java-Python bridge), which pip installs for you. +// REMOVE WHEN: the old arcadedb-embedded* shims are retired and two releases have shipped with the two-package layout allow-old-name +NOTE: *Transition note.* The first release of this layout is 26.11.1. Until `{native-pkg-name}` passes its readiness bar, it may appear only as pre-releases, `26.11.1.devN`. Install it with `pip install --pre {native-pkg-name}` or `uv add --prerelease allow {native-pkg-name}`. + +// REMOVE WHEN: the old arcadedb-embedded* shims are retired and two releases have shipped with the two-package layout allow-old-name +NOTE: *Old names.* The JVM package was formerly `arcadedb-embedded`. It is now `{jvm-pkg-name}`, the same build. The native package was formerly `arcadedb-embedded-native`. It is now `{native-pkg-name}`. See xref:which-package.adoc#old-names[Old names]. + +The JVM package is self-contained. It carries a Java 25 runtime for its platform, so you do not install Java. Its only Python dependency is `JPype1` 1.5.0 or later (the Java-Python bridge), which pip installs for you. The native package needs no Java and no JPype. Optional extras install the columnar helpers of the native package: `[numpy]`, `[arrow]`, `[pandas]` and `[polars]`. == Requirements * Python 3.11 to 3.14. The wheels' CI runs all four versions on every supported platform for `main`, the nightly run and each release. A pull request runs a smaller selection (xref:development/ci.adoc[Continuous integration]). -* One of the four platforms below. +* One of the four platforms below. The Windows wheel of the native build is the newest, and its test run is the least proven of the four. -Every wheel is platform-specific, built on a native runner of its platform, and carries a JRE for that platform, so pip needs no hint to pick the right one. +Every wheel is platform-specific and built on a native runner of its platform, so pip needs no hint to pick the right one. The JVM wheel also carries a JRE for its platform. The native wheel is one file for each platform, and it serves every Python version because it loads its library with `ctypes`. [cols="2,3"] |=== -| Platform | Wheel tag (Python 3.12) +| Platform | Wheel tag (JVM wheel, Python 3.12) | Linux x86_64 | `manylinux_2_34_x86_64` @@ -52,20 +63,24 @@ Every wheel is platform-specific, built on a native runner of its platform, and * The ArcadeDB engine with all models (graph, document, key/value, vector, time series) and both query languages, SQL and openCypher. * Vector indexes, and data import for CSV, XML and ArcadeDB JSONL. * The optional in-process HTTP server and, from 26.11.1, the Studio web UI. They cost nothing at run time until you call `create_server()`. -* A bundled Java 25 runtime, trimmed with `jlink`. +* In the JVM package: a bundled Java 25 runtime, trimmed with `jlink`. In the native package: the engine as a GraalVM native-image shared library. Engine modules the bindings do not use are left out to keep the wheel small. These include the gRPC and MongoDB wire protocols, Raft HA, and Gremlin. NOTE: *JavaScript is not available.* The GraalJS jars are left out, so `DEFINE FUNCTION ... LANGUAGE js` fails. Define functions in SQL or openCypher. Registering a Python function is planned in https://github.com/ArcadeData/arcadedb-python/pull/29[#29]. -NOTE: *The Studio web UI is in the wheels from 26.11.1.* Studio used to bundle ApexCharts, which is no longer open source (https://github.com/ArcadeData/arcadedb/issues/10579[ArcadeData/arcadedb#10579]). https://github.com/ArcadeData/arcadedb/pull/10627[ArcadeData/arcadedb#10627] replaced it with Apache ECharts. The earlier 26.10.x wheels on PyPI do not contain Studio. The HTTP API of the in-process server works in all of them. +// REMOVE WHEN: the old arcadedb-embedded* shims are retired and two releases have shipped with the two-package layout allow-old-name +NOTE: *The Studio web UI is in the JVM wheels from 26.11.1.* Studio used to bundle ApexCharts, which is no longer open source (https://github.com/ArcadeData/arcadedb/issues/10579[ArcadeData/arcadedb#10579]). https://github.com/ArcadeData/arcadedb/pull/10627[ArcadeData/arcadedb#10627] replaced it with Apache ECharts. The earlier 26.10.x wheels on PyPI do not contain Studio. The HTTP API of the in-process server works in all of them. == Verify the installation +The script works with both packages. `arcadedb.__backend__` tells you which one runs: `native` or `jvm`. + [source,python,subs="attributes+"] ---- {import-stmt} print(f"ArcadeDB Python bindings version: {arcadedb.__version__}") +print(f"Backend: {arcadedb.__backend__}") # Test database creation with arcadedb.create_database("./test") as db: @@ -73,20 +88,21 @@ with arcadedb.create_database("./test") as db: print(f"Database working: {result.first().get('test') == 1}") ---- -Run in an empty directory, the script prints (the version matches what you installed): +Run in an empty directory, the script prints (the version and the backend match what you installed; this output is from the JVM package): [source,text] ---- ArcadeDB Python bindings version: 26.10.2.post1 +Backend: jvm Database working: True ---- The engine also writes log lines to the standard error stream. The output on this page shows the standard output only. [[jvm-options]] -== JVM options +== JVM options (`{jvm-pkg-name}` only) -Configure the bundled JVM inside Python, before the first database is created: +The native package has no JVM, so `start_jvm`, `jvm_kwargs` and the flags below do not exist there. With `{jvm-pkg-name}`, configure the bundled JVM inside Python, before the first database is created: [source,python,subs="attributes+"] ---- @@ -129,18 +145,12 @@ WARNING: *One JVM configuration per process.* JVM options are locked after the J Each Python process starts its own JVM, and each JVM sees the whole container limit. With several Python processes in one container, give each process its own `heap_size`. -== Native build (preview) - -`{native-pkg-name}` is the same engine compiled to a GraalVM native-image shared library. It needs no JVM and no JPype, so the JVM options above do not apply to it. It is a preview of alpha quality, and its versions are pre-releases: - -[source,shell,subs="attributes+"] ----- -pip install --pre {native-pkg-name} ----- +== Native build notes (`{native-pkg-name}`) -It imports as `{native-import-name}`, the same name as `{pkg-name}`, so install only one of the two in an environment. If both are installed, importing `{import-name}` prints a `RuntimeWarning`, because pip overwrites one with the other without a message. Optional extras install the columnar helpers: `[numpy]`, `[arrow]`, `[pandas]` and `[polars]`. Report problems in the {repo-url}/issues[issue tracker] and say that you use the native wheel. +The native package is the same engine compiled to a GraalVM native-image shared library. It needs no JVM and no JPype, so the JVM options above do not apply to it. A few API calls raise `NotOnNativeYet` (a `NotImplementedError`) instead of misbehaving. The list is in xref:which-package.adoc[Which package?]. Report problems in the {repo-url}/issues[issue tracker] and say which package you use. == Next steps * xref:quickstart.adoc[Quickstart] * xref:how-to/databases-and-queries.adoc[Databases and queries] +* xref:which-package.adoc[Which package?] diff --git a/docs/modules/ROOT/pages/packages.adoc b/docs/modules/ROOT/pages/packages.adoc deleted file mode 100644 index 49f5563..0000000 --- a/docs/modules/ROOT/pages/packages.adoc +++ /dev/null @@ -1,56 +0,0 @@ -= Which package for which mode -:description: Pick between the embedded JVM package, the embedded native package, the official server drivers and the community driver of the same name. - -Several PyPI packages mention ArcadeDB, and two of them look alike. This page says which is which and which one to install for your process. - -== The short answer - -* One Python process owns the database and you want the simplest setup: install `{pkg-name}`. It runs the engine on the JVM, inside your process, and bundles its own Java runtime. -* A short-lived process, or one where memory and start-up time matter: install `{native-pkg-name}`. It runs the engine as a GraalVM native image. It is a preview. -* Several processes, or several services, share one database: run an ArcadeDB server and talk to it with a driver (see <>). - -== Rule of thumb - -[cols="2,3"] -|=== -| Your process | Use - -| Short-lived and footprint-sensitive: a CLI, a function that runs per request, a test run, a notebook cell that opens a database and exits -| The native build, `{native-pkg-name}`. It starts faster and holds less memory than the JVM build. - -| Long-lived, with heavy repeated scans or big loads -| The JVM build, `{pkg-name}`. The JVM warms up and then runs long scans and bulk loads faster than the native build. - -| Several processes or services read and write the same database -| The server, with a driver. An embedded database is owned by one process; see xref:index.adoc#embedded-vs-http[Embedded bindings or the HTTP driver]. -|=== - -Which build is ahead depends on whether you measure the first pass after a start or the warm passes, so measure with your own data before you commit to one. The two embedded packages expose the same API; xref:index.adoc#jvm-vs-native[JVM build or native build] compares them. - -== The embedded packages - -[cols="2,2,3"] -|=== -| Package | Kind | What it is - -| `{pkg-name}` -| Embedded, JVM build -| The engine inside your Python process, through https://jpype.readthedocs.io/[JPype], with a bundled Java runtime. Import it as `{import-name}`. This is the package this component documents. - -| `{native-pkg-name}` -| Embedded, GraalVM native build (preview) -| The same API with the engine compiled ahead of time: no JVM and no JPype. Import it as `{native-import-name}`. - -| `{jvm-pkg-name}` -| Embedded, JVM build -| A reserved name for the JVM build. It installs `{pkg-name}`. -|=== - -[[drivers]] -== Packages that talk to a server - -None of these contains the engine. They connect to a running ArcadeDB server. - -* `{driver-pkg-name}`: the official HTTP client. See xref:arcadedb::how-to/connectivity/drivers/python-http.adoc[Python -- HTTP driver]. -* `{grpc-driver-pkg-name}`: the official gRPC client. See xref:arcadedb::how-to/connectivity/drivers/python-grpc.adoc[Python -- gRPC driver]. -* `{community-driver-pkg-name}`: a community HTTP driver by Steve Reiner and others ({community-repo-url}[stevereiner/arcadedb-python]). It is not related to this repository, whose name is also `arcadedb-python`. Check which one a tutorial installs before you copy its code. diff --git a/docs/modules/ROOT/pages/quickstart.adoc b/docs/modules/ROOT/pages/quickstart.adoc index 77a7cb4..8017c33 100644 --- a/docs/modules/ROOT/pages/quickstart.adoc +++ b/docs/modules/ROOT/pages/quickstart.adoc @@ -3,7 +3,22 @@ // Source: bindings docs getting-started/quickstart.md. -This page builds a small database from scratch. You need the package from xref:installation.adoc[Installation]. Each script runs against the published `{pkg-name}` wheel; the output shown is what it printed. +This page builds a small database from scratch. You need one of the two packages from xref:installation.adoc[Installation]. Both import as `{import-name}`. + +[cols="1,2,2"] +|=== +| Package | Install | Pick this if + +| `{native-pkg-name}` (native, the default) +| `pip install {native-pkg-name}` +| Scripts, notebooks, tests, CLIs, serverless functions, agents. + +| `{jvm-pkg-name}` (JVM) +| `pip install {jvm-pkg-name}` +| Heavy, long-running loads, or the Java-object APIs. +|=== + +See xref:which-package.adoc[Which package?] if you are not sure. The scripts on this page ran against the published `{jvm-pkg-name}` wheel, and the output shown is what it printed. The code is the same for both packages. The wording of an error message can differ on `{native-pkg-name}`. == Embedded mode @@ -136,7 +151,10 @@ with arcadedb.create_server("./databases", root_password="password123") as serve HTTP port: 2480 ---- -The server runs only while your Python process runs. From 26.11.1 it also serves the Studio web UI, and `get_studio_url()` returns its address. The earlier 26.10.x wheels do not contain Studio: see the note in xref:installation.adoc#what-is-included[What is included]. See xref:index.adoc#embedded-vs-http[Embedded bindings or the HTTP driver] for when to run the regular ArcadeDB server instead. +The server runs only while your Python process runs. See xref:index.adoc#embedded-vs-http[Embedded bindings or the HTTP driver] for when to run the regular ArcadeDB server (the official Docker image `arcadedata/arcadedb`) instead. + +// REMOVE WHEN: the old arcadedb-embedded* shims are retired and two releases have shipped with the two-package layout allow-old-name +In the `{jvm-pkg-name}` wheels from 26.11.1 the server also serves the Studio web UI, and `get_studio_url()` returns its address. The earlier 26.10.x wheels do not contain Studio: see the note in xref:installation.adoc#what-is-included[What is included]. == Next steps diff --git a/docs/modules/ROOT/pages/which-package.adoc b/docs/modules/ROOT/pages/which-package.adoc new file mode 100644 index 0000000..75bf4c8 --- /dev/null +++ b/docs/modules/ROOT/pages/which-package.adoc @@ -0,0 +1,145 @@ += Which package? +:description: Two PyPI packages carry the ArcadeDB engine for Python, arcadedb (native) and arcadedb-jvm (JVM). Both import as arcadedb. This page says which one to install. + +Two packages give you the ArcadeDB engine inside your Python process. Both use the import name `{import-name}`, and they have the same public API. Install one of them, not both. + +[cols="1,2,3"] +|=== +| Package | Engine | Pick it for + +| `{native-pkg-name}` (the default) +| The native build. The engine is compiled by GraalVM Community Edition into a shared library, which Python loads with `ctypes`. No JVM, no JPype, no bundled Java runtime. +| Embedded use in scripts, notebooks, tests, CLIs, serverless functions and agents. It starts in milliseconds and holds little memory. + +| `{jvm-pkg-name}` +| The JVM build. The Java engine runs in your process through https://jpype.readthedocs.io/[JPype], on a bundled Java 25 runtime. +| Heavy, long-running loads, where the JIT compiler makes the engine faster over time. Also the Java-object APIs (`get_java_database` and similar), JVM options, `jvm_kwargs` and `start_jvm`. +|=== + +[source,shell,subs="attributes+"] +---- +pip install {native-pkg-name} # native, the default +pip install {jvm-pkg-name} # JVM, for heavy long-running loads +---- + +If you are not sure, install `{native-pkg-name}`. You can switch later: your code does not change (see <>). + +// REMOVE WHEN: the old arcadedb-embedded* shims are retired and two releases have shipped with the two-package layout allow-old-name +NOTE: *Transition note.* The first release of this layout is 26.11.1. Until the native build passes its readiness bar, `{native-pkg-name}` may appear only as pre-releases, `26.11.1.devN`. Then you install it with `pip install --pre {native-pkg-name}` (or `uv add --prerelease allow {native-pkg-name}`). The readiness bar is listed in xref:development/release.adoc[Release]. + +A third name is planned: `arcadedb-native`. It is an alias of `{native-pkg-name}`, with the same version, and it only installs `{native-pkg-name}`. It is not published yet. + +[[which-one-do-i-have]] +== Check which one you have + +[source,shell,subs="attributes+"] +---- +python -c "import {import-name}; print({import-name}.__backend__)" +---- + +It prints `native` or `jvm`. You can also ask pip: + +[source,shell,subs="attributes+"] +---- +pip show {native-pkg-name} {jvm-pkg-name} +---- + +`pip show` prints a block for each package that is installed, and a warning for the one that is not. In a correct environment exactly one of the two is installed. + +== What changes between them + +The API is the same. The runtime is not. These are the differences you can plan for. + +[cols="2,3,3"] +|=== +| | `{native-pkg-name}` (native) | `{jvm-pkg-name}` (JVM) + +| Start-up +| Milliseconds. There is no JVM to start. +| About one second or more, to start the JVM. + +| Memory +| Small footprint. +| Larger. The JVM keeps a heap, and each Python process starts its own JVM. + +| Throughput on long loads +| No JIT warm-up. Speed over a long run depends on the workload. +| The JIT compiler makes long-running loads faster over time. + +| Speed on your data +| Measure on your own workload. +| Measure on your own workload. + +| Wheel size +| Smaller. It carries no Java runtime. +| Larger. It carries a Java 25 runtime for its platform. + +| Python versions +| 3.11 to 3.14 +| 3.11 to 3.14 + +| Platforms +| Linux x86_64 (glibc 2.34 or later), Linux aarch64, macOS arm64, Windows x64. The Windows wheel is the newest: it is built in CI, and its test run is the least proven of the four. +| Linux x86_64 (glibc 2.34 or later), Linux aarch64, macOS arm64, Windows x64. + +| Java-object APIs (`get_java_database`, `get_java_*`) +| Not available. +| Available. + +| JVM options (`jvm_kwargs`, `start_jvm`, `-X` and `-D` flags) +| Do not exist. There is no JVM. +| Available. See xref:installation.adoc#jvm-options[JVM options]. + +| A few API calls +| Raise `NotOnNativeYet` (a `NotImplementedError`): `AsyncExecutor.transaction`, the approximate search `find_nearest_approximate` over product quantization, `get_metadata` of a foreign index, and `get_stats`. +| Work. +|=== + +Both packages run the same ArcadeDB engine, with the same SQL and openCypher and the same database format. + +[[switching]] +== Switch from one to the other + +The two wheels refuse to work together. If both are installed, `import arcadedb` raises an `ImportError` that names both packages. The reason is that both wheels write into the same `arcadedb` directory, and pip would overwrite files of one with files of the other without a message. So remove both and install the one you want: + +[source,shell,subs="attributes+"] +---- +pip uninstall {native-pkg-name} {jvm-pkg-name} +pip install {jvm-pkg-name} +---- + +There is no extra to select a build: `{native-pkg-name}[jvm]` does not exist. Your code stays as it is, because both import as `{import-name}`. Only code that uses a JVM-only API (see the table above) needs a change when you move to the native build. + +== Server use + +Neither package is a server deployment. They embed the engine in one Python process. For HTTP access, Studio, many processes, other programming languages or high availability, run the official Docker image `arcadedata/arcadedb` and connect with a driver (see <>). The xref:index.adoc#embedded-vs-http[Embedded bindings or the HTTP driver] section compares the two ways. + +// REMOVE WHEN: the old arcadedb-embedded* shims are retired and two releases have shipped with the two-package layout allow-old-name +[[old-names]] +== Old names + +The packages had other names until 26.10.x. Search results and old tutorials may still use them. + +[cols="2,3,3"] +|=== +| New package | Old name | Note + +| `{jvm-pkg-name}` +| Formerly `arcadedb-embedded`, imported as `arcadedb_embedded` +| The same build. Nothing changes silently in the runtime. The import name is now `{import-name}`. + +| `{native-pkg-name}` +| Formerly `arcadedb-embedded-native`, imported as `arcadedb_native` +| The same build, now with the import name `{import-name}`. The old preview versions were 26.10.x pre-releases. +|=== + +The old packages will get a deprecation shim that points to the new names. Releases of the new layout start with 26.11.1. The 26.10.x wheels were built in a separate repository of the humemai organisation. + +[[drivers]] +== Packages that talk to a server + +None of these contains the engine. They connect to a running ArcadeDB server. + +* `{driver-pkg-name}`: the official HTTP client. See xref:arcadedb::how-to/connectivity/drivers/python-http.adoc[Python -- HTTP driver]. +* `{grpc-driver-pkg-name}`: the official gRPC client. See xref:arcadedb::how-to/connectivity/drivers/python-grpc.adoc[Python -- gRPC driver]. +* `{community-driver-pkg-name}`: a community HTTP driver by Steve Reiner and others ({community-repo-url}[stevereiner/arcadedb-python]). It is not related to this repository, whose name is also `arcadedb-python`. Check which one a tutorial installs before you copy its code. diff --git a/native/wheel/README.md b/native/wheel/README.md new file mode 100644 index 0000000..b147b00 --- /dev/null +++ b/native/wheel/README.md @@ -0,0 +1,69 @@ +# arcadedb (native build) + +This is `arcadedb`, the native build of the ArcadeDB Python bindings, and the default choice for scripts, notebooks, tests, CLIs, serverless functions and agents. Its sibling `arcadedb-jvm` runs the Java engine on a bundled JVM: use `arcadedb-jvm` for heavy, long-running loads, where the JIT compiler makes the engine faster over time, and when you need the Java-object APIs. Both packages `import arcadedb` and have the same public API. Install one, not both. [Which package?](https://github.com/ArcadeData/arcadedb-python/blob/main/docs/modules/ROOT/pages/which-package.adoc) + +ArcadeDB (SQL, openCypher, document, graph, vector, time series) runs in your Python process **without a JVM and without JPype**. The engine is compiled ahead of time with GraalVM Community Edition into a shared library (`libarcadedb.so`, `.dylib` or `.dll`), and Python loads it with `ctypes`. It starts in milliseconds and has a small memory footprint. + + +> Transition note: the first release of this layout is 26.11.1. Until `arcadedb` passes its readiness bar, it may appear only as pre-releases, `26.11.1.devN`. Install those with `pip install --pre arcadedb`. Report problems at and say that you use the native build. + +## Install + +```bash +pip install arcadedb # or: uv add arcadedb +``` + +Optional extras for the columnar helpers: `[numpy]`, `[arrow]`, `[pandas]`, `[polars]`. + + +Old names: this package was formerly `arcadedb-embedded-native`, imported as `arcadedb_native`. The old package will get a deprecation shim that points here. + +If `arcadedb-jvm` is installed in the same environment, `import arcadedb` raises an `ImportError` that names both packages. Run `pip uninstall arcadedb arcadedb-jvm`, then install the one you want. + +## Use + +```python +import arcadedb + +print(arcadedb.__backend__) # "native" + +with arcadedb.create_database("/tmp/demo") as db: + db.schema.create_vertex_type("Person") + with db.transaction(): + db.command("sql", "CREATE VERTEX Person SET name = ?", "Ada") + print(db.query("sql", "SELECT FROM Person").to_list()) +``` + +`arcadedb.__backend__` is `native` for this package and `jvm` for `arcadedb-jvm`. `arcadedb.__version__` is the package version. + +## What is in the wheel + +- The shared library with the ArcadeDB engine: SQL and openCypher, the document, graph, key/value, vector and time series models, ACID transactions, and the in-process server with its HTTP API and the Postgres, Redis and Bolt wire protocols. +- The Python API on top of it, loaded with `ctypes`. +- Not in the image: gRPC, the MongoDB wire protocol, Raft/HA and Gremlin. +- No polyglot scripting languages (`js` or `python` as query languages, script triggers, polyglot functions). The polyglot engine is switched off at start-up (`arcadedb.polyglotEngineEnabled=false`), so the first database open writes nothing at WARNING. + +## Platforms + +CPython 3.11 to 3.14. One wheel per platform serves every Python version, because the library is loaded with `ctypes` and is not compiled against CPython. The images are built for the baseline CPU of the platform (`-march=compatibility`), so they do not need a special instruction set. + +| Platform | Wheel tag | Note | +| --- | --- | --- | +| Linux x86-64, glibc 2.34+ | `manylinux_2_34_x86_64` | | +| Linux arm64, glibc 2.34+ | `manylinux_2_34_aarch64` | | +| macOS arm64 | `macosx__arm64` | | +| Windows x64 | `win_amd64` | The newest wheel. It is built in CI, and its test run is the least proven of the four | + +Intel macOS and Windows arm64 are not built. + +## Not available or not yet + +- **JVM-only APIs do not exist**: `get_java_database()` and the other `get_java_*` calls, `start_jvm()`, `jvm_kwargs` and JVM options. There is no JVM. Use `arcadedb-jvm` if you need them. +- **A few API calls raise `NotOnNativeYet`** (a `NotImplementedError`) instead of misbehaving: `AsyncExecutor.transaction`, `find_nearest_approximate` over product quantization, `get_metadata` of a foreign index, and `get_stats`. +- Free-threaded Python builds are not in the CI matrix. + +The bindings' own tests run against the installed wheel (the parity run). What does not pass is the list above, or JVM-only by nature. + +## Licence + +Apache-2.0 for ArcadeDB and these bindings. The library also contains the GraalVM Community Edition runtime and JDK class library code, licensed GPLv2 **with the Classpath Exception**, plus the Apache/BSD/MIT/UPL/EDL third-party Java libraries listed in `THIRD_PARTY_NOTICES.md`. All texts and the source locations are in the wheel's `*.dist-info/licenses/`. From a7c0f1e4a0471ed6489f84b58dbb49f4ee40756e Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 20:36:31 +0900 Subject: [PATCH 06/33] Native tests: find the Java glue in native/, skip the docs site test in the parity run test_native_startup.py looked for the glue under native-spike/ and skipped when it was missing; a skip fails CI in this repository, so it now asserts the file exists at native/src/main/java. parity_wheel.py names test files on the command line, which bypasses collect_ignore, so it applies the docs-site rule of tests/conftest.py itself. Co-Authored-By: Claude Sonnet 5.5 --- native/wheel/parity_wheel.py | 3 +++ tests/test_native_startup.py | 11 +++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/native/wheel/parity_wheel.py b/native/wheel/parity_wheel.py index 6a85347..0788cec 100644 --- a/native/wheel/parity_wheel.py +++ b/native/wheel/parity_wheel.py @@ -52,6 +52,9 @@ sys.platform == "win32" ): # conftest.py's collect_ignore does not apply to a file named on the command line files = [f for f in files if os.path.basename(f) not in win_files] +if not os.path.isfile(os.path.join(pkg, "docs", "index.md")): + # tests/conftest.py collect-ignores test_docs_examples.py without the Markdown docs site; a file named on the command line is collected anyway + files = [f for f in files if os.path.basename(f) != "test_docs_examples.py"] if a.files: want = set(a.files.split(",")) files = [f for f in files if os.path.basename(f) in want] diff --git a/tests/test_native_startup.py b/tests/test_native_startup.py index 15b107f..9c3f2cf 100644 --- a/tests/test_native_startup.py +++ b/tests/test_native_startup.py @@ -3,11 +3,9 @@ import re from pathlib import Path -import pytest - ARCADE_NATIVE = ( - Path(__file__).resolve().parents[3] - / "native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java" + Path(__file__).resolve().parents[1] + / "native/src/main/java/org/humemai/nativelib/ArcadeNative.java" ) @@ -16,8 +14,9 @@ def test_the_image_glue_switches_the_polyglot_engine_off_before_any_entry_point( Static check (no native build needed): the glue class that holds the entry points must set the option in a static initializer. """ - if not ARCADE_NATIVE.is_file(): - pytest.skip("the repository checkout is not available") + assert ( + ARCADE_NATIVE.is_file() + ), f"{ARCADE_NATIVE} is missing: run the tests from a checkout of the repository" source = ARCADE_NATIVE.read_text() assert re.search( r"static\s*\{[^{}]*GlobalConfiguration\.POLYGLOT_ENGINE_ENABLED\.setValue\(false\);[^{}]*\}", From 9ae92b370d1c86bc51acc63b1544bc8a26ac3ec2 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 20:40:53 +0900 Subject: [PATCH 07/33] Native: positional-count refusal, RID value surface, run_in_transaction never replays a partly committed block Three Python-only ports from humemai/arcadedb-embedded-python native-preview (#537, #531, #527): - NativeDatabase._exec calls refuse_surplus_positionals (already shared with the JVM build, PR #52): more positional values than the SQL text can hold is a TypeError, not a silently dropped value. - The native RID class offers the Java RID surface (ordering, getBucketId, getPosition, toString, isValid, equals, compareTo) and Result.get_rid() reads a projected @rid. - run_in_transaction: the guard that refuses to replay a block whose BATCH boundary or commit() already made part of its work durable sat inside a try whose own except swallowed the raise. The raise moves outside it. New test tests/test_run_in_transaction_partial_commit.py runs on both builds. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/codec.py | 65 +++++++++- src/arcadedb/_native/database.py | 21 ++- .../test_run_in_transaction_partial_commit.py | 120 ++++++++++++++++++ 3 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 tests/test_run_in_transaction_partial_commit.py diff --git a/src/arcadedb/_native/codec.py b/src/arcadedb/_native/codec.py index c57504f..c362956 100644 --- a/src/arcadedb/_native/codec.py +++ b/src/arcadedb/_native/codec.py @@ -123,7 +123,13 @@ def _dump(obj, binary): class RID: - """A record id as the engine returns it in a row; passed back as a parameter it is a link, not a string.""" + """A record id as the engine returns it in a row; passed back as a parameter it is a link, not a string. + + The JVM build hands out the engine's own ``RID`` in the same places, so this class offers what user code written + against that one uses: ``str(rid)`` is the text (``"#10:5"``), two RIDs of one record are equal and hash alike, + RIDs sort by bucket then position, and ``getBucketId()``, ``getPosition()``, ``toString()``, ``isValid()``, + ``equals()`` and ``compareTo()`` answer like the Java methods. Unlike the Java RID, an RID here also equals its text. + """ __slots__ = ("text",) @@ -142,6 +148,63 @@ def __eq__(self, other): def __hash__(self): return hash(self.text) + def _numbers(self): + """(bucket id, position) from the text; (-1, -1) for a text that is not ``#bucket:position``.""" + bucket, sep, position = self.text.lstrip("#").partition(":") + try: + return (int(bucket), int(position)) if sep else (-1, -1) + except ValueError: + return (-1, -1) + + def __lt__(self, other): + return ( + self._numbers() < other._numbers() + if isinstance(other, RID) + else NotImplemented + ) + + def __le__(self, other): + return ( + self._numbers() <= other._numbers() + if isinstance(other, RID) + else NotImplemented + ) + + def __gt__(self, other): + return ( + self._numbers() > other._numbers() + if isinstance(other, RID) + else NotImplemented + ) + + def __ge__(self, other): + return ( + self._numbers() >= other._numbers() + if isinstance(other, RID) + else NotImplemented + ) + + # the Java RID's methods, by their Java names (the JVM build's RID is that object) + def getBucketId(self): + return self._numbers()[0] + + def getPosition(self): + return self._numbers()[1] + + def toString(self): + return self.text + + def isValid(self): + bucket, position = self._numbers() + return bucket > -1 and position > -1 + + def equals(self, other): + return isinstance(other, RID) and self.text == other.text + + def compareTo(self, other): + mine, theirs = self._numbers(), other._numbers() + return (mine > theirs) - (mine < theirs) + def _restore_float(text): return float(text) diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index a487f2a..12cee0d 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -9,6 +9,7 @@ from datetime import date, datetime from typing import Any, Dict, Iterator, List, Optional +from .._params import refuse_surplus_positionals from ..exceptions import ArcadeDBError from ..transactions import TransactionContext from ..type_conversion import json_bulk_dumps @@ -83,7 +84,11 @@ def get_raw(self, name: str) -> Any: return self.get(name) def get_rid(self) -> Optional[str]: - return self._meta[0] if self._meta else None + if self._meta: + return self._meta[0] + # a projection of @rid (not an alias of it) is a row with that identity, as on the JVM build + rid = self._props.get("@rid") + return None if rid is None else str(rid) def _record(self, kinds): from .records import _wrap @@ -468,6 +473,10 @@ def _exec( self, flags: int, prefix: str, language: str, command: str, args ) -> NativeResultSet: self._check_not_closed() + if args: + refuse_surplus_positionals( + "command" if flags & 1 else "query", language, command, args + ) if flags & 1 and _LAZY_COMMAND_ROWS and lib.CMD_PATH: # phase 8: the cursor id comes back as a long (no reply text to free and decode); the rows are read late or never text, buf, count = encode_params(args, binary=True) @@ -614,11 +623,15 @@ def run_in_transaction(self, fn, retries: int = 12, backoff_s: float = 0.005): raise duplicated_key_retried = True if commits_at_start is not None: + # the rollback above cannot take back what a BATCH boundary or a commit() inside the block already made + # durable, and a replay would apply it twice. Only the READ of the witness may fail quietly (no witness: + # retry, as before); the raise sits outside that try, or its own except would swallow it. try: - if self._transaction_commit_mark() != commits_at_start: - raise + commits_now = self._transaction_commit_mark() except ArcadeDBError: - pass + commits_now = commits_at_start + if commits_now != commits_at_start: + raise _time.sleep( max(backoff_s * (attempt + 1), reply.get("retry_ms", 0) / 1000.0) ) diff --git a/tests/test_run_in_transaction_partial_commit.py b/tests/test_run_in_transaction_partial_commit.py new file mode 100644 index 0000000..1295846 --- /dev/null +++ b/tests/test_run_in_transaction_partial_commit.py @@ -0,0 +1,120 @@ +""" +``Database.run_in_transaction`` never replays a block that already committed part of its work (#7916), on both wheels. + +A statement with a ``BATCH n`` boundary, or a ``commit()`` inside the block, makes part of the block's work durable. When the block +then hits a retryable conflict, the rollback cannot take that part back, and a replay would apply it twice and report clean success. +The conflict has to be raised instead. The conflict here is a real one (a second thread commits to a record the block also wrote), so +the same test runs against the JPype wheel and the native wheel, and it needs neither a Java class nor a fabricated error. + +The native wheel's guard once sat in a ``try`` whose own ``except ArcadeDBError: pass`` swallowed the ``raise`` it was written to +make, so a native block that had committed part of its work was replayed. +""" + +import threading + +import pytest + +import arcadedb as arcadedb +from arcadedb import ArcadeDBError + + +@pytest.fixture +def db(temp_db_path): + database = arcadedb.create_database(temp_db_path) + database.command("sql", "CREATE DOCUMENT TYPE R") + database.command("sql", "CREATE PROPERTY R.k INTEGER") + database.command("sql", "CREATE PROPERTY R.v INTEGER") + database.command("sql", "CREATE DOCUMENT TYPE C") + database.command("sql", "CREATE PROPERTY C.n INTEGER") + with database.transaction(): + database.command("sql", "INSERT INTO C SET n = 0") + yield database + database.drop() + + +def _count(db, where="true"): + sql = f"SELECT count(*) AS c FROM R WHERE {where}" # nosec B608 - test-owned predicate text, no user input + return db.query("sql", sql).first().get("c") + + +def _commit_from_another_thread(db): + """Commit a write to the record the block also wrote, so the block's own commit meets a ConcurrentModificationException.""" + errors = [] + + def other(): + try: + with db.transaction(): + db.command("sql", "UPDATE C SET n = n + 100") + except BaseException as e: # reported on the calling thread + errors.append(e) + + thread = threading.Thread(target=other) + thread.start() + thread.join() + assert errors == [] + + +def _commit_inside_the_block(db): + """Make the rows written so far durable with an explicit commit, then carry on in a new transaction.""" + db.command("sql", "UPDATE R SET v = v + 1") + db.commit() + db.begin() + + +def _batch_boundary(db): + """The rows the BATCH boundaries committed (20 of 25) are durable; the remaining 5 sit in the open transaction.""" + db.command("sql", "UPDATE R SET v = v + 1 BATCH 10") + + +@pytest.mark.parametrize( + "publish, durable_v1", + [ + pytest.param(_batch_boundary, 20, id="batch-boundary"), + pytest.param(_commit_inside_the_block, 25, id="commit-inside-the-block"), + ], +) +def test_a_block_that_committed_part_of_its_work_is_not_replayed( + db, publish, durable_v1 +): + with db.transaction(): + for _ in range(25): + db.command("sql", "INSERT INTO R SET k = 0, v = 0") + attempts = [] + + def fn(): + attempts.append(1) + publish(db) + db.command("sql", "UPDATE C SET n = n + 1") + if len(attempts) == 1: + _commit_from_another_thread(db) + return "done" + + with pytest.raises(ArcadeDBError, match="ConcurrentModification"): + db.run_in_transaction(fn, retries=5, backoff_s=0) + + assert ( + len(attempts) == 1 + ), "the block was replayed after it had committed part of its work" + # a replay would have raised every durable row to v = 2 and reported success + assert _count(db, "v = 2") == 0 + assert _count(db, "v = 1") == durable_v1 + assert not db.is_transaction_active() + + +def test_a_block_that_committed_nothing_is_still_retried(db): + """The guard must not turn every conflict into a refusal: a block with no durable part is replayed, once, and succeeds.""" + attempts = [] + + def fn(): + attempts.append(1) + db.command("sql", "INSERT INTO R SET k = 1, v = 0") + db.command("sql", "UPDATE C SET n = n + 1") + if len(attempts) == 1: + _commit_from_another_thread(db) + return "done" + + assert db.run_in_transaction(fn, retries=5, backoff_s=0) == "done" + assert len(attempts) == 2 + assert _count(db, "k = 1") == 1 + # the first attempt was rolled back; the other thread added 100 and the second attempt 1 + assert db.query("sql", "SELECT n FROM C").first().get("n") == 101 From 5cfc2ef45988960189394c7fded4609c320e1dbd Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 20:50:54 +0900 Subject: [PATCH 08/33] Native tests: skips come from explicit markers and NotOnNativeYet, never from failure text Port of humemai/arcadedb-embedded-python#535, adapted to this repository's conftest. The native build used to turn a failing test into an NA-JVM or NOTYET skip when its failure text matched a list of strings ("JVMNotRunning", "assert (None is not None)", "KeyError: 'flush'"), which is also what a real bug looks like. Now: - NATIVE_NA_JVM_TESTS is keyed ::[::] and lists only tests that reach into the JVM on purpose, with the reason. Rows were added for the tests this repository has that the upstream table did not know (temporal writers, XML limits, to_java_rid, db._java_db patching). - pytest_runtest_makereport reports NOTYET only for NotOnNativeYet, raised by the native code. - to_java_int_array and to_java_byte_array return NumPy arrays on the native build, so the sparse vector tests no longer import jpype.types. - tests/README.md says how a test may be skipped on the native build. Co-Authored-By: Claude Sonnet 5.5 --- native/wheel/parity_wheel.py | 2 +- src/arcadedb/vector.py | 41 ++++- tests/README.md | 9 + tests/conftest.py | 194 +++++++++++++--------- tests/test_resultset_read_errors.py | 5 + tests/test_sparse_quantization_compact.py | 7 +- tests/test_vector_sql.py | 5 +- 7 files changed, 177 insertions(+), 86 deletions(-) diff --git a/native/wheel/parity_wheel.py b/native/wheel/parity_wheel.py index 0788cec..ec03dad 100644 --- a/native/wheel/parity_wheel.py +++ b/native/wheel/parity_wheel.py @@ -5,7 +5,7 @@ A run per test file in its own process (a hang or crash costs one file), junit XML per file, then one table: passed / failed / error / NOTYET (the test needs a Java-object API, reported by tests/conftest.py as a skip whose reason starts -with NOTYET) / other skips / TIMEOUT-or-crash. Files in tests/conftest.py's NATIVE_NOT_APPLICABLE are not collected (JVM-only). +with NOTYET, raised as NotOnNativeYet) / NA-JVM (a row of NATIVE_NA_JVM_TESTS: the test reaches into the JVM on purpose) / other skips / TIMEOUT-or-crash. Files in tests/conftest.py's NATIVE_NOT_APPLICABLE are not collected (JVM-only). """ import argparse diff --git a/src/arcadedb/vector.py b/src/arcadedb/vector.py index c1ed3cb..6c6acf3 100644 --- a/src/arcadedb/vector.py +++ b/src/arcadedb/vector.py @@ -92,8 +92,24 @@ def to_java_int_array(vector): vector: Array-like object containing integer values Returns: - Java int array + Java int array (a NumPy int32 array on the native backend, which has no JVM to build one in: like + :func:`to_java_float_array`, it binds as a SQL parameter in place of the Java array) """ + if _native_enabled(): + values = ( + vector if isinstance(vector, _np.ndarray) else _np.asarray(list(vector)) + ) + if values.size == 0: + return values.astype(_np.int32) + if values.dtype.kind not in "iu": + raise TypeError("to_java_int_array takes integer values") + bounds = _np.iinfo(_np.int32) + if values.min() < bounds.min or values.max() > bounds.max: + raise OverflowError( + "Cannot convert value to Java int: an element does not fit in 32 bits" + ) + return values.astype(_np.int32) + if _np is not None and isinstance(vector, _np.ndarray): if vector.dtype.kind in "iu" and vector.size: bounds = _np.iinfo(_np.int32) @@ -125,8 +141,29 @@ def to_java_byte_array(vector): vector: Array-like object containing byte values Returns: - Java byte array compatible with INT8-encoded ArcadeDB vector indexes + Java byte array compatible with INT8-encoded ArcadeDB vector indexes (a NumPy int8 array on the native + backend, which has no JVM to build one in) """ + if _native_enabled(): + if isinstance(vector, (bytes, bytearray, memoryview)): + return _np.frombuffer(bytes(vector), dtype=_np.int8).copy() + if ( + isinstance(vector, _np.ndarray) + and vector.dtype.kind in "iu" + and vector.dtype.itemsize == 1 + ): + return _np.ascontiguousarray(vector).view(_np.int8).copy() + native_bytes = [] + for value in vector.tolist() if isinstance(vector, _np.ndarray) else vector: + byte_value = int(value) + if byte_value < -128 or byte_value > 255: + raise ArcadeDBError( + "Byte vectors must contain values in the signed byte range " + "[-128, 127] or unsigned byte range [0, 255]" + ) + native_bytes.append(byte_value - 256 if byte_value > 127 else byte_value) + return _np.asarray(native_bytes, dtype=_np.int8) + try: import numpy as np diff --git a/tests/README.md b/tests/README.md index 5296e1f..723c95e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -32,6 +32,15 @@ uv run pytest -m "not server" The server tests do not skip when the server stack is absent: the wheel always ships it, and a test that skips on a missing feature cannot notice the feature going missing (in 26.7.2 the server JARs were dropped from the wheel and the guarded tests skipped, so the suite stayed green while the feature was gone). `test_server_packaging.py` states the packaging requirement and fails without the JARs. CI also fails any skip that `scripts/check_test_skips.py` does not list, and that list is empty. +## The native build + +The same suite runs against the native `arcadedb` wheel (`native/wheel/parity_wheel.py`). A test is reported as a skip on the native build in two explicit ways, never by what its failure says: + +- `NotOnNativeYet`: the native code raised it because the feature needs a Java object API the library does not expose. The report says `NOTYET`, and a final release allows none. +- A row of `NATIVE_NA_JVM_TESTS` in `tests/conftest.py`, keyed `::[::]`: the test reaches into the JVM on purpose (`jpype.JClass`, a Java array, `db._java_db`, counted bridge crossings). The report says `NA-JVM`. Add a row only for such a test, and give the reason. When the behaviour is shared, write a native test beside it (`test_native_*.py`, collected on the native build only). + +A test that fails on the native build for any other reason is a failure. Fix the native code, or mark a known engine bug with a strict `xfail`. Do not add a row to hide it. + ## Need Help? - **Found a bug?** [Open an issue](https://github.com/ArcadeData/arcadedb-python/issues) diff --git a/tests/conftest.py b/tests/conftest.py index 21549b9..365d268 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,6 +41,8 @@ collect_ignore.append("test_native_udf.py") # the native library's start-up output (#477) can only be checked with the native library loaded collect_ignore.append("test_native_startup_runtime.py") + # the XML entity limits of an import, lifted by the native glue (the JVM build lifts them with JVM system properties) + collect_ignore.append("test_native_xml_limits.py") if not os.path.isfile( os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md") ): @@ -78,85 +80,143 @@ collect_ignore.extend(NATIVE_NOT_APPLICABLE) -# Failure texts that mean the test is about the JPype bridge, not about database behaviour. -NATIVE_JVM_MARKERS = ( - "JVMNotRunning", - "without JVM", - "Java Virtual Machine is not running", - "'_java_db'", - "'_java_result'", - "bridge jar", - "'nextRows'", - "getClass", - "DID NOT RAISE KeyboardInterrupt", - "Cannot drop the database in transaction", # teardown after an injected KeyboardInterrupt - "assert (None is not None)", # fixtures that need a bridge class - "KeyError: 'flush'", - "KeyError: 'max_connections'", - "cause is ", # the Java exception as __cause__ - "findNeighborsFromVector", # the test calls the Java vector index directly - "to_java_rid", +# Tests that exercise the JVM itself, so the native backend (no JVM, no jars) has nothing to run. Skipped before they start, at +# collection time, with the reason (reported as "NA-JVM: "). The key is "::[::]": every parameter +# of the function, or one parameter id as in "test_x.py::test_y[java]". A test is listed only when it reaches into the JVM on purpose: +# it builds or reads a Java object (jpype.JClass, a Java array, db._java_db, _java_index, the bridge classes), counts JPype bridge +# crossings, or patches a hook of the JPype path. A failure text never puts a test on this list, and a failing test is never turned +# into a skip: a test whose behaviour native shares has to pass on native, and one that fails there is a bug to file (an +# xfail(strict=True) naming the BACKLOG-bindings.md row, so it turns red when fixed), not an entry here. +# A feature the native wheel lacks on purpose raises NotOnNativeYet from the native code, which pytest_runtest_makereport reports as NOTYET. +JAVA_CLASS = "builds a java.* object through JPype" +JAVA_ARRAY = "builds a Java array through JPype (jpype.JArray)" +JAVA_DB = "reads the Java database behind the wrapper (db._java_db)" +JAVA_INDEX = "reads the Java index behind the wrapper (_java_index: getClass, metadata)" +BRIDGE_COUNT = ( + "counts or intercepts the JPype bridge crossings (an internal of the JPype path)" ) - - -# Failure texts that mean the test needs a Java-object wrapper (Document, Vertex, Edge) or the exporter's Java path. -NATIVE_NOTYET_MARKERS = ( - "'NativeResult' object is not iterable", - "assert 'NativeResult' ==", -) - - -# Tests that count the JPype parameter conversions (an internal of the JPype path). -NATIVE_NA_TESTS = { - "test_non_scalars_keep_the_general_path", - "test_a_subclass_of_a_scalar_is_not_assumed_to_be_one", - "test_lsm_vector_search_looks_up_all_hits_in_one_bridge_call", # counts GraphCalls bridge crossings +NATIVE_NA_JVM_TESTS = { + # JVM start-up, signals and settings + "test_sigint.py::test_ctrl_c_raises_keyboard_interrupt_and_runs_cleanup[java]": "a Java call blocked in JPype's SIGINT handler (#179)", + "test_sigint.py::test_interrupt_true_keeps_the_jvm_handling_sigint": "start_jvm(interrupt=True): the JVM handles SIGINT", + "test_docs_examples.py::test_docs_installation_jvm_examples": "documents start_jvm() and jvm_kwargs", + "test_server_wire_protocols.py::test_the_host_reaches_exactly_the_listener_bind_settings": "reads the engine's GlobalConfiguration through the JVM", + "test_server_wire_protocols.py::test_wire_listeners_stay_on_loopback_by_default": "reads the server's GlobalConfiguration through the JVM", + "test_server_wire_protocols.py::test_host_applies_to_every_wire_listener": "reads the server's GlobalConfiguration through the JVM", + "test_server_wire_protocols.py::test_a_protocol_host_overrides_the_server_host_for_that_protocol_only": "reads the server's GlobalConfiguration through the JVM", + "test_sql_constraints_and_maintenance.py::test_backup_check_and_rebuild_index_statements": "sets GlobalConfiguration.SERVER_BACKUP_DIRECTORY through the JVM", + "test_vector_params_verification.py::TestVectorParams::test_jvm_heap_check": "reads java.lang.Runtime through JPype", + "test_vector_params_verification.py::TestVectorParams::test_wrapper_default_matches_engine_default": "reads the engine default through java.lang.String and GlobalConfiguration", + "test_vector_params_verification.py::TestVectorParams::test_omitted_max_connections_reaches_the_index": "reads the engine default through java.lang.String and GlobalConfiguration", + "test_vector_sql.py::TestVectorSearchOptionsSql::test_where_map_and_condition": "picks the expected behaviour by com.arcadedb.Constants.getRawVersion() through JPype", + "test_vector_sql.py::TestVectorSearchOptionsSql::test_where_combines_with_filter": "picks the expected behaviour by com.arcadedb.Constants.getRawVersion() through JPype", + # Java objects built or read by the test + "test_conversion_fixes.py::test_java_instants_convert_without_losing_microseconds": JAVA_CLASS, + "test_type_conversion.py::test_offset_datetime_conversion": JAVA_CLASS, + "test_type_conversion.py::test_scalar_list_crosses_as_one_array_with_the_same_types": "asserts the converted value is a java.util.ArrayList", + "test_type_conversion.py::TestPrimitiveArrayFormats::test_int_array": JAVA_ARRAY, + "test_type_conversion.py::TestPrimitiveArrayFormats::test_long_array": JAVA_ARRAY, + "test_percall_calls.py::test_float_array_conversion_gives_a_float_array": 'asserts the converted value is a Java float[] (getClass() == "[F")', + "test_typed_rows.py::test_java_values_sql_cannot_produce": JAVA_CLASS, + "test_typed_rows.py::test_value_python_cannot_hold_raises_as_before": JAVA_CLASS, + "test_typed_rows.py::test_object_array_raises_as_before": JAVA_ARRAY, + "test_typed_rows.py::test_iter_dicts_reads_one_row_at_a_time": "builds a Java InternalResultSet", + "test_typed_rows.py::test_vertex_edges_match_the_engine_iteration": "the oracle is the Java Vertex.getEdges iteration", + "test_typed_rows.py::test_vector_search_hits_match_the_per_hit_path": "the oracle is the Java vector index (findNeighborsFromVector)", + "test_typed_rows.py::test_a_batch_of_wide_rows_ends_at_a_few_megabytes": "calls the bridge's typed_rows.nextRows on the Java result set", + "test_typed_rows.py::test_the_batch_says_whether_the_result_set_is_drained": "calls the bridge's typed_rows.nextRows on the Java result set", + "test_graph_api.py::test_document_wrap_static_method": "wraps the Java object behind a result (_java_result)", + "test_legacy_temporal_implementations.py::test_the_engine_hands_back_the_configured_class_holding_the_utc_instant": JAVA_DB, + "test_graph_batch.py::test_graph_batch_leaves_the_callers_wal_setting_alone": JAVA_DB, + "test_transaction_config.py::test_set_wal_flush_is_database_wide": JAVA_DB, + "test_run_in_transaction_retry.py::test_contended_commits_spend_their_budget_before_failing": JAVA_DB, + "test_float_vector_bulk.py::TestAllowedRids::test_rid_objects_and_text_mixed_still_work": "builds a Java RID (db.to_java_rid) and reads the Java RID set of the index", + "test_xml_limits.py::test_import_hitting_a_limit_names_the_option": "reads the JDK's system properties through JPype (native: test_native_xml_limits.py)", + "test_xml_limits.py::test_per_call_option_lifts_the_limits_for_that_call_only": "reads the JDK's system properties through JPype (native: test_native_xml_limits.py)", + "test_xml_limits.py::test_per_call_option_restores_a_value_that_was_already_set": "sets and reads a JDK system property through JPype", + "test_xml_limits.py::test_limits_are_restored_when_the_import_fails": "reads the JDK's system properties through JPype (native: test_native_xml_limits.py)", + "test_temporal_writers_agree.py::test_the_json_and_columnar_writers_write_the_same_number_for_every_temporal": JAVA_CLASS, + "test_temporal_writers_agree.py::test_the_json_writer_is_the_engines_generic_writer_for_every_temporal": JAVA_CLASS, + "test_python_function_interrupt.py::test_jpypes_fatal_error_during_a_statement_is_a_keyboard_interrupt": "replaces db._java_db with an object that raises JPype's fatal error", + # Java exceptions thrown and engine hooks set from the test + "test_run_in_transaction_retry.py::test_every_need_retry_exception_is_retried": "throws Java exceptions (com.arcadedb.exception.*) from the test", + "test_run_in_transaction_retry.py::test_the_budget_is_spent_before_the_conflict_reaches_the_caller": "throws Java exceptions (com.arcadedb.exception.*) from the test", + "test_run_in_transaction_retry.py::test_the_wait_the_refusing_side_asked_for_is_honored": "throws Java exceptions (com.arcadedb.exception.*) from the test", + "test_run_in_transaction_retry.py::test_the_wait_is_capped_by_retry_after_max_wait": "throws Java exceptions (com.arcadedb.exception.*) from the test", + "test_run_in_transaction_retry.py::test_a_block_that_committed_part_of_its_work_is_not_replayed": "throws a Java ConcurrentModificationException from the test (native: test_run_in_transaction_partial_commit.py)", + "test_graph_batch.py::test_graph_batch_outside_the_callers_transactions": "sets GraphBatch.TEST_BEFORE_VERTEX_COMMIT_HOOK, a Java static, through JPype", + "test_graph_batch.py::test_graph_batch_create_vertices_refuses_before_any_commit_attempt": "sets GraphBatch.TEST_BEFORE_VERTEX_COMMIT_HOOK, a Java static, through JPype", + "test_graph_batch.py::test_graph_batch_create_vertices_keyboard_interrupt_rolls_back": "patches GraphBatch._create_vertices_json_bulk, the JPype path's bulk call", + # parameter conversion and bridge crossings of the JPype path + "test_parameter_binding.py::test_each_parameter_shape_reaches_one_java_overload": "inspects the Java Database overloads through JPype", + "test_parameter_binding.py::test_async_parameters_reach_one_java_overload": "inspects the Java DatabaseAsyncExecutor overloads through JPype", + "test_parameter_binding.py::TestScalarParametersTakeTheShortPath::test_a_java_array_value_is_not_converted_and_binds_as_before": JAVA_ARRAY, + "test_parameter_binding.py::TestScalarParametersTakeTheShortPath::test_non_scalars_keep_the_general_path": "counts JPype parameter conversions", + "test_parameter_binding.py::TestScalarParametersTakeTheShortPath::test_a_subclass_of_a_scalar_is_not_assumed_to_be_one": "counts JPype parameter conversions", + "test_resultset.py::test_to_dict_one_crossing_matches_the_per_property_path": "needs the bridge jar's RowAccess class", + "test_resultset.py::TestSmallResultsCostOneBridgeCall::test_to_list_makes_one_call_for_a_short_result": BRIDGE_COUNT, + "test_resultset.py::TestSmallResultsCostOneBridgeCall::test_to_list_of_an_empty_result_makes_one_call": BRIDGE_COUNT, + "test_resultset.py::TestSmallResultsCostOneBridgeCall::test_to_json_list_makes_one_call_for_a_short_result": BRIDGE_COUNT, + "test_resultset.py::TestSmallResultsCostOneBridgeCall::test_a_full_batch_still_ends_on_the_next_call": BRIDGE_COUNT, + "test_resultset.py::TestSmallResultsCostOneBridgeCall::test_a_drained_set_reads_as_empty_and_is_not_an_error": BRIDGE_COUNT, + "test_resultset.py::TestSmallResultsCostOneBridgeCall::test_python_does_not_cross_the_bridge_to_close_a_drained_set": BRIDGE_COUNT, + "test_resultset.py::TestSmallResultsCostOneBridgeCall::test_the_bridge_closes_what_it_drains_and_only_that": BRIDGE_COUNT, + "test_vector.py::TestLSMVectorIndex::test_lsm_vector_search_looks_up_all_hits_in_one_bridge_call": BRIDGE_COUNT, + "test_vector.py::TestLSMVectorIndex::test_lsm_vector_search_uses_database_rid_conversion": "intercepts Database.to_java_rid, the JPype path's RID conversion", + # the Java index behind the wrapper + "test_vector.py::TestLSMVectorIndex::test_create_vector_index_with_pq_params": JAVA_INDEX, + "test_vector.py::TestLSMVectorIndex::test_create_vector_index_with_custom_id_property": JAVA_INDEX, + "test_vector.py::TestLSMVectorIndex::test_lsm_vector_search_approximate_product": JAVA_INDEX, + "test_vector.py::TestLSMVectorIndex::test_lsm_vector_search_approximate_typeindex": JAVA_INDEX, + **{ + f"test_vector_params_verification.py::TestVectorParams::{name}": JAVA_INDEX + for name in ( + "test_quantization_param", + "test_store_vectors_in_graph_param", + "test_encoding_param", + "test_add_hierarchy_param", + "test_per_index_cache_params", + "test_quantization_none", + "test_quantization_binary", + "test_quantization_product", + ) + }, } -# Tests that exercise the JVM itself, so the native backend (no JVM, no jars) has nothing to run. Skipped before they start, -# with the reason; a test whose behaviour native shares (Ctrl-C in a Python loop, the zone of to_json_list) is not listed. -NATIVE_NA_JVM_TESTS = { - "test_ctrl_c_raises_keyboard_interrupt_and_runs_cleanup[java]": "a Java call blocked in JPype's SIGINT handler (#179)", - "test_interrupt_true_keeps_the_jvm_handling_sigint": "start_jvm(interrupt=True): the JVM handles SIGINT", - "test_docs_installation_jvm_examples": "documents start_jvm() and jvm_kwargs", - "test_to_java_int_array_keeps_values_that_fit": "to_java_int_array builds a Java int[] (JPype)", - "test_to_java_int_array_refuses_values_beyond_32_bits_in_an_array": "to_java_int_array builds a Java int[] (JPype)", - "test_to_java_byte_array_accepts_bytes_like_fast_paths": "to_java_byte_array builds a Java byte[] (JPype)", - "test_to_java_int_array": "to_java_int_array builds a Java int[] (JPype)", - "test_to_java_int_array_accepts_numpy_directly": "to_java_int_array builds a Java int[] (JPype)", - "test_to_java_int_array_round_trips_through_a_sparse_vector": "to_java_int_array builds a Java int[] (JPype)", - "test_the_host_reaches_exactly_the_listener_bind_settings": "reads the engine's GlobalConfiguration through the JVM", -} +def _native(): + return BACKEND == "native" def pytest_collection_modifyitems(config, items): - if BACKEND != "native": + if not _native(): return for item in items: - reason = NATIVE_NA_JVM_TESTS.get(item.name) + # "::[::][]", the file without its directory + test_id = f"{item.path.name}::{item.nodeid.split('::', 1)[-1]}" + reason = NATIVE_NA_JVM_TESTS.get(test_id) or NATIVE_NA_JVM_TESTS.get( + test_id.split("[", 1)[0] + ) if reason: item.add_marker(pytest.mark.skip(reason=f"NA-JVM: {reason}")) @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item, call): + """On the native backend, report a test as NOTYET when the native code raised NotOnNativeYet. + + That is the one explicit signal: the native code path names a feature the wheel does not have (yet) by raising a dedicated + exception, the exception itself or an exception raised from it. Nothing else is turned into a skip. In particular no failure text + is matched: a text such as "assert None is not None" or "KeyError: 'flush'" is also what a real bug looks like. + """ outcome = yield - if BACKEND != "native": + if not _native(): return report = outcome.get_result() if call.excinfo is None or report.passed: return from arcadedb._native import NotOnNativeYet - if item.name in NATIVE_NA_TESTS: - report.outcome = "skipped" - report.longrepr = ( - str(item.fspath), - item.location[1] or 0, - "NA-JVM: counts JPype conversions", - ) - return exc, seen = call.excinfo.value, 0 while exc is not None and seen < 8: if isinstance(exc, NotOnNativeYet): @@ -167,25 +227,7 @@ def pytest_runtest_makereport(item, call): f"NOTYET: {exc}", ) return - text = f"{type(exc).__name__}: {exc}" - if any(marker in text for marker in NATIVE_NOTYET_MARKERS): - report.outcome = "skipped" - report.longrepr = ( - str(item.fspath), - item.location[1] or 0, - "NOTYET: Java-object wrapper or exporter path", - ) - return - if any(marker in text for marker in NATIVE_JVM_MARKERS): - # the test reaches into the JVM directly (jpype.JClass, Java arrays, bridge classes): about the JPype path - report.outcome = "skipped" - report.longrepr = ( - str(item.fspath), - item.location[1] or 0, - "NA-JVM: the test uses the JVM directly", - ) - return - exc, seen = exc.__cause__ or exc.__context__, seen + 1 + exc, seen = exc.__cause__, seen + 1 @pytest.hookimpl(trylast=True) diff --git a/tests/test_resultset_read_errors.py b/tests/test_resultset_read_errors.py index 782088b..d3986fb 100644 --- a/tests/test_resultset_read_errors.py +++ b/tests/test_resultset_read_errors.py @@ -76,6 +76,11 @@ def db(): def _assert_wraps_the_java_error(error): + if arcadedb.__backend__ == "native": + # no Java exception object to chain: the engine's error is named in the message, which is what the other backend's cause adds + assert str(error).startswith("Reading the result set failed: ") + assert "Exception: " in str(error), str(error) + return cause = error.__cause__ assert isinstance(cause, jpype.JException), f"cause is {type(cause)!r}" assert isinstance(cause, jpype.JClass("java.lang.RuntimeException")) diff --git a/tests/test_sparse_quantization_compact.py b/tests/test_sparse_quantization_compact.py index 98b3008..9704f34 100644 --- a/tests/test_sparse_quantization_compact.py +++ b/tests/test_sparse_quantization_compact.py @@ -2,7 +2,6 @@ beam argument: three engine features the vector guide documents and the benchmark harness depends on (guide/vectors.md, 2026-09-07).""" -import jpype.types as jtypes import pytest import arcadedb @@ -38,7 +37,7 @@ def _neighbours(db, toks, wts, k=5): rows = db.query( "sql", "SELECT id FROM (SELECT expand(`vector.sparseNeighbors`('SDoc[tokens,weights]', ?, ?, ?)))", - jtypes.JArray(jtypes.JInt)(toks), + arcadedb.to_java_int_array(toks), arcadedb.to_java_float_array(wts), k, ).to_list() @@ -120,7 +119,7 @@ def _scores_against_exact(db, type_name, meta): "sql", f"INSERT INTO {type_name} SET id = ?, tokens = ?, weights = ?", i, - jtypes.JArray(jtypes.JInt)(toks), + arcadedb.to_java_int_array(toks), arcadedb.to_java_float_array(wts), ) db.command("sql", f"COMPACT INDEX `{type_name}[tokens,weights]`") @@ -129,7 +128,7 @@ def _scores_against_exact(db, type_name, meta): "sql", "SELECT id, score FROM (SELECT expand(`vector.sparseNeighbors`(" f"'{type_name}[tokens,weights]', ?, ?, ?)))", # nosec B608 - test type name - jtypes.JArray(jtypes.JInt)(list(query)), + arcadedb.to_java_int_array(list(query)), arcadedb.to_java_float_array(list(query.values())), 5, ).to_list() diff --git a/tests/test_vector_sql.py b/tests/test_vector_sql.py index 1fbdb66..54f10a7 100644 --- a/tests/test_vector_sql.py +++ b/tests/test_vector_sql.py @@ -7,7 +7,6 @@ import math -import jpype.types as jtypes import pytest import arcadedb @@ -565,7 +564,7 @@ def test_sparse_vector_neighbors_sql(self, test_db): "(SELECT expand(`vector.sparseNeighbors`('SparseDoc[tokens,weights]', ?, ?, ?))) " "ORDER BY score DESC" ), - jtypes.JArray(jtypes.JInt)([5]), + arcadedb.to_java_int_array([5]), arcadedb.to_java_float_array([1.0]), 5, ).to_list() @@ -1338,7 +1337,7 @@ def _search(db, options): "SELECT id, score FROM " # nosec B608 - test-owned option literals "(SELECT expand(`vector.sparseNeighbors`('SparseOpt[tokens,weights]', " # nosec B608 f"?, ?, 5, {options}))) ORDER BY score DESC", # nosec B608 - jtypes.JArray(jtypes.JInt)([5]), + arcadedb.to_java_int_array([5]), arcadedb.to_java_float_array([1.0]), ).to_list() From 99a9dc2afcedac00d5a9eaa8f0784b70c541a9a5 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 20:50:54 +0900 Subject: [PATCH 09/33] Native glue: graph batches go through db.batch() (P-04), parallel insert_many into a vertex type creates vertices (P-01b) Port of humemai/arcadedb-embedded-python#538. gbCreate called GraphBatch.builder(db), which skips the engine's single-batch guard, so two concurrent batches on one database lost edges. It now calls db.batch(), as the JPype path does. insert_parallel looked at no schema and always called newDocument, which the JVM build refuses for a vertex type; it now looks the type up once per call: a vertex type gets vertices, an edge type is refused. Tests: TestInsertManyVertexType, the second-batch refusal and the two-writers-one-hub test (the last two run on both builds). Needs a rebuilt native library. Co-Authored-By: Claude Sonnet 5.5 --- .../org/humemai/nativelib/NativeExtras.java | 31 +++++- tests/test_bulk_insert.py | 98 +++++++++++++++++++ tests/test_graph_batch.py | 87 ++++++++++++++++ 3 files changed, 215 insertions(+), 1 deletion(-) diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index dfc04d2..8b86316 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -574,6 +574,33 @@ public void onError(final Exception e) { // ---- import and export + private static final String[] XML_LIMIT_PROPERTIES = { "jdk.xml.maxGeneralEntitySizeLimit", "jdk.xml.entityExpansionLimit", + "jdk.xml.totalEntitySizeLimit" }; + private static final Object XML_LIMITS_LOCK = new Object(); + + /** + * Runs the import with the JDK's XML entity limits lifted and puts the previous values back afterwards. The lock serializes lifted imports + * with each other; the properties are global, so an XML parse started by another thread inside the call sees the lifted limits too. + */ + private static Map loadWithoutXmlLimits(final Importer importer) { + synchronized (XML_LIMITS_LOCK) { + final String[] saved = new String[XML_LIMIT_PROPERTIES.length]; + for (int i = 0; i < saved.length; i++) { + saved[i] = System.getProperty(XML_LIMIT_PROPERTIES[i]); + System.setProperty(XML_LIMIT_PROPERTIES[i], "0"); + } + try { + return importer.load(); + } finally { + for (int i = 0; i < saved.length; i++) + if (saved[i] == null) + System.clearProperty(XML_LIMIT_PROPERTIES[i]); + else + System.setProperty(XML_LIMIT_PROPERTIES[i], saved[i]); + } + } + } + /** settings: {key: string} as the Python importer builds them; the reply is {"ok":true,"stats":{..}|null}. */ @CEntryPoint(name = "arcadedb_n_import") public static CCharPointer importDocuments(final IsolateThread thread, final long dbh, final CCharPointer settingsJson) { @@ -582,9 +609,11 @@ public static CCharPointer importDocuments(final IsolateThread thread, final lon final Map settings = new HashMap<>(); for (final Map.Entry e : args(settingsJson).entrySet()) settings.put(e.getKey(), String.valueOf(e.getValue())); + // "__disableXmlLimits" is the binding's own key (the JDK's XML entity limits are system properties): lifted around this one load + final boolean liftXmlLimits = "true".equals(settings.remove("__disableXmlLimits")); final Importer importer = new Importer(db, null); importer.setSettings(settings); - final Map stats = importer.load(); + final Map stats = liftXmlLimits ? loadWithoutXmlLimits(importer) : importer.load(); final StringBuilder sb = new StringBuilder("{\"ok\":true,\"stats\":"); if (stats == null) sb.append("null"); diff --git a/tests/test_bulk_insert.py b/tests/test_bulk_insert.py index 124c427..444a185 100644 --- a/tests/test_bulk_insert.py +++ b/tests/test_bulk_insert.py @@ -246,6 +246,104 @@ def test_a_missing_type_still_raises(self, temp_db): temp_db.insert_many("Nope", [{"id": 1}]) +class TestInsertManyVertexType: + """insert_many and insert_columns into a VERTEX type store real vertices. + + The JVM wheel raised "Cannot create a document of type ... because is not a + document type" (the Java helper called newDocument), while the native wheel + accepted the same call (found 2026-10-25 by a parity probe between the two + wheels). A record counts as a vertex when an edge can join it: CREATE EDGE + refuses a record that is not one. + """ + + ROWS = 25 + + def _vertex_db(self, db, name="Vtx", buckets=None): + suffix = f" BUCKETS {buckets}" if buckets else "" + db.command("sql", f"CREATE VERTEX TYPE {name}{suffix}") + db.command("sql", "CREATE EDGE TYPE Knows") + + def _assert_vertices(self, db, name, expected): + assert _count(db, name) == expected + with db.transaction(): + db.command( + "sql", + f"CREATE EDGE Knows FROM (SELECT FROM {name} WHERE id = 0) " # nosec B608 - test-owned type name, no user input + f"TO (SELECT FROM {name} WHERE id = 1)", + ) + assert _count(db, "Knows") == 1 + out = db.query( + "sql", + f"SELECT out('Knows').size() AS n FROM {name} WHERE id = 0", # nosec B608 - test-owned type name, no user input + ).to_list()[0]["n"] + assert out == 1 + + def test_insert_many(self, temp_db): + self._vertex_db(temp_db) + rows = [{"id": i, "name": f"v{i}", "tags": ["a"]} for i in range(self.ROWS)] + assert temp_db.insert_many("Vtx", rows, commit_every=10) == self.ROWS + self._assert_vertices(temp_db, "Vtx", self.ROWS) + got = temp_db.query("sql", "SELECT FROM Vtx WHERE id = 7").to_list()[0] + assert got["name"] == "v7" + assert list(got["tags"]) == ["a"] + + def test_insert_many_parallel(self, temp_db): + self._vertex_db(temp_db, buckets=2) + rows = [{"id": i} for i in range(2000)] + assert temp_db.insert_many("Vtx", rows, parallel=True) == 2000 + self._assert_vertices(temp_db, "Vtx", 2000) + + def test_insert_many_non_json_fallback(self, temp_db): + self._vertex_db(temp_db) + rows = [ + {"id": i, "when": datetime.datetime(2026, 7, 25, 12, 0, i)} + for i in range(3) + ] + assert temp_db.insert_many("Vtx", rows) == 3 + self._assert_vertices(temp_db, "Vtx", 3) + + def test_insert_many_inside_a_caller_transaction(self, temp_db): + self._vertex_db(temp_db) + with pytest.raises(RuntimeError): + with temp_db.transaction(): + temp_db.insert_many("Vtx", [{"id": 0}, {"id": 1}], commit_every=1) + raise RuntimeError("roll back") + assert _count(temp_db, "Vtx") == 0 + with temp_db.transaction(): + temp_db.insert_many("Vtx", [{"id": 0}, {"id": 1}], commit_every=1) + self._assert_vertices(temp_db, "Vtx", 2) + + def test_insert_columns(self, temp_db): + import numpy as np + + self._vertex_db(temp_db) + assert temp_db.insert_columns("Vtx", {"id": np.arange(5, dtype="int64")}) == 5 + self._assert_vertices(temp_db, "Vtx", 5) + + def test_insert_columns_parallel(self, temp_db): + import numpy as np + + self._vertex_db(temp_db, buckets=2) + n = 1000 + assert ( + temp_db.insert_columns( + "Vtx", {"id": np.arange(n, dtype="int64")}, parallel=True + ) + == n + ) + self._assert_vertices(temp_db, "Vtx", n) + + def test_an_edge_type_is_refused(self, temp_db): + self._vertex_db(temp_db) + with pytest.raises(arcadedb.ArcadeDBError, match="Knows"): + temp_db.insert_many("Knows", [{"id": 1}]) + assert _count(temp_db, "Knows") == 0 + + def test_a_missing_type_still_raises(self, temp_db): + with pytest.raises(arcadedb.ArcadeDBError, match="Nope"): + temp_db.insert_many("Nope", [{"id": 1}]) + + class TestInsertManyParallelReportsFailures: """A record the parallel writers fail to store must fail the call. diff --git a/tests/test_graph_batch.py b/tests/test_graph_batch.py index c197a9c..618121c 100644 --- a/tests/test_graph_batch.py +++ b/tests/test_graph_batch.py @@ -6,6 +6,8 @@ import arcadedb from arcadedb.exceptions import ArcadeDBError +_NATIVE = arcadedb.__backend__ == "native" + def test_graph_batch_creates_vertices_and_edges(temp_db_path): with arcadedb.create_database(temp_db_path) as db: @@ -525,6 +527,11 @@ def interrupt(*_args, **_kwargs): # untouched transaction, so a change that commits quietly again, or one that stops refusing # without leaving the transaction alone, fails them. def _is_the_engines_refusal(error): + if _NATIVE: + # no Java exception object to chain: the engine's class is named in the message + return isinstance(error, ArcadeDBError) and "IllegalStateException" in str( + error + ) import jpype return isinstance(error, ArcadeDBError) and isinstance( @@ -726,6 +733,86 @@ def test_graph_batch_close_refused_in_the_callers_transaction_can_be_retried( db.graph_batch(parallel_flush=False).close() +def test_a_second_graph_batch_is_refused_while_one_is_open(temp_db_path): + """#5666: two concurrent GraphBatch instances on one database silently lose edges, so the engine refuses the second.""" + with arcadedb.create_database(temp_db_path) as db: + _vertex_edge_note_types(db) + batch = db.graph_batch(parallel_flush=False) + try: + with pytest.raises(ArcadeDBError, match="already in progress"): + db.graph_batch(parallel_flush=False) + finally: + batch.close() + db.graph_batch(parallel_flush=False).close() + + +def test_two_concurrent_batches_on_one_hub_lose_no_edge(temp_db_path): + """#5666 on both wheels: a batch that finds another open is refused, so two writers that + load edges onto the same hub vertex take turns and every edge lands (counts exact). + + The first writer holds its batch open until the second has tried to open one. If that + attempt were not refused, both batches would be open at once and the loser's edges on the + hub would be orphaned at close (P-04 on the native wheel before the glue went through db.batch()). + """ + import threading + import time + + per_writer = 300 + with arcadedb.create_database(temp_db_path) as db: + _vertex_edge_note_types(db) + with db.graph_batch(parallel_flush=False) as seed: + hub = seed.create_vertices("V", 1)[0] + spokes = [seed.create_vertices("V", per_writer) for _ in range(2)] + first_open = threading.Event() + second_tried = threading.Event() + refusals, errors = [], [] + + def writer(index): + try: + if index == 1: + first_open.wait(30) + batch = None + while batch is None: + try: + batch = db.graph_batch(parallel_flush=False) + except ArcadeDBError as e: + assert "already in progress" in str(e), repr(e) + refusals.append(index) + time.sleep(0.01) + if index == 1: + second_tried.set() + if index == 0: + first_open.set() + assert second_tried.wait( + 30 + ), "the second writer never tried to open a batch" + try: + for dst in spokes[index]: + batch.new_edge(hub, "E", dst) + finally: + batch.close() + except BaseException as e: # reported by the main thread + errors.append(repr(e)) + first_open.set() + second_tried.set() + + threads = [threading.Thread(target=writer, args=(i,)) for i in range(2)] + for t in threads: + t.start() + for t in threads: + t.join(120) + assert not any(t.is_alive() for t in threads) + assert errors == [] + assert db.count_type("E") == 2 * per_writer + out = db.query( + "sql", f"SELECT out('E').size() AS n FROM {hub}" + ).to_list() # nosec B608 - a RID from this test + assert out[0]["n"] == 2 * per_writer + assert ( + 1 in refusals + ), "the second batch was not refused while the first was open" + + def test_graph_batch_leaves_the_callers_wal_setting_alone(temp_db_path): """#9242: with the batch open, a transaction of the caller's commits with the caller's WAL setting. Before the fix the batch's own setting (no WAL, by default) stayed on the From 0e266d74b6a9d87eddc22366769fbaab922f3dd7 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 20:50:54 +0900 Subject: [PATCH 10/33] Native parity: PQ settings, float vector bulk, XML limits, example 29 on an engine without the where option - create_vector_index takes rerank_oversample and fused_pq (CREATE INDEX .. METADATA rerankOversample / fusedPQ) and get_metadata reports them. - insert_columns: only a float32 matrix is a column of vectors (float64 and N x 0 are refused with the one-dimensional ValueError), and the non-fast and parallel paths accept the float32 matrix. A float64 NumPy array in a row of insert_many is the list it would have been, not a float[]. - import_documents takes disable_xml_limits. The glue lifts the three jdk.xml limits around that one load and restores them; the XMLMessages bundle goes into the image so that an entity-limit error names JAXP0001000x instead of "Could not load any resource bundle". New tests/test_native_xml_limits.py (native build only). - example 29: the where form is asserted refused ("Unknown option 'where'") on an engine that predates 26.11.1, like has_pq_rerank does for the PQ settings. The XML limit and bundle changes need a rebuilt native library. Co-Authored-By: Claude Sonnet 5.5 --- .../native-image.properties | 4 +- src/arcadedb/_native/bulk.py | 4 +- src/arcadedb/_native/codec.py | 14 +++++ src/arcadedb/_native/database.py | 26 +++++++-- src/arcadedb/_native/schema_objects.py | 21 ++++++- tests/test_example29_hybrid_memory.py | 31 +++++++++- tests/test_native_xml_limits.py | 58 +++++++++++++++++++ 7 files changed, 149 insertions(+), 9 deletions(-) create mode 100644 tests/test_native_xml_limits.py diff --git a/native/src/main/resources/META-INF/native-image/org.humemai/arcadedb-native-lib/native-image.properties b/native/src/main/resources/META-INF/native-image/org.humemai/arcadedb-native-lib/native-image.properties index 021c1ee..2234167 100644 --- a/native/src/main/resources/META-INF/native-image/org.humemai/arcadedb-native-lib/native-image.properties +++ b/native/src/main/resources/META-INF/native-image/org.humemai/arcadedb-native-lib/native-image.properties @@ -4,4 +4,6 @@ # and a search from 1.07 ms to 0.89 ms. The dotProduct and cosine entries are the same scalar counterparts for DOT_PRODUCT and COSINE # indexes (not profiled; the same getters sit in their loops). A filter without a signature matches every overload. This file is read from # the glue jar on the image build classpath; preview-build.sh fails the build when the log does not show the option as picked up. -Args = -H:PriorityForceInline=io.github.jbellis.jvector.vector.ArrayVectorFloat.length,io.github.jbellis.jvector.vector.ArrayVectorFloat.get,io.github.jbellis.jvector.vector.VectorUtil.squareL2Distance,io.github.jbellis.jvector.vector.VectorUtil.dotProduct,io.github.jbellis.jvector.vector.VectorUtil.cosine,io.github.jbellis.jvector.vector.DefaultVectorUtilSupport.squareDistance,io.github.jbellis.jvector.vector.DefaultVectorUtilSupport.squareDistanceUnrolled,io.github.jbellis.jvector.vector.DefaultVectorUtilSupport.dotProduct,io.github.jbellis.jvector.vector.DefaultVectorUtilSupport.cosine +# The JDK's XML parser builds its error messages (JAXP00010001, entity limits and every syntax error) from this bundle; without it in the image a parse error +# reads "Could not load any resource bundle by ...XMLMessages" and the entity-limit code is lost (test_native_xml_limits.py). +Args = -H:PriorityForceInline=io.github.jbellis.jvector.vector.ArrayVectorFloat.length,io.github.jbellis.jvector.vector.ArrayVectorFloat.get,io.github.jbellis.jvector.vector.VectorUtil.squareL2Distance,io.github.jbellis.jvector.vector.VectorUtil.dotProduct,io.github.jbellis.jvector.vector.VectorUtil.cosine,io.github.jbellis.jvector.vector.DefaultVectorUtilSupport.squareDistance,io.github.jbellis.jvector.vector.DefaultVectorUtilSupport.squareDistanceUnrolled,io.github.jbellis.jvector.vector.DefaultVectorUtilSupport.dotProduct,io.github.jbellis.jvector.vector.DefaultVectorUtilSupport.cosine -H:IncludeResourceBundles=com.sun.org.apache.xerces.internal.impl.msg.XMLMessages diff --git a/src/arcadedb/_native/bulk.py b/src/arcadedb/_native/bulk.py index e8510a3..9cae198 100644 --- a/src/arcadedb/_native/bulk.py +++ b/src/arcadedb/_native/bulk.py @@ -45,7 +45,9 @@ def insert_columns_fast(db, type_name, columns, commit_every): kind, arr = "d", _np.ascontiguousarray(values, dtype=_np.float64) elif values.ndim == 1 and k == "b": kind, arr = "b", _np.ascontiguousarray(values, dtype=_np.uint8) - elif values.ndim == 2 and k == "f" and values.shape[1] > 0: + elif ( + values.ndim == 2 and values.dtype == _np.float32 and values.shape[1] > 0 + ): # only a float32 matrix is a column of vectors, as on the JPype path kind, arr, dim = ( "f", _np.ascontiguousarray(values, dtype=_np.float32), diff --git a/src/arcadedb/_native/codec.py b/src/arcadedb/_native/codec.py index c362956..211c91f 100644 --- a/src/arcadedb/_native/codec.py +++ b/src/arcadedb/_native/codec.py @@ -122,6 +122,20 @@ def _dump(obj, binary): return text, buf, state[1] +def row_default(value): + """json.dumps hook for the rows of a bulk load: a NumPy float array that is not float32 is the list it would have been + (a float64 array stored on an undeclared property keeps its doubles, as on the JPype path); the rest as :func:`_default`. + """ + if ( + _np is not None + and isinstance(value, _np.ndarray) + and value.dtype.kind == "f" + and value.dtype != _np.float32 + ): + return value.tolist() + return _default(value) + + class RID: """A record id as the engine returns it in a row; passed back as a parameter it is a link, not a string. diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 12cee0d..d640e8a 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -14,13 +14,17 @@ from ..transactions import TransactionContext from ..type_conversion import json_bulk_dumps from . import lib, not_yet -from .codec import _default as codec_default from .codec import _prep as _prep_row -from .codec import encode_params, json_native, loads +from .codec import encode_params, json_native, loads, row_default _LAZY_COMMAND_ROWS = os.environ.get("ARCADEDB_NATIVE_LAZY_CMD", "1") != "0" _BINARY_REPLY = os.environ.get("ARCADEDB_NATIVE_BIN_REPLY", "1") != "0" _INSERT_MANY_CHUNK = 10_000 +_XML_LIMIT_ERROR = re.compile(r"JAXP0001000[134]") # the JDK's XML entity limits +_XML_LIMIT_HINT = ( + " The JDK's XML entity limits stopped this import. Pass disable_xml_limits=True to import_documents to lift them for this" + " call only." +) _RID_RE = re.compile(r"#?-?\d+:-?\d+") _FIRST_BATCH = 256 _EPOCH_DAY_ORDINAL = date(1970, 1, 1).toordinal() @@ -744,7 +748,7 @@ def _command_many(self, language: str, command: str, rows) -> int: all in one crossing and in the open transaction; the cursors are drained and closed. Not part of the public API. """ self._check_not_closed() - payload = json.dumps([_prep_row(r) for r in rows], default=codec_default) + payload = json.dumps([_prep_row(r) for r in rows], default=row_default) reply = json.loads( lib.call( "arcadedb_n_exec_many", @@ -904,6 +908,8 @@ def create_vector_index( pq_clusters=None, pq_center_globally=None, pq_training_limit=None, + rerank_oversample=None, + fused_pq=None, build_graph_now=True, ): """A vector index (``CREATE INDEX .. LSM_VECTOR``) and its :class:`VectorIndex` wrapper; arguments as the JPype path.""" @@ -936,6 +942,8 @@ def create_vector_index( pq_clusters, pq_center_globally, pq_training_limit, + rerank_oversample, + fused_pq, build_graph_now, ) @@ -958,6 +966,7 @@ def import_documents( trim_text=None, on_row_error=None, extra_settings=None, + disable_xml_limits=False, ): """Import document-shaped files (CSV, JSON, ...) through the engine's importer; same arguments as the JPype path.""" from ..importer import ( @@ -987,6 +996,9 @@ def import_documents( on_row_error=on_row_error, extra_settings=extra_settings, ) + if disable_xml_limits: + # the glue lifts the JDK's XML entity limits around this one import and restores them after it + settings = {**settings, "__disableXmlLimits": "true"} try: stats = extras.run_import(self, settings, parallel, commit_every, wal) return ImportResult( @@ -996,7 +1008,10 @@ def import_documents( statistics=stats or {}, ) except Exception as e: - raise ArcadeDBError(f"Document import failed: {e}") from e + message = f"Document import failed: {e}" + if not disable_xml_limits and _XML_LIMIT_ERROR.search(str(e)): + message += _XML_LIMIT_HINT + raise ArcadeDBError(message) from e def export_database( self, @@ -1144,6 +1159,9 @@ def _column_values(name, values): else: # a pandas extension dtype (Int64, string, boolean, category): its is a null values = values.to_numpy(dtype=object, na_value=None) if np is not None and isinstance(values, np.ndarray): + if values.ndim == 2 and values.dtype == np.float32 and values.shape[1] > 0: + # a column of vectors, as the JPype path takes it; float32 widens to the same doubles in the JSON + return values.tolist() if values.ndim != 1: raise ValueError( f"column {name!r} is {values.ndim}-dimensional; a column is one-dimensional" diff --git a/src/arcadedb/_native/schema_objects.py b/src/arcadedb/_native/schema_objects.py index eb14755..455b6fc 100644 --- a/src/arcadedb/_native/schema_objects.py +++ b/src/arcadedb/_native/schema_objects.py @@ -239,6 +239,8 @@ def get_metadata(self): "pqClusters": "pq_clusters", "pqCenterGlobally": "pq_center_globally", "pqTrainingLimit": "pq_training_limit", + "rerankOversample": "rerank_oversample", + "fusedPQ": "fused_pq", } buckets = [ r["name"] @@ -278,10 +280,19 @@ def _check_vector_arguments( pq_clusters, pq_center_globally, pq_training_limit, + rerank_oversample=None, + fused_pq=None, ): if any( v is not None - for v in (pq_subspaces, pq_clusters, pq_center_globally, pq_training_limit) + for v in ( + pq_subspaces, + pq_clusters, + pq_center_globally, + pq_training_limit, + rerank_oversample, + fused_pq, + ) ): if not quantization or quantization.upper() != "PRODUCT": raise ValueError("PQ parameters require quantization='PRODUCT'") @@ -315,6 +326,8 @@ def create_vector_index( pq_clusters, pq_center_globally, pq_training_limit, + rerank_oversample, + fused_pq, build_graph_now, ): """``CREATE INDEX .. LSM_VECTOR METADATA {..}`` with the JPype builder's defaults and checks; returns the wrapper.""" @@ -326,6 +339,8 @@ def create_vector_index( pq_clusters, pq_center_globally, pq_training_limit, + rerank_oversample, + fused_pq, ) except ValueError as e: raise ArcadeDBError(f"Failed to create vector index: {e}") from e @@ -349,6 +364,10 @@ def create_vector_index( ): if value is not None: meta[key] = bool(value) if key == "pqCenterGlobally" else int(value) + if rerank_oversample is not None: + meta["rerankOversample"] = int(rerank_oversample) + if fused_pq is not None: + meta["fusedPQ"] = bool(fused_pq) if store_vectors_in_graph: meta["storeVectorsInGraph"] = True if add_hierarchy is not None: diff --git a/tests/test_example29_hybrid_memory.py b/tests/test_example29_hybrid_memory.py index 50e5f14..04e2ca9 100644 --- a/tests/test_example29_hybrid_memory.py +++ b/tests/test_example29_hybrid_memory.py @@ -33,9 +33,30 @@ def memory(example29, tmp_path_factory): yield db, centroids +@pytest.fixture(scope="module") +def has_where(example29, memory): + """True when the engine takes the ``where`` option of ``vector.neighbors`` (ArcadeDB 26.11.1). + + The native build links the newest engine RELEASE, which predates it until 26.11.1 ships: the form is then refused with the engine's + own message, and the tests below assert that answer instead of the new one. When the native build links 26.11.1, delete this + fixture and the old-answer branches. + """ + db, centroids = memory + try: + example29.run(db, centroids, "where") + except arcadedb.ArcadeDBError as e: + assert "Unknown option 'where'" in str(e), e + return False + return True + + @pytest.mark.parametrize("form", ["filter", "where"]) -def test_hybrid_rows_and_hops_stay_in_the_tenant(example29, memory, form): +def test_hybrid_rows_and_hops_stay_in_the_tenant(example29, memory, form, has_where): db, centroids = memory + if form == "where" and not has_where: + with pytest.raises(arcadedb.ArcadeDBError, match="Unknown option 'where'"): + example29.run(db, centroids, form) + return out = example29.run(db, centroids, form) tenant = example29.REQUESTED_TENANT @@ -56,9 +77,15 @@ def test_filtering_after_the_top_k_loses_the_rows(example29, memory): assert len(out["post_filtered"]) < example29.K # the pre-filter returns K -def test_the_forms_agree(example29, memory): +def test_the_forms_agree(example29, memory, has_where): db, centroids = memory by_filter = example29.run(db, centroids, "filter") + if ( + not has_where + ): # the engine predates the option: the other form is refused (see has_where) + with pytest.raises(arcadedb.ArcadeDBError, match="Unknown option 'where'"): + example29.run(db, centroids, "where") + return by_where = example29.run(db, centroids, "where") assert [r["fid"] for r in by_filter["hybrid"]] == [ r["fid"] for r in by_where["hybrid"] diff --git a/tests/test_native_xml_limits.py b/tests/test_native_xml_limits.py new file mode 100644 index 0000000..2706dcc --- /dev/null +++ b/tests/test_native_xml_limits.py @@ -0,0 +1,58 @@ +"""The native build lifts the JDK's XML entity limits for one import, and only when asked (the JVM build does the same with JVM +system properties, tests/test_xml_limits.py).""" + +import pytest + +import arcadedb +from arcadedb.exceptions import ArcadeDBError + + +@pytest.fixture +def db(tmp_path): + database = arcadedb.create_database(str(tmp_path / "xml_limits_db")) + database.command("sql", "CREATE DOCUMENT TYPE Item") + yield database + if database.is_transaction_active(): + database.rollback() + database.drop() + + +@pytest.fixture +def big_xml(tmp_path): + """One text node of 1,000,000 characters (200,000 entity references).""" + path = tmp_path / "big.xml" + path.write_text( + "" + "&" * 200_000 + "" + "small" + ) + return path + + +def test_import_hitting_a_limit_names_the_option(db, big_xml): + with pytest.raises(ArcadeDBError) as excinfo: + db.import_documents(big_xml, document_type="Item", file_type="xml") + assert "JAXP0001000" in str(excinfo.value) + assert "disable_xml_limits=True" in str(excinfo.value) + + +def test_per_call_option_lifts_the_limits_for_that_call_only(db, big_xml): + result = db.import_documents( + big_xml, document_type="Item", file_type="xml", disable_xml_limits=True + ) + assert result.result == "OK" + assert result.get("parsedRecords") == 2 + # the limits are back after the call: the same file is refused again + with pytest.raises(ArcadeDBError, match="JAXP0001000"): + db.import_documents(big_xml, document_type="Item", file_type="xml") + + +def test_limits_are_restored_when_the_import_fails(db, big_xml, tmp_path): + missing = tmp_path / "missing.xml" + with pytest.raises(ArcadeDBError) as excinfo: + db.import_documents( + missing, document_type="Item", file_type="xml", disable_xml_limits=True + ) + assert "disable_xml_limits=True" not in str(excinfo.value) + # the failed lifted import put the limits back: the big file is refused again + with pytest.raises(ArcadeDBError, match="JAXP0001000"): + db.import_documents(big_xml, document_type="Item", file_type="xml") From a28fc0f5c62f87350a27147416ea0276f3860305 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 20:51:25 +0900 Subject: [PATCH 11/33] CI: the native dev-release job publishes through the environment pypi, like the final job arcadedb now has one PyPI trusted publisher (native.yml with the environment pypi) for dev, final and post releases. The environment pypi-prerelease is gone (deleted on GitHub), and the workflow header and docs/modules/ROOT/pages/development/release.adoc say so. The readiness gate (ARCADEDB_NATIVE_READINESS_BAR_MET and the full matrix for a final or post release) is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/native.yml | 4 ++-- docs/modules/ROOT/pages/development/release.adoc | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml index 2a4c5df..8078c74 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/native.yml @@ -21,7 +21,7 @@ # besides the counted NOTYET and NA-JVM kinds (tests/conftest.py). # # Publishing (tags only; PyPI trusted publishing, no token; a trusted publisher on the PyPI project `arcadedb` for this workflow file): -# A.B.C.devN environment `pypi-prerelease`. Needs the parity run of linux-x86_64, linux-aarch64 and macos-arm64 on every Python. +# A.B.C.devN environment `pypi` (one trusted publisher for dev, final and post releases). Needs the parity run of linux-x86_64, linux-aarch64 and macos-arm64 on every Python. # Windows is published too when it passed, and is informational otherwise. # A.B.C, A.B.C.postN environment `pypi`, ONLY when the readiness bar is met: the repository variable ARCADEDB_NATIVE_READINESS_BAR_MET # is `true` (a maintainer sets it after checking the correctness fixes and the fault and concurrency probes), the parity run @@ -641,7 +641,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 environment: - name: pypi-prerelease + name: pypi url: https://pypi.org/project/arcadedb/ permissions: id-token: write # PyPI trusted publishing (OIDC); no API token anywhere diff --git a/docs/modules/ROOT/pages/development/release.adoc b/docs/modules/ROOT/pages/development/release.adoc index a487bbe..5c5898c 100644 --- a/docs/modules/ROOT/pages/development/release.adoc +++ b/docs/modules/ROOT/pages/development/release.adoc @@ -84,6 +84,8 @@ The two packages release from two workflows with two PyPI environments: The version rule above holds for both packages. +`arcadedb` has one trusted publisher on PyPI (`native.yml` with the environment `pypi`) for dev, final and post releases. There is no separate pre-release environment; the readiness gate below is what keeps a final release back. + * A native pre-release (`26.11.1.devN`) publishes when the builds on `linux-x86_64`, `linux-aarch64` and `macos-arm64` pass. * A FINAL native release (`A.B.C` or `A.B.C.postN`) needs two more things: ** The repository variable `ARCADEDB_NATIVE_READINESS_BAR_MET` is `true`. A maintainer sets it after checking the readiness bar: the correctness fixes are merged; the full suite is green on every platform, Windows included, and on Python 3.11 to 3.14; the fault and concurrency probes pass; and the documented API raises no `NotOnNativeYet`. From b55e7c4967399ad4762a684a7c60490866de0687 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 21:05:00 +0900 Subject: [PATCH 12/33] CI: one test and one release workflow per build (test-jvm, release-jvm, test-native, release-native) The workflow files now follow one scheme: - test-python-bindings.yml -> test-jvm.yml (the job names `Test arcadedb (, Python )` do not change) - release.yml -> release-jvm.yml (publishes arcadedb-jvm through the environment pypi-jvm) - native.yml -> test-native.yml (native build and parity on pull requests that touch native paths, on main and nightly; it never publishes) - release-native.yml (new): on a version tag it validates the tag, calls test-native.yml with `release: true` (4 platforms x Python 3.11 to 3.14), applies the readiness gate ARCADEDB_NATIVE_READINESS_BAR_MET, and publishes arcadedb through the environment pypi in one job for dev, final and post - test-python-examples.yml stays The trusted publishers on PyPI are keyed on the release file names: arcadedb-jvm <- release-jvm.yml / pypi-jvm, arcadedb <- release-native.yml / pypi. select_native_matrix.py: the `tag` event is now `release` and the `publish` output is gone (publishing is decided in release-native.yml). select_matrix.py: the `bindings` workflow key is now `jvm`. Paths filters, the actionlint config, the tests that read workflow files, README badges (JVM and native), CLAUDE.md and the CI and Release docs follow. Co-Authored-By: Claude Sonnet 5.5 --- .github/actionlint.yaml | 2 +- .../{release.yml => release-jvm.yml} | 12 +- .github/workflows/release-native.yml | 218 ++++++++++++++++++ ...{test-python-bindings.yml => test-jvm.yml} | 20 +- .../workflows/{native.yml => test-native.yml} | 194 ++++------------ .github/workflows/test-python-examples.yml | 14 +- CLAUDE.md | 8 +- README.md | 3 +- docs/modules/ROOT/pages/development/ci.adoc | 27 ++- .../ROOT/pages/development/release.adoc | 21 +- native/scripts/build-image.sh | 2 +- scripts/after_tag_verify.sh | 4 +- scripts/select_matrix.py | 10 +- scripts/select_native_matrix.py | 20 +- tests/test_check_version.py | 2 +- tests/test_select_matrix.py | 14 +- tests/test_select_native_matrix.py | 12 +- 17 files changed, 358 insertions(+), 225 deletions(-) rename .github/workflows/{release.yml => release-jvm.yml} (97%) create mode 100644 .github/workflows/release-native.yml rename .github/workflows/{test-python-bindings.yml => test-jvm.yml} (97%) rename .github/workflows/{native.yml => test-native.yml} (75%) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index f0a6d38..22c17e2 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -4,7 +4,7 @@ # ArcadeData/arcadedb, scoped to the exact file and rule. They are style findings in `run:` scripts, not # schema errors, and the list is meant to shrink: fix the script and delete its entry. paths: - .github/workflows/test-python-bindings.yml: + .github/workflows/test-jvm.yml: ignore: - 'shellcheck reported issue in this script: SC(2001|2010|2012|2086|2129):' .github/workflows/test-python-examples.yml: diff --git a/.github/workflows/release.yml b/.github/workflows/release-jvm.yml similarity index 97% rename from .github/workflows/release.yml rename to .github/workflows/release-jvm.yml index dc84ffe..66e11ee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release-jvm.yml @@ -1,15 +1,15 @@ -name: Release to PyPI +name: Release JVM (arcadedb-jvm) # Pushing a tag that is the bare version (26.11.1, 26.11.1.post1, 26.11.1.dev0; no "v", same as # ArcadeData/arcadedb) builds the wheels, tests them, checks them and publishes them to PyPI. # Versions run A.B.C.devN -> A.B.C -> A.B.C.postN; never aN, bN or rcN. devN starts at 0 and # restarts at 0 on a new base A.B.C, postN starts at 1; scripts/check_version.py enforces the -# counting in the first job (a later native job passes --tag-prefix native-v for its own tags). +# counting in the first job. The same tag also starts release-native.yml; each workflow checks the tag against its own PyPI project. # # PyPI keys a trusted publisher on this FILENAME and on the job's environment, so do not rename # either without changing the publisher on PyPI at the same time: -# project arcadedb-jvm <- ArcadeData/arcadedb-python, workflow release.yml, environment pypi-jvm -# project arcadedb <- ArcadeData/arcadedb-python, workflow native.yml, environment pypi (the native build) +# project arcadedb-jvm <- ArcadeData/arcadedb-python, workflow release-jvm.yml, environment pypi-jvm +# project arcadedb <- ArcadeData/arcadedb-python, workflow release-native.yml, environment pypi (the native build) # Both GitHub environments only accept deployments from tags that match [0-9]*. # # Nothing here stores a token: the publish job authenticates with PyPI trusted publishing (OIDC), @@ -18,7 +18,7 @@ name: Release to PyPI # pyproject.toml names the distribution arcadedb-jvm (tracker #19). The wheel file is arcadedb_jvm--... # (PEP 427 turns the dash into an underscore). The guard in the publish job refuses to publish anything # that is not `arcadedb-jvm`, so this workflow cannot publish the native `arcadedb` by accident; the -# native build has its own workflow, native.yml, with the same kind of guard. +# native build has its own workflow, release-native.yml, with the same kind of guard. on: push: @@ -97,7 +97,7 @@ jobs: test: name: Unit tests and wheels needs: validate-version - uses: ./.github/workflows/test-python-bindings.yml + uses: ./.github/workflows/test-jvm.yml with: arcadedb-tag: ${{ needs.validate-version.outputs.arcadedb-tag }} build-version: ${{ needs.validate-version.outputs.version }} diff --git a/.github/workflows/release-native.yml b/.github/workflows/release-native.yml new file mode 100644 index 0000000..65c3d8e --- /dev/null +++ b/.github/workflows/release-native.yml @@ -0,0 +1,218 @@ +name: Release native (arcadedb) + +# Pushing a tag that is the bare version (26.11.1.dev0, 26.11.1, 26.11.1.post1; no "v", never aN/bN/rcN) builds the native wheels, runs +# the parity suite against them on every platform and Python, checks them and publishes them to PyPI as `arcadedb`. The same tag also +# starts release-jvm.yml, which publishes `arcadedb-jvm`; the two workflows do not wait for each other. +# +# The build and the tests are test-native.yml, called with `release: true` (the full matrix: linux-x86_64, linux-aarch64, +# macos-arm64 and windows-x64, each on Python 3.11 to 3.14). A native image build takes about 1 to 2.5 hours per platform; this workflow adds +# no build of its own. The way the JVM release calls test-jvm.yml is the model. +# +# PyPI keys a trusted publisher on this FILENAME and on the job's environment, so do not rename either without changing the publisher +# on PyPI at the same time: +# project arcadedb <- ArcadeData/arcadedb-python, workflow release-native.yml, environment pypi +# project arcadedb-jvm <- ArcadeData/arcadedb-python, workflow release-jvm.yml, environment pypi-jvm +# The GitHub environment `pypi` only accepts deployments from tags that match [0-9]*. There is one environment and one publish job for +# dev, final and post releases. +# +# What is published (the `verify` job decides, the `publish` job uploads): +# A.B.C.devN the parity run of linux-x86_64, linux-aarch64 and macos-arm64 passed on every Python. windows-x64 is published too when it +# passed on every Python, and is informational otherwise. +# A.B.C, A.B.C.postN ONLY when the readiness bar is met: the repository variable ARCADEDB_NATIVE_READINESS_BAR_MET is `true` (a maintainer +# sets it after checking the correctness fixes and the fault and concurrency probes), the parity run of ALL four platforms +# on ALL four Pythons passed, and no test is a NOTYET skip (test-native.yml passes --max-notyet 0 for a final version). +# Speed parity is not part of the bar. The bar is written down in docs/modules/ROOT/pages/development/release.adoc. +# +# Nothing here stores a token: the publish job authenticates with PyPI trusted publishing (OIDC), and only that job has the id-token +# permission. The guard in the publish job refuses to publish anything that is not `arcadedb`, so this workflow cannot publish the +# JVM `arcadedb-jvm` by accident. + +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]+*" # 26.11.1.dev0, 26.11.1, 26.11.1.post1; validate-version rejects the rest + +permissions: + contents: read + +# One release per tag at a time. A running release is never cancelled half way through an upload. The group differs from the one of +# test-native.yml (test-native-...), which this workflow calls. +concurrency: + group: release-native-${{ github.ref_name }} + cancel-in-progress: false + +env: + # The only distribution this workflow may publish. See the guard in the publish job. + DIST_NAME: arcadedb + +jobs: + # The tag must be a bare PEP 440 version of the allowed kinds, and the next number of its base. + validate-version: + name: Validate tag + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + version: ${{ steps.validate.outputs.version }} + final: ${{ steps.validate.outputs.final }} + steps: + - name: Validate tag + id: validate + env: + TAG: ${{ github.ref_name }} + run: | + if ! [[ "$TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then + echo "::error::Tag '$TAG' is not A.B.C, A.B.C.devN or A.B.C.postN (bare version, no v, no aN/bN/rcN)" + exit 1 + fi + # a .devN wheel is a pre-release; an A.B.C or A.B.C.postN wheel is final and needs the readiness bar + if [[ "$TAG" == *.dev* ]]; then final=false; else final=true; fi + { echo "version=$TAG"; echo "final=$final"; } >> "$GITHUB_OUTPUT" + echo "Release $TAG (final=$final)" + + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # The counting rule is on the docs page development/release.adoc. The script only checks and fails closed: exit 1 is a tag that + # breaks the rule, exit 2 is a source (git tags, PyPI) it could not read, and either stops the release before anything is built. + # The tag being released is left out of the git tags; running the whole workflow again for a version that is already on PyPI + # fails here, so re-run only the publish job instead. Its tests are in tests/test_check_version.py. + - name: Tag must be the next version + env: + TAG: ${{ github.ref_name }} + run: python3 -I scripts/check_version.py "$TAG" --git-remote origin --pypi "$DIST_NAME" + + # Builds a wheel per platform, runs the parity suite on every Python and uploads the wheels (native-wheel-) and one marker per + # passed leg (native-passed--py). + test: + name: Native build and parity + needs: validate-version + uses: ./.github/workflows/test-native.yml + with: + version: ${{ needs.validate-version.outputs.version }} + release: true + + # What may be published, decided once: the wheels whose platform passed on every required Python, and the readiness bar for a final + # release. Runs when `test` failed too, because a platform that is only informational may fail; the markers say which legs passed. + verify: + name: Verify what may be published + needs: [validate-version, test] + if: ${{ !cancelled() && needs.validate-version.result == 'success' }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + outputs: + ok: ${{ steps.verify.outputs.ok }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: native-passed-* + path: passed + merge-multiple: true + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: native-wheel-* + path: wheels + merge-multiple: true + + - name: Every required platform passed on every Python, and the wheels are the ones that were tested + id: verify + env: + VERSION: ${{ needs.validate-version.outputs.version }} + FINAL: ${{ needs.validate-version.outputs.final }} + BAR: ${{ vars.ARCADEDB_NATIVE_READINESS_BAR_MET }} + run: | + set -euo pipefail + mkdir -p passed dist + ls -l passed wheels + pythons="3.11 3.12 3.13 3.14" + if [ "$FINAL" = true ]; then + required="linux-x86_64 linux-aarch64 macos-arm64 windows-x64" + if [ "$BAR" != true ]; then + echo "::error::$VERSION is a final release of arcadedb (native), and the readiness bar is not marked as met (repository variable ARCADEDB_NATIVE_READINESS_BAR_MET): nothing is published. See docs/modules/ROOT/pages/development/release.adoc" >&2 + exit 1 + fi + else + required="linux-x86_64 linux-aarch64 macos-arm64" + fi + for key in $required; do + for py in $pythons; do + [ -f "passed/$key-py$py.ok" ] || { echo "::error::required leg $key on Python $py did not pass its parity run: nothing is published" >&2; exit 1; } + done + done + # every platform that passed on every Python is published (Windows is informational for a pre-release) + for key in linux-x86_64 linux-aarch64 macos-arm64 windows-x64; do + all=true + for py in $pythons; do [ -f "passed/$key-py$py.ok" ] || all=false; done + [ "$all" = true ] || continue + w="$(cat "passed/$key-py3.12.ok")" + case "$w" in + "arcadedb-${VERSION}-py3-none-"*.whl) ;; + *) echo "::error::unexpected wheel $w for version $VERSION" >&2; exit 1 ;; + esac + [ -f "wheels/$w" ] || { echo "::error::wheel $w is missing" >&2; exit 1; } + cp "wheels/$w" dist/ + done + ls -l dist + echo "ok=true" >> "$GITHUB_OUTPUT" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: native-release-dist + path: dist/*.whl + if-no-files-found: error + retention-days: 7 + + # One job for A.B.C.devN, A.B.C and A.B.C.postN. `verify` already refused a final version unless ARCADEDB_NATIVE_READINESS_BAR_MET is true. + publish: + name: Publish to PyPI + needs: [validate-version, verify] + if: ${{ !cancelled() && needs.verify.result == 'success' }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + environment: + name: pypi + url: https://pypi.org/project/arcadedb/ + permissions: + id-token: write # PyPI trusted publishing (OIDC); no API token anywhere + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: native-release-dist + path: dist + + # GUARD: this workflow publishes the distribution `arcadedb` and nothing else. A wheel with another name stops the job here, + # before anything reaches PyPI. Two checks, so neither can drift alone: the wheel file name and the Name field of its METADATA, + # both must be DIST_NAME. + - name: Refuse unless every wheel is the distribution ${{ env.DIST_NAME }} + run: | + python3 -I - <<'PY' + import os, pathlib, re, sys, zipfile + + want = os.environ["DIST_NAME"] + norm = lambda s: re.sub(r"[-_.]+", "-", s).lower() + wheels = sorted(pathlib.Path("dist").glob("*.whl")) + if not wheels: + sys.exit("no wheels in dist/") + bad = [] + for w in wheels: + file_dist = w.name.split("-")[0] + with zipfile.ZipFile(w) as zf: + meta = next(n for n in zf.namelist() if n.endswith(".dist-info/METADATA")) + meta_dist = re.search(r"^Name: (.+)$", zf.read(meta).decode(), re.M).group(1) + if norm(file_dist) != want or norm(meta_dist) != want: + bad.append(f"{w.name}: file name says {file_dist}, METADATA says {meta_dist}") + if bad: + print("\n".join(f"::error::{b}" for b in bad)) + sys.exit(f"Refusing to publish: this workflow publishes only the distribution '{want}'.") + print(f"OK: {len(wheels)} wheels, all distribution '{want}'") + PY + + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + packages-dir: dist diff --git a/.github/workflows/test-python-bindings.yml b/.github/workflows/test-jvm.yml similarity index 97% rename from .github/workflows/test-python-bindings.yml rename to .github/workflows/test-jvm.yml index 3a160f7..cf3f911 100644 --- a/.github/workflows/test-python-bindings.yml +++ b/.github/workflows/test-jvm.yml @@ -1,4 +1,6 @@ -name: Test Python Bindings +# The JVM build: the `arcadedb-jvm` distribution. release-jvm.yml calls this workflow on a version tag and publishes the wheels it builds. +# The native build, `arcadedb`, is test-native.yml / release-native.yml. +name: Test JVM (arcadedb-jvm) on: push: @@ -16,9 +18,10 @@ on: - "CLAUDE.md" - "examples/README.md" - "LICENSE*" - # the native build has its own workflow (native.yml); nothing under native/ is in the JVM wheel + # the native build has its own workflows; nothing under native/ is in the JVM wheel - "native/**" - - ".github/workflows/native.yml" + - ".github/workflows/test-native.yml" + - ".github/workflows/release-native.yml" pull_request: # ready_for_review: draft PRs skip the platform matrices (see the job conditions), # so marking one ready is what starts them. @@ -30,9 +33,10 @@ on: - "CLAUDE.md" - "examples/README.md" - "LICENSE*" - # the native build has its own workflow (native.yml); nothing under native/ is in the JVM wheel + # the native build has its own workflows; nothing under native/ is in the JVM wheel - "native/**" - - ".github/workflows/native.yml" + - ".github/workflows/test-native.yml" + - ".github/workflows/release-native.yml" # A new upstream snapshot is published as arcadedata/arcadedb:latest without # any change here, so build and test against it every night as well. The tag @@ -42,7 +46,7 @@ on: schedule: - cron: "0 3 * * *" - # Allow being called by other workflows (e.g., release workflow) + # Allow being called by other workflows (release-jvm.yml) workflow_call: inputs: arcadedb-tag: @@ -75,7 +79,7 @@ on: # the workflow literally: github.workflow is the CALLER's name in a # workflow_call, which would put this workflow and its sibling in one group. concurrency: - group: test-python-bindings-${{ github.event.pull_request.number || github.run_id }} + group: test-jvm-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: @@ -210,7 +214,7 @@ jobs: env: EVENT_NAME: ${{ github.event_name }} run: | - matrix="$(python3 scripts/select_matrix.py bindings --event "$EVENT_NAME")" + matrix="$(python3 scripts/select_matrix.py jvm --event "$EVENT_NAME")" echo "$matrix" echo "matrix=$matrix" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/native.yml b/.github/workflows/test-native.yml similarity index 75% rename from .github/workflows/native.yml rename to .github/workflows/test-native.yml index 8078c74..30fb575 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/test-native.yml @@ -1,13 +1,15 @@ -# The native build: the `arcadedb` distribution (a GraalVM native-image shared library loaded with ctypes). The JVM build, -# `arcadedb-jvm`, is release.yml / test-python-bindings.yml. +# The native build: the `arcadedb` distribution (a GraalVM native-image shared library loaded with ctypes). This workflow builds the +# wheels and runs the parity suite against them; release-native.yml calls it on a version tag and publishes. The JVM build, +# `arcadedb-jvm`, is test-jvm.yml / release-jvm.yml. # # A native image build takes about 1 to 2.5 hours per platform and the ArcadeData organisation shares a runner cap with # upstream ArcadeDB, so the load is selective (scripts/select_native_matrix.py holds the legs, tests/test_select_native_matrix.py pins them): # pull request only when a native path changed (the `paths` filter below); linux-x86_64, Python 3.12; not for a draft # push to main only when a native path changed; all four platforms, Python 3.12 and 3.14 # nightly all four platforms, Python 3.12 and 3.14, but only when something besides docs changed in the last 25 hours (a full run on Mondays) -# dispatch all four platforms, Python 3.12 and 3.14; never publishes -# tag A.B.C.devN | A.B.C | A.B.C.postN all four platforms, Python 3.11 to 3.14; then publishes (see below) +# dispatch all four platforms, Python 3.12 and 3.14 +# workflow_call from release-native.yml with `release: true`: all four platforms, Python 3.11 to 3.14 +# This workflow never publishes: it has no id-token permission and no environment. # # One wheel per platform, py3-none-: # linux-x86_64 manylinux_2_34 container manylinux_2_34_x86_64 libarcadedb.so + libjvector-x86_64.so @@ -20,14 +22,9 @@ # test suite of this repository run against the INSTALLED wheel (native/wheel/parity_wheel.py): no failure, error, crash, timeout, or skip # besides the counted NOTYET and NA-JVM kinds (tests/conftest.py). # -# Publishing (tags only; PyPI trusted publishing, no token; a trusted publisher on the PyPI project `arcadedb` for this workflow file): -# A.B.C.devN environment `pypi` (one trusted publisher for dev, final and post releases). Needs the parity run of linux-x86_64, linux-aarch64 and macos-arm64 on every Python. -# Windows is published too when it passed, and is informational otherwise. -# A.B.C, A.B.C.postN environment `pypi`, ONLY when the readiness bar is met: the repository variable ARCADEDB_NATIVE_READINESS_BAR_MET -# is `true` (a maintainer sets it after checking the correctness fixes and the fault and concurrency probes), the parity run -# of ALL four platforms on ALL four Pythons passed, and no test is a NOTYET skip. Speed parity is not part of the bar. -# The bar is written down in docs/modules/ROOT/pages/development/release.adoc. -name: Native wheel (arcadedb) +# A leg that passes uploads a marker (native-passed--py) and the wheel (native-wheel-); release-native.yml +# reads them to decide what may be published. Do not rename these artifacts without changing release-native.yml. +name: Test native (arcadedb) on: pull_request: @@ -39,18 +36,33 @@ on: - "tests/conftest.py" - "scripts/select_native_matrix.py" - "tests/test_select_native_matrix.py" - - ".github/workflows/native.yml" + - ".github/workflows/test-native.yml" push: branches: [main] paths: *native_paths - tags: - - "[0-9]+.[0-9]+.[0-9]+*" # 26.11.1.dev0, 26.11.1, 26.11.1.post1; the plan job rejects the rest schedule: - cron: "40 3 * * *" + # Called by release-native.yml on a version tag. + workflow_call: + inputs: + version: + description: "Wheel version to build (PEP 440: A.B.C.devN, A.B.C or A.B.C.postN); the release passes the tag" + required: true + type: string + engine: + description: "Engine release (Maven Central) the image links; empty = the release segment of a final version, else the newest release" + required: false + default: "" + type: string + release: + description: "The release legs: all four platforms and Python 3.11 to 3.14 (otherwise the small main/nightly legs)" + required: false + default: false + type: boolean workflow_dispatch: inputs: version: - description: "Wheel version to build (PEP 440: A.B.C.devN, A.B.C or A.B.C.postN); a tag push takes it from the tag" + description: "Wheel version to build (PEP 440: A.B.C.devN, A.B.C or A.B.C.postN)" required: false default: "26.11.1.dev0" type: string @@ -60,9 +72,11 @@ on: default: "" type: string +# The group names the workflow literally: github.workflow is the CALLER's name in a workflow_call, and the group of release-native.yml +# must differ from this one or the two would wait for each other. concurrency: - group: native-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} - # A dry run is superseded by a newer one; a tag run is never cancelled (a half-done release is worse than a queued one). + group: test-native-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} + # A dry run is superseded by a newer one; a run for a tag is never cancelled (a half-done release is worse than a queued one). cancel-in-progress: ${{ github.ref_type != 'tag' }} permissions: @@ -73,7 +87,7 @@ env: GRAALVM_BASE: https://github.com/graalvm/graalvm-ce-builds/releases/download/graal-25.4.4.1.1 UV_VERSION: "0.12.0" PYTHONUTF8: "1" # Windows defaults to cp1252; the notices contain non-ASCII author names - # The version and engine of a run that is not a tag. The newest engine release on Maven Central is what an A.B.C.devN wheel links + # The version and engine of a run that does not pass them (pull request, push, nightly). The newest engine release on Maven Central is what an A.B.C.devN wheel links # (the A.B.C release does not exist yet); raise it when a newer release is out. DEFAULT_VERSION: "26.11.1.dev0" DEV_ENGINE: "26.10.2" @@ -81,7 +95,7 @@ env: jobs: plan: name: Plan the legs - # a draft pull request starts the platform legs when it is marked ready, as in test-python-bindings.yml + # a draft pull request starts the platform legs when it is marked ready, as in test-jvm.yml if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} runs-on: ubuntu-24.04 timeout-minutes: 10 @@ -92,7 +106,6 @@ jobs: final: ${{ steps.ver.outputs.final }} build: ${{ steps.plan.outputs.build }} test: ${{ steps.plan.outputs.test }} - publish: ${{ steps.plan.outputs.publish }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -102,18 +115,12 @@ jobs: - name: Resolve the version and the engine id: ver env: - REF_TYPE: ${{ github.ref_type }} - REF_NAME: ${{ github.ref_name }} INPUT_VERSION: ${{ inputs.version }} INPUT_ENGINE: ${{ inputs.engine }} run: | set -euo pipefail - if [ "$REF_TYPE" = "tag" ]; then - version="$REF_NAME" - [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]] || { echo "::error::Tag '$version' is not A.B.C, A.B.C.devN or A.B.C.postN (bare version, no v, no aN/bN/rcN)"; exit 1; } - else - version="${INPUT_VERSION:-$DEFAULT_VERSION}" - fi + version="${INPUT_VERSION:-$DEFAULT_VERSION}" + [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]] || { echo "::error::Version '$version' is not A.B.C, A.B.C.devN or A.B.C.postN (no aN/bN/rcN)"; exit 1; } base="${version%%.dev*}" base="${base%%.post*}" if [[ "$version" == *.dev* ]]; then @@ -128,24 +135,18 @@ jobs: { echo "version=$version"; echo "engine=$engine"; echo "final=$final"; } >> "$GITHUB_OUTPUT" echo "wheel version $version, engine $engine, final=$final" - - name: Tag must be the next version - if: ${{ github.ref_type == 'tag' }} - env: - TAG: ${{ github.ref_name }} - run: python3 -I scripts/check_version.py "$TAG" --git-remote origin --pypi arcadedb - - name: Choose the legs id: plan env: EVENT: ${{ github.event_name }} - REF_TYPE: ${{ github.ref_type }} + RELEASE: ${{ inputs.release }} run: | set -euo pipefail run=true event="$EVENT" - if [ "$REF_TYPE" = "tag" ]; then event=tag; fi + if [ "$RELEASE" = true ]; then event=release; fi if [ "$EVENT" = "schedule" ]; then - changed="$(git log --since='25 hours ago' --format=%h -- src native tests scripts pyproject.toml .github/workflows/native.yml | wc -l)" + changed="$(git log --since='25 hours ago' --format=%h -- src native tests scripts pyproject.toml .github/workflows/test-native.yml | wc -l)" monday="$(date -u +%u)" if [ "$changed" = 0 ] && [ "$monday" != 1 ]; then run=false @@ -534,7 +535,7 @@ jobs: if (skipped + 0 != 0) { print "skipped tests that are neither NOTYET nor NA-JVM: " skipped > "/dev/stderr"; exit 1 } }' "$tsv" | tee -a "$GITHUB_STEP_SUMMARY" - # Reached only when the gate passed: the publish jobs read these markers (one per platform and Python, holding the wheel file name). + # Reached only when the gate passed: release-native.yml reads these markers (one per platform and Python, holding the wheel file name). - name: Mark this leg as passed run: | set -euo pipefail @@ -559,118 +560,3 @@ jobs: ${{ runner.temp }}/parity/parity-wheel-xml/ if-no-files-found: warn retention-days: 30 - - # What may be published, decided once for both publish jobs: a list of the wheels whose platform passed on every required Python. - verify: - name: Verify what may be published - needs: [plan, build, test] - if: ${{ !cancelled() && needs.plan.outputs.publish == 'true' }} - runs-on: ubuntu-24.04 - timeout-minutes: 15 - outputs: - ok: ${{ steps.verify.outputs.ok }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - pattern: native-passed-* - path: passed - merge-multiple: true - - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - pattern: native-wheel-* - path: wheels - merge-multiple: true - - - name: Every required platform passed on every Python, and the wheels are the ones that were tested - id: verify - env: - VERSION: ${{ needs.plan.outputs.version }} - FINAL: ${{ needs.plan.outputs.final }} - BAR: ${{ vars.ARCADEDB_NATIVE_READINESS_BAR_MET }} - run: | - set -euo pipefail - mkdir -p passed dist - ls -l passed wheels - pythons="3.11 3.12 3.13 3.14" - if [ "$FINAL" = true ]; then - required="linux-x86_64 linux-aarch64 macos-arm64 windows-x64" - if [ "$BAR" != true ]; then - echo "::error::$VERSION is a final release of arcadedb (native), and the readiness bar is not marked as met (repository variable ARCADEDB_NATIVE_READINESS_BAR_MET): nothing is published. See docs/modules/ROOT/pages/development/release.adoc" >&2 - exit 1 - fi - else - required="linux-x86_64 linux-aarch64 macos-arm64" - fi - for key in $required; do - for py in $pythons; do - [ -f "passed/$key-py$py.ok" ] || { echo "::error::required leg $key on Python $py did not pass its parity run: nothing is published" >&2; exit 1; } - done - done - # every platform that passed on every Python is published (Windows is informational for a pre-release) - for key in linux-x86_64 linux-aarch64 macos-arm64 windows-x64; do - all=true - for py in $pythons; do [ -f "passed/$key-py$py.ok" ] || all=false; done - [ "$all" = true ] || continue - w="$(cat "passed/$key-py3.12.ok")" - case "$w" in - "arcadedb-${VERSION}-py3-none-"*.whl) ;; - *) echo "::error::unexpected wheel $w for version $VERSION" >&2; exit 1 ;; - esac - [ -f "wheels/$w" ] || { echo "::error::wheel $w is missing" >&2; exit 1; } - cp "wheels/$w" dist/ - done - ls -l dist - echo "ok=true" >> "$GITHUB_OUTPUT" - - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: native-release-dist - path: dist/*.whl - if-no-files-found: error - retention-days: 7 - - publish-prerelease: - name: Publish a pre-release to PyPI - needs: [plan, verify] - if: ${{ !cancelled() && needs.verify.result == 'success' && needs.plan.outputs.final == 'false' }} - runs-on: ubuntu-24.04 - timeout-minutes: 15 - environment: - name: pypi - url: https://pypi.org/project/arcadedb/ - permissions: - id-token: write # PyPI trusted publishing (OIDC); no API token anywhere - steps: - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: native-release-dist - path: dist - - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 - with: - packages-dir: dist - - # The first FINAL arcadedb (native) release. `verify` already refused it unless ARCADEDB_NATIVE_READINESS_BAR_MET is true. - publish: - name: Publish to PyPI - needs: [plan, verify] - if: ${{ !cancelled() && needs.verify.result == 'success' && needs.plan.outputs.final == 'true' }} - runs-on: ubuntu-24.04 - timeout-minutes: 15 - environment: - name: pypi - url: https://pypi.org/project/arcadedb/ - permissions: - id-token: write - steps: - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: native-release-dist - path: dist - - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 - with: - packages-dir: dist diff --git a/.github/workflows/test-python-examples.yml b/.github/workflows/test-python-examples.yml index 5a5a77c..295ffb9 100644 --- a/.github/workflows/test-python-examples.yml +++ b/.github/workflows/test-python-examples.yml @@ -16,9 +16,10 @@ on: - "CLAUDE.md" - "examples/README.md" - "LICENSE*" - # the native build has its own workflow (native.yml); nothing under native/ is in the JVM wheel + # the native build has its own workflows; nothing under native/ is in the JVM wheel - "native/**" - - ".github/workflows/native.yml" + - ".github/workflows/test-native.yml" + - ".github/workflows/release-native.yml" pull_request: # ready_for_review: draft PRs skip the platform matrices (see the job conditions), # so marking one ready is what starts them. @@ -30,9 +31,10 @@ on: - "CLAUDE.md" - "examples/README.md" - "LICENSE*" - # the native build has its own workflow (native.yml); nothing under native/ is in the JVM wheel + # the native build has its own workflows; nothing under native/ is in the JVM wheel - "native/**" - - ".github/workflows/native.yml" + - ".github/workflows/test-native.yml" + - ".github/workflows/release-native.yml" # A new upstream snapshot is published as arcadedata/arcadedb:latest without # any change here, so build and test against it every night as well. The tag @@ -42,7 +44,7 @@ on: schedule: - cron: "0 4 * * *" - # Allow being called by other workflows (e.g., release workflow) + # Allow being called by other workflows (release-jvm.yml) workflow_call: inputs: arcadedb-tag: @@ -72,7 +74,7 @@ on: required: false default: "0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py" -# SUPERSEDE, do not race (same rule in test-python-bindings.yml). +# SUPERSEDE, do not race (same rule in test-jvm.yml). # # A force-push to a PR branch left the OLD run going alongside the new one: on # #6889 both 118b347ce and 75293c7a2 ran at once, and the stale one reported a diff --git a/CLAUDE.md b/CLAUDE.md index 720b5ef..73e419a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -69,6 +69,10 @@ python -m bandit -c pyproject.toml -r examples --severity-level medium --confide ## CI -`.github/workflows/test-python-bindings.yml` builds wheels for linux/amd64, linux/arm64, darwin/arm64, windows/amd64 and tests Python 3.11–3.14; also runs nightly against `arcadedata/arcadedb:latest`. The org shares a runner cap with upstream, so a pull request runs a small matrix (bindings: linux/amd64 on all four Pythons, the other three platforms on 3.11 and 3.14 = 10 legs; examples: all four platforms on 3.14 = 4 legs, 20 checks in all), while push to main, the nightly run, dispatch and the release run all 16 legs of each workflow (38 checks). `scripts/select_matrix.py` holds both lists (`PULL_REQUEST_LEGS`), the first job of each workflow runs it, and `tests/test_select_matrix.py` pins the full matrix; docs page `development/ci.adoc`. Keep the job names `Test arcadedb (, Python )` unchanged. Both workflows accept an `arcadedb-tag` input to pin the image. `test-python-examples.yml` runs `examples/`. A dependency-floor audit resolves extras with `--resolution lowest-direct` for each Python classifier and runs pip-audit, so raise lower bounds when a floor has an advisory. +Each package has one test and one release workflow: `test-jvm.yml` and `release-jvm.yml` for `arcadedb-jvm`, `test-native.yml` and `release-native.yml` for `arcadedb` (the release workflows call the test workflows with `workflow_call`; the PyPI trusted publishers are keyed on the release file names and the environments `pypi-jvm` and `pypi`, so never rename them without changing PyPI first). `test-python-examples.yml` runs `examples/`. -`.github/workflows/release.yml` publishes on a bare-version tag (`26.11.1.dev0`, `26.11.1`, `26.11.1.post1`; no `v`, never `aN`/`bN`/`rcN`). Its first job runs `scripts/check_version.py` (the devN/postN counting rule, read-only, fails closed); `scripts/after_tag_verify.sh` checks the finished release. The rule is stated once, on the docs page `development/release.adoc`. +`.github/workflows/test-jvm.yml` builds wheels for linux/amd64, linux/arm64, darwin/arm64, windows/amd64 and tests Python 3.11–3.14; also runs nightly against `arcadedata/arcadedb:latest`. The org shares a runner cap with upstream, so a pull request runs a small matrix (jvm: linux/amd64 on all four Pythons, the other three platforms on 3.11 and 3.14 = 10 legs; examples: all four platforms on 3.14 = 4 legs, 20 checks in all), while push to main, the nightly run, dispatch and the release run all 16 legs of each workflow (38 checks). `scripts/select_matrix.py` holds both lists (`PULL_REQUEST_LEGS`, keyed `jvm` and `examples`), the first job of each workflow runs it, and `tests/test_select_matrix.py` pins the full matrix; docs page `development/ci.adoc`. Keep the job names `Test arcadedb (, Python )` unchanged. Both workflows accept an `arcadedb-tag` input to pin the image. `test-python-examples.yml` runs `examples/`. A dependency-floor audit resolves extras with `--resolution lowest-direct` for each Python classifier and runs pip-audit, so raise lower bounds when a floor has an advisory. + +`.github/workflows/release-jvm.yml` publishes `arcadedb-jvm` on a bare-version tag (`26.11.1.dev0`, `26.11.1`, `26.11.1.post1`; no `v`, never `aN`/`bN`/`rcN`). Its first job runs `scripts/check_version.py` (the devN/postN counting rule, read-only, fails closed); `scripts/after_tag_verify.sh` checks the finished release. The rule is stated once, on the docs page `development/release.adoc`. + +`test-native.yml` builds the GraalVM native image and runs the parity suite against the installed wheel (selective: scripts/select_native_matrix.py, 1 to 2.5 hours per platform, do not add runs lightly). `.github/workflows/release-native.yml` publishes `arcadedb` on the same tag: it calls `test-native.yml` with `release: true` (4 platforms x Python 3.11 to 3.14), then the readiness gate (the repository variable `ARCADEDB_NATIVE_READINESS_BAR_MET`, for a final or post version), then one publish job through the environment `pypi`. diff --git a/README.md b/README.md index 3ca0caa..e4168f7 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ [![PyPI arcadedb](https://img.shields.io/pypi/v/arcadedb?label=arcadedb)](https://pypi.org/project/arcadedb/) [![PyPI arcadedb-jvm](https://img.shields.io/pypi/v/arcadedb-jvm?label=arcadedb-jvm)](https://pypi.org/project/arcadedb-jvm/) [![Python versions](https://img.shields.io/pypi/pyversions/arcadedb-jvm)](https://pypi.org/project/arcadedb-jvm/) -[![Tests](https://github.com/ArcadeData/arcadedb-python/actions/workflows/test-python-bindings.yml/badge.svg)](https://github.com/ArcadeData/arcadedb-python/actions/workflows/test-python-bindings.yml) +[![Tests JVM](https://github.com/ArcadeData/arcadedb-python/actions/workflows/test-jvm.yml/badge.svg)](https://github.com/ArcadeData/arcadedb-python/actions/workflows/test-jvm.yml) +[![Tests native](https://github.com/ArcadeData/arcadedb-python/actions/workflows/test-native.yml/badge.svg)](https://github.com/ArcadeData/arcadedb-python/actions/workflows/test-native.yml) [![License](https://img.shields.io/badge/license-Apache--2.0-blue)](https://github.com/ArcadeData/arcadedb-python/blob/main/LICENSE) Python bindings for [ArcadeDB](https://arcadedb.com), the multi-model database (graph, document, key/value, vector and time series, queried with SQL or openCypher). The engine runs inside your Python process, with no server and no network hop. This repository builds two packages, and both import as `arcadedb`. `arcadedb-jvm` has a sibling, `arcadedb`, the native build: pick `arcadedb` for scripts, notebooks, tests, CLIs, serverless functions and agents, and `arcadedb-jvm` for heavy, long-running loads. Status: beta. diff --git a/docs/modules/ROOT/pages/development/ci.adoc b/docs/modules/ROOT/pages/development/ci.adoc index d6c467e..9a4c289 100644 --- a/docs/modules/ROOT/pages/development/ci.adoc +++ b/docs/modules/ROOT/pages/development/ci.adoc @@ -1,7 +1,22 @@ = Continuous integration :description: Which platforms and Python versions the CI runs for a pull request, and which for main, the nightly run and a release. -Two workflows test the `arcadedb-jvm` package: `test-python-bindings.yml` builds a wheel and runs the unit tests, and `test-python-examples.yml` runs the scripts in `examples/`. Both build the wheels on a native runner of each platform. A third workflow, `native.yml`, builds and tests the `arcadedb` (native) package; see <<_native_build,Native build>>. +Each package has one test workflow and one release workflow: + +[cols="2,2,2"] +|=== +| Package | Test | Release + +| `arcadedb-jvm` (the JVM build) +| `test-jvm.yml` +| `release-jvm.yml` + +| `arcadedb` (the native build) +| `test-native.yml` +| `release-native.yml` +|=== + +The release workflows call the test workflows (`workflow_call`) and then publish; see xref:development/release.adoc[Release]. Two workflows test the `arcadedb-jvm` package: `test-jvm.yml` builds a wheel and runs the unit tests, and `test-python-examples.yml` runs the scripts in `examples/`. Both build the wheels on a native runner of each platform. `test-native.yml` builds and tests the `arcadedb` (native) package; see <<_native_build,Native build>>. The package supports four platforms and four Python versions (3.11 to 3.14). That is 16 combinations for each workflow. A pull request does not run all of them, because the organisation shares one runner limit with ArcadeData/arcadedb. Each macOS or Windows job on a pull request delays the upstream jobs too. @@ -26,7 +41,7 @@ A pull request runs 20 checks: 14 test jobs (10 and 4), the two `Download Arcade The names of the test jobs do not change: `Test arcadedb (, Python )`. A check keeps its history when a pull request runs fewer of them. -A release always runs the full matrix, because `release.yml` expects one wheel for each platform and Python version. The nightly run also tests against the newest `arcadedata/arcadedb:latest` image. +A release always runs the full matrix, because `release-jvm.yml` expects one wheel for each platform and Python version. The nightly run also tests against the newest `arcadedata/arcadedb:latest` image. == Other rules @@ -36,11 +51,11 @@ A release always runs the full matrix, because `release.yml` expects one wheel f == Native build -The workflow `native.yml` builds the native image of the `arcadedb` package. It does not follow the table above. +The workflow `test-native.yml` builds the native image of the `arcadedb` package and runs the test suite against the installed wheel. It does not follow the table above. It never publishes; `release-native.yml` calls it on a version tag. -* A pull request runs it only when native paths change: `native/**`, `src/arcadedb/_native/**`, the native tests, or `native.yml` itself. It then runs on `linux-x86_64` with Python 3.12 only. -* A push to `main`, the nightly run, a manual run (dispatch) and a release tag run the full platform matrix: `linux-x86_64`, `linux-aarch64`, `macos-arm64` and `windows-x64`. -* A native image build takes about 1 to 2.5 hours for each platform. The organisation shares one runner limit with upstream ArcadeDB, so do not add runs lightly. Change the paths or the matrix of `native.yml` only for a reason that you can write down. +* A pull request runs it only when native paths change: `native/**`, `src/arcadedb/_native/**`, the native tests, or `test-native.yml` itself. It then runs on `linux-x86_64` with Python 3.12 only. +* A push to `main`, the nightly run, a manual run (dispatch) run all four platforms: `linux-x86_64`, `linux-aarch64`, `macos-arm64` and `windows-x64` (Python 3.12 and 3.14). A release (`release-native.yml` calls the workflow with `release: true`) runs the same four platforms on all four Python versions, 16 legs. +* A native image build takes about 1 to 2.5 hours for each platform. The organisation shares one runner limit with upstream ArcadeDB, so do not add runs lightly. Change the paths or the matrix of `test-native.yml` only for a reason that you can write down. == Change the selection diff --git a/docs/modules/ROOT/pages/development/release.adoc b/docs/modules/ROOT/pages/development/release.adoc index 5c5898c..3bc648d 100644 --- a/docs/modules/ROOT/pages/development/release.adoc +++ b/docs/modules/ROOT/pages/development/release.adoc @@ -1,7 +1,7 @@ = Release :description: How a release of the arcadedb-jvm and arcadedb (native) packages is numbered, tagged, checked and published to PyPI. -A release is a Git tag. Pushing the tag starts the release workflows. This repository releases two packages: `arcadedb-jvm` (the JVM build) with `release.yml`, and `arcadedb` (the native build) with `native.yml`. They build the wheels, test them, check them and publish them to PyPI with trusted publishing. The workflows store no token. The sections below describe `release.yml`; the native release is in <<_native_release,Native release>>. +A release is a Git tag. Pushing the tag starts the release workflows. This repository releases two packages: `arcadedb-jvm` (the JVM build) with `release-jvm.yml`, and `arcadedb` (the native build) with `release-native.yml`. Each release workflow calls the test workflow of its build (`test-jvm.yml`, `test-native.yml`), checks the wheels and publishes them to PyPI with trusted publishing. The workflows store no token. The sections below describe `release-jvm.yml`; the native release is in <<_native_release,Native release>>. == Version numbering @@ -42,13 +42,13 @@ python3 scripts/check_version.py 26.11.1.dev0 --git-remote origin --pypi arcaded It exits 0 when the version is the correct next one, 1 when it is not, and 2 when a source cannot be read. Exit 2 is a failure, not a pass: the check fails closed. -The first job of `release.yml` runs the same command on the pushed tag. When it fails, the workflow builds and publishes nothing. Run it by hand before you push the tag. Running the whole workflow again for a version that is already on PyPI fails the check on purpose. To retry a failed upload, re-run only the publish job. +The first job of each release workflow runs the same command on the pushed tag. When it fails, the workflow builds and publishes nothing. Run it by hand before you push the tag. Running the whole workflow again for a version that is already on PyPI fails the check on purpose. To retry a failed upload, re-run only the publish job. The tests of the script are in `tests/test_check_version.py`. == After the tag push -`scripts/after_tag_verify.sh` checks the finished release. It only reads from GitHub and PyPI: +`scripts/after_tag_verify.sh` checks the finished `arcadedb-jvm` release (the run of `release-jvm.yml`). It only reads from GitHub and PyPI: [source,bash] ---- @@ -74,17 +74,24 @@ The two packages release from two workflows with two PyPI environments: | Package | Workflow | GitHub environment | `arcadedb-jvm` -| `release.yml` +| `release-jvm.yml` | `pypi-jvm` | `arcadedb` (native) -| `native.yml` +| `release-native.yml` | `pypi` |=== -The version rule above holds for both packages. +The version rule above holds for both packages. One tag starts both release workflows, and each one checks the tag against its own PyPI project. -`arcadedb` has one trusted publisher on PyPI (`native.yml` with the environment `pypi`) for dev, final and post releases. There is no separate pre-release environment; the readiness gate below is what keeps a final release back. +`release-native.yml` has four jobs: + +. `Validate tag` checks that the tag is a bare version and the next one (`scripts/check_version.py`). +. `test` calls `test-native.yml` with the full matrix: four platforms, each on Python 3.11 to 3.14. It builds the native image and the wheel on each platform and runs the test suite of this repository against the installed wheel. +. `Verify what may be published` reads which legs passed, applies the readiness gate below and collects the wheels. +. `Publish to PyPI` uploads the wheels through the environment `pypi`. It refuses any wheel that is not the distribution `arcadedb`. + +`arcadedb` has one trusted publisher on PyPI (`release-native.yml` with the environment `pypi`) for dev, final and post releases, and one publish job in the workflow. `arcadedb-jvm` has one trusted publisher (`release-jvm.yml` with the environment `pypi-jvm`). PyPI matches the workflow file name, so renaming either file means changing the publisher on PyPI first. There is no separate pre-release environment; the readiness gate below is what keeps a final release back. * A native pre-release (`26.11.1.devN`) publishes when the builds on `linux-x86_64`, `linux-aarch64` and `macos-arm64` pass. * A FINAL native release (`A.B.C` or `A.B.C.postN`) needs two more things: diff --git a/native/scripts/build-image.sh b/native/scripts/build-image.sh index 3ccdbb0..e454c07 100755 --- a/native/scripts/build-image.sh +++ b/native/scripts/build-image.sh @@ -5,7 +5,7 @@ # MARCH (default compatibility = baseline x86-64, what a manylinux wheel promises, PEP 600; x86-64-v3 is the GraalVM CE 25.4 default and the faster reference). # usage: [MARCH=x86-64-v3] [XMX=12g] [NI_EXTRA="-H:NumberOfThreads=2"] build-image.sh [OUTDIR] A laptop with other jobs runs it through its job gate; the CI calls it directly. # Platforms: Linux (libarcadedb.so; on x86-64 also libjvector-x86_64.so, jvector ships the Highway library for x86-64 only here), macOS -# (libarcadedb.dylib) and Windows in Git Bash with the MSVC environment loaded (libarcadedb.dll). The CI (native.yml) caps the +# (libarcadedb.dylib) and Windows in Git Bash with the MSVC environment loaded (libarcadedb.dll). The CI (test-native.yml) caps the # builder heap and threads on the 7 GB macOS runner (XMX=5g, NI_EXTRA=-H:NumberOfThreads=2). # e.g. NEED_GB=13 JOB_MEM_GB=14 quiet_build.sh bash native/scripts/build-image.sh set -eu diff --git a/scripts/after_tag_verify.sh b/scripts/after_tag_verify.sh index 6a4e16f..a029de2 100755 --- a/scripts/after_tag_verify.sh +++ b/scripts/after_tag_verify.sh @@ -1,6 +1,6 @@ #!/bin/bash # After the tag push: verify the release end to end. Read-only against GitHub and PyPI (no writes). -# Ported from humemai/arcadedb-embedded-python#479 and adapted to .github/workflows/release.yml. +# Ported from humemai/arcadedb-embedded-python#479 and adapted to .github/workflows/release-jvm.yml (the arcadedb-jvm release; the artifact it reads is release-dist). # # usage: after_tag_verify.sh [] [ ] # example: after_tag_verify.sh 26.11.1 26.10.1 @@ -64,7 +64,7 @@ want={u["filename"].replace(pv,v) for u in prev["urls"] if kept(u["filename"])} have={u["filename"]:u for u in new["urls"]} gaps=0 if pv=="-": - # No release to compare with: one wheel per supported Python version and platform, as release.yml counts them. + # No release to compare with: one wheel per supported Python version and platform, as release-jvm.yml counts them. pythons=[c.rsplit(".",1)[1] for c in project["classifiers"] if c.startswith("Programming Language :: Python :: 3.")] plats=[r"manylinux.*x86_64",r"manylinux.*aarch64",r"macosx.*arm64",r"win_amd64"] for py in pythons: diff --git a/scripts/select_matrix.py b/scripts/select_matrix.py index 6ef9bf8..3774036 100644 --- a/scripts/select_matrix.py +++ b/scripts/select_matrix.py @@ -9,13 +9,13 @@ - Every other event runs the full matrix: a push to main, the nightly schedule, a manual dispatch, and the release workflow (which calls the workflows as `workflow_call`; the event name seen there is the caller's, a tag push). The release needs one wheel per platform and - Python version, and `release.yml` counts them. + Python version, and `release-jvm.yml` counts them. The selection fails open to the full matrix: an event name this script does not know runs everything, never less. usage: - select_matrix.py bindings [--event NAME] # test-python-bindings.yml + select_matrix.py jvm [--event NAME] # test-jvm.yml select_matrix.py examples [--event NAME] # test-python-examples.yml `--event` defaults to $GITHUB_EVENT_NAME. The output is one line of JSON for @@ -48,7 +48,7 @@ # The legs of a pull request, per workflow: platform -> Python versions. # -# bindings: all Pythons on linux/amd64 (the cheap, fast runner and the one most contributors +# jvm: all Pythons on linux/amd64 (the cheap, fast runner and the one most contributors # use), the oldest and the newest on the three other platforms. A bug that depends on the # interpreter shows on the ends of the range, and one that depends on the platform shows on # any version of it. @@ -58,7 +58,7 @@ # examples 07 to 12) and per-platform steps in the workflow (a CPU torch index on macOS, UTF-8 # on Windows), so all four platforms stay. PULL_REQUEST_LEGS = { - "bindings": { + "jvm": { "linux/amd64": PYTHONS, "linux/arm64": (OLDEST, NEWEST), "darwin/arm64": (OLDEST, NEWEST), @@ -77,7 +77,7 @@ def _legs(per_platform: dict[str, tuple[str, ...]]) -> list[dict[str, str]]: def select(workflow: str, event: str) -> dict[str, list[dict[str, str]]]: - """The `strategy.matrix` of `workflow` ("bindings" or "examples") for `event`.""" + """The `strategy.matrix` of `workflow` ("jvm" or "examples") for `event`.""" if workflow not in PULL_REQUEST_LEGS: raise ValueError( f"unknown workflow {workflow!r}; use one of {sorted(PULL_REQUEST_LEGS)}" diff --git a/scripts/select_native_matrix.py b/scripts/select_native_matrix.py index 03ae360..d027718 100644 --- a/scripts/select_native_matrix.py +++ b/scripts/select_native_matrix.py @@ -1,15 +1,16 @@ #!/usr/bin/env python3 -"""Choose the legs of the native workflow (.github/workflows/native.yml) for one event. +"""Choose the legs of the native test workflow (.github/workflows/test-native.yml) for one event. A native image build takes about 1 to 2.5 hours per platform and the ArcadeData organisation shares a runner cap with upstream ArcadeDB, so a pull request builds one platform (linux-x86_64) and tests one Python (3.12), and only when native paths changed (the workflow's `paths` filter). Push to main, the nightly run and a dispatch build all four -platforms and test Python 3.12 and 3.14 on each. A release tag tests every supported Python (3.11 to 3.14) on all four. +platforms and test Python 3.12 and 3.14 on each. A release (release-native.yml calls the workflow with `release: true`) tests every +supported Python (3.11 to 3.14) on all four. -usage: select_native_matrix.py EVENT [--tag-kind final|dev|none] EVENT: pull_request | push | schedule | workflow_dispatch | tag +usage: select_native_matrix.py EVENT EVENT: pull_request | push | schedule | workflow_dispatch | release -Prints `build=`, `test=` and `publish=` lines (GITHUB_OUTPUT format). `publish` is true for a -tag only. The GraalVM CE archives are pinned by sha256 per platform (GraalVM Community Edition only, never Oracle GraalVM). +Prints `build=` and `test=` lines (GITHUB_OUTPUT format). Publishing is not decided here: release-native.yml +does it. The GraalVM CE archives are pinned by sha256 per platform (GraalVM Community Edition only, never Oracle GraalVM). """ from __future__ import annotations @@ -82,7 +83,7 @@ ALL_PYTHONS = ["3.11", "3.12", "3.13", "3.14"] # Platforms whose parity run must pass before a pre-release (A.B.C.devN) is published. Windows is the newest wheel and is -# informational for a pre-release; a FINAL release needs all four (see ALL_PLATFORMS and native.yml). +# informational for a pre-release; a FINAL release needs all four (see ALL_PLATFORMS and release-native.yml). PRERELEASE_REQUIRED = ["linux-x86_64", "linux-aarch64", "macos-arm64"] ALL_PLATFORMS = list(PLATFORMS) @@ -96,7 +97,7 @@ def select(event: str) -> dict: platforms, pythons = PULL_REQUEST_PLATFORMS, PULL_REQUEST_PYTHONS elif event in ("push", "schedule", "workflow_dispatch"): platforms, pythons = ALL_PLATFORMS, MAIN_PYTHONS - elif event == "tag": + elif event == "release": platforms, pythons = ALL_PLATFORMS, ALL_PYTHONS else: raise ValueError(f"unknown event {event!r}") @@ -111,20 +112,19 @@ def select(event: str) -> dict: for k in platforms for py in pythons ] - return {"build": build, "test": test, "publish": event == "tag"} + return {"build": build, "test": test} def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser() ap.add_argument( "event", - choices=["pull_request", "push", "schedule", "workflow_dispatch", "tag"], + choices=["pull_request", "push", "schedule", "workflow_dispatch", "release"], ) a = ap.parse_args(argv) chosen = select(a.event) print(f"build={json.dumps(chosen['build'], separators=(',', ':'))}") print(f"test={json.dumps(chosen['test'], separators=(',', ':'))}") - print(f"publish={'true' if chosen['publish'] else 'false'}") return 0 diff --git a/tests/test_check_version.py b/tests/test_check_version.py index cb712b7..56c8559 100644 --- a/tests/test_check_version.py +++ b/tests/test_check_version.py @@ -13,7 +13,7 @@ SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "check_version.py" PYPROJECT_FIELD = re.compile(r'^version = "([^"]+)"', re.MULTILINE) -RELEASE_WORKFLOW = SCRIPT_PATH.parents[1] / ".github" / "workflows" / "release.yml" +RELEASE_WORKFLOW = SCRIPT_PATH.parents[1] / ".github" / "workflows" / "release-jvm.yml" def _load(): diff --git a/tests/test_select_matrix.py b/tests/test_select_matrix.py index fd45ed9..cc95fb0 100644 --- a/tests/test_select_matrix.py +++ b/tests/test_select_matrix.py @@ -15,7 +15,7 @@ ROOT = Path(__file__).resolve().parents[1] SCRIPT_PATH = ROOT / "scripts" / "select_matrix.py" WORKFLOWS = { - "bindings": ROOT / ".github" / "workflows" / "test-python-bindings.yml", + "jvm": ROOT / ".github" / "workflows" / "test-jvm.yml", "examples": ROOT / ".github" / "workflows" / "test-python-examples.yml", } @@ -61,7 +61,7 @@ def _pairs(matrix): return set(pairs) -@pytest.mark.parametrize("workflow", ["bindings", "examples"]) +@pytest.mark.parametrize("workflow", ["jvm", "examples"]) @pytest.mark.parametrize("event", NON_PULL_REQUEST_EVENTS) def test_every_event_but_pull_request_runs_the_full_matrix(workflow, event): matrix = sm.select(workflow, event) @@ -69,8 +69,8 @@ def test_every_event_but_pull_request_runs_the_full_matrix(workflow, event): assert _pairs(matrix) == FULL -def test_pull_request_bindings_matrix(): - pairs = _pairs(sm.select("bindings", "pull_request")) +def test_pull_request_jvm_matrix(): + pairs = _pairs(sm.select("jvm", "pull_request")) assert {v for p, v in pairs if p == "linux/amd64"} == set(PYTHONS) for platform in ("linux/arm64", "darwin/arm64", "windows/amd64"): assert {v for p, v in pairs if p == platform} == {"3.11", "3.14"}, platform @@ -82,7 +82,7 @@ def test_pull_request_examples_matrix(): assert pairs == {(p, "3.14") for p in PLATFORMS} -@pytest.mark.parametrize("workflow", ["bindings", "examples"]) +@pytest.mark.parametrize("workflow", ["jvm", "examples"]) def test_pull_request_matrix_is_part_of_the_full_matrix(workflow): assert _pairs(sm.select(workflow, "pull_request")) < FULL @@ -93,7 +93,7 @@ def test_unknown_workflow_is_refused(): def test_pythons_match_the_classifiers_of_pyproject(): - # release.yml expects one wheel per classifier Python and platform from the full matrix; a + # release-jvm.yml expects one wheel per classifier Python and platform from the full matrix; a # Python added to one place and not the other would fail the release, not a pull request. project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))[ "project" @@ -112,7 +112,7 @@ def test_pythons_match_the_classifiers_of_pyproject(): def test_command_line_prints_one_line_of_matrix_json(): for event, count in (("pull_request", 10), ("push", 16)): out = subprocess.run( # nosec B603 - fixed argument list, no untrusted input - [sys.executable, "-I", str(SCRIPT_PATH), "bindings", "--event", event], + [sys.executable, "-I", str(SCRIPT_PATH), "jvm", "--event", event], capture_output=True, text=True, check=True, diff --git a/tests/test_select_native_matrix.py b/tests/test_select_native_matrix.py index 16e575b..609aea7 100644 --- a/tests/test_select_native_matrix.py +++ b/tests/test_select_native_matrix.py @@ -1,4 +1,4 @@ -"""The legs of the native workflow per event: a pull request must stay small (the org shares a runner cap with upstream).""" +"""The legs of the native test workflow per event: a pull request must stay small (the org shares a runner cap with upstream).""" import importlib.util import json @@ -21,7 +21,6 @@ def test_pull_request_builds_one_platform_and_tests_one_python(): assert [(t["key"], t["python"]) for t in chosen["test"]] == [ ("linux-x86_64", "3.12") ] - assert chosen["publish"] is False def test_main_nightly_and_dispatch_build_all_four_platforms(): @@ -34,15 +33,13 @@ def test_main_nightly_and_dispatch_build_all_four_platforms(): "windows-x64", ] assert len(chosen["test"]) == 4 * len(sel.MAIN_PYTHONS) - assert chosen["publish"] is False -def test_a_release_tag_tests_every_supported_python_on_every_platform(): - chosen = sel.select("tag") +def test_a_release_tests_every_supported_python_on_every_platform(): + chosen = sel.select("release") assert len(chosen["build"]) == 4 assert len(chosen["test"]) == 4 * 4 assert {t["python"] for t in chosen["test"]} == {"3.11", "3.12", "3.13", "3.14"} - assert chosen["publish"] is True def test_pythons_are_the_classifiers_of_the_native_wheel(): @@ -70,6 +67,5 @@ def test_the_script_prints_github_output_lines(): text=True, check=True, ).stdout.splitlines() - assert [line.split("=", 1)[0] for line in out] == ["build", "test", "publish"] + assert [line.split("=", 1)[0] for line in out] == ["build", "test"] assert json.loads(out[0].split("=", 1)[1])[0]["key"] == "linux-x86_64" - assert out[2] == "publish=false" From 56aab395a21bc75cd60d2e7b28829de938372541 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 21:05:00 +0900 Subject: [PATCH 13/33] Tests: put the degree query of the graph batch test in a variable so the nosec B608 marker applies Bandit reports B608 on the line of the f-string, and the marker sat on a later line. The statement is now one line with its marker; the interpolated value is a RID created by the test. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_graph_batch.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_graph_batch.py b/tests/test_graph_batch.py index 618121c..43b52ed 100644 --- a/tests/test_graph_batch.py +++ b/tests/test_graph_batch.py @@ -804,9 +804,8 @@ def writer(index): assert not any(t.is_alive() for t in threads) assert errors == [] assert db.count_type("E") == 2 * per_writer - out = db.query( - "sql", f"SELECT out('E').size() AS n FROM {hub}" - ).to_list() # nosec B608 - a RID from this test + degree_sql = f"SELECT out('E').size() AS n FROM {hub}" # nosec B608 - a RID from this test, no user input + out = db.query("sql", degree_sql).to_list() assert out[0]["n"] == 2 * per_writer assert ( 1 in refusals From 1871c1ea26ffe06c55873dde1969d9b672292af6 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 21:21:12 +0900 Subject: [PATCH 14/33] Native: write the engine log to the per-user log directory, never the working directory The native library opened ./log/arcadedb.log in the current working directory. The loader now creates the log directory (ARCADEDB_LOG_DIR, else the state directory, like the JVM build), writes logging.properties there and passes arcadedb.server.logsDirectory and java.util.logging.config.file as isolate arguments, which are in place before the engine's first log call. A user's own isolate argument wins. The log-directory logic moves from jvm.py to _logdir.py so both builds share it. Tests: the subprocess tests of test_log_dir.py run on both builds, test_native_log_dir.py covers the isolate-argument paths, and a JVM-side test compares the native copy of the engine's log settings with the engine jar. Co-Authored-By: Claude Sonnet 5.5 --- docs/modules/ROOT/nav.adoc | 1 + docs/modules/ROOT/pages/how-to/logs.adoc | 43 ++++++++ src/arcadedb/_logdir.py | 133 +++++++++++++++++++++++ src/arcadedb/_native/lib.py | 35 +++++- src/arcadedb/jvm.py | 95 ++-------------- tests/conftest.py | 5 + tests/test_log_dir.py | 73 ++++++++++++- tests/test_native_log_dir.py | 42 +++++++ 8 files changed, 335 insertions(+), 92 deletions(-) create mode 100644 docs/modules/ROOT/pages/how-to/logs.adoc create mode 100644 src/arcadedb/_logdir.py create mode 100644 tests/test_native_log_dir.py diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index ce55e5c..79c9979 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -5,6 +5,7 @@ * How-to ** xref:how-to/databases-and-queries.adoc[Databases and queries] ** xref:how-to/concurrency.adoc[Concurrency, processes and crashes] +** xref:how-to/logs.adoc[Where the logs go] * Development ** xref:development/ci.adoc[Continuous integration] ** xref:development/release.adoc[Release] diff --git a/docs/modules/ROOT/pages/how-to/logs.adoc b/docs/modules/ROOT/pages/how-to/logs.adoc new file mode 100644 index 0000000..4019bbd --- /dev/null +++ b/docs/modules/ROOT/pages/how-to/logs.adoc @@ -0,0 +1,43 @@ += Where the logs go +:description: Where the engine writes its log file on the arcadedb and arcadedb-jvm packages, how to move it, and what the packages never write. + +Neither package writes a `log` directory into your working directory. The engine's own default is `./log/arcadedb.log`, relative to wherever the process started. Both packages replace it with a per-user directory, so a read-only working directory or a notebook that starts anywhere leaves no files behind. + +== Default location + +The log directory is `arcadedb/log` under your state directory: + +[cols="1,3"] +|=== +| Platform | Directory + +| Linux +| `$XDG_STATE_HOME/arcadedb/log`, or `~/.local/state/arcadedb/log` when `XDG_STATE_HOME` is not set to an absolute path + +| macOS +| `~/Library/Logs/arcadedb/log` + +| Windows +| `%LOCALAPPDATA%\arcadedb\log` +|=== + +When the directory cannot be created or written, the package logs a warning and uses a new `arcadedb-log-*` directory under the system temp directory. + +The directory holds the engine log (`arcadedb.log.0`, `arcadedb.log.1` and so on, up to 10 files of 100 MB), and the `logging.properties` that the package writes for the engine. The `arcadedb-jvm` package also keeps the JVM crash file `hs_err_pid.log` there. + +== Choose the directory + +Set `ARCADEDB_LOG_DIR` before the first database is opened. The first call that starts the engine reads it, so setting it later has no effect in the same process. + +[source,bash] +---- +export ARCADEDB_LOG_DIR=/var/log/my-app/arcadedb +---- + +== Settings that win over `ARCADEDB_LOG_DIR` + +The packages pass two Java system properties to the engine: `arcadedb.server.logsDirectory` (the directory) and `java.util.logging.config.file` (the `logging.properties` in it). If you set one of them yourself, yours is kept and the other default still applies. + +`arcadedb-jvm`:: Add the property to `jvm_kwargs={"jvm_args": [...]}` or to the environment variable `ARCADEDB_JVM_ARGS`, for example `-Darcadedb.server.logsDirectory=/var/log/arcadedb`. + +`arcadedb` (native):: Add the property to the environment variable `ARCADEDB_NATIVE_ISOLATE_ARGS`. It takes space-separated isolate arguments, so a directory in it cannot contain a space. For example `ARCADEDB_NATIVE_ISOLATE_ARGS=-Darcadedb.server.logsDirectory=/var/log/arcadedb`. Use `ARCADEDB_LOG_DIR` for a directory with a space in its name. diff --git a/src/arcadedb/_logdir.py b/src/arcadedb/_logdir.py new file mode 100644 index 0000000..777d7b0 --- /dev/null +++ b/src/arcadedb/_logdir.py @@ -0,0 +1,133 @@ +"""Where the engine writes its log, for both builds (the JVM build and the native build). + +The engine's own ``arcadedb-log.properties`` writes ``./log/arcadedb.log`` into the working directory. Both builds move the file +log to a per-user directory instead: ``ARCADEDB_LOG_DIR`` if set, else ``arcadedb/log`` under the user's state directory. They +write a ``logging.properties`` there whose file pattern is ``${arcadedb.server.logsDirectory}/arcadedb.log`` and point two +system properties at it: ``arcadedb.server.logsDirectory`` (the directory) and ``java.util.logging.config.file`` (that file). +The JVM build sets them as JVM arguments (``jvm.py``), the native build as isolate arguments (``_native/lib.py``). The engine +reads both when it first logs, so they have to be in place before the first engine call. This module has no JVM and no JPype. +""" + +import os +import re +import sys +import tempfile +from pathlib import Path + +from ._logging import get_logger + +_LOGGER = get_logger(__name__) + +LOG_DIR_ENV = "ARCADEDB_LOG_DIR" +LOGGING_CONFIG_NAME = "logging.properties" + +# The engine's arcadedb-log.properties without its file pattern, for the native build: the image holds the engine's copy as a +# resource, but an isolate argument has to name a real file, and there is no jar to read it from. tests/test_log_dir.py compares +# this text with the file in the engine jar, so an engine upgrade that changes the file fails there until this is updated. +NATIVE_ENGINE_LOG_PROPERTIES = """\ +handlers = java.util.logging.ConsoleHandler, java.util.logging.FileHandler +.level = INFO +io.undertow.level=WARNING +org.xnio.level=WARNING +org.jboss.level=WARNING +org.apache.ratis.grpc.server.GrpcLogAppender.level = SEVERE +com.arcadedb.level = INFO +java.util.logging.ConsoleHandler.level = INFO +java.util.logging.ConsoleHandler.formatter = com.arcadedb.utility.AnsiLogFormatter +java.util.logging.FileHandler.level = INFO +java.util.logging.FileHandler.formatter = com.arcadedb.log.LogFormatter +java.util.logging.FileHandler.limit=100000000 +java.util.logging.FileHandler.count=10 +""" + + +def state_base_dir() -> Path: + """Per-user base directory for state and logs, by platform (stdlib only). + + The home directory is read only where the platform needs it, and when it + cannot be determined (no HOME or USERPROFILE, as in a stripped-down service + environment) the system temp directory is used, so this never raises. + """ + if sys.platform == "win32": + local = os.environ.get("LOCALAPPDATA") + if local: + return Path(local) + elif sys.platform != "darwin": + xdg = os.environ.get("XDG_STATE_HOME") + # The XDG spec ignores a relative value. + if xdg and os.path.isabs(xdg): + return Path(xdg) + try: + home = Path.home() + except (RuntimeError, KeyError, OSError): + return Path(tempfile.gettempdir()) + if sys.platform == "win32": + return home / "AppData" / "Local" + if sys.platform == "darwin": + return home / "Library" / "Logs" + return home / ".local" / "state" + + +def default_log_dir() -> Path: + """Where the engine log, the crash log and logging.properties go. + + ``ARCADEDB_LOG_DIR`` if set, else ``arcadedb/log`` under the + per-user state directory. Never the working directory. + """ + override = os.environ.get(LOG_DIR_ENV) + if override: + return Path(os.path.abspath(os.path.expanduser(override))) + return state_base_dir() / "arcadedb" / "log" + + +def render_logging_config(engine_text: str) -> str: + """The engine's logging config with the file log in ``logsDirectory``. + + The jar's own file hard-codes ``./log/arcadedb.log``, which lands in the + working directory. The engine resolves ``${arcadedb.server.logsDirectory}`` + in the pattern, and the argument that sets it is added by the caller. + """ + lines = [ + line + for line in engine_text.splitlines() + if not re.match(r"\s*java\.util\.logging\.FileHandler\.pattern\s*[=:]", line) + ] + lines.append( + "java.util.logging.FileHandler.pattern=" + "${arcadedb.server.logsDirectory}/arcadedb.log" + ) + return "\n".join(lines) + "\n" + + +def usable_log_dir(log_dir: Path) -> Path: + """Create the log directory; return it, or a fresh directory under the system temp directory when it cannot be created or + written (a read-only home, for example), so that nothing is ever written to the working directory. + """ + try: + log_dir.mkdir(parents=True, exist_ok=True) + if not os.access(log_dir, os.W_OK | os.X_OK): + raise OSError(f"{log_dir} is not writable") + except OSError as exc: + fallback = Path(tempfile.mkdtemp(prefix="arcadedb-log-")) + _LOGGER.warning( + "Cannot use %s for ArcadeDB logs (%s); using %s. Set %s to choose one.", + log_dir, + exc, + fallback, + LOG_DIR_ENV, + ) + return fallback + return log_dir + + +def write_logging_config(log_dir: Path, engine_text: str) -> None: + """Write ``logging.properties`` in ``log_dir`` (replaced atomically, and only when its text changed).""" + config = log_dir / LOGGING_CONFIG_NAME + rendered = render_logging_config(engine_text) + try: + if not config.exists() or config.read_text("iso-8859-1") != rendered: + tmp = log_dir / f"{LOGGING_CONFIG_NAME}.{os.getpid()}.tmp" + tmp.write_text(rendered, encoding="iso-8859-1", newline="\n") + os.replace(tmp, config) + except OSError as exc: + _LOGGER.warning("Cannot write %s: %s", config, exc) diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index b2f16ac..55a0c44 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -7,6 +7,7 @@ import threading from pathlib import Path +from .. import _logdir from ..exceptions import ArcadeDBError _LIB_NAMES = { @@ -29,6 +30,9 @@ # ARCADEDB_NATIVE_EXEC_MANY: the rows of an insert_many chunk that JSON text cannot carry unchanged go in one crossing (arcadedb_n_exec_many), not one call each EXEC_MANY = os.environ.get("ARCADEDB_NATIVE_EXEC_MANY", "1") != "0" +# the directory the engine log, logging.properties and the crash files go to (None until the library is loaded) +LOG_DIR_USED = None + _lock = threading.Lock() _lib = None _isolate = None @@ -61,6 +65,32 @@ def __del__(self): pass +def _log_isolate_args(user_args, log_dir) -> list: + """The isolate arguments that put the engine log in ``log_dir``; each one the user already gave is left out (theirs wins). + + The engine reads ``arcadedb.server.logsDirectory`` and ``java.util.logging.config.file`` when it first logs, and the log file it + opens by default is ``./log/arcadedb.log`` in the working directory. An isolate argument is in place before any Java code runs. + """ + posix = Path(log_dir).as_posix() + args = [] + if not any(a.startswith("-Darcadedb.server.logsDirectory=") for a in user_args): + args.append(f"-Darcadedb.server.logsDirectory={posix}") + if not any(a.startswith("-Djava.util.logging.config.file=") for a in user_args): + args.append( + f"-Djava.util.logging.config.file={posix}/{_logdir.LOGGING_CONFIG_NAME}" + ) + return args + + +def _prepare_logging(user_args) -> list: + """Create the log directory, write its logging.properties and return the isolate arguments for it (see :func:`_log_isolate_args`).""" + global LOG_DIR_USED + log_dir = _logdir.usable_log_dir(_logdir.default_log_dir()) + _logdir.write_logging_config(log_dir, _logdir.NATIVE_ENGINE_LOG_PROPERTIES) + LOG_DIR_USED = log_dir + return _log_isolate_args(user_args, log_dir) + + def _check_cpu() -> None: """An image built for x86-64-v3 (AVX2, BMI2, FMA) needs them: fail with a clear error instead of SIGILL on an older CPU.""" if sys.platform != "linux" or os.environ.get("ARCADEDB_NATIVE_SKIP_CPU_CHECK"): @@ -112,9 +142,12 @@ def _load(): ] lib.graal_create_isolate.restype = ctypes.c_int params = None + # run-time image options for the isolate (e.g. "-XX:InitialCollectionPolicy=Adaptive -XX:MaximumYoungGenerationSizePercent=20") iso_args = os.environ.get("ARCADEDB_NATIVE_ISOLATE_ARGS", "").split() + # the log directory: always set, so that nothing is written to the working directory (the same rule as the JVM build) + iso_args += _prepare_logging(iso_args) if iso_args: - # run-time image options for the isolate (e.g. "-XX:InitialCollectionPolicy=Adaptive -XX:MaximumYoungGenerationSizePercent=20") + class _Params(ctypes.Structure): _fields_ = [ ("version", ctypes.c_int), diff --git a/src/arcadedb/jvm.py b/src/arcadedb/jvm.py index d13750f..77810df 100644 --- a/src/arcadedb/jvm.py +++ b/src/arcadedb/jvm.py @@ -8,12 +8,10 @@ import glob import os import platform -import re import shlex import shutil import signal import sys -import tempfile import threading import zipfile from pathlib import Path @@ -23,6 +21,12 @@ import jpype.imports from . import aot as _aot +from ._logdir import LOG_DIR_ENV as _LOG_DIR_ENV +from ._logdir import LOGGING_CONFIG_NAME as _LOGGING_CONFIG_NAME +from ._logdir import default_log_dir as _default_log_dir +from ._logdir import render_logging_config as _render_logging_config +from ._logdir import usable_log_dir as _usable_log_dir +from ._logdir import write_logging_config as _write_logging_config from ._logging import get_logger from .exceptions import ArcadeDBError @@ -48,8 +52,6 @@ _JVM_INTERRUPT = None # the `interrupt` the running JVM was started with _LOG_DIR_USED = None # the log directory the running JVM was started with -_LOG_DIR_ENV = "ARCADEDB_LOG_DIR" -_LOGGING_CONFIG_NAME = "logging.properties" _ENGINE_LOG_RESOURCE = "arcadedb-log.properties" @@ -697,45 +699,6 @@ def _dedupe_max_heap(args: list[str]) -> list[str]: return filtered -def _state_base_dir() -> Path: - """Per-user base directory for state and logs, by platform (stdlib only). - - The home directory is read only where the platform needs it, and when it - cannot be determined (no HOME or USERPROFILE, as in a stripped-down service - environment) the system temp directory is used, so this never raises. - """ - if sys.platform == "win32": - local = os.environ.get("LOCALAPPDATA") - if local: - return Path(local) - elif sys.platform != "darwin": - xdg = os.environ.get("XDG_STATE_HOME") - # The XDG spec ignores a relative value. - if xdg and os.path.isabs(xdg): - return Path(xdg) - try: - home = Path.home() - except (RuntimeError, KeyError, OSError): - return Path(tempfile.gettempdir()) - if sys.platform == "win32": - return home / "AppData" / "Local" - if sys.platform == "darwin": - return home / "Library" / "Logs" - return home / ".local" / "state" - - -def _default_log_dir() -> Path: - """Where the engine log, the crash log and logging.properties go. - - ``ARCADEDB_LOG_DIR`` if set, else ``arcadedb/log`` under the - per-user state directory. Never the working directory. - """ - override = os.environ.get(_LOG_DIR_ENV) - if override: - return Path(os.path.abspath(os.path.expanduser(override))) - return _state_base_dir() / "arcadedb" / "log" - - def _read_engine_log_properties(jar_files: Iterable[str]) -> Optional[str]: """Read the engine's bundled ``arcadedb-log.properties`` from its jar.""" for jar in sorted(jar_files, key=lambda j: "engine" not in os.path.basename(j)): @@ -747,26 +710,6 @@ def _read_engine_log_properties(jar_files: Iterable[str]) -> Optional[str]: return None -def _render_logging_config(engine_text: str) -> str: - """The engine's logging config with the file log in ``logsDirectory``. - - The jar's own file hard-codes ``./log/arcadedb.log``, which lands in the - working directory. The engine resolves ``${arcadedb.server.logsDirectory}`` - in the pattern, and the JVM argument that sets it is added in - ``_build_jvm_args``. - """ - lines = [ - line - for line in engine_text.splitlines() - if not re.match(r"\s*java\.util\.logging\.FileHandler\.pattern\s*[=:]", line) - ] - lines.append( - "java.util.logging.FileHandler.pattern=" - "${arcadedb.server.logsDirectory}/arcadedb.log" - ) - return "\n".join(lines) + "\n" - - def _prepare_log_dir(log_dir: Path, jar_files: Iterable[str]) -> Path: """Create the log directory and write its logging.properties. @@ -775,32 +718,10 @@ def _prepare_log_dir(log_dir: Path, jar_files: Iterable[str]) -> Path: read-only home, for example), so that nothing is ever written to the working directory. """ - try: - log_dir.mkdir(parents=True, exist_ok=True) - if not os.access(log_dir, os.W_OK | os.X_OK): - raise OSError(f"{log_dir} is not writable") - except OSError as exc: - fallback = Path(tempfile.mkdtemp(prefix="arcadedb-log-")) - _LOGGER.warning( - "Cannot use %s for ArcadeDB logs (%s); using %s. Set %s to choose one.", - log_dir, - exc, - fallback, - _LOG_DIR_ENV, - ) - log_dir = fallback - + log_dir = _usable_log_dir(log_dir) engine_text = _read_engine_log_properties(jar_files) if engine_text is not None: - config = log_dir / _LOGGING_CONFIG_NAME - rendered = _render_logging_config(engine_text) - try: - if not config.exists() or config.read_text("iso-8859-1") != rendered: - tmp = log_dir / f"{_LOGGING_CONFIG_NAME}.{os.getpid()}.tmp" - tmp.write_text(rendered, encoding="iso-8859-1", newline="\n") - os.replace(tmp, config) - except OSError as exc: - _LOGGER.warning("Cannot write %s: %s", config, exc) + _write_logging_config(log_dir, engine_text) return log_dir diff --git a/tests/conftest.py b/tests/conftest.py index 365d268..4e3f5dc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,6 +43,8 @@ collect_ignore.append("test_native_startup_runtime.py") # the XML entity limits of an import, lifted by the native glue (the JVM build lifts them with JVM system properties) collect_ignore.append("test_native_xml_limits.py") + # where the native library writes its log (isolate arguments); the JVM build is tested in test_log_dir.py + collect_ignore.append("test_native_log_dir.py") if not os.path.isfile( os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md") ): @@ -97,6 +99,9 @@ ) NATIVE_NA_JVM_TESTS = { # JVM start-up, signals and settings + "test_log_dir.py::test_jvm_args_can_move_the_engine_log": "passes JVM arguments through jvm_kwargs (native: test_native_log_dir.py)", + "test_log_dir.py::test_the_engine_jar_carries_the_log_config_the_render_starts_from": "reads the engine jar, which the native wheel does not ship", + "test_log_dir.py::test_native_engine_log_config_matches_the_engine_jar": "reads the engine jar, which the native wheel does not ship", "test_sigint.py::test_ctrl_c_raises_keyboard_interrupt_and_runs_cleanup[java]": "a Java call blocked in JPype's SIGINT handler (#179)", "test_sigint.py::test_interrupt_true_keeps_the_jvm_handling_sigint": "start_jvm(interrupt=True): the JVM handles SIGINT", "test_docs_examples.py::test_docs_installation_jvm_examples": "documents start_jvm() and jvm_kwargs", diff --git a/tests/test_log_dir.py b/tests/test_log_dir.py index eba86e5..8119f8a 100644 --- a/tests/test_log_dir.py +++ b/tests/test_log_dir.py @@ -1,4 +1,7 @@ -"""Logs and the JVM crash file go to a per-user directory, never the working directory.""" +"""Logs (and, on the JVM build, the JVM crash file) go to a per-user directory, never the working directory. + +The subprocess tests run on both builds; the JVM-only ones are listed in conftest.NATIVE_NA_JVM_TESTS and +test_native_log_dir.py has their native counterparts.""" import os import subprocess # nosec B404 - test-controlled child process @@ -9,19 +12,29 @@ import pytest -from arcadedb import jvm +from arcadedb import _logdir, jvm +from arcadedb._native import lib as native_lib from arcadedb.jvm import _build_jvm_args, _default_log_dir CHILD = """ import sys import arcadedb from arcadedb import jvm +from arcadedb._native import enabled jvm_args = sys.argv[2:] -db = arcadedb.create_database(sys.argv[1], jvm_kwargs={"jvm_args": jvm_args}) +# the JVM build takes its settings as jvm_kwargs; the native build has none (its log settings are tested in test_native_log_dir.py) +kwargs = {} if enabled() else {"jvm_kwargs": {"jvm_args": jvm_args}} +db = arcadedb.create_database(sys.argv[1], **kwargs) db.command("sql", "CREATE DOCUMENT TYPE Doc") db.close() -print("LOGDIR=" + str(jvm._LOG_DIR_USED)) +if enabled(): + from arcadedb._native import lib + + used = lib.LOG_DIR_USED +else: + used = jvm._LOG_DIR_USED +print("LOGDIR=" + str(used)) """ @@ -204,3 +217,55 @@ def test_arg_values_use_forward_slashes_on_every_platform(tmp_path): ) assert f"-Darcadedb.server.logsDirectory={tmp_path.as_posix()}" in args assert f"-XX:ErrorFile={tmp_path.as_posix()}/hs_err_pid%p.log" in args + + +def test_native_isolate_arguments_name_the_log_dir(tmp_path): + args = native_lib._log_isolate_args([], tmp_path) + assert f"-Darcadedb.server.logsDirectory={tmp_path.as_posix()}" in args + config = f"-Djava.util.logging.config.file={tmp_path.as_posix()}/logging.properties" + assert config in args + assert not any("./log" in a for a in args) + + +def test_native_user_isolate_arguments_win_over_the_log_defaults(tmp_path): + user = [ + "-Darcadedb.server.logsDirectory=/x/logs", + "-Djava.util.logging.config.file=/x/my.properties", + ] + assert native_lib._log_isolate_args(user, tmp_path) == [] + only_dir = native_lib._log_isolate_args(user[:1], tmp_path) + assert len(only_dir) == 1 and only_dir[0].startswith( + "-Djava.util.logging.config.file=" + ) + + +def test_native_engine_log_config_renders_like_the_jvm_one(): + rendered = _logdir.render_logging_config(_logdir.NATIVE_ENGINE_LOG_PROPERTIES) + assert "./log" not in rendered + assert rendered.count("FileHandler.pattern") == 1 + assert "${arcadedb.server.logsDirectory}/arcadedb.log" in rendered + assert "java.util.logging.FileHandler.count=10" in rendered + assert "handlers = java.util.logging.ConsoleHandler" in rendered + + +def _settings(text): + """The key/value pairs of a .properties text, comments and spacing around = dropped.""" + pairs = {} + for line in text.splitlines(): + line = line.strip() + if line and not line.startswith(("#", "!")): + key, _, value = line.partition("=") + pairs[key.strip()] = value.strip() + return pairs + + +def test_native_engine_log_config_matches_the_engine_jar(): + # The native image has no jar to read, so its log settings are a copy of the engine's arcadedb-log.properties (without + # the file pattern, which the render sets). This is the guard that fails when an engine upgrade changes that file. + import glob + + jars = glob.glob(os.path.join(jvm.get_jar_path(), "*.jar")) + engine = _settings(jvm._read_engine_log_properties(jars)) + engine.pop("java.util.logging.FileHandler.pattern") + copy = _settings(_logdir.NATIVE_ENGINE_LOG_PROPERTIES) + assert copy == engine diff --git a/tests/test_native_log_dir.py b/tests/test_native_log_dir.py new file mode 100644 index 0000000..7d4076a --- /dev/null +++ b/tests/test_native_log_dir.py @@ -0,0 +1,42 @@ +"""Native build: the isolate writes its log where the log-dir settings say, and never into the working directory. + +The JVM build gets the same settings as JVM arguments (test_log_dir.py). The native library is a GraalVM isolate: the glue passes +the two system properties as isolate arguments, because the engine reads them at its first log call and an isolate argument is +the only setting that is in place before it. ARCADEDB_NATIVE_ISOLATE_ARGS takes the user's own arguments, which win. +""" + +from arcadedb import _logdir + +from .test_log_dir import _run_child + + +def test_isolate_argument_moves_the_engine_log(tmp_path): + target = tmp_path / "from_isolate_args" + cwd, _, _ = _run_child( + tmp_path, + { + "ARCADEDB_NATIVE_ISOLATE_ARGS": f"-Darcadedb.server.logsDirectory={target.as_posix()}" + }, + ) + assert list(cwd.iterdir()) == [] + assert list(target.glob("arcadedb.log*")) + + +def test_own_logging_config_wins(tmp_path): + mine = tmp_path / "mine" + mine.mkdir() + config = tmp_path / "my.properties" + rendered = _logdir.render_logging_config(_logdir.NATIVE_ENGINE_LOG_PROPERTIES) + rendered = rendered.replace( + "${arcadedb.server.logsDirectory}/arcadedb.log", f"{mine.as_posix()}/own.log" + ) + config.write_text(rendered, encoding="iso-8859-1") + cwd, log_dir, _ = _run_child( + tmp_path, + { + "ARCADEDB_NATIVE_ISOLATE_ARGS": f"-Djava.util.logging.config.file={config.as_posix()}" + }, + ) + assert list(cwd.iterdir()) == [] + assert list(mine.glob("own.log*")) + assert not list(log_dir.glob("arcadedb.log*")) From 7bf78db8d0b9e791aad9ee5dfe6d0d4d131026c9 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 20:47:23 +0900 Subject: [PATCH 15/33] Native: Python functions in SQL and openCypher, record events, Ctrl-C, and the async writer crash Python functions (register_function with replace, deterministic and inferred argument counts, unregister_function) and record events (Database.on) run on the native backend through the existing Java-to-Python upcall. The engine side is a new entry point, arcadedb_n_cb, whose checks, wording, veto and failure rules are those of the JPype bridge. A record argument or event record reaches Python as a Document, Vertex or Edge wrapper. A KeyboardInterrupt in a callback, or one that arrives as the callback is entered, aborts the statement and reaches the caller as a KeyboardInterrupt chained from the ArcadeDBError. A Java thread that runs a Python callback (an async writer) is attached to the isolate already; the thread handle for it is no longer an owned one that detaches the thread when the callback's Python thread state ends. That detach aborted the process in tests/test_async_executor.py (a callback that calls the database from a writer thread). Co-Authored-By: Claude Sonnet 5.5 --- docs/modules/ROOT/pages/installation.adoc | 2 +- .../org/humemai/nativelib/NativeExtras.java | 325 ++++++++++++++++-- src/arcadedb/_native/__init__.py | 2 +- src/arcadedb/_native/database.py | 49 ++- src/arcadedb/_native/extras.py | 215 ++++++++++-- src/arcadedb/_native/interrupts.py | 33 ++ src/arcadedb/_native/lib.py | 11 +- src/arcadedb/callbacks.py | 52 ++- tests/test_native_udf.py | 10 +- tests/test_python_callbacks.py | 13 +- 10 files changed, 624 insertions(+), 88 deletions(-) create mode 100644 src/arcadedb/_native/interrupts.py diff --git a/docs/modules/ROOT/pages/installation.adoc b/docs/modules/ROOT/pages/installation.adoc index b972406..733a505 100644 --- a/docs/modules/ROOT/pages/installation.adoc +++ b/docs/modules/ROOT/pages/installation.adoc @@ -67,7 +67,7 @@ Every wheel is platform-specific and built on a native runner of its platform, s Engine modules the bindings do not use are left out to keep the wheel small. These include the gRPC and MongoDB wire protocols, Raft HA, and Gremlin. -NOTE: *JavaScript is not available.* The GraalJS jars are left out, so `DEFINE FUNCTION ... LANGUAGE js` fails. Define functions in SQL or openCypher. Registering a Python function is planned in https://github.com/ArcadeData/arcadedb-python/pull/29[#29]. +NOTE: *JavaScript is not available.* The GraalJS jars are left out, so `DEFINE FUNCTION ... LANGUAGE js` fails. Define functions in SQL or openCypher, or register a Python function with `Database.register_function` (it works on both packages). // REMOVE WHEN: the old arcadedb-embedded* shims are retired and two releases have shipped with the two-package layout allow-old-name NOTE: *The Studio web UI is in the JVM wheels from 26.11.1.* Studio used to bundle ApexCharts, which is no longer open source (https://github.com/ArcadeData/arcadedb/issues/10579[ArcadeData/arcadedb#10579]). https://github.com/ArcadeData/arcadedb/pull/10627[ArcadeData/arcadedb#10627] replaced it with Apache ECharts. The earlier 26.10.x wheels on PyPI do not contain Studio. The HTTP API of the in-process server works in all of them. diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index 8b86316..ebbddf5 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -20,14 +20,27 @@ import com.arcadedb.database.DocumentCallback; import com.arcadedb.database.Identifiable; import com.arcadedb.database.MutableDocument; +import com.arcadedb.database.Record; +import com.arcadedb.database.RecordEvents; import com.arcadedb.database.RID; import com.arcadedb.database.async.DatabaseAsyncExecutor; import com.arcadedb.database.async.ErrorCallback; import com.arcadedb.database.async.NewEdgeCallback; import com.arcadedb.engine.ErrorRecordCallback; import com.arcadedb.engine.WALFile; +import com.arcadedb.event.AfterRecordCreateListener; +import com.arcadedb.event.AfterRecordDeleteListener; +import com.arcadedb.event.AfterRecordUpdateListener; +import com.arcadedb.event.BeforeRecordCreateListener; +import com.arcadedb.event.BeforeRecordDeleteListener; +import com.arcadedb.event.BeforeRecordUpdateListener; +import com.arcadedb.exception.DatabaseOperationException; +import com.arcadedb.function.CypherBuiltinFunctions; +import com.arcadedb.function.CypherFunctionRegistry; +import com.arcadedb.function.FunctionRegistry; import com.arcadedb.function.sql.DefaultSQLFunctionFactory; import com.arcadedb.function.sql.SQLFunctionAbstract; +import com.arcadedb.graph.Edge; import com.arcadedb.graph.GraphBatch; import com.arcadedb.graph.MutableVertex; import com.arcadedb.graph.Vertex; @@ -35,6 +48,9 @@ import com.arcadedb.integration.importer.Importer; import com.arcadedb.query.sql.executor.CommandContext; import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.EdgeType; +import com.arcadedb.schema.VertexType; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CFunctionPointer; @@ -240,7 +256,9 @@ public static CCharPointer gbCreate(final IsolateThread thread, final long dbh, try { final Database db = ArcadeNative.db(dbh); final Map o = args(options); - GraphBatch.Builder b = GraphBatch.builder(db); + // db.batch(), as the JPype path: it hands the builder the engine's single-batch guard (LocalDatabase.batchStarted, taken by build() and + // released by close()), which GraphBatch.builder(db) bypasses; a second batch while one is open is refused instead of losing edges (P-04) + GraphBatch.Builder b = db.batch(); for (final Map.Entry e : o.entrySet()) { final Object v = e.getValue(); final int iv = intOf(v, 0); @@ -385,6 +403,19 @@ private static Object[] objects(final Object list) { return ((List) list).toArray(); } + /** + * Whether a bulk load into the type creates vertices (true) or documents (false). + * + * @throws IllegalArgumentException for an edge type: an edge cannot be created without its two endpoints + */ + private static boolean isVertexType(final Database db, final String typeName) { + final DocumentType type = db.getSchema().getType(typeName); + if (type instanceof EdgeType) + throw new IllegalArgumentException("Cannot bulk-insert into '" + typeName + + "' because it is an edge type; load the vertices, then create the edges (graph_batch().new_edges() or CREATE EDGE)"); + return type instanceof VertexType; + } + /** * ops: get | set {key,value} | wait {timeout_ms} | kill | create_record {type, kind, props, cb, err} | command / query {lang, text, params, * cb, err} | on_ok {cb} | on_error {cb} | append_samples {type, ts, cols} | insert_parallel {type, rows} | failures {fid}. @@ -539,8 +570,12 @@ public void onError(final Exception e) { }; final String type = (String) a.get("type"); try { - for (final Object row : (List) a.get("rows")) { - final MutableDocument doc = db.newDocument(type); + final List rows = (List) a.get("rows"); + // the type is looked up once per call, as DocumentBatcher does on the JPype path (humemai #519): a vertex type gets vertices, + // a document type documents, an edge type is refused (an edge needs both endpoints); a missing type raises the schema's own error + final boolean vertex = !rows.isEmpty() && isVertexType(db, type); + for (final Object row : rows) { + final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); for (final Map.Entry e : ((Map) row).entrySet()) doc.set(e.getKey(), e.getValue()); ex.createRecord(doc, null, onError); @@ -647,39 +682,109 @@ public static CCharPointer exportDatabase(final IsolateThread thread, final long } } - // ---- Python functions in SQL: arcadedb_n_udf_register(name, callbackId, minArgs, maxArgs) + // ---- Python functions in SQL and record events: arcadedb_n_cb(op, json) + + /* + * One entry point for the callback registry of the Python side (functions and record events), JSON in and out like the + * rest: {"ok":true,..} or {"ok":true,"refused":"text"} when the request is understood but not allowed (Python raises the + * text as an ArcadeDBError). The engine logic is the one of the JPype bridge (src/java/com/arcadedb/python/PythonFunctions + * and PythonRecordEvents): same checks, same wording, same veto and failure rules; only the call into Python differs (the + * upcall, with JSON). Keep the two in step. + * + * ops: fn_register {name,cb,min,max,det,replace} fn_unregister {h} ev_register {db,type,event,cb} ev_remove {h} + * A handle (h) names a registration for the unregister call; the reply to a replace or an unregister carries the callback id + * of the registration that went away ("dropped"), so Python can forget it. + */ + + private static final Object REGISTRY = new Object(); + private static final Map HANDLES = new ConcurrentHashMap<>(); + private static final AtomicLong NEXT_HANDLE = new AtomicLong(1); + + static String describe(final Throwable t) { + final String message = t.getMessage(); + return message == null || message.isEmpty() ? t.getClass().getName() : message; + } + + /** The reply of a Python call that must return a value: a missing reply or an {"error"} reply fails with Python's text. */ + @SuppressWarnings("unchecked") + private static Map answer(final String reply) { + if (reply == null) + throw new IllegalStateException("the call was interrupted before Python returned a value"); + final Map r = (Map) Json.parse(reply); + if (r.get("error") != null) + throw new IllegalStateException(String.valueOf(r.get("error"))); + return r; + } + + /** A record as a tagged dict with its category, for Python to wrap as a Document, Vertex or Edge. */ + static void appendRecord(final StringBuilder sb, final Document doc) { + sb.append('{'); + if (doc.getIdentity() != null) { + sb.append("\"@rid\":"); + NativeCodec.quote(sb, doc.getIdentity().toString()); + sb.append(','); + } + sb.append("\"@type\":"); + NativeCodec.quote(sb, doc.getTypeName()); + sb.append(",\"@cat\":\"").append(doc instanceof Vertex ? 'v' : doc instanceof Edge ? 'e' : 'd').append('"'); + for (final String name : doc.getPropertyNames()) { + sb.append(','); + NativeCodec.quote(sb, name); + sb.append(':'); + NativeCodec.append(sb, doc.get(name)); + } + sb.append('}'); + } /** * A SQL function whose body is a Python callable. The engine thread that runs the statement calls the upcall with - * {"k":"fn","args":[..]} (tagged values, as parameters) and expects {"v":} or {"error":"text"}; an error is thrown - * as a CommandExecutionException from the function, so the statement fails with the Python message. + * {"k":"fn","args":[..]} (tagged values as parameters, a record argument as a dict with @rid, @type and @cat) and expects + * {"v":} or {"error":"text"}; any failure is thrown as a CommandExecutionException from the function, so the + * statement fails with the Python message. */ static final class PythonFunction extends SQLFunctionAbstract { - private final long cb; + final long handle = NEXT_HANDLE.getAndIncrement(); + final long cb; private final int min, max; + private final boolean deterministic; + volatile boolean revoked; - PythonFunction(final String name, final long cb, final int min, final int max) { + PythonFunction(final String name, final long cb, final int min, final int max, final boolean deterministic) { super(name); this.cb = cb; this.min = min; this.max = max; + this.deterministic = deterministic; } @Override public Object execute(final Object self, final Identifiable record, final Object result, final Object[] params, final CommandContext ctx) { - final StringBuilder sb = new StringBuilder("{\"k\":\"fn\",\"args\":["); - for (int i = 0; i < params.length; i++) { - if (i > 0) - sb.append(','); - NativeCodec.append(sb, params[i]); + final String name = getName(); + if (revoked) + throw new com.arcadedb.exception.CommandExecutionException("Python function '" + name + "' was unregistered"); + if (params.length < min || params.length > max) + throw new com.arcadedb.exception.CommandExecutionException( + "Python function '" + name + "' takes " + arity() + " argument(s), got " + params.length); + try { + final StringBuilder sb = new StringBuilder("{\"k\":\"fn\",\"args\":["); + for (int i = 0; i < params.length; i++) { + if (i > 0) + sb.append(','); + if (params[i] instanceof Document doc) + appendRecord(sb, doc); + else + NativeCodec.append(sb, params[i]); + } + return answer(upcall(cb, sb.append("]}").toString())).get("v"); + } catch (final Throwable t) { + throw new com.arcadedb.exception.CommandExecutionException("Python function '" + name + "' failed: " + describe(t), t); } - final String reply = upcall(cb, sb.append("]}").toString()); - if (reply == null) - return null; - @SuppressWarnings("unchecked") final Map r = (Map) Json.parse(reply); - if (r.get("error") != null) - throw new com.arcadedb.exception.CommandExecutionException(String.valueOf(r.get("error"))); - return r.get("v"); + } + + private String arity() { + if (min == max) + return String.valueOf(min); + return max == Integer.MAX_VALUE ? "at least " + min : min + " to " + max; } @Override @@ -692,17 +797,187 @@ public int getMaxArgs() { return max; } + @Override + public boolean isDeterministic() { + return deterministic; + } + @Override public String getSyntax() { - return getName() + "(*)"; + return getName() + "(" + (max == 0 ? "" : "") + ") [Python]"; + } + } + + private static boolean takenByEngine(final String key) { + final DefaultSQLFunctionFactory factory = DefaultSQLFunctionFactory.getInstance(); + if (factory.hasFunction(key)) + return !(factory.getFunctionInstance(key) instanceof PythonFunction); + return CypherBuiltinFunctions.isBuiltin(key) || CypherFunctionRegistry.hasFunction(key) || FunctionRegistry.hasFunction(key); + } + + private static PythonFunction pythonFunction(final String key) { + final DefaultSQLFunctionFactory factory = DefaultSQLFunctionFactory.getInstance(); + return factory.hasFunction(key) && factory.getFunctionInstance(key) instanceof PythonFunction f ? f : null; + } + + /** Revokes and unregisters a function; the callback id it had, or 0 when it was not the registered one any more. */ + private static long removeFunction(final PythonFunction function) { + function.revoked = true; + HANDLES.remove(function.handle); + final String key = function.getName(); + final DefaultSQLFunctionFactory factory = DefaultSQLFunctionFactory.getInstance(); + if (factory.hasFunction(key) && factory.getFunctionInstance(key) == function) { + factory.unregister(key); + FunctionRegistry.unregister(key); + } + return function.cb; + } + + /** The listener of one event of a Python callback: a before_* callback may veto, any failure fails the operation. */ + static final class PythonListener implements BeforeRecordCreateListener, BeforeRecordUpdateListener, BeforeRecordDeleteListener, + AfterRecordCreateListener, AfterRecordUpdateListener, AfterRecordDeleteListener { + final long handle = NEXT_HANDLE.getAndIncrement(); + final long cb; + final String event; + final RecordEvents events; + + PythonListener(final RecordEvents events, final String event, final long cb) { + this.events = events; + this.event = event; + this.cb = cb; + } + + private boolean dispatch(final Record record) { + if (!(record instanceof Document doc)) + return true; + try { + final StringBuilder sb = new StringBuilder("{\"k\":\"ev\",\"e\":\"").append(event).append("\",\"rec\":"); + appendRecord(sb, doc); + return !Boolean.FALSE.equals(answer(upcall(cb, sb.append('}').toString())).get("v")); + } catch (final Throwable t) { + throw new DatabaseOperationException("Python '" + event + "' callback failed: " + describe(t), t); + } } + + @Override + public boolean onBeforeCreate(final Record record) { + return dispatch(record); + } + + @Override + public boolean onBeforeUpdate(final Record record) { + return dispatch(record); + } + + @Override + public boolean onBeforeDelete(final Record record) { + return dispatch(record); + } + + @Override + public void onAfterCreate(final Record record) { + dispatch(record); + } + + @Override + public void onAfterUpdate(final Record record) { + dispatch(record); + } + + @Override + public void onAfterDelete(final Record record) { + dispatch(record); + } + + void register() { + switch (event) { + case "before_create" -> events.registerListener((BeforeRecordCreateListener) this); + case "before_update" -> events.registerListener((BeforeRecordUpdateListener) this); + case "before_delete" -> events.registerListener((BeforeRecordDeleteListener) this); + case "after_create" -> events.registerListener((AfterRecordCreateListener) this); + case "after_update" -> events.registerListener((AfterRecordUpdateListener) this); + case "after_delete" -> events.registerListener((AfterRecordDeleteListener) this); + default -> throw new IllegalArgumentException("Unknown record event: " + event); + } + } + + void unregister() { + switch (event) { + case "before_create" -> events.unregisterListener((BeforeRecordCreateListener) this); + case "before_update" -> events.unregisterListener((BeforeRecordUpdateListener) this); + case "before_delete" -> events.unregisterListener((BeforeRecordDeleteListener) this); + case "after_create" -> events.unregisterListener((AfterRecordCreateListener) this); + case "after_update" -> events.unregisterListener((AfterRecordUpdateListener) this); + case "after_delete" -> events.unregisterListener((AfterRecordDeleteListener) this); + default -> throw new IllegalArgumentException("Unknown record event: " + event); + } + } + } + + private static CCharPointer refused(final String text) { + final StringBuilder sb = new StringBuilder("{\"ok\":true,\"refused\":"); + NativeCodec.quote(sb, text); + return ArcadeNative.out(sb.append('}').toString()); + } + + private static long longOf(final Object v) { + return ((Number) v).longValue(); } - @CEntryPoint(name = "arcadedb_n_udf_register") - public static CCharPointer udfRegister(final IsolateThread thread, final CCharPointer name, final long cb, final int min, final int max) { + @CEntryPoint(name = "arcadedb_n_cb") + public static CCharPointer callbackRegistry(final IsolateThread thread, final CCharPointer op, final CCharPointer json) { try { - DefaultSQLFunctionFactory.getInstance().register(new PythonFunction(ArcadeNative.str(name), cb, min, max)); - return ArcadeNative.OK(); + final Map a = args(json); + switch (ArcadeNative.str(op)) { + case "fn_register": { + final String name = String.valueOf(a.get("name")); + final String key = name.toLowerCase(java.util.Locale.ENGLISH); + synchronized (REGISTRY) { + if (takenByEngine(key)) + return refused("Cannot register '" + name + "': it is already a function of the engine"); + final PythonFunction old = pythonFunction(key); + if (old != null && !Boolean.TRUE.equals(a.get("replace"))) + return refused("A Python function '" + name + "' is already registered (pass replace=True to replace it)"); + final long dropped = old == null ? 0 : removeFunction(old); + final PythonFunction function = new PythonFunction(key, longOf(a.get("cb")), (int) longOf(a.get("min")), (int) longOf(a.get("max")), + Boolean.TRUE.equals(a.get("det"))); + DefaultSQLFunctionFactory.getInstance().register(function); + HANDLES.put(function.handle, function); + return ArcadeNative.out("{\"ok\":true,\"h\":" + function.handle + ",\"dropped\":" + dropped + "}"); + } + } + case "fn_unregister": { + synchronized (REGISTRY) { + final Object handle = HANDLES.get(longOf(a.get("h"))); + final long dropped = handle instanceof PythonFunction f ? removeFunction(f) : 0; + return ArcadeNative.out("{\"ok\":true,\"dropped\":" + dropped + "}"); + } + } + case "ev_register": { + final Database db = ArcadeNative.db(longOf(a.get("db"))); + final String type = (String) a.get("type"); + final RecordEvents events; + if (type == null) + events = db.getEvents(); + else { + if (!db.getSchema().existsType(type)) + return refused("Type '" + type + "' does not exist"); + events = db.getSchema().getType(type).getEvents(); + } + final PythonListener listener = new PythonListener(events, String.valueOf(a.get("event")), longOf(a.get("cb"))); + listener.register(); + HANDLES.put(listener.handle, listener); + return ArcadeNative.out("{\"ok\":true,\"h\":" + listener.handle + "}"); + } + case "ev_remove": { + final Object handle = HANDLES.remove(longOf(a.get("h"))); + if (handle instanceof PythonListener listener) + listener.unregister(); + return ArcadeNative.out("{\"ok\":true,\"dropped\":" + (handle instanceof PythonListener l ? l.cb : 0) + "}"); + } + default: + throw new IllegalArgumentException("unknown callback op " + ArcadeNative.str(op)); + } } catch (final Throwable t) { return ArcadeNative.err(t); } diff --git a/src/arcadedb/_native/__init__.py b/src/arcadedb/_native/__init__.py index 27820cb..b413293 100644 --- a/src/arcadedb/_native/__init__.py +++ b/src/arcadedb/_native/__init__.py @@ -5,7 +5,7 @@ library (``libarcadedb``) that Python loads with ``ctypes``: no JVM, no JPype. The surface is the DSL-first core: databases, ``query``/``command`` for SQL and openCypher, transactions, results and the in-process server. -What is not on native yet (each raises :class:`NotOnNativeYet`): Python UDFs and record callbacks, ``GraphBatch``, the +What is not on native yet (each raises :class:`NotOnNativeYet`): ``GraphBatch``, the async executor, the Java-object APIs (``Document``/``Vertex``/``Edge`` wrappers, ``new_vertex``, ``new_document``, ``schema``, ``get_java_database``), the columnar/Arrow readers other than a simple row conversion, importers and exporters that go through Java objects, and every JVM setting (``jvm_kwargs``, ``start_jvm``). diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index d640e8a..9646cbb 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -13,7 +13,7 @@ from ..exceptions import ArcadeDBError from ..transactions import TransactionContext from ..type_conversion import json_bulk_dumps -from . import lib, not_yet +from . import interrupts, lib, not_yet from .codec import _prep as _prep_row from .codec import encode_params, json_native, loads, row_default @@ -39,7 +39,9 @@ def _error(prefix: str, reply: dict) -> ArcadeDBError: exc = ArcadeDBError(f"{prefix}: {head}" if prefix else head) exc.native = reply exc.java_chain = chain - return exc + return interrupts.convert( + exc + ) # a Ctrl-C in a Python function is a KeyboardInterrupt chained from this error def _read_error(reply: dict) -> ArcadeDBError: @@ -48,7 +50,7 @@ def _read_error(reply: dict) -> ArcadeDBError: ) exc.native = reply exc.java_chain = reply.get("chain") or [] - return exc + return interrupts.convert(exc) def _call(prefix: str, name: str, *args) -> dict: @@ -392,6 +394,7 @@ def __init__(self, handle: int, owned_by_server: bool = False): self._closed = False self._server_owned = owned_by_server self._async_executors: list = [] + self._registry = None # Python functions and event callbacks, once one is registered (extras.Registrations) self._vector_meta: dict = {} self._vector_dimensions: dict = {} self._vector_id_properties: dict = {} @@ -412,6 +415,8 @@ def close(self): if self._closed: return self._close_async_executors() + if self._registry is not None: + self._registry.clear() # before the engine closes: the functions are process-wide, the listeners die with the database if self._server_owned: # owned by the server: the engine refuses the close, and this handle is done either way self._closed = True @@ -481,6 +486,7 @@ def _exec( refuse_surplus_positionals( "command" if flags & 1 else "query", language, command, args ) + interrupts.clear() if flags & 1 and _LAZY_COMMAND_ROWS and lib.CMD_PATH: # phase 8: the cursor id comes back as a long (no reply text to free and decode); the rows are read late or never text, buf, count = encode_params(args, binary=True) @@ -876,13 +882,40 @@ def async_executor(self): self._async_executors.append(executor) return executor - def register_function(self, name, fn, min_args=0, max_args=-1): - """Make a Python callable available to SQL (process-wide, see ``extras.register_function``).""" - from .extras import register_function + def _registrations(self): + from .extras import Registrations - register_function(name, fn, min_args, max_args) + if self._registry is None: + self._registry = Registrations(self) + return self._registry + + def register_function( + self, + name, + fn, + *, + min_args=None, + max_args=None, + deterministic=False, + replace=False, + ): + """Make a Python callable available to SQL and openCypher (``Database.register_function``: process-wide, the Python + function is called from the engine thread that runs the statement).""" + self._check_not_closed() + self._registrations().register_function( + name, fn, min_args, max_args, deterministic, replace + ) + + def unregister_function(self, name): + """Remove a Python function registered through this database.""" + self._check_not_closed() + self._registrations().unregister_function(name) + + def on(self, event, fn, type_name=None): + """Call a Python function when a record is created, updated or deleted (``Database.on``).""" + self._check_not_closed() + return self._registrations().on(event, fn, type_name) - @property def schema(self): """DDL helpers over SQL statements (the DSL-first subset); everything else is not on native yet.""" self._check_not_closed() diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py index 69c6789..96873ef 100644 --- a/src/arcadedb/_native/extras.py +++ b/src/arcadedb/_native/extras.py @@ -1,6 +1,6 @@ """Native backend coverage beyond the DSL core: GraphBatch, the async executor, database settings, import/export and Python functions in SQL, over the ``arcadedb_n_gb*``, ``arcadedb_n_async``, ``arcadedb_n_dbset``, ``arcadedb_n_import``, -``arcadedb_n_export`` and ``arcadedb_n_udf_register`` entry points (``NativeExtras.java``). +``arcadedb_n_export`` and ``arcadedb_n_cb`` entry points (``NativeExtras.java``). Callbacks (async completions, error handlers, Python functions called from SQL) go through ONE ctypes function pointer that ``_ensure_upcall`` registers with the library: the engine thread (or an async writer thread) calls ``_dispatch(id, json)``, @@ -10,13 +10,17 @@ import ctypes import itertools import json +import sys import threading +import weakref from collections.abc import Mapping from typing import Any, Callable, Iterable, Optional +from ..callbacks import EventHandle as _BaseEventHandle +from ..callbacks import check_event, check_function from ..exceptions import ArcadeDBError from ..graph_batch import GraphBatch as _BaseGraphBatch -from . import lib, not_yet +from . import interrupts, lib, not_yet from .codec import _PLAIN, _default, _prep, loads # -------------------------------------------------------------------------------------------------------- upcall @@ -43,6 +47,7 @@ def _dispatch(cid, raw): reply = fn(message) if fn is not None else None data = None if reply is None else _enc(reply).encode("utf-8") except BaseException as exc: # nosec B110 - an exception must not cross the C frame + interrupts.note(exc) data = json.dumps({"error": f"{type(exc).__name__}: {exc}"}).encode("utf-8") if data is None: return None @@ -52,10 +57,32 @@ def _dispatch(cid, raw): return ctypes.addressof(buf) +def _install_unraisable_hook() -> None: + """Keep a Ctrl-C that arrives as the callback is entered. + + CPython runs a pending signal handler at the first bytecode of a function, so a KeyboardInterrupt can be raised before + ``_dispatch`` reaches its ``try``: ctypes then reports it as unraisable and returns NULL to the engine. The hook records it + as the interrupt of this thread (the engine fails the statement on a NULL reply, and the error becomes the + KeyboardInterrupt); every other unraisable exception goes to the hook that was there. + """ + previous = sys.unraisablehook + + def hook(unraisable): + if unraisable.object is _dispatch and isinstance( + unraisable.exc_value, KeyboardInterrupt + ): + interrupts.note(unraisable.exc_value) + else: + previous(unraisable) + + sys.unraisablehook = hook + + def _ensure_upcall() -> None: global _upcall_ref if _upcall_ref is not None: return + _install_unraisable_hook() lib.call_void( "arcadedb_n_set_upcall", ctypes.cast(_make_pointer(), ctypes.c_void_p) ) @@ -67,6 +94,13 @@ def _make_pointer(): return _upcall_ref +def _forget(cid) -> None: + """Drop a callback the engine no longer holds (an unregistered function, a removed listener, a replaced function).""" + if cid: + with _callbacks_lock: + _callbacks.pop(cid, None) + + def _register(fn: Callable) -> int: _ensure_upcall() with _callbacks_lock: @@ -789,31 +823,164 @@ def export_database( )["stats"] -# -------------------------------------------------------------------------------------------------------- Python functions in SQL +# -------------------------------------------------------------------------------------------------------- Python functions in SQL and record events +_MAX_LONG = 2**63 -def register_function( - name: str, fn: Callable, min_args: int = 0, max_args: int = -1 -) -> None: - """Make ``fn`` callable from SQL as ``name(..)``: ``SELECT name(a, b) FROM T``. - ``fn`` receives the argument values as Python values and returns a value of a type a parameter can carry (str, int, float, - bool, None, list, dict, Decimal, datetime). An exception raised by ``fn`` fails the statement with its text. Registration is - process-wide (the engine's function factory), like any engine function; ``max_args`` < 0 means any number. - """ +def _check_result(value) -> None: + """A function result the engine can carry: an int past a Java long is an OverflowError, as on the JPype path.""" + t = type(value) + if t is int: + if not -_MAX_LONG <= value < _MAX_LONG: + raise OverflowError("Python int too large to convert to Java long") + elif t is list or t is tuple: + for item in value: + _check_result(item) + elif t is dict: + for item in value.values(): + _check_result(item) + + +def _record_or_value(database, value): + """A record the engine sent as a dict with ``@rid``, ``@type`` and ``@cat`` becomes a Document, Vertex or Edge.""" + if type(value) is dict and "@type" in value and "@cat" in value: + from .records import _wrap_dict + + return _wrap_dict(database, value) + return value + + +def _function_handler(fn: Callable, database_ref) -> Callable: + def handler(message): + database = database_ref() + result = fn(*[_record_or_value(database, a) for a in message.get("args", [])]) + _check_result(result) + return {"v": result} + + return handler + + +def _event_handler(event: str, fn: Callable, database_ref) -> Callable: + is_before = event.startswith("before_") def handler(message): + from .records import _wrap_dict + + result = fn(_wrap_dict(database_ref(), message["rec"])) + return {"v": not (is_before and result is False)} + + return handler + + +class NativeEventHandle(_BaseEventHandle): + """A registered record event callback; ``remove()`` stops it (also a context manager, as on the JPype backend).""" + + def __init__(self, registrations, event, type_name, handle, cid): + super().__init__(None, event, type_name, handle, None) + self._registrations = registrations + self._cid = cid + + def remove(self) -> None: + if not self._active: + return + self._active = False + self._registrations._events.discard(self) try: - return {"v": fn(*message.get("args", []))} - except Exception as exc: - return {"error": f"{type(exc).__name__}: {exc}"} - - cid = _register(handler) - _call( - "Failed to register the function", - "arcadedb_n_udf_register", - lib.b(name), - cid, - int(min_args), - int(max_args if max_args >= 0 else 2**31 - 1), - ) + _call( + f"Failed to remove '{self.event}' callback", + "arcadedb_n_cb", + lib.b("ev_remove"), + lib.b(json.dumps({"h": self._java_handle})), + ) + finally: + _forget(self._cid) + + +class Registrations: + """The Python functions and event callbacks one ``NativeDatabase`` registered, to remove them when it closes. + + The engine keeps functions for the whole process, so each database removes its own; the checks and the replies are those + of the JPype backend (``arcadedb.callbacks``), the engine side is ``arcadedb_n_cb`` in ``NativeExtras.java``. + """ + + def __init__(self, database): + # weak: the callbacks hold the database only for wrapping records, and the registry must not keep it alive + self._database_ref = weakref.ref(database) + self._h = database._h + self._functions: dict = {} # lower-case name -> (engine handle, callback id) + self._events: set = set() + + def _cb(self, prefix: str, op: str, **args) -> dict: + reply = _call(prefix, "arcadedb_n_cb", lib.b(op), lib.b(json.dumps(args))) + if "refused" in reply: + raise ArcadeDBError(reply["refused"]) + return reply + + def register_function( + self, name, fn, min_args, max_args, deterministic, replace + ) -> None: + min_args, max_args = check_function(name, fn, min_args, max_args) + cid = _register(_function_handler(fn, self._database_ref)) + try: + reply = self._cb( + f"Failed to register function '{name}'", + "fn_register", + name=name, + cb=cid, + min=min_args, + max=min(max_args, 2**31 - 1), + det=bool(deterministic), + replace=bool(replace), + ) + except BaseException: + _forget(cid) + raise + _forget(reply.get("dropped")) # the function this one replaced + self._functions[name.lower()] = (reply["h"], cid) + + def unregister_function(self, name: str) -> None: + entry = self._functions.pop(str(name).lower(), None) + if entry is None: + raise ArcadeDBError( + f"No Python function '{name}' is registered through this database" + ) + try: + self._cb(f"Failed to unregister '{name}'", "fn_unregister", h=entry[0]) + finally: + _forget(entry[1]) + + def on(self, event: str, fn: Callable, type_name: Optional[str]): + check_event(event, fn) + cid = _register(_event_handler(event, fn, self._database_ref)) + try: + reply = self._cb( + f"Failed to register '{event}' callback", + "ev_register", + db=self._h, + type=type_name, + event=event, + cb=cid, + ) + except BaseException: + _forget(cid) + raise + handle = NativeEventHandle(self, event, type_name, reply["h"], cid) + self._events.add(handle) + return handle + + def clear(self) -> None: + """Remove everything this database registered (called when it closes); a failure here must not stop the close.""" + functions, self._functions = self._functions, {} + events, self._events = self._events, set() + for handle, cid in functions.values(): + try: + self._cb("", "fn_unregister", h=handle) + except Exception: # nosec B110 - the library may be shutting down + pass + _forget(cid) + for event in events: + try: + event.remove() + except Exception: # nosec B110 + pass diff --git a/src/arcadedb/_native/interrupts.py b/src/arcadedb/_native/interrupts.py new file mode 100644 index 0000000..c3bfcaa --- /dev/null +++ b/src/arcadedb/_native/interrupts.py @@ -0,0 +1,33 @@ +"""Ctrl-C inside a Python function or callback that the engine called, on the native backend. + +A ``KeyboardInterrupt`` cannot cross the engine as it is: it ends the statement as an engine error. It is remembered per +thread when the upcall catches it (``note``), and the place that turns the statement's failure into an ``ArcadeDBError`` +raises the ``KeyboardInterrupt`` instead (chained from that error, ``convert``), so Ctrl-C reaches the caller as a +``KeyboardInterrupt`` wherever the signal landed: in the function, in the engine between two calls, or on the way into the +callback (the same rule as the JPype backend, see ``arcadedb.callbacks``). +""" + +import threading + +_PENDING = threading.local() + + +def note(error: BaseException) -> None: + """Remember a ``KeyboardInterrupt`` raised in a callback on this thread.""" + if isinstance(error, KeyboardInterrupt): + _PENDING.interrupt = error + + +def clear() -> None: + """Forget an interrupt an earlier statement left (called when a statement starts).""" + _PENDING.interrupt = None + + +def convert(error: BaseException) -> BaseException: + """The ``KeyboardInterrupt`` to raise in place of ``error`` if a callback left one on this thread, else ``error``.""" + interrupt = getattr(_PENDING, "interrupt", None) + if interrupt is None: + return error + _PENDING.interrupt = None + interrupt.__cause__ = error + return interrupt diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index b2f16ac..d0c05d5 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -146,6 +146,8 @@ class _Params(ctypes.Structure): ctypes.POINTER(ctypes.c_void_p), ] lib.graal_attach_thread.restype = ctypes.c_int + lib.graal_get_current_thread.argtypes = [ctypes.c_void_p] + lib.graal_get_current_thread.restype = ctypes.c_void_p lib.graal_detach_thread.argtypes = [ctypes.c_void_p] lib.graal_detach_thread.restype = ctypes.c_int # a Java long is 64 bits on every platform; C long is 32 bits on Windows, so L is long long (the same size as c_long elsewhere) @@ -176,7 +178,7 @@ class _Params(ctypes.Structure): "arcadedb_n_async": [P, L, S, S], "arcadedb_n_import": [P, L, S], "arcadedb_n_export": [P, L, S, S, I, S], - "arcadedb_n_udf_register": [P, S, L, I, I], + "arcadedb_n_cb": [P, S, S], "arcadedb_n_set_upcall": [P, P], "arcadedb_free": [P, P], } @@ -222,6 +224,13 @@ def loaded() -> bool: def _thread_handle(): t = getattr(_local, "thread", None) if t is None: + # A Java thread that runs a Python callback (an async writer, an engine thread) is attached to the isolate already. + # Attaching it again would hand Python an owned handle that detaches the thread, while it still runs Java code, as + # soon as the callback's Python thread state ends: the process aborts. + current = _lib.graal_get_current_thread(_isolate) + if current: + t = _local.thread = _Thread(ctypes.c_void_p(current), owned=False) + return t.handle thread = ctypes.c_void_p() rc = _lib.graal_attach_thread(_isolate, ctypes.byref(thread)) if rc != 0: diff --git a/src/arcadedb/callbacks.py b/src/arcadedb/callbacks.py index 62274ac..17904a8 100644 --- a/src/arcadedb/callbacks.py +++ b/src/arcadedb/callbacks.py @@ -153,6 +153,38 @@ def _arity(fn: Callable) -> Tuple[int, int]: return minimum, maximum +def check_function( + name: str, fn: Callable, min_args: Optional[int], max_args: Optional[int] +) -> Tuple[int, int]: + """Validate the arguments of ``register_function`` and return the (min, max) argument counts to register. + + Shared by both backends (the JPype ``FunctionRegistration`` and the native one), so a bad call fails the same way. + """ + if not isinstance(name, str) or not _NAME_RE.match(name): + raise ValueError( + f"Invalid function name {name!r}: use letters, digits and underscores " + "(not starting with a digit), optionally joined by dots like 'py.slugify'" + ) + if not callable(fn): + raise TypeError("fn must be callable") + found_min, found_max = _arity(fn) + min_args = found_min if min_args is None else min_args + max_args = found_max if max_args is None else max_args + if min_args < 0 or max_args < min_args: + raise ValueError( + f"Invalid argument counts: min_args={min_args}, max_args={max_args}" + ) + return min_args, max_args + + +def check_event(event: str, fn: Callable) -> None: + """Validate the arguments of ``Database.on`` (shared by both backends).""" + if event not in EVENTS: + raise ValueError(f"Invalid event {event!r}. Must be one of: {list(EVENTS)}") + if not callable(fn): + raise TypeError("fn must be callable") + + # The proxy classes are defined once, on first use (the JVM must be running, so not at import): # a @JImplements class defined on every registration leaked about 6 KB of Python heap per # registration, which neither the engine's unregister nor gc gave back. State is per instance. @@ -273,20 +305,7 @@ def register( deterministic: bool, replace: bool, ) -> None: - if not isinstance(name, str) or not _NAME_RE.match(name): - raise ValueError( - f"Invalid function name {name!r}: use letters, digits and underscores " - "(not starting with a digit), optionally joined by dots like 'py.slugify'" - ) - if not callable(fn): - raise TypeError("fn must be callable") - found_min, found_max = _arity(fn) - min_args = found_min if min_args is None else min_args - max_args = found_max if max_args is None else max_args - if min_args < 0 or max_args < min_args: - raise ValueError( - f"Invalid argument counts: min_args={min_args}, max_args={max_args}" - ) + min_args, max_args = check_function(name, fn, min_args, max_args) functions = _bridge_class("com.arcadedb.python.PythonFunctions") try: @@ -377,10 +396,7 @@ def __repr__(self) -> str: def register_event(database, event: str, fn: Callable, type_name: Optional[str]): """Register ``fn`` for ``event`` on the database, or on one type, and return its handle.""" - if event not in EVENTS: - raise ValueError(f"Invalid event {event!r}. Must be one of: {list(EVENTS)}") - if not callable(fn): - raise TypeError("fn must be callable") + check_event(event, fn) java_db = database.get_java_database() try: if type_name is None: diff --git a/tests/test_native_udf.py b/tests/test_native_udf.py index e5dc83b..234794a 100644 --- a/tests/test_native_udf.py +++ b/tests/test_native_udf.py @@ -9,7 +9,7 @@ def test_a_python_function_in_a_select(temp_db): - temp_db.register_function("py_double", lambda x: x * 2, 1, 1) + temp_db.register_function("py_double", lambda x: x * 2, min_args=1, max_args=1) row = temp_db.query("sql", "SELECT py_double(21) AS v").first() assert row.get("v") == 42 @@ -19,7 +19,9 @@ def test_a_python_function_reads_columns_and_returns_text(temp_db): with temp_db.transaction(): for i in range(3): temp_db.command("sql", "INSERT INTO T SET n = ?, s = ?", i, f"x{i}") - temp_db.register_function("py_tag", lambda n, s: f"{s}:{n + 1}", 2, 2) + temp_db.register_function( + "py_tag", lambda n, s: f"{s}:{n + 1}", min_args=2, max_args=2 + ) rows = temp_db.query("sql", "SELECT py_tag(n, s) AS v FROM T ORDER BY n").to_list() assert [r["v"] for r in rows] == ["x0:1", "x1:2", "x2:3"] @@ -28,7 +30,7 @@ def test_an_exception_in_the_function_fails_the_statement_with_its_text(temp_db) def boom(x): raise ValueError("no way") - temp_db.register_function("py_boom", boom, 1, 1) + temp_db.register_function("py_boom", boom, min_args=1, max_args=1) with pytest.raises(ArcadeDBError, match="no way"): temp_db.query("sql", "SELECT py_boom(1) AS v").first() @@ -41,7 +43,7 @@ def test_a_function_may_call_back_into_the_database(temp_db): def count_c(): return temp_db.query("sql", "SELECT count(*) AS c FROM C").first().get("c") - temp_db.register_function("py_count_c", count_c, 0, 0) + temp_db.register_function("py_count_c", count_c, min_args=0, max_args=0) assert temp_db.query("sql", "SELECT py_count_c() AS v").first().get("v") == 1 diff --git a/tests/test_python_callbacks.py b/tests/test_python_callbacks.py index 061293b..d3e167b 100644 --- a/tests/test_python_callbacks.py +++ b/tests/test_python_callbacks.py @@ -219,17 +219,18 @@ def test_a_numpy_array_is_a_return_value(db): strict=False, ) def test_registering_again_and_again_does_not_grow_the_python_heap(db): - """The JPype proxy class is defined once, not per registration (it leaked about 6 KB each).""" + """The JPype proxy class is defined once, not per registration (it leaked about 6 KB each); on the native backend the callback table must not keep a removed callback.""" import gc import time import tracemalloc - import jpype - def settle(): - # JPype frees a proxy's Python object once the JVM has collected the Java side - jpype.JClass("java.lang.System").gc() - time.sleep(0.3) + if arcadedb.__backend__ == "jvm": + import jpype + + # JPype frees a proxy's Python object once the JVM has collected the Java side + jpype.JClass("java.lang.System").gc() + time.sleep(0.3) gc.collect() def cycle(n): From bab98173c97c2e2dfdaf406a25737a3e18d1edfb Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 20:49:09 +0900 Subject: [PATCH 16/33] Native: a callback reply lives in library memory that Java frees A callback on a Java thread (an async writer) ends its Python thread state when it returns, and the thread-local buffer that held the reply went with it: the engine read freed memory ("bad JSON value at 0" for a Python function called from an async command). The reply is now allocated by the library (arcadedb_n_alloc), and Java copies the text and frees it. Co-Authored-By: Claude Sonnet 5.5 --- .../org/humemai/nativelib/NativeExtras.java | 23 ++++++++++++++++--- src/arcadedb/_native/extras.py | 9 +++++++- src/arcadedb/_native/lib.py | 12 ++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index ebbddf5..df6e216 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -6,8 +6,8 @@ * Same reply convention as ArcadeNative: a malloc'd JSON text, {"ok":true,...} or {"ok":false,"error":..,"cls":..,"chain":..}. * * Upcalls. Python registers ONE ctypes function pointer with arcadedb_n_set_upcall: CCharPointer fn(long id, CCharPointer json). - * `id` names a Python-side callback, `json` is the message, the return value is a JSON reply that Python keeps alive until its - * next call on the same thread (Java copies it at once). The call is a transition-to-native C function pointer call, so the + * `id` names a Python-side callback, `json` is the message, the return value is a JSON reply in memory from arcadedb_n_alloc + * (NULL for no reply); Java copies the text and frees it. The call is a transition-to-native C function pointer call, so the * isolate can run a safepoint or a GC while the Python callback holds the GIL and calls back into the library on its thread. * ctypes takes the GIL itself (PyGILState_Ensure) when a foreign Java thread (an async writer) or the engine thread inside a * query calls the pointer. @@ -52,6 +52,7 @@ import com.arcadedb.schema.EdgeType; import com.arcadedb.schema.VertexType; import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.function.InvokeCFunctionPointer; @@ -93,10 +94,26 @@ static String upcall(final long id, final String json) { final PyCallback fn = WordFactory.pointer(raw); try (CTypeConversion.CCharPointerHolder in = CTypeConversion.toCString(json)) { final CCharPointer reply = fn.invoke(id, in.get()); - return reply.isNull() ? null : CTypeConversion.toJavaString(reply); + if (reply.isNull()) + return null; + try { + return CTypeConversion.toJavaString(reply); + } finally { + UnmanagedMemory.free(reply); + } } } + /** + * Memory for the reply of a Python callback (see upcall: Java copies the text and frees it). Python cannot keep the reply in + * its own memory: a callback on a Java thread (an async writer) ends its Python thread state when it returns, and that frees + * what the state held before Java has read the text. + */ + @CEntryPoint(name = "arcadedb_n_alloc") + public static CCharPointer alloc(final IsolateThread thread, final long size) { + return UnmanagedMemory.malloc(WordFactory.unsigned(size)); + } + /** A callback failure that must not stop an async writer: reported on stderr, not lost. */ static void swallowed(final String where, final Throwable t) { System.err.println("[arcadedb-native] " + where + " failed: " + t); diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py index 96873ef..f5c02c2 100644 --- a/src/arcadedb/_native/extras.py +++ b/src/arcadedb/_native/extras.py @@ -51,7 +51,14 @@ def _dispatch(cid, raw): data = json.dumps({"error": f"{type(exc).__name__}: {exc}"}).encode("utf-8") if data is None: return None - # kept alive until this thread's next callback: Java copies the text before it returns + size = len(data) + 1 + pointer = lib.alloc(size) + if pointer: + # library memory: Java copies the text and frees it (on a Java thread this callback's Python thread state, and + # everything a thread-local held, is gone when it returns) + ctypes.memmove(pointer, data + b"\0", size) + return pointer + # a library older than arcadedb_n_alloc: kept alive until this thread's next callback buf = ctypes.create_string_buffer(data) _reply.buf = buf return ctypes.addressof(buf) diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index d0c05d5..c02616a 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -179,6 +179,7 @@ class _Params(ctypes.Structure): "arcadedb_n_import": [P, L, S], "arcadedb_n_export": [P, L, S, S, I, S], "arcadedb_n_cb": [P, S, S], + "arcadedb_n_alloc": [P, L], "arcadedb_n_set_upcall": [P, P], "arcadedb_free": [P, P], } @@ -277,6 +278,17 @@ def call_bin(name: str, *args): return raw.decode("utf-8"), floats +def alloc(size: int): + """Memory from the library's allocator, or None when the library is older than ``arcadedb_n_alloc``. + + The reply of a callback goes through it: Java copies the text and frees the memory, so the reply outlives the callback's + Python thread state (a callback on an async writer thread ends that state as it returns). + """ + lib = _lib or _load() + fn = getattr(lib, "arcadedb_n_alloc", None) + return None if fn is None else fn(_thread_handle(), size) + + def call_void(name: str, *args) -> None: """Run an entry point that returns nothing (``arcadedb_n_set_upcall``).""" lib = _lib or _load() From bae8dfc5cfde0d790ac66cc8aa8765e0131e84a7 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 21:13:36 +0900 Subject: [PATCH 17/33] Native: a vetoed save is not an error; an after_delete event of an unloaded record reports only its identity save() of a new record whose before_create callback vetoed it returns the unsaved record, as on the JPype backend, instead of raising "the statement returned no record". The after_delete event of a record that was not loaded (a delete by RID) found no record to read the properties from and failed the delete; it sends the identity only now (Java change, in the next native image). Co-Authored-By: Claude Sonnet 5.5 --- .../org/humemai/nativelib/NativeExtras.java | 17 +++++++++++------ src/arcadedb/_native/extras.py | 13 ++++++++++++- src/arcadedb/_native/records.py | 5 +++++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index df6e216..aa54e4b 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -744,13 +744,18 @@ static void appendRecord(final StringBuilder sb, final Document doc) { sb.append("\"@type\":"); NativeCodec.quote(sb, doc.getTypeName()); sb.append(",\"@cat\":\"").append(doc instanceof Vertex ? 'v' : doc instanceof Edge ? 'e' : 'd').append('"'); - for (final String name : doc.getPropertyNames()) { - sb.append(','); - NativeCodec.quote(sb, name); - sb.append(':'); - NativeCodec.append(sb, doc.get(name)); + final StringBuilder props = new StringBuilder(); + try { + for (final String name : doc.getPropertyNames()) { + props.append(','); + NativeCodec.quote(props, name); + props.append(':'); + NativeCodec.append(props, doc.get(name)); + } + } catch (final com.arcadedb.exception.RecordNotFoundException e) { + props.setLength(0); // an after_delete event of a record that was not loaded: it is gone, only its identity is left to report } - sb.append('}'); + sb.append(props).append('}'); } /** diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py index f5c02c2..a580c99 100644 --- a/src/arcadedb/_native/extras.py +++ b/src/arcadedb/_native/extras.py @@ -868,6 +868,14 @@ def handler(message): return handler +_veto = threading.local() + + +def veto_count() -> int: + """How many ``before_*`` callbacks vetoed a change on this thread so far (``save`` tells a veto from a failure by it).""" + return getattr(_veto, "count", 0) + + def _event_handler(event: str, fn: Callable, database_ref) -> Callable: is_before = event.startswith("before_") @@ -875,7 +883,10 @@ def handler(message): from .records import _wrap_dict result = fn(_wrap_dict(database_ref(), message["rec"])) - return {"v": not (is_before and result is False)} + vetoed = is_before and result is False + if vetoed: + _veto.count = veto_count() + 1 + return {"v": not vetoed} return handler diff --git a/src/arcadedb/_native/records.py b/src/arcadedb/_native/records.py index 7f03d2f..dcdbd1b 100644 --- a/src/arcadedb/_native/records.py +++ b/src/arcadedb/_native/records.py @@ -99,9 +99,14 @@ def set(self, name: str, value: Any): def save(self): db = self._db if self._rid is None: + from .extras import veto_count + + vetoes = veto_count() row = db.command( "sql", self._CREATE.format(t=_q(self._type)), {"row": self._props} ).first() + if row is None and veto_count() != vetoes: + return self # a before_create callback vetoed it: nothing is written, no error (as on the JPype backend) self._adopt(row) else: db.command( From fbd8664fa74d3527b51e0f0bbf76039977377e2a Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 21:22:10 +0900 Subject: [PATCH 18/33] Native: the reply of a callback goes through an out slot, never a return value A Ctrl-C raised as Python enters the callback (before its try) left the C return value unset, and the engine took whatever the return register held: the pointer of an earlier reply that Java had freed already. Java then freed it again and the process aborted in free(). The callback is void now and writes its reply to a slot that Java zeroes before the call, so a callback that ends early leaves NULL; the failed statement becomes a KeyboardInterrupt. Python writes only library memory into the slot (the fallback to a Python-owned buffer is gone), and the unraisablehook is not used. A test runs the SIGINT loop in a fresh process and fails on a nonzero exit. Co-Authored-By: Claude Sonnet 5.5 --- .../org/humemai/nativelib/NativeExtras.java | 21 +++++--- src/arcadedb/_native/extras.py | 49 +++++------------ src/arcadedb/_native/interrupts.py | 12 ++++- src/arcadedb/_native/lib.py | 7 ++- tests/test_python_function_interrupt.py | 54 +++++++++++++++++++ 5 files changed, 96 insertions(+), 47 deletions(-) diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index aa54e4b..9903789 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -5,9 +5,9 @@ * database settings, import and export, and the Java->Python upcall (async callbacks, Python functions in SQL). * Same reply convention as ArcadeNative: a malloc'd JSON text, {"ok":true,...} or {"ok":false,"error":..,"cls":..,"chain":..}. * - * Upcalls. Python registers ONE ctypes function pointer with arcadedb_n_set_upcall: CCharPointer fn(long id, CCharPointer json). - * `id` names a Python-side callback, `json` is the message, the return value is a JSON reply in memory from arcadedb_n_alloc - * (NULL for no reply); Java copies the text and frees it. The call is a transition-to-native C function pointer call, so the + * Upcalls. Python registers ONE ctypes function pointer with arcadedb_n_set_upcall: void fn(long id, CCharPointer json, void **out). + * `id` names a Python-side callback, `json` is the message, the JSON reply is written to *out as memory from arcadedb_n_alloc + * (left NULL for no reply); Java copies the text and frees it. The call is a transition-to-native C function pointer call, so the * isolate can run a safepoint or a GC while the Python callback holds the GIL and calls back into the library on its thread. * ctypes takes the GIL itself (PyGILState_Ensure) when a foreign Java thread (an async writer) or the engine thread inside a * query calls the pointer. @@ -58,6 +58,7 @@ import org.graalvm.nativeimage.c.function.InvokeCFunctionPointer; import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; +import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; import java.util.ArrayList; @@ -76,7 +77,7 @@ private NativeExtras() { interface PyCallback extends CFunctionPointer { @InvokeCFunctionPointer - CCharPointer invoke(long id, CCharPointer json); + void invoke(long id, CCharPointer json, Pointer out); } private static volatile long upcallPointer; @@ -92,15 +93,23 @@ static String upcall(final long id, final String json) { if (raw == 0) throw new IllegalStateException("no Python callback is registered (arcadedb_n_set_upcall)"); final PyCallback fn = WordFactory.pointer(raw); + // The reply comes back through *out, which is zeroed here, never through the return value of the call: a callback that + // fails before it writes (a Ctrl-C raised as Python enters it) leaves a NULL, where a C return value would be whatever the + // return register held. + final Pointer out = UnmanagedMemory.malloc(8); + out.writeLong(0, 0L); try (CTypeConversion.CCharPointerHolder in = CTypeConversion.toCString(json)) { - final CCharPointer reply = fn.invoke(id, in.get()); + fn.invoke(id, in.get(), out); + final CCharPointer reply = out.readWord(0); if (reply.isNull()) return null; try { return CTypeConversion.toJavaString(reply); } finally { - UnmanagedMemory.free(reply); + UnmanagedMemory.free(reply); // memory of arcadedb_n_alloc, the only kind Python writes into *out } + } finally { + UnmanagedMemory.free(out); } } diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py index a580c99..072184e 100644 --- a/src/arcadedb/_native/extras.py +++ b/src/arcadedb/_native/extras.py @@ -10,7 +10,6 @@ import ctypes import itertools import json -import sys import threading import weakref from collections.abc import Mapping @@ -25,13 +24,13 @@ # -------------------------------------------------------------------------------------------------------- upcall +# (Java long id, char *json, void **out): the reply goes through *out, not through a return value (see _dispatch) _UPCALL_TYPE = ctypes.CFUNCTYPE( - ctypes.c_void_p, ctypes.c_longlong, ctypes.c_char_p -) # (Java long id, char *json) + None, ctypes.c_longlong, ctypes.c_char_p, ctypes.c_void_p +) _callbacks: dict = {} _callback_ids = itertools.count(1) _callbacks_lock = threading.Lock() -_reply = threading.local() _upcall_ref = None @@ -40,7 +39,15 @@ def _enc(value) -> str: return json.dumps(_prep(value), default=_default) -def _dispatch(cid, raw): +def _dispatch(cid, raw, out): + """Run callback ``cid`` with the message ``raw`` and write the JSON reply to ``*out`` (Java zeroed it before the call). + + The reply never travels as a return value. A C callback that returns without setting its return value hands the engine + whatever the return register held, here the pointer of an earlier reply, which Java had already freed: an abort in + ``free()``. That is what happens when a Ctrl-C is raised as this function is entered, before its ``try``. Through ``*out`` + the engine sees NULL then, fails the statement, and the error becomes a KeyboardInterrupt (``interrupts.convert``). + The memory comes from the library (``arcadedb_n_alloc``): Java copies the text and frees it, so it must be nothing else. + """ try: fn = _callbacks.get(cid) message = loads(raw.decode("utf-8")) if raw else {} @@ -50,46 +57,18 @@ def _dispatch(cid, raw): interrupts.note(exc) data = json.dumps({"error": f"{type(exc).__name__}: {exc}"}).encode("utf-8") if data is None: - return None + return size = len(data) + 1 pointer = lib.alloc(size) if pointer: - # library memory: Java copies the text and frees it (on a Java thread this callback's Python thread state, and - # everything a thread-local held, is gone when it returns) ctypes.memmove(pointer, data + b"\0", size) - return pointer - # a library older than arcadedb_n_alloc: kept alive until this thread's next callback - buf = ctypes.create_string_buffer(data) - _reply.buf = buf - return ctypes.addressof(buf) - - -def _install_unraisable_hook() -> None: - """Keep a Ctrl-C that arrives as the callback is entered. - - CPython runs a pending signal handler at the first bytecode of a function, so a KeyboardInterrupt can be raised before - ``_dispatch`` reaches its ``try``: ctypes then reports it as unraisable and returns NULL to the engine. The hook records it - as the interrupt of this thread (the engine fails the statement on a NULL reply, and the error becomes the - KeyboardInterrupt); every other unraisable exception goes to the hook that was there. - """ - previous = sys.unraisablehook - - def hook(unraisable): - if unraisable.object is _dispatch and isinstance( - unraisable.exc_value, KeyboardInterrupt - ): - interrupts.note(unraisable.exc_value) - else: - previous(unraisable) - - sys.unraisablehook = hook + ctypes.c_void_p.from_address(out).value = pointer def _ensure_upcall() -> None: global _upcall_ref if _upcall_ref is not None: return - _install_unraisable_hook() lib.call_void( "arcadedb_n_set_upcall", ctypes.cast(_make_pointer(), ctypes.c_void_p) ) diff --git a/src/arcadedb/_native/interrupts.py b/src/arcadedb/_native/interrupts.py index c3bfcaa..cfe990f 100644 --- a/src/arcadedb/_native/interrupts.py +++ b/src/arcadedb/_native/interrupts.py @@ -23,11 +23,19 @@ def clear() -> None: _PENDING.interrupt = None +# What the engine says when a callback returned no reply (NativeExtras.answer): the callback ended before it could write one. +# That happens when a Ctrl-C is raised as Python enters the callback, before its code (and its try) runs. +_NO_REPLY = "was interrupted before Python returned a value" + + def convert(error: BaseException) -> BaseException: - """The ``KeyboardInterrupt`` to raise in place of ``error`` if a callback left one on this thread, else ``error``.""" + """The ``KeyboardInterrupt`` to raise in place of ``error``: one a callback left on this thread, or the Ctrl-C that + ended a callback before it could reply. Else ``error`` itself.""" interrupt = getattr(_PENDING, "interrupt", None) if interrupt is None: - return error + if _NO_REPLY not in str(error): + return error + interrupt = KeyboardInterrupt("interrupted while a Python function was called") _PENDING.interrupt = None interrupt.__cause__ = error return interrupt diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index c02616a..95cc7a3 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -279,14 +279,13 @@ def call_bin(name: str, *args): def alloc(size: int): - """Memory from the library's allocator, or None when the library is older than ``arcadedb_n_alloc``. + """Memory from the library's allocator (``arcadedb_n_alloc``), for the reply of a callback. - The reply of a callback goes through it: Java copies the text and frees the memory, so the reply outlives the callback's + Java copies the text and frees the memory, so it must come from here and from nowhere else; it also outlives the callback's Python thread state (a callback on an async writer thread ends that state as it returns). """ lib = _lib or _load() - fn = getattr(lib, "arcadedb_n_alloc", None) - return None if fn is None else fn(_thread_handle(), size) + return lib.arcadedb_n_alloc(_thread_handle(), size) def call_void(name: str, *args) -> None: diff --git a/tests/test_python_function_interrupt.py b/tests/test_python_function_interrupt.py index c2a3332..0bcc87b 100644 --- a/tests/test_python_function_interrupt.py +++ b/tests/test_python_function_interrupt.py @@ -11,6 +11,8 @@ import os import signal +import subprocess +import sys import threading import time @@ -82,3 +84,55 @@ def test_a_statement_that_fails_otherwise_is_still_an_arcadedb_error(db): with pytest.raises(arcadedb.ArcadeDBError) as raised: db.query("sql", "SELECT FROM NoSuchType") assert not isinstance(raised.value, KeyboardInterrupt) + + +_SIGINT_LOOP = r""" +import os, signal, tempfile, threading, time +import numpy as np +import arcadedb + +with tempfile.TemporaryDirectory() as directory: + db = arcadedb.create_database(os.path.join(directory, "db")) + db.command("sql", "CREATE DOCUMENT TYPE T") + db.insert_columns("T", {"n": np.arange(150_000, dtype=np.int64)}) + db.register_function("plus_one", lambda x: x + 1) + interrupted = 0 + for delay in (0.05, 0.15, 0.25, 0.35, 0.45, 0.55): + timer = threading.Timer(delay, os.kill, (os.getpid(), signal.SIGINT)) + timer.start() + try: + db.query("sql", "SELECT sum(plus_one(n)) AS s FROM T").first() + done = True + except KeyboardInterrupt: + done = False + interrupted += 1 + finally: + timer.cancel() + timer.join() + if done: + try: + time.sleep(0.05) + except KeyboardInterrupt: + interrupted += 1 + assert db.count_type("T") == 150_000 + db.close() + print("interrupted", interrupted) +""" + + +@pytest.mark.skipif( + sys.platform == "win32", reason="a signal cannot be sent to the own process here" +) +def test_sigint_loop_in_a_fresh_process_ends_cleanly(): + """The same loop in its own process. A Ctrl-C raised as the Python callback is entered used to leave the engine a stale + reply pointer, and the process aborted in ``free()`` (nonzero exit); pytest's own hooks hid it inside a test run. + """ + result = subprocess.run( + [sys.executable, "-c", _SIGINT_LOOP], + capture_output=True, + text=True, + timeout=600, + check=False, + ) + assert result.returncode == 0, result.stderr[-2000:] + assert int(result.stdout.split()[-1]) >= 1, result.stdout From a561129bbe54eb2665278eefca69e60dc65ff36f Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 21:24:34 +0900 Subject: [PATCH 19/33] Native parity: install setuptools in the parity env, pass SYSTEMROOT to the Windows child test_wheel_packages builds with setup.py build_py; the uv parity env had no setuptools, so the test failed on windows-x64 and macos-arm64 and passed on Linux only because the runner's Python ships it (humemai/arcadedb-embedded-python#539). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test-native.yml | 2 +- tests/test_wheel_packages.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index 30fb575..fea41d0 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -512,7 +512,7 @@ jobs: cores="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)" limit=() if [ "$FINAL" = true ]; then limit=(--max-notyet 0); fi - uv run --no-project --python "$PY" --with "$w" --with pytest --with numpy --with requests --with jpype1 --with pyarrow \ + uv run --no-project --python "$PY" --with "$w" --with pytest --with setuptools --with numpy --with requests --with jpype1 --with pyarrow \ --with polars --with pandas --with duckdb --with networkx --with "psycopg[binary]" --with adbc-driver-postgresql \ --with redis --with neo4j --with ladybug \ python native/wheel/parity_wheel.py --repo "$GITHUB_WORKSPACE" \ diff --git a/tests/test_wheel_packages.py b/tests/test_wheel_packages.py index af217db..6bbf34f 100644 --- a/tests/test_wheel_packages.py +++ b/tests/test_wheel_packages.py @@ -61,6 +61,8 @@ def test_backend_selection_imports_without_native_library(tmp_path): """A fresh interpreter, as the wheel's ``test_install.py`` runs: the JVM backend is the default.""" root = _built_root(tmp_path) env = {"PATH": os.environ["PATH"]} + if "SYSTEMROOT" in os.environ: # a Windows child without it cannot import socket + env["SYSTEMROOT"] = os.environ["SYSTEMROOT"] if root is not None: env["PYTHONPATH"] = str(root) code = "import arcadedb.core, arcadedb._native as n;" "assert n.enabled() is False" From 702e49c39602fe1a2c1fdac45b32bad21cded54c Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 21:28:16 +0900 Subject: [PATCH 20/33] Native: restore the schema property; the SIGINT subprocess test installs the default handler NativeDatabase.on() took the @property line of schema with it when register_function was replaced; db.schema was a bound method again. The fresh-process SIGINT test sets the default Ctrl-C handler in the child: a parent started as a background job ignores SIGINT, and Python then installs no handler. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/database.py | 1 + tests/test_python_function_interrupt.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 9646cbb..c8695aa 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -916,6 +916,7 @@ def on(self, event, fn, type_name=None): self._check_not_closed() return self._registrations().on(event, fn, type_name) + @property def schema(self): """DDL helpers over SQL statements (the DSL-first subset); everything else is not on native yet.""" self._check_not_closed() diff --git a/tests/test_python_function_interrupt.py b/tests/test_python_function_interrupt.py index 0bcc87b..4a0d031 100644 --- a/tests/test_python_function_interrupt.py +++ b/tests/test_python_function_interrupt.py @@ -91,6 +91,8 @@ def test_a_statement_that_fails_otherwise_is_still_an_arcadedb_error(db): import numpy as np import arcadedb +# a parent started as a background job ignores SIGINT, and a child inherits that: Python then installs no handler +signal.signal(signal.SIGINT, signal.default_int_handler) with tempfile.TemporaryDirectory() as directory: db = arcadedb.create_database(os.path.join(directory, "db")) db.command("sql", "CREATE DOCUMENT TYPE T") From 7f88779cffc7c4e258eed0759bf01411ca08ff6b Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 20:59:33 +0900 Subject: [PATCH 21/33] Native CI: a cheap Python-version job on the built linux-x86_64 wheel (3.11, 3.13, 3.14, 3.14t) select_native_matrix.py prints a pyver list: the supported Pythons the full parity run of the event leaves out, plus the free-threaded 3.14t. The new pyver job of test-native.yml installs the already built linux-x86_64 wheel (py3-none, no rebuild) into each and runs 13 test files (14 where polars installs) through native/wheel/parity_wheel.py --files, with the same gate as the test job. The job is informational: release-native.yml reads only the native-passed-* markers and the native-wheel-* wheels, so a red pyver leg never blocks a release (a test pins that). 3.14t runs without polars, which has no free-threaded wheel, and so without test_dataframe_ingest.py. A step reports whether the GIL stays off. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/test-native.yml | 128 ++++++++++++++++++++ docs/modules/ROOT/pages/development/ci.adoc | 1 + scripts/select_native_matrix.py | 28 ++++- tests/test_select_native_matrix.py | 60 ++++++++- 4 files changed, 213 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index fea41d0..daeadd9 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -11,6 +11,11 @@ # workflow_call from release-native.yml with `release: true`: all four platforms, Python 3.11 to 3.14 # This workflow never publishes: it has no id-token permission and no environment. # +# Beside the full parity runs, the `pyver` job installs the already built linux-x86_64 wheel (py3-none: no rebuild) into the Pythons the full +# runs of the event leave out (a pull request: 3.11, 3.13, 3.14; main, nightly, dispatch: 3.11, 3.13; a release: none) and into the free-threaded +# 3.14t, and runs a small subset of the tests there (about 20 s of tests per Python). It is informational: release-native.yml does not need it. +# scripts/select_native_matrix.py decides its legs. +# # One wheel per platform, py3-none-: # linux-x86_64 manylinux_2_34 container manylinux_2_34_x86_64 libarcadedb.so + libjvector-x86_64.so # linux-aarch64 manylinux_2_34 aarch64 container manylinux_2_34_aarch64 libarcadedb.so @@ -106,6 +111,7 @@ jobs: final: ${{ steps.ver.outputs.final }} build: ${{ steps.plan.outputs.build }} test: ${{ steps.plan.outputs.test }} + pyver: ${{ steps.plan.outputs.pyver }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -522,6 +528,7 @@ jobs: run: | set -euo pipefail tsv="$T/parity/parity-wheel.tsv" + # the pyver job has a copy of this awk: keep the two in sync awk -F'\t' ' NR == 1 { next } $1 == "TOTAL" { passed = $2; failed = $3; err = $4; notyet = $5; na = $6; skipped = $7; tot = 1; next } @@ -560,3 +567,124 @@ jobs: ${{ runner.temp }}/parity/parity-wheel-xml/ if-no-files-found: warn retention-days: 30 + + # The cheap Python-version coverage of the wheel of linux-x86_64 (py3-none: the same file for every Python, nothing is rebuilt): a subset of + # the test files, not the whole suite. The legs are the supported Pythons the full parity runs of this event leave out, plus the + # free-threaded 3.14t (scripts/select_native_matrix.py). + # INFORMATIONAL for now: release-native.yml (verify and the publish jobs) reads only the native-passed-* markers and the native-wheel-* wheels, + # never an artifact of this job, so a red leg never blocks a release. Promote it once it has been green for a while. + # 3.14t runs without polars, and so without test_dataframe_ingest.py (it imports polars at module level): polars ships abi3 wheels only + # (polars and polars-runtime-32 have no cp314t wheel, and an abi3 wheel cannot be installed on a free-threaded build). The other + # dependencies of the subset (numpy, pandas, pyarrow, jpype1, requests) all have cp314t wheels. polars is an optional extra of the wheel. + pyver: + name: Python ${{ matrix.python }} on the wheel (linux-x86_64) + needs: [plan, build] + # The linux-x86_64 build may have failed: then there is no wheel to download and this shows red, as the test job does. + if: ${{ !cancelled() && needs.plan.outputs.run == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.pyver) }} + env: + KEY: linux-x86_64 + PY: ${{ matrix.python }} + WITH_POLARS: ${{ matrix.polars }} + FREE_THREADED: ${{ matrix.freethreaded }} + # The subset: ctypes layer and database lifecycle (test_core, test_native_udf), queries and results (test_resultset), parameters + # (test_parameter_binding), type conversion (test_type_conversion), transactions (test_transaction_config), async (test_async_executor), + # threads (test_concurrency, test_free_threading), vectors (test_vector), numpy (test_numpy_support), arrow (test_resultset_arrow), pandas + # and arrow readers (test_columnar_readers). test_dataframe_ingest (pandas, polars and arrow ingest) is added where polars installs. + SUBSET: test_core.py,test_resultset.py,test_type_conversion.py,test_parameter_binding.py,test_transaction_config.py,test_async_executor.py,test_vector.py,test_numpy_support.py,test_resultset_arrow.py,test_columnar_readers.py,test_native_udf.py,test_concurrency.py,test_free_threading.py + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Paths + run: echo "T=$RUNNER_TEMP" >> "$GITHUB_ENV" + + - name: Install uv + run: | + installer="$(mktemp)" + curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" "https://astral.sh/uv/${UV_VERSION}/install.sh" + sh "$installer" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + # The wheel of the build job, not a rebuild: the very file the test job installs. + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: native-wheel-linux-x86_64 + path: ${{ runner.temp }}/wheel + + - name: Run the subset against the installed wheel + run: | + set -euo pipefail + w="$(ls "$T"/wheel/*.whl)" + mkdir -p "$T/parity" + files="$SUBSET" + with_polars=() + if [ "$WITH_POLARS" = true ]; then + files="$files,test_dataframe_ingest.py" + with_polars=(--with polars) + fi + echo "python $PY, files: $files" + cores="$(nproc)" + uv run --no-project --python "$PY" --with "$w" --with pytest --with numpy --with requests --with jpype1 --with pyarrow \ + --with pandas ${with_polars[@]+"${with_polars[@]}"} \ + python native/wheel/parity_wheel.py --repo "$GITHUB_WORKSPACE" --files "$files" \ + --out "$T/parity/parity-wheel.tsv" --cores "0-$((cores - 1))" | tee "$T/parity/parity-wheel.run.log" + + # A free-threaded leg only means something if the GIL stays off once the wheel and the extension modules are loaded. + - name: Report whether the GIL is enabled + run: | + set -euo pipefail + w="$(ls "$T"/wheel/*.whl)" + uv run --no-project --python "$PY" --with "$w" --with numpy --with pandas --with pyarrow --with jpype1 python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY" + import os + import sys + + import arcadedb # noqa: F401 + import jpype # noqa: F401 + import numpy # noqa: F401 + import pandas # noqa: F401 + import pyarrow # noqa: F401 + + gil = sys._is_gil_enabled() if hasattr(sys, "_is_gil_enabled") else True + print(f"python {sys.version.split()[0]}, GIL enabled after loading the wheel and its dependencies: {gil}") + if os.environ["FREE_THREADED"] == "true" and gil: + print("::warning::the free-threaded leg re-enabled the GIL: it does not test free threading") + PY + + - name: "Gate: no failure, error, crash, timeout or uncounted skip" + run: | + set -euo pipefail + tsv="$T/parity/parity-wheel.tsv" + # the same awk as the gate of the test job: keep the two in sync + awk -F'\t' ' + NR == 1 { next } + $1 == "TOTAL" { passed = $2; failed = $3; err = $4; notyet = $5; na = $6; skipped = $7; tot = 1; next } + $8 != "" { bad = bad " " $1 "(" $8 ")" } + END { + if (!tot) { print "no TOTAL row" > "/dev/stderr"; exit 1 } + printf "parity %s py%s: passed=%s failed=%s error=%s notyet=%s na_jvm=%s skipped=%s\n", ENVIRON["KEY"], ENVIRON["PY"], passed, failed, err, notyet, na, skipped + if (bad != "") { print "files with a status:" bad > "/dev/stderr"; exit 1 } + if (passed + 0 == 0 || failed + 0 != 0 || err + 0 != 0) exit 1 + # this repository accepts no skips; NOTYET and NA-JVM are the two counted kinds of the native build + if (skipped + 0 != 0) { print "skipped tests that are neither NOTYET nor NA-JVM: " skipped > "/dev/stderr"; exit 1 } + }' "$tsv" | tee -a "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: ${{ !cancelled() }} + with: + name: native-parity-pyver-py${{ matrix.python }} + path: | + ${{ runner.temp }}/parity/parity-wheel.tsv + ${{ runner.temp }}/parity/parity-wheel.run.log + ${{ runner.temp }}/parity/parity-wheel-xml/ + if-no-files-found: warn + retention-days: 30 diff --git a/docs/modules/ROOT/pages/development/ci.adoc b/docs/modules/ROOT/pages/development/ci.adoc index 9a4c289..d1ba2ae 100644 --- a/docs/modules/ROOT/pages/development/ci.adoc +++ b/docs/modules/ROOT/pages/development/ci.adoc @@ -55,6 +55,7 @@ The workflow `test-native.yml` builds the native image of the `arcadedb` package * A pull request runs it only when native paths change: `native/**`, `src/arcadedb/_native/**`, the native tests, or `test-native.yml` itself. It then runs on `linux-x86_64` with Python 3.12 only. * A push to `main`, the nightly run, a manual run (dispatch) run all four platforms: `linux-x86_64`, `linux-aarch64`, `macos-arm64` and `windows-x64` (Python 3.12 and 3.14). A release (`release-native.yml` calls the workflow with `release: true`) runs the same four platforms on all four Python versions, 16 legs. +* The `pyver` job installs the already built `linux-x86_64` wheel (no rebuild) into the Pythons that the full parity runs of the event leave out (3.11, 3.13 and 3.14 on a pull request; 3.11 and 3.13 on a push to `main`, the nightly run and a dispatch; none on a release) and into the free-threaded 3.14t, and runs 13 test files there (14 where polars installs), about 20 seconds of tests per Python. It is informational: `release-native.yml` reads only the `native-passed-*` markers and the `native-wheel-*` wheels, so a red `pyver` leg never blocks a release. The 3.14t leg has no polars (no free-threaded wheel exists), so it skips `test_dataframe_ingest.py`. `scripts/select_native_matrix.py` decides its legs. * A native image build takes about 1 to 2.5 hours for each platform. The organisation shares one runner limit with upstream ArcadeDB, so do not add runs lightly. Change the paths or the matrix of `test-native.yml` only for a reason that you can write down. == Change the selection diff --git a/scripts/select_native_matrix.py b/scripts/select_native_matrix.py index d027718..be4cb22 100644 --- a/scripts/select_native_matrix.py +++ b/scripts/select_native_matrix.py @@ -9,8 +9,10 @@ usage: select_native_matrix.py EVENT EVENT: pull_request | push | schedule | workflow_dispatch | release -Prints `build=` and `test=` lines (GITHUB_OUTPUT format). Publishing is not decided here: release-native.yml -does it. The GraalVM CE archives are pinned by sha256 per platform (GraalVM Community Edition only, never Oracle GraalVM). +Prints `build=`, `test=` and `pyver=` lines (GITHUB_OUTPUT format). Publishing is not decided here: +release-native.yml does it. `pyver` is the cheap Python-version job (see PYVER_NOTE): a subset of the tests against the built +linux-x86_64 wheel on the Pythons the full parity run of the event does not cover, and always on the free-threaded 3.14t. It never +gates a release. The GraalVM CE archives are pinned by sha256 per platform (GraalVM Community Edition only, never Oracle GraalVM). """ from __future__ import annotations @@ -87,11 +89,30 @@ PRERELEASE_REQUIRED = ["linux-x86_64", "linux-aarch64", "macos-arm64"] ALL_PLATFORMS = list(PLATFORMS) +# The `pyver` job of test-native.yml installs the already built linux-x86_64 wheel (py3-none, no rebuild) into every Python below that the +# full parity run of the event does not cover, and runs a subset of the test files there. 3.14t is the free-threaded build of 3.14: +# it is not a classifier of the wheel and has no polars (polars ships abi3 wheels only, which a free-threaded Python cannot install). +FREE_THREADED_PYTHONS = ["3.14t"] +PYVER_NOTE = "informational: release-native.yml does not read it" + PULL_REQUEST_PLATFORMS = ["linux-x86_64"] PULL_REQUEST_PYTHONS = ["3.12"] MAIN_PYTHONS = ["3.12", "3.14"] +def pyver_legs(covered: list[str]) -> list[dict]: + """The legs of the cheap Python-version job: every supported Python that `covered` (the full parity run) leaves out.""" + return [ + { + "python": py, + "freethreaded": py in FREE_THREADED_PYTHONS, + "polars": py not in FREE_THREADED_PYTHONS, + } + for py in ALL_PYTHONS + FREE_THREADED_PYTHONS + if py not in covered + ] + + def select(event: str) -> dict: if event == "pull_request": platforms, pythons = PULL_REQUEST_PLATFORMS, PULL_REQUEST_PYTHONS @@ -112,7 +133,7 @@ def select(event: str) -> dict: for k in platforms for py in pythons ] - return {"build": build, "test": test} + return {"build": build, "test": test, "pyver": pyver_legs(pythons)} def main(argv: list[str] | None = None) -> int: @@ -125,6 +146,7 @@ def main(argv: list[str] | None = None) -> int: chosen = select(a.event) print(f"build={json.dumps(chosen['build'], separators=(',', ':'))}") print(f"test={json.dumps(chosen['test'], separators=(',', ':'))}") + print(f"pyver={json.dumps(chosen['pyver'], separators=(',', ':'))}") return 0 diff --git a/tests/test_select_native_matrix.py b/tests/test_select_native_matrix.py index 609aea7..d015daf 100644 --- a/tests/test_select_native_matrix.py +++ b/tests/test_select_native_matrix.py @@ -67,5 +67,63 @@ def test_the_script_prints_github_output_lines(): text=True, check=True, ).stdout.splitlines() - assert [line.split("=", 1)[0] for line in out] == ["build", "test"] + assert [line.split("=", 1)[0] for line in out] == ["build", "test", "pyver"] assert json.loads(out[0].split("=", 1)[1])[0]["key"] == "linux-x86_64" + assert [leg["python"] for leg in json.loads(out[2].split("=", 1)[1])] == [ + "3.11", + "3.13", + "3.14", + "3.14t", + ] + + +def _pyver(event): + return [ + (leg["python"], leg["freethreaded"], leg["polars"]) + for leg in sel.select(event)["pyver"] + ] + + +def test_the_python_version_job_covers_what_the_full_parity_run_leaves_out(): + # a pull request tests 3.12 only: the other three Pythons and the free-threaded one run the cheap subset + assert _pyver("pull_request") == [ + ("3.11", False, True), + ("3.13", False, True), + ("3.14", False, True), + ("3.14t", True, False), + ] + # main, nightly and dispatch test 3.12 and 3.14 in full + for event in ("push", "schedule", "workflow_dispatch"): + assert _pyver(event) == [ + ("3.11", False, True), + ("3.13", False, True), + ("3.14t", True, False), + ] + # a release tests 3.11 to 3.14 in full: only the free-threaded build is left + assert _pyver("release") == [("3.14t", True, False)] + + +def test_the_python_version_job_never_repeats_a_full_parity_leg_and_has_no_polars_on_a_free_threaded_python(): + for event in ("pull_request", "push", "schedule", "workflow_dispatch", "release"): + chosen = sel.select(event) + full = {t["python"] for t in chosen["test"]} + assert not full & {leg["python"] for leg in chosen["pyver"]}, event + assert any(leg["freethreaded"] for leg in chosen["pyver"]), event + for leg in chosen["pyver"]: + assert leg["polars"] is (not leg["freethreaded"]), leg + + +def test_the_release_does_not_need_the_python_version_job(): + workflows = ROOT / ".github" / "workflows" + test_native = (workflows / "test-native.yml").read_text() + release = (workflows / "release-native.yml").read_text() + assert re.search(r"^ pyver:\n", test_native, re.M) + # no job of either workflow waits for it + for text in (test_native, release): + for needs in re.findall(r"^ needs: (.*)$", text, re.M): + assert "pyver" not in needs, needs + # release-native.yml reads the markers and the wheels only, never the artifacts of the pyver job + patterns = re.findall(r"^\s+pattern: (.*)$", release, re.M) + assert patterns and all( + p in ("native-passed-*", "native-wheel-*") for p in patterns + ), patterns From 648c62b87309a46544eab42bbd3cc2a646ffba00 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 21:29:57 +0900 Subject: [PATCH 22/33] Tests: nosec markers for the fresh-process SIGINT test's subprocess Co-Authored-By: Claude Opus 5.5 --- tests/test_python_function_interrupt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_python_function_interrupt.py b/tests/test_python_function_interrupt.py index 4a0d031..0ca4ddd 100644 --- a/tests/test_python_function_interrupt.py +++ b/tests/test_python_function_interrupt.py @@ -11,7 +11,7 @@ import os import signal -import subprocess +import subprocess # nosec B404 - test-controlled child process import sys import threading import time @@ -129,7 +129,7 @@ def test_sigint_loop_in_a_fresh_process_ends_cleanly(): """The same loop in its own process. A Ctrl-C raised as the Python callback is entered used to leave the engine a stale reply pointer, and the process aborted in ``free()`` (nonzero exit); pytest's own hooks hid it inside a test run. """ - result = subprocess.run( + result = subprocess.run( # nosec B603 - fixed argv, no shell, test-owned code string [sys.executable, "-c", _SIGINT_LOOP], capture_output=True, text=True, From a815a57a62877f2b6c85c75dc7dbd3ae0a97257f Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 21:32:16 +0900 Subject: [PATCH 23/33] Native: get_stats, get_metadata and PQ search work, AsyncExecutor.transaction runs its block; the module docstring says what is true VectorIndex.get_stats reads the schema:index: row (the counters, not the row's own columns). get_metadata and get_quantization ask the engine through one new glue entry point (arcadedb_n_vindex, op metadata), so an index this session did not create reports the same keys as on the JVM build, defaults included. find_nearest_approximate runs the engine's PQ search through the same entry point (op approximate): no statement runs it. warm_up runs the first search that loads a persisted graph. AsyncExecutor.transaction is a new op of arcadedb_n_async: the engine begins the transaction on a writer thread, Python runs the block in the upcall, and a Python exception is thrown on the Java side so the writer reports it. The block runs in the upcall of #67, and a Python exception comes back through its out slot as {"error": ...}, which the Java side throws so the writer rolls back and reports it. Tests first, shared by both builds: tests/test_vector_index_introspection.py and tests/test_async_transaction.py (18 tests on the JVM build, 17 of 18 on native, see the PR). The two get_java_* calls stay JVM-only and are documented as such. The module docstring of _native/__init__.py listed far more as missing than is true. Co-Authored-By: Claude Sonnet 5.5 --- docs/modules/ROOT/pages/installation.adoc | 2 +- docs/modules/ROOT/pages/which-package.adoc | 6 +- .../org/humemai/nativelib/NativeExtras.java | 108 +++++++++ src/arcadedb/_native/__init__.py | 24 +- src/arcadedb/_native/extras.py | 25 +- src/arcadedb/_native/lib.py | 1 + src/arcadedb/_native/schema_objects.py | 191 ++++++++++----- tests/test_async_transaction.py | 97 ++++++++ tests/test_vector_index_introspection.py | 227 ++++++++++++++++++ 9 files changed, 608 insertions(+), 73 deletions(-) create mode 100644 tests/test_async_transaction.py create mode 100644 tests/test_vector_index_introspection.py diff --git a/docs/modules/ROOT/pages/installation.adoc b/docs/modules/ROOT/pages/installation.adoc index 733a505..4f9456f 100644 --- a/docs/modules/ROOT/pages/installation.adoc +++ b/docs/modules/ROOT/pages/installation.adoc @@ -147,7 +147,7 @@ Each Python process starts its own JVM, and each JVM sees the whole container li == Native build notes (`{native-pkg-name}`) -The native package is the same engine compiled to a GraalVM native-image shared library. It needs no JVM and no JPype, so the JVM options above do not apply to it. A few API calls raise `NotOnNativeYet` (a `NotImplementedError`) instead of misbehaving. The list is in xref:which-package.adoc[Which package?]. Report problems in the {repo-url}/issues[issue tracker] and say which package you use. +The native package is the same engine compiled to a GraalVM native-image shared library. It needs no JVM and no JPype, so the JVM options above do not apply to it. The calls that hand out a Java object (`get_java_database`, `get_java_document`) raise `NotOnNativeYet` (a `NotImplementedError`) instead of misbehaving. The list is in xref:which-package.adoc[Which package?]. Report problems in the {repo-url}/issues[issue tracker] and say which package you use. == Next steps diff --git a/docs/modules/ROOT/pages/which-package.adoc b/docs/modules/ROOT/pages/which-package.adoc index 75bf4c8..4cb20af 100644 --- a/docs/modules/ROOT/pages/which-package.adoc +++ b/docs/modules/ROOT/pages/which-package.adoc @@ -82,8 +82,8 @@ The API is the same. The runtime is not. These are the differences you can plan | Linux x86_64 (glibc 2.34 or later), Linux aarch64, macOS arm64, Windows x64. The Windows wheel is the newest: it is built in CI, and its test run is the least proven of the four. | Linux x86_64 (glibc 2.34 or later), Linux aarch64, macOS arm64, Windows x64. -| Java-object APIs (`get_java_database`, `get_java_*`) -| Not available. +| Java-object APIs (`get_java_database`, `get_java_document`, `get_java_*`) +| Not available. They raise `NotOnNativeYet`. | Available. | JVM options (`jvm_kwargs`, `start_jvm`, `-X` and `-D` flags) @@ -91,7 +91,7 @@ The API is the same. The runtime is not. These are the differences you can plan | Available. See xref:installation.adoc#jvm-options[JVM options]. | A few API calls -| Raise `NotOnNativeYet` (a `NotImplementedError`): `AsyncExecutor.transaction`, the approximate search `find_nearest_approximate` over product quantization, `get_metadata` of a foreign index, and `get_stats`. +| None, except the Java-object calls above, which raise `NotOnNativeYet` (a `NotImplementedError`). | Work. |=== diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index 9903789..7fc3797 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -25,6 +25,7 @@ import com.arcadedb.database.RID; import com.arcadedb.database.async.DatabaseAsyncExecutor; import com.arcadedb.database.async.ErrorCallback; +import com.arcadedb.database.async.OkCallback; import com.arcadedb.database.async.NewEdgeCallback; import com.arcadedb.engine.ErrorRecordCallback; import com.arcadedb.engine.WALFile; @@ -39,6 +40,11 @@ import com.arcadedb.function.CypherFunctionRegistry; import com.arcadedb.function.FunctionRegistry; import com.arcadedb.function.sql.DefaultSQLFunctionFactory; +import com.arcadedb.index.IndexInternal; +import com.arcadedb.index.TypeIndex; +import com.arcadedb.index.vector.LSMVectorIndex; +import com.arcadedb.schema.LSMVectorIndexMetadata; +import com.arcadedb.utility.Pair; import com.arcadedb.function.sql.SQLFunctionAbstract; import com.arcadedb.graph.Edge; import com.arcadedb.graph.GraphBatch; @@ -63,8 +69,10 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -269,6 +277,81 @@ public static CCharPointer dbSet(final IsolateThread thread, final long h, final } } + // ---- vector indexes: what no statement reports (the engine's configuration of an index) and the PQ search no statement runs + + private static String q(final String text) { + return text == null ? "null" : "\"" + text.replace("\\", "\\\\").replace("\"", "\\\"") + "\""; + } + + /** + * ops: metadata {type, prop} | approximate {type, prop, vector, k, allowed}. The index is the one the type has on the property; its bucket + * indexes are searched in turn (the caller merges). metadata describes the first bucket index, as the JPype wrapper does. + */ + @CEntryPoint(name = "arcadedb_n_vindex") + @SuppressWarnings("unchecked") + public static CCharPointer vindex(final IsolateThread thread, final long dbh, final CCharPointer op, final CCharPointer argsJson) { + try { + final Database db = ArcadeNative.db(dbh); + final Map a = args(argsJson); + final String type = (String) a.get("type"), prop = (String) a.get("prop"); + final TypeIndex typeIndex = db.getSchema().getType(type).getPolymorphicIndexByProperties(prop); + if (typeIndex == null) + throw new IllegalArgumentException("No vector index on " + type + "[" + prop + "]"); + final List parts = new ArrayList<>(); + for (final IndexInternal index : typeIndex.getIndexesOnBuckets()) + if (index instanceof LSMVectorIndex lsm) + parts.add(lsm); + if (parts.isEmpty()) + throw new IllegalArgumentException("Underlying index is not an LSM vector index"); + final StringBuilder sb = new StringBuilder("{\"ok\":true"); + switch (ArcadeNative.str(op)) { + case "metadata" -> { + final LSMVectorIndex first = parts.get(0); + final LSMVectorIndexMetadata m = first.getMetadata(); + sb.append(",\"bucket\":").append(q(first.getName())).append(",\"type_name\":").append(q(first.getTypeName())); + sb.append(",\"dimensions\":").append(m.dimensions).append(",\"similarity_function\":").append(q(String.valueOf(m.similarityFunction))); + sb.append(",\"id_property\":").append(q(m.idPropertyName)).append(",\"quantization\":").append(q(String.valueOf(m.quantizationType))); + sb.append(",\"max_connections\":").append(m.maxConnections).append(",\"beam_width\":").append(m.beamWidth); + sb.append(",\"ef_search\":").append(m.efSearch).append(",\"graph_build_cache_size\":").append(m.graphBuildCacheSize); + sb.append(",\"mutations_before_rebuild\":").append(m.mutationsBeforeRebuild).append(",\"store_vectors_in_graph\":").append(m.storeVectorsInGraph); + sb.append(",\"add_hierarchy\":").append(m.addHierarchy).append(",\"build_state\":").append(q(m.buildState)); + sb.append(",\"pq_subspaces\":").append(m.pqSubspaces).append(",\"pq_clusters\":").append(m.pqClusters); + sb.append(",\"pq_center_globally\":").append(m.pqCenterGlobally).append(",\"pq_training_limit\":").append(m.pqTrainingLimit); + } + case "approximate" -> { + final List vector = (List) a.get("vector"); + final float[] query = new float[vector.size()]; + for (int i = 0; i < query.length; i++) + query[i] = ((Number) vector.get(i)).floatValue(); + final int k = intOf(a.get("k"), 10); + Set allowed = null; + if (a.get("allowed") instanceof List rids) { + allowed = new HashSet<>(); + for (final Object rid : rids) + allowed.add(new RID(String.valueOf(rid))); + } + sb.append(",\"hits\":["); + boolean firstHit = true; + for (final LSMVectorIndex lsm : parts) { + final List> hits = allowed != null ? lsm.findNeighborsFromVectorApproximate(query, k, allowed) + : lsm.findNeighborsFromVectorApproximate(query, k); + for (final Pair hit : hits) { + if (!firstHit) + sb.append(','); + firstHit = false; + sb.append("[\"").append(hit.getFirst()).append("\",").append(hit.getSecond()).append(']'); + } + } + sb.append(']'); + } + default -> throw new IllegalArgumentException("unknown vector index op " + ArcadeNative.str(op)); + } + return ArcadeNative.out(sb.append('}').toString()); + } catch (final Throwable t) { + return ArcadeNative.err(t); + } + } + // ---- GraphBatch private record BatchEntry(GraphBatch batch, Database db) { @@ -543,6 +626,31 @@ public void onError(final Exception e) { else ex.query(lang, text, callback); } + case "transaction" -> { + // a block that runs on an async writer inside a transaction: Python runs it in the upcall; its failure is thrown here so the + // writer rolls back, retries a NeedRetryException and reports to the error callback (ok / err: per-call callbacks, slot: one writer) + final long okCb = a.get("ok") instanceof Number n ? n.longValue() : 0L; + final Database.TransactionScope scope = () -> { + final String reply = upcall(cb, event("tx", "")); + if (reply != null && reply.startsWith("{\"error\"")) + throw new RuntimeException(String.valueOf(((Map) Json.parse(reply)).get("error"))); + }; + final int retries = intOf(a.get("retries"), 1); + final OkCallback ok = okCb == 0 ? null : () -> { + try { + fire("transaction ok callback", okCb, event("ok", "")); + } catch (final Throwable e) { + swallowed("transaction ok callback", e); // the writer goes on + } + }; + final ErrorCallback error = errCb == 0 ? null : errorCallback(errCb); + if (a.get("retries") == null && ok == null && error == null && a.get("slot") == null) + ex.transaction(scope); + else if (a.get("slot") instanceof Number slot) + ex.transaction(scope, retries, ok, error, slot.intValue()); + else + ex.transaction(scope, retries, ok, error); + } case "on_ok" -> ex.onOk(() -> { try { fire("on_ok callback", cb, event("ok", "")); diff --git a/src/arcadedb/_native/__init__.py b/src/arcadedb/_native/__init__.py index b413293..47939e3 100644 --- a/src/arcadedb/_native/__init__.py +++ b/src/arcadedb/_native/__init__.py @@ -1,21 +1,25 @@ -"""The native backend: the engine as a GraalVM native-image library, no JVM and no JPype. +"""The native backend: the engine as a GraalVM native-image shared library (``libarcadedb``) that Python loads with ``ctypes``. No JVM, no JPype. -It is the only backend of the `arcadedb` wheel. In a source checkout or in the `arcadedb-jvm` wheel, ``ARCADEDB_BACKEND=native`` -selects it (with ``ARCADEDB_NATIVE_LIB`` pointing at a library you built); the default there is JPype. The engine runs as a GraalVM native-image shared -library (``libarcadedb``) that Python loads with ``ctypes``: no JVM, no JPype. The surface is the DSL-first core: -databases, ``query``/``command`` for SQL and openCypher, transactions, results and the in-process server. +It is the only backend of the ``arcadedb`` wheel. In a source checkout or in the ``arcadedb-jvm`` wheel, ``ARCADEDB_BACKEND=native`` selects it +(with ``ARCADEDB_NATIVE_LIB`` pointing at a library you built); the default there is JPype. The public API is the JVM build's, and the same +test suite (``tests/``) runs against both. -What is not on native yet (each raises :class:`NotOnNativeYet`): ``GraphBatch``, the -async executor, the Java-object APIs (``Document``/``Vertex``/``Edge`` wrappers, ``new_vertex``, ``new_document``, -``schema``, ``get_java_database``), the columnar/Arrow readers other than a simple row conversion, importers and -exporters that go through Java objects, and every JVM setting (``jvm_kwargs``, ``start_jvm``). +On native: databases and the factory, ``query``/``command`` (SQL and openCypher), transactions and ``run_in_transaction``, results as rows, +columns and Arrow, pandas, polars and numpy frames, the ``Document``/``Vertex``/``Edge`` wrappers, the schema (types, properties, indexes), +vector indexes (``find_nearest``, the PQ ``find_nearest_approximate``, ``get_metadata``, ``get_stats``), ``insert_many``, ``GraphBatch``, +the ``AsyncExecutor`` (``transaction`` included), Python functions in SQL and record callbacks, importers and exporters, and the in-process +server. + +Not on native, by design (each raises :class:`NotOnNativeYet`): the Java objects of the JVM build. These are ``get_java_database``, +``get_java_document``, and the Java object a ``Schema``, type or index method returns where the native wrapper has no such method. +There is no JVM to start or tune: ``jvm_kwargs`` is ignored with a warning. """ import os class NotOnNativeYet(NotImplementedError): - """The feature needs the Java object API, which the native backend does not expose (yet).""" + """The call hands out a Java object of the JVM build (``get_java_database``, ``get_java_document``, a Schema method's Java object), which the native build does not have.""" # The backend used when ARCADEDB_BACKEND is unset. "jpype" here, which is the arcadedb-jvm wheel; the build of the native diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py index 072184e..b1042e6 100644 --- a/src/arcadedb/_native/extras.py +++ b/src/arcadedb/_native/extras.py @@ -693,8 +693,29 @@ def on_error(message): "scan_type", type=type_name, polymorphic=bool(polymorphic), cb=cb, err=err ) - def transaction(self, *args, **kwargs): - not_yet("AsyncExecutor.transaction") + def transaction( + self, + tx_block: Callable[[], None], + retries: Optional[int] = None, + ok_callback: Optional[Callable[[], None]] = None, + error_callback: Optional[Callable[[Exception], None]] = None, + slot: Optional[int] = None, + ): + """Run ``tx_block`` on an async writer inside a transaction (the engine begins it and commits or rolls it back, retrying a conflict ``retries`` times).""" + + def run(message): + tx_block() + + args = {"cb": _register(run)} + if retries is not None: + args["retries"] = int(retries) + if ok_callback is not None: + args["ok"] = _register(lambda message: ok_callback()) + if error_callback is not None: + args["err"] = self._error_callback(error_callback) + if slot is not None: + args["slot"] = int(slot) + self._op("transaction", **args) def wait_completion(self, timeout_ms: Optional[int] = None): if timeout_ms is None: diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index 95cc7a3..1683f58 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -176,6 +176,7 @@ class _Params(ctypes.Structure): "arcadedb_n_gb_create": [P, L, S], "arcadedb_n_gb": [P, L, S, S], "arcadedb_n_async": [P, L, S, S], + "arcadedb_n_vindex": [P, L, S, S], "arcadedb_n_import": [P, L, S], "arcadedb_n_export": [P, L, S, S, I, S], "arcadedb_n_cb": [P, S, S], diff --git a/src/arcadedb/_native/schema_objects.py b/src/arcadedb/_native/schema_objects.py index 455b6fc..3589ad1 100644 --- a/src/arcadedb/_native/schema_objects.py +++ b/src/arcadedb/_native/schema_objects.py @@ -101,14 +101,30 @@ def getTypeName(self): def getPropertyNames(self): return [self._prop] + def engine_info(self) -> dict: + """The engine's configuration of the index (the defaults it chose included), read once: it does not change after the index exists.""" + info = self.__dict__.get("_engine_info") + if info is None: + from . import lib + from .extras import _call, _enc + + info = self._engine_info = _call( + "Failed to read the vector index metadata", + "arcadedb_n_vindex", + self._db._h, + lib.b("metadata"), + lib.b(_enc({"type": self._type, "prop": self._prop})), + ) + return info + def getMetadata(self): - """Just what the PRODUCT-readiness check reads: the session's quantization and pq_clusters (engine default 256).""" + """Just what the PRODUCT-readiness check reads: the index's quantization and pq_clusters.""" from types import SimpleNamespace - meta = self._db._vector_meta.get((self._type, self._prop), {}) + info = self.engine_info() return SimpleNamespace( - quantizationType=str(meta.get("quantization") or "NONE"), - pqClusters=int(meta.get("pqClusters", 256)), + quantizationType=str(info["quantization"] or "NONE"), + pqClusters=int(info["pq_clusters"]), ) def getIdPropertyName(self): @@ -120,14 +136,15 @@ def countEntries(self): return int(row.get("c")) def warmUp(self): - return None + # the engine loads a persisted graph on the first search: run that search now + self.buildVectorGraphNow() def buildVectorGraphNow(self): # the engine builds the graph on the first search: one search now, on a vector of the index's dimension - dims = self._db._vector_dimensions.get((self._type, self._prop)) + dims = self.engine_info()["dimensions"] if dims: self._db.query( - "sql", "SELECT vectorNeighbors(?, ?, 1)", self._name, [0.0] * dims + "sql", "SELECT vectorNeighbors(?, ?, 1)", self._name, [1.0] * dims ).close() @@ -151,7 +168,7 @@ def _run_search( self, *, query_vector, k, allowed_rids, approximate, ef_search=None ): if approximate: - not_yet("VectorIndex.find_nearest_approximate over PRODUCT quantization") + return self._run_approximate(query_vector, k, allowed_rids) from .records import _wrap_dict vector = ( @@ -195,11 +212,59 @@ def _run_search( out.append((record, float(row.get("distance")))) return self._sort_results(out)[:k] + def _run_approximate(self, query_vector, k, allowed_rids): + """The PQ-scored search of the engine (no statement runs it): the engine returns (RID, distance), the records are loaded here.""" + from . import lib + from .extras import _call, _enc + + vector = ( + query_vector.tolist() + if hasattr(query_vector, "tolist") + else list(query_vector) + ) + reply = _call( + "Approximate search failed", + "arcadedb_n_vindex", + self._database._h, + lib.b("approximate"), + lib.b( + _enc( + { + "type": self._shim._type, + "prop": self._shim._prop, + "vector": vector, + "k": int(k), + "allowed": ( + [_rid_text(r) for r in allowed_rids] + if allowed_rids + else None + ), + } + ) + ), + ) + out = [ + (self._database.lookup_by_rid(rid), float(distance)) + for rid, distance in reply["hits"] + ] + return self._sort_results(out)[:k] + def find_nearest_approximate(self, query_vector, k=10, allowed_rids=None): - """PQ approximate search needs PRODUCT quantization; the search itself has no statement form (not on native yet).""" + """PQ approximate search over PRODUCT quantization, the engine's own (``findNeighborsFromVectorApproximate``).""" if self.get_quantization() != "PRODUCT": raise ArcadeDBError("Approximate search requires quantization=PRODUCT (PQ)") - not_yet("VectorIndex.find_nearest_approximate over PRODUCT quantization") + self._ensure_product_quantization_ready() + try: + return self._run_search( + query_vector=query_vector, + k=k, + allowed_rids=allowed_rids, + approximate=True, + ) + except ArcadeDBError: + raise + except Exception as e: + raise ArcadeDBError(f"Approximate search failed: {e}") from e def _lookup_query_vector_by_key(self, key): type_name, prop = self._shim._type, self._shim._prop @@ -217,60 +282,72 @@ def _lookup_query_vector_by_key(self, key): return row.get("query_vector") def get_stats(self): - not_yet("VectorIndex.get_stats") + """The engine's live counters of the primary bucket index, from its ``schema:index:`` row (the row's own columns left out).""" + bucket = self._shim.engine_info()["bucket"] + rows = self._database.query( + "sql", + f"SELECT FROM schema:index:`{bucket}`", # nosec B608 - a backtick-quoted index name the engine reported + ).to_list() + if not rows: + raise ArcadeDBError( + f"Failed to read vector index stats: no index {bucket!r}" + ) + return {k: v for k, v in rows[0].items() if k not in _INDEX_ROW_COLUMNS} def get_metadata(self): - """The configuration this session created the index with (no engine-chosen values: the image has no accessor for them).""" - meta = self._database._vector_meta.get((self._shim._type, self._shim._prop)) - if meta is None: - not_yet("VectorIndex.get_metadata of an index this session did not create") - names = { - "dimensions": "dimensions", - "similarity": "similarity_function", - "idPropertyName": "id_property", - "quantization": "quantization", - "maxConnections": "max_connections", - "beamWidth": "beam_width", - "graphBuildCacheSize": "graph_build_cache_size", - "mutationsBeforeRebuild": "mutations_before_rebuild", - "storeVectorsInGraph": "store_vectors_in_graph", - "addHierarchy": "add_hierarchy", - "pqSubspaces": "pq_subspaces", - "pqClusters": "pq_clusters", - "pqCenterGlobally": "pq_center_globally", - "pqTrainingLimit": "pq_training_limit", - "rerankOversample": "rerank_oversample", - "fusedPQ": "fused_pq", - } - buckets = [ - r["name"] - for r in self._database.query( - "sql", "SELECT FROM schema:indexes WHERE indexType = 'LSM_VECTOR'" - ).to_list() - if r["typeName"] == self._shim._type - and r["properties"] == [[self._shim._prop]] - and "fileId" in r - ] - out = { + """The engine's configuration of the index: what this session created and what an index opened from disk carries, defaults included.""" + info = self._shim.engine_info() + return { "index_name": self._shim._name, - "bucket_index_name": buckets[0] if buckets else None, - "type_name": self._shim._type, + "bucket_index_name": info["bucket"], + "type_name": info["type_name"], "vector_property": self._shim._prop, + **{k: info[k] for k in _METADATA_KEYS}, } - out.update({names[k]: v for k, v in meta.items() if k in names}) - if "similarity_function" in out: - out["similarity_function"] = str(out["similarity_function"]).upper() - if "quantization" in out: - out["quantization"] = str(out["quantization"]).upper() - return out def get_quantization(self): - meta = self._database._vector_meta.get((self._shim._type, self._shim._prop)) - if meta is None: - not_yet( - "VectorIndex.get_quantization of an index this session did not create" - ) - return str(meta.get("quantization") or "NONE") + return str(self._shim.engine_info()["quantization"] or "NONE") + + +# the descriptive columns of a schema:index row; every other column is a counter of IndexInternal.getStats() +_INDEX_ROW_COLUMNS = frozenset( + { + "name", + "indexType", + "typeName", + "properties", + "keyTypes", + "unique", + "automatic", + "compacting", + "valid", + "supportsOrderedIterations", + "nullStrategy", + "associatedBucketId", + "fileId", + "size", + "upgradeWarning", + "typeIndexName", + } +) +_METADATA_KEYS = ( + "dimensions", + "similarity_function", + "id_property", + "quantization", + "max_connections", + "beam_width", + "ef_search", + "graph_build_cache_size", + "mutations_before_rebuild", + "store_vectors_in_graph", + "add_hierarchy", + "build_state", + "pq_subspaces", + "pq_clusters", + "pq_center_globally", + "pq_training_limit", +) def _check_vector_arguments( diff --git a/tests/test_async_transaction.py b/tests/test_async_transaction.py new file mode 100644 index 0000000..94c70cf --- /dev/null +++ b/tests/test_async_transaction.py @@ -0,0 +1,97 @@ +"""``AsyncExecutor.transaction``: a block that runs on an async writer inside a transaction, on the JVM wheel and the native wheel.""" + +import threading + +import pytest + +import arcadedb + +TIMEOUT = 30 + + +@pytest.fixture +def items_db(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE Item") + temp_db.command("sql", "CREATE PROPERTY Item.n INTEGER") + return temp_db + + +def _count(db): + return int(db.query("sql", "SELECT count(*) AS c FROM Item").first().get("c")) + + +def _insert(db, n): + db.command("sql", "INSERT INTO Item SET n = ?", n) + + +def test_the_block_runs_in_a_transaction_and_commits(items_db): + async_exec = items_db.async_executor() + ran_on = [] + + def block(): + ran_on.append(threading.current_thread()) + _insert(items_db, 1) + _insert(items_db, 2) + + async_exec.transaction(block) + async_exec.wait_completion(TIMEOUT * 1000) + async_exec.close() + + assert _count(items_db) == 2 + # an async writer ran it, not the calling thread + assert len(ran_on) == 1 and ran_on[0] is not threading.current_thread() + + +def test_the_ok_callback_fires_after_the_commit(items_db): + async_exec = items_db.async_executor() + done = threading.Event() + seen_count = [] + + async_exec.transaction( + lambda: _insert(items_db, 7), + retries=3, + ok_callback=lambda: (seen_count.append(_count(items_db)), done.set()), + ) + assert done.wait(TIMEOUT) + async_exec.wait_completion(TIMEOUT * 1000) + async_exec.close() + + assert seen_count == [1] + assert _count(items_db) == 1 + + +def test_a_failing_block_rolls_back_and_reaches_the_error_callback(items_db): + async_exec = items_db.async_executor() + failed = threading.Event() + errors = [] + + def block(): + _insert(items_db, 1) + raise ValueError("boom") + + def on_error(exc): + errors.append(exc) + failed.set() + + async_exec.transaction(block, retries=1, error_callback=on_error) + assert failed.wait(TIMEOUT) + async_exec.wait_completion(TIMEOUT * 1000) + async_exec.close() + + assert len(errors) == 1 and isinstance(errors[0], Exception) + assert _count(items_db) == 0 + + +def test_blocks_on_one_slot_run_in_submission_order(items_db): + async_exec = items_db.async_executor() + order = [] + + for i in range(5): + async_exec.transaction( + lambda i=i: (order.append(i), _insert(items_db, i)), retries=1, slot=0 + ) + async_exec.wait_completion(TIMEOUT * 1000) + async_exec.close() + + assert order == [0, 1, 2, 3, 4] + assert _count(items_db) == 5 diff --git a/tests/test_vector_index_introspection.py b/tests/test_vector_index_introspection.py new file mode 100644 index 0000000..d6f58a9 --- /dev/null +++ b/tests/test_vector_index_introspection.py @@ -0,0 +1,227 @@ +"""Vector index introspection and PRODUCT-quantization search: the same behaviour on the JVM wheel and the native wheel. + +``get_stats`` (live counters), ``get_metadata`` and ``get_quantization`` of an index that this session did not create (a +reopened database), and ``find_nearest_approximate`` over PRODUCT quantization. The tests use no Java object, so they run +on both backends. +""" + +import pytest + +import arcadedb + +# keys the metadata dict has on both wheels (the engine's configuration of the index) +METADATA_KEYS = { + "index_name", + "bucket_index_name", + "type_name", + "vector_property", + "dimensions", + "similarity_function", + "id_property", + "quantization", + "max_connections", + "beam_width", + "ef_search", + "graph_build_cache_size", + "mutations_before_rebuild", + "store_vectors_in_graph", + "add_hierarchy", + "build_state", + "pq_subspaces", + "pq_clusters", + "pq_center_globally", + "pq_training_limit", +} + + +def _make_type(db): + db.command("sql", "CREATE VERTEX TYPE Doc") + db.command("sql", "CREATE PROPERTY Doc.name STRING") + db.command("sql", "CREATE PROPERTY Doc.embedding ARRAY_OF_FLOATS") + + +def _insert(db, rows): + with db.transaction(): + for name, vector in rows: + db.command( + "sql", + "INSERT INTO Doc SET name = ?, embedding = ?", + name, + arcadedb.to_java_float_array(vector), + ) + + +UNIT_VECTORS = [("x", [1.0, 0.0, 0.0]), ("y", [0.0, 1.0, 0.0]), ("z", [0.0, 0.0, 1.0])] +# as many vectors as PQ clusters: each vector is its own centroid, so the approximate answer is the exact one +PQ_VECTORS = UNIT_VECTORS + [("w", [-1.0, 0.0, 0.0])] + + +class TestGetStats: + def test_counters_start_at_zero_and_track_inserts(self, temp_db): + _make_type(temp_db) + index = temp_db.create_vector_index("Doc", "embedding", dimensions=3) + + stats = index.get_stats() + assert isinstance(stats, dict) and stats + assert stats["totalVectors"] == 0 + assert isinstance(stats["dimensions"], int) + assert stats["vectorFetchFromDocuments"] >= 0 + + _insert(temp_db, UNIT_VECTORS[:2]) + assert index.get_stats()["totalVectors"] == 2 + + def test_values_are_plain_scalars_and_the_listing_columns_are_not_counters( + self, temp_db + ): + _make_type(temp_db) + index = temp_db.create_vector_index("Doc", "embedding", dimensions=3) + _insert(temp_db, UNIT_VECTORS) + + stats = index.get_stats() + assert all( + v is None or isinstance(v, (int, float, bool, str)) for v in stats.values() + ) + # a schema:index row also names the index and describes it; get_stats is the counters only + for descriptive in ("name", "indexType", "typeName", "properties", "fileId"): + assert descriptive not in stats + + def test_searches_move_the_search_counter(self, temp_db): + _make_type(temp_db) + index = temp_db.create_vector_index("Doc", "embedding", dimensions=3) + _insert(temp_db, UNIT_VECTORS) + + before = index.get_stats()["searchOperations"] + index.find_nearest([1.0, 0.0, 0.0], k=1) + assert index.get_stats()["searchOperations"] > before + + +class TestGetMetadataOfAReopenedIndex: + @staticmethod + def _create(path, **kwargs): + with arcadedb.create_database(path) as db: + _make_type(db) + index = db.create_vector_index( + "Doc", "embedding", dimensions=4, id_property="name", **kwargs + ) + _insert( + db, + [ + ("a", [1.0, 0.0, 0.0, 0.0]), + ("b", [0.0, 1.0, 0.0, 0.0]), + ("c", [0.0, 0.0, 1.0, 0.0]), + ], + ) + return index.get_metadata() + + def test_metadata_after_reopen_equals_the_creating_session(self, temp_db_path): + created = self._create( + temp_db_path, + distance_function="EUCLIDEAN", + max_connections=16, + beam_width=64, + add_hierarchy=False, + ) + assert set(created) == METADATA_KEYS + + with arcadedb.open_database(temp_db_path) as db: + index = db.schema.get_vector_index("Doc", "embedding") + reopened = index.get_metadata() + + assert set(reopened) == METADATA_KEYS + assert reopened["dimensions"] == 4 + assert reopened["similarity_function"] == "EUCLIDEAN" + assert reopened["id_property"] == "name" + assert reopened["max_connections"] == 16 + assert reopened["beam_width"] == 64 + assert reopened["add_hierarchy"] is False + # the build state is runtime state; everything else is the index's configuration + created.pop("build_state") + reopened.pop("build_state") + assert reopened == created + + def test_product_quantization_settings_survive_a_reopen(self, temp_db_path): + self._create(temp_db_path, quantization="PRODUCT", pq_clusters=2) + + with arcadedb.open_database(temp_db_path) as db: + index = db.schema.get_vector_index("Doc", "embedding") + assert index.get_quantization() == "PRODUCT" + meta = index.get_metadata() + assert meta["quantization"] == "PRODUCT" + assert meta["pq_clusters"] == 2 + + @pytest.mark.parametrize("quantization", ["NONE", "INT8", "BINARY"]) + def test_quantization_of_a_reopened_index(self, temp_db_path, quantization): + self._create(temp_db_path, quantization=quantization) + with arcadedb.open_database(temp_db_path) as db: + index = db.schema.get_vector_index("Doc", "embedding") + assert index.get_quantization() == quantization + assert index.get_metadata()["quantization"] == quantization + + def test_the_default_quantization_is_the_same_after_a_reopen(self, temp_db_path): + created = self._create(temp_db_path) + with arcadedb.open_database(temp_db_path) as db: + index = db.schema.get_vector_index("Doc", "embedding") + assert index.get_quantization() == created["quantization"] + + +class TestFindNearestApproximate: + @staticmethod + def _pq_index(db, rows): + _make_type(db) + index = db.create_vector_index( + "Doc", "embedding", dimensions=3, quantization="PRODUCT", pq_clusters=4 + ) + _insert(db, rows) + index.build_graph_now() + return index + + def test_returns_the_nearest_record_and_its_distance(self, temp_db): + index = self._pq_index(temp_db, PQ_VECTORS) + + results = index.find_nearest_approximate([0.9, 0.1, 0.0], k=1) + + assert len(results) == 1 + record, distance = results[0] + assert record.get("name") == "x" + assert isinstance(distance, float) + + def test_k_results_come_back_nearest_first(self, temp_db): + index = self._pq_index(temp_db, PQ_VECTORS) + + results = index.find_nearest_approximate([0.9, 0.1, 0.0], k=3) + + assert [r.get("name") for r, _ in results][0] == "x" + assert len(results) == 3 + distances = [d for _, d in results] + assert distances == sorted(distances) + + def test_allowed_rids_restrict_the_answer(self, temp_db): + index = self._pq_index(temp_db, PQ_VECTORS) + rids = { + row.get("name"): row.get("rid") + for row in temp_db.query("sql", "SELECT name, @rid AS rid FROM Doc") + } + + results = index.find_nearest_approximate( + [0.9, 0.1, 0.0], k=3, allowed_rids=[str(rids["y"]), str(rids["z"])] + ) + + names = {r.get("name") for r, _ in results} + assert names <= {"y", "z"} and names + + def test_a_typeindex_wrapper_from_the_schema_searches_too(self, temp_db): + self._pq_index(temp_db, PQ_VECTORS) + index = temp_db.schema.get_vector_index("Doc", "embedding") + + results = index.find_nearest_approximate([0.0, 0.9, 0.1], k=1) + + assert [r.get("name") for r, _ in results] == ["y"] + + def test_without_product_quantization_it_raises(self, temp_db): + _make_type(temp_db) + index = temp_db.create_vector_index("Doc", "embedding", dimensions=3) + _insert(temp_db, UNIT_VECTORS) + index.build_graph_now() + + with pytest.raises(arcadedb.ArcadeDBError): + index.find_nearest_approximate([0.9, 0.1, 0.0], k=1) From 0af1225e6a4e8cd966366967b2fca00a2f56a41c Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 21:32:16 +0900 Subject: [PATCH 24/33] Native: AsyncExecutor.transaction releases its callbacks once the executor has finished its work AsyncExecutor.transaction registers up to three callbacks per call (block, ok, error). They are kept in the executor's _cb_ids (declared, never used so far) and forgotten when wait_completion has returned or the executor is closed, because a retry calls the block again. The other async operations still register callbacks that are never released; that is unchanged. The reply of a failing block needs nothing new here: with #67 it is written into the out slot as library memory, which outlives the Python thread state of an async writer, so the Java side reads {"error": ...} and the writer rolls back and reaches the error callback. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/extras.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py index b1042e6..e27a1c7 100644 --- a/src/arcadedb/_native/extras.py +++ b/src/arcadedb/_native/extras.py @@ -713,10 +713,18 @@ def run(message): args["ok"] = _register(lambda message: ok_callback()) if error_callback is not None: args["err"] = self._error_callback(error_callback) + # a retry calls the block again, so the callbacks live until the executor has finished its work (wait_completion, close) + self._cb_ids.extend(v for k, v in args.items() if k in ("cb", "ok", "err")) if slot is not None: args["slot"] = int(slot) self._op("transaction", **args) + def _release_callbacks(self): + """Forget the callbacks of the finished ``transaction`` calls.""" + ids, self._cb_ids = self._cb_ids, [] + for cid in ids: + _forget(cid) + def wait_completion(self, timeout_ms: Optional[int] = None): if timeout_ms is None: self._op("wait") @@ -729,6 +737,7 @@ def wait_completion(self, timeout_ms: Optional[int] = None): raise TimeoutError( f"Async operations did not complete within {timeout_ms}ms" ) + self._release_callbacks() # nothing is pending any more def on_ok(self, callback: Callable[[], None]): self._op("on_ok", cb=_register(lambda message: callback())) @@ -745,6 +754,7 @@ def close(self): self._op("close") finally: self._closed = True + self._release_callbacks() owner, self._owner = self._owner, None if owner is not None: owner._discard_async_executor(self) From ae9cf586d98031f8bedc7ecedad6f1d51975fadb Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 21:32:16 +0900 Subject: [PATCH 25/33] Native get_metadata: rerank_oversample and fused_pq follow the engine, as on the JVM build The base added these two keys to the JVM get_metadata when the engine has them (hasattr). The image links engine 26.10.2, which has not, so the native dict carries them only when the glue reports them; the shared test accepts them as optional. The glue reports them with the move to 26.11.1. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/schema_objects.py | 3 +++ tests/test_vector_index_introspection.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/arcadedb/_native/schema_objects.py b/src/arcadedb/_native/schema_objects.py index 3589ad1..3bd9046 100644 --- a/src/arcadedb/_native/schema_objects.py +++ b/src/arcadedb/_native/schema_objects.py @@ -303,6 +303,8 @@ def get_metadata(self): "type_name": info["type_name"], "vector_property": self._shim._prop, **{k: info[k] for k in _METADATA_KEYS}, + # present when the engine the image links has them, as on the JVM build (26.10.2 has not; the glue emits them from 26.11.1 on) + **{k: info[k] for k in _OPTIONAL_METADATA_KEYS if k in info}, } def get_quantization(self): @@ -330,6 +332,7 @@ def get_quantization(self): "typeIndexName", } ) +_OPTIONAL_METADATA_KEYS = ("rerank_oversample", "fused_pq") _METADATA_KEYS = ( "dimensions", "similarity_function", diff --git a/tests/test_vector_index_introspection.py b/tests/test_vector_index_introspection.py index d6f58a9..89a254d 100644 --- a/tests/test_vector_index_introspection.py +++ b/tests/test_vector_index_introspection.py @@ -56,6 +56,10 @@ def _insert(db, rows): PQ_VECTORS = UNIT_VECTORS + [("w", [-1.0, 0.0, 0.0])] +# the engine reports these two only from 26.11.1 on, on both builds +OPTIONAL_METADATA_KEYS = {"rerank_oversample", "fused_pq"} + + class TestGetStats: def test_counters_start_at_zero_and_track_inserts(self, temp_db): _make_type(temp_db) @@ -121,13 +125,13 @@ def test_metadata_after_reopen_equals_the_creating_session(self, temp_db_path): beam_width=64, add_hierarchy=False, ) - assert set(created) == METADATA_KEYS + assert set(created) - OPTIONAL_METADATA_KEYS == METADATA_KEYS with arcadedb.open_database(temp_db_path) as db: index = db.schema.get_vector_index("Doc", "embedding") reopened = index.get_metadata() - assert set(reopened) == METADATA_KEYS + assert set(reopened) - OPTIONAL_METADATA_KEYS == METADATA_KEYS assert reopened["dimensions"] == 4 assert reopened["similarity_function"] == "EUCLIDEAN" assert reopened["id_property"] == "name" From 3f783a2816666a91c797301aadf47ac792139054 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 21:38:22 +0900 Subject: [PATCH 26/33] CI: match auditwheel's wrapped report on one line; the wheel-check test names the arcadedb_jvm wheel auditwheel wraps 'is consistent with the following platform tag' across lines, so the native image gate failed on a good wheel. test_build_wheel_check still wrote an arcadedb- wheel after the JVM dist became arcadedb-jvm, so build.sh's check found none (3 failures on every JVM leg). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test-native.yml | 8 +++++--- tests/test_build_wheel_check.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index fea41d0..9c444a6 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -360,9 +360,11 @@ jobs: set -euo pipefail w="$(ls "$T"/wheel/dist/*.whl)" uvx auditwheel show "$w" | tee "$T/auditwheel.txt" - grep -F 'consistent with the following platform tag' "$T/auditwheel.txt" - grep -F "\"$PLAT\"" "$T/auditwheel.txt" - if grep -Eiq 'instruction set|x86_64_v[0-9]' "$T/auditwheel.txt"; then + # auditwheel wraps its report to the terminal width, so match on one line with the whitespace collapsed + tr -s '[:space:]' ' ' < "$T/auditwheel.txt" > "$T/auditwheel.flat" + grep -F 'consistent with the following platform tag' "$T/auditwheel.flat" + grep -F "\"$PLAT\"" "$T/auditwheel.flat" + if grep -Eiq 'instruction set|x86_64_v[0-9]' "$T/auditwheel.flat"; then echo "auditwheel reports an ISA warning: the image is not baseline x86-64" >&2 exit 1 fi diff --git a/tests/test_build_wheel_check.py b/tests/test_build_wheel_check.py index 7516a85..60f84fc 100644 --- a/tests/test_build_wheel_check.py +++ b/tests/test_build_wheel_check.py @@ -42,7 +42,7 @@ def test_wheel_check_finds_this_builds_wheel(tmp_path, engine, wheel_version): (tmp_path / "local-jars" / "lib").mkdir(parents=True) (tmp_path / "local-jars" / "lib" / jar_name).write_bytes(b"integration") (tmp_path / "dist").mkdir() - wheel = tmp_path / "dist" / f"arcadedb-{wheel_version}-py3-none-any.whl" + wheel = tmp_path / "dist" / f"arcadedb_jvm-{wheel_version}-py3-none-any.whl" with zipfile.ZipFile(wheel, "w") as zf: zf.writestr(f"arcadedb/jars/{jar_name}", b"integration") From a7813c3e9c7aaaa8842b41896c95c7639745ad01 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 21:55:21 +0900 Subject: [PATCH 27/33] Native glue: apply the P-04 and P-01b hunks 99a9dc2 dropped (graph batches via db.batch(), parallel insert_many into a vertex type creates vertices) Commit 99a9dc2 was meant to port humemai/arcadedb-embedded-python#538, and its tests came across, but its NativeExtras.java change is the XML-limits helper (loadWithoutXmlLimits) and nothing else: the two hunks of #538 never reached the native glue, so the native build still failed 5 tests (test_insert_many_parallel and test_insert_columns_parallel: "Cannot create a document of type 'Vtx' because is not a document type"; test_graph_batch_close_refused_in_the_callers_transaction_can_be_retried and test_a_second_graph_batch_is_refused_while_one_is_open: DID NOT RAISE; test_two_concurrent_batches_on_one_hub_lose_no_edge: "Slot rebase not possible" on close). P-04 (data loss): arcadedb_n_gb_create built its GraphBatch with GraphBatch.builder(db), which takes no part in LocalDatabase.batchStarted. It now opens the batch with db.batch(), as the JPype path does, so build() takes the guard and close() gives it back. The Python API is unchanged. P-01b: the async insert_parallel helper called newDocument for every row. It now looks the type up once per call (the rule of humemai #519 / DocumentBatcher): a vertex type gets newVertex, a document type newDocument, an edge type is refused with a message naming it. Needs a rebuilt native library. Checked on a fresh image built from this tree (native/scripts/build-image.sh, engine 26.10.2): test_bulk_insert.py and test_graph_batch.py 87 passed, 4 skipped (all NA-JVM), test_core.py 39 passed, test_concurrency.py 5 passed. Co-Authored-By: Claude Sonnet 5.5 --- .../org/humemai/nativelib/NativeExtras.java | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index 8b86316..aa8879d 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -35,6 +35,9 @@ import com.arcadedb.integration.importer.Importer; import com.arcadedb.query.sql.executor.CommandContext; import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.EdgeType; +import com.arcadedb.schema.VertexType; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CFunctionPointer; @@ -240,7 +243,9 @@ public static CCharPointer gbCreate(final IsolateThread thread, final long dbh, try { final Database db = ArcadeNative.db(dbh); final Map o = args(options); - GraphBatch.Builder b = GraphBatch.builder(db); + // db.batch(), as the JPype path: it hands the builder the engine's single-batch guard (LocalDatabase.batchStarted, taken by build() and + // released by close()), which GraphBatch.builder(db) bypasses; a second batch while one is open is refused instead of losing edges (P-04) + GraphBatch.Builder b = db.batch(); for (final Map.Entry e : o.entrySet()) { final Object v = e.getValue(); final int iv = intOf(v, 0); @@ -385,6 +390,19 @@ private static Object[] objects(final Object list) { return ((List) list).toArray(); } + /** + * Whether a bulk load into the type creates vertices (true) or documents (false). + * + * @throws IllegalArgumentException for an edge type: an edge cannot be created without its two endpoints + */ + private static boolean isVertexType(final Database db, final String typeName) { + final DocumentType type = db.getSchema().getType(typeName); + if (type instanceof EdgeType) + throw new IllegalArgumentException("Cannot bulk-insert into '" + typeName + + "' because it is an edge type; load the vertices, then create the edges (graph_batch().new_edges() or CREATE EDGE)"); + return type instanceof VertexType; + } + /** * ops: get | set {key,value} | wait {timeout_ms} | kill | create_record {type, kind, props, cb, err} | command / query {lang, text, params, * cb, err} | on_ok {cb} | on_error {cb} | append_samples {type, ts, cols} | insert_parallel {type, rows} | failures {fid}. @@ -539,8 +557,12 @@ public void onError(final Exception e) { }; final String type = (String) a.get("type"); try { - for (final Object row : (List) a.get("rows")) { - final MutableDocument doc = db.newDocument(type); + final List rows = (List) a.get("rows"); + // the type is looked up once per call, as DocumentBatcher does on the JPype path (humemai #519): a vertex type gets vertices, + // a document type documents, an edge type is refused (an edge needs both endpoints); a missing type raises the schema's own error + final boolean vertex = !rows.isEmpty() && isVertexType(db, type); + for (final Object row : rows) { + final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); for (final Map.Entry e : ((Map) row).entrySet()) doc.set(e.getKey(), e.getValue()); ex.createRecord(doc, null, onError); From 6fd16246271be7cb4be90c1d207c77408cb3c5fc Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 22:03:10 +0900 Subject: [PATCH 28/33] Native: columnar bulk insert by address for strings, nulls and vectors (no JSON text) Port of humemai/arcadedb-embedded-python#514 onto the P-04 port fix. - arcadedb_n_insert_colp: the image reads whole columns in place (int64, float64, bool, int32, float32, UTF-8 strings as Arrow offsets plus bytes, float32 vector matrices, an Arrow validity bitmap per column), no JSON text and no per-value crossing. - _native/columnar.py lays the columns out (numpy, lists of one value type with None as null, lists of float32 arrays, pyarrow arrays); _native/bulk.py sends them; insert_many chunks and the DataFrame/Arrow document load use the entry when every column maps to a supported kind, any other batch keeps the JSON path. - Only a float32 matrix (or a list of float32 arrays) is a column of vectors, as on the JPype path (tests/test_float_vector_bulk.py); #514 took any float matrix. The pack test says so. - New tests: test_native_columnar_pack.py (no native library needed, runs on both backends) and test_native_columnar_insert.py (compares the stored records of the JSON and the columnar entry points, native backend only, collect_ignore otherwise). - The two new test files use plain imports of numpy, pandas and pyarrow, not importorskip (the repo's no-skips policy). Needs a rebuilt native library (BulkOps.java, ArcadeNative.java). Co-Authored-By: Claude Sonnet 5.5 --- .../org/humemai/nativelib/ArcadeNative.java | 27 +- .../java/org/humemai/nativelib/BulkOps.java | 162 ++++-- src/arcadedb/_native/bulk.py | 139 ++--- src/arcadedb/_native/columnar.py | 373 +++++++++++++ src/arcadedb/_native/database.py | 13 +- src/arcadedb/_native/lib.py | 1 + src/arcadedb/dataframe.py | 65 ++- tests/conftest.py | 2 + tests/test_native_columnar_insert.py | 488 ++++++++++++++++++ tests/test_native_columnar_pack.py | 301 +++++++++++ 10 files changed, 1433 insertions(+), 138 deletions(-) create mode 100644 src/arcadedb/_native/columnar.py create mode 100644 tests/test_native_columnar_insert.py create mode 100644 tests/test_native_columnar_pack.py diff --git a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java index 49a102f..2cbd978 100644 --- a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java +++ b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java @@ -29,6 +29,7 @@ import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CFloatPointer; import org.graalvm.nativeimage.c.type.CLongPointer; +import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; import org.graalvm.nativeimage.c.type.CTypeConversion; @@ -510,13 +511,33 @@ private static CCharPointer replyOf(final ResultSet rs, final int max, final int return p; } - /** Columnar insert (see BulkOps): header JSON, one buffer holding the columns. Returns {"ok":true,"n":rows}. */ + /** Columnar insert (see BulkOps): header JSON, one buffer holding the columns back to back. Returns {"ok":true,"n":rows}. */ @CEntryPoint(name = "arcadedb_n_insert_cols") public static CCharPointer insertCols(final IsolateThread thread, final long h, final CCharPointer type, final CCharPointer header, final CCharPointer buf, final long bufBytes) { try { - final java.nio.ByteBuffer bb = CTypeConversion.asByteBuffer(buf, (int) bufBytes); - return out("{\"ok\":true,\"n\":" + BulkOps.insertColumns(db(h), str(type), str(header), bb) + "}"); + return out("{\"ok\":true,\"n\":" + BulkOps.insertColumns(db(h), str(type), str(header), (Pointer) buf, bufBytes) + "}"); + } catch (final Throwable t) { + return err(t); + } + } + + /** + * Columnar insert by pointer: no copy of the columns into one buffer. spec points at 6 longs per column (data address and bytes, aux + * address and bytes, null-bitmap address and bytes, see BulkOps.insertColumns) and nspec is their count. Kinds q d b f as + * arcadedb_n_insert_cols, plus i (int32), e (float32) and s (Arrow UTF-8 string: int32 offsets and the bytes they index), and null bitmaps. + * The caller keeps every buffer alive and unchanged until the call returns. Returns {"ok":true,"n":rows}. + */ + @CEntryPoint(name = "arcadedb_n_insert_colp") + public static CCharPointer insertColp(final IsolateThread thread, final long h, final CCharPointer type, final CCharPointer header, + final CCharPointer spec, final int nspec) { + try { + if (nspec < 0 || nspec % BulkOps.SLOTS != 0) + throw new IllegalArgumentException("address table of " + nspec + " entries is not a multiple of " + BulkOps.SLOTS); + final long[] table = new long[nspec]; + for (int i = 0; i < nspec; i++) + table[i] = ((Pointer) spec).readLong(WordFactory.signed(8L * i)); + return out("{\"ok\":true,\"n\":" + BulkOps.insertColumns(db(h), str(type), str(header), table) + "}"); } catch (final Throwable t) { return err(t); } diff --git a/native/src/main/java/org/humemai/nativelib/BulkOps.java b/native/src/main/java/org/humemai/nativelib/BulkOps.java index a3042bb..37cedf9 100644 --- a/native/src/main/java/org/humemai/nativelib/BulkOps.java +++ b/native/src/main/java/org/humemai/nativelib/BulkOps.java @@ -2,8 +2,9 @@ * SPDX-License-Identifier: Apache-2.0 * * Phase 7: columnar bulk insert through one crossing. The documents are built here from whole columns that Python lays out - * in one native-endian buffer (int64, float64, bool bytes, float32 vector matrix), as DocumentBatcher.insertColumns does on - * the JPype path from long[], double[], boolean[] and Object[] columns. + * in native-endian buffers (int64, float64, bool bytes, int32, float32, float32 vector matrix, UTF-8 strings as Arrow offsets plus + * bytes, optional Arrow null bitmaps), as DocumentBatcher.insertColumns does on the JPype path from long[], double[], boolean[] and + * Object[] columns. Values are read straight from the caller's buffers; no JSON text, no parser, no intermediate arrays. */ package org.humemai.nativelib; @@ -11,8 +12,12 @@ import com.arcadedb.database.MutableDocument; import com.arcadedb.schema.VertexType; +import org.graalvm.nativeimage.c.type.CTypeConversion; +import org.graalvm.word.Pointer; +import org.graalvm.word.WordFactory; + import java.nio.ByteBuffer; -import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; @@ -20,57 +25,118 @@ final class BulkOps { private BulkOps() { } + /** Slots per column in the address table: data (address, bytes), aux (address, bytes), nulls (address, bytes). */ + static final int SLOTS = 6; + /** - * header: {"n":rows,"commit":every,"cols":[[name,kind,dim],..]} with kind q (int64), d (float64), b (bool byte) or f - * (float32 matrix, dim floats per row); the buffer holds the columns in that order. Same transaction contract as - * DocumentBatcher.insertColumns: batch commits and the rollback on failure apply only to a transaction opened here. + * The original single-buffer form (arcadedb_n_insert_cols): the columns of kind q, d, b or f lie back to back in one buffer, in + * header order. The address table of the pointer form is derived from the layout, so one loop builds the documents for both. */ @SuppressWarnings("unchecked") - static long insertColumns(final Database db, final String type, final String header, final ByteBuffer raw) { + static long insertColumns(final Database db, final String type, final String header, final Pointer buf, final long bufBytes) { final Map h = (Map) Json.parse(header); - final int n = ((Number) h.get("n")).intValue(); + final long n = ((Number) h.get("n")).longValue(); + final List cols = (List) h.get("cols"); + final long[] spec = new long[SLOTS * cols.size()]; + long pos = 0; + for (int c = 0; c < cols.size(); c++) { + final List col = (List) cols.get(c); + final char kind = ((String) col.get(1)).charAt(0); + final long dim = ((Number) col.get(2)).longValue(); + final long bytes = switch (kind) { + case 'q', 'd' -> 8 * n; + case 'b' -> n; + case 'f' -> 4 * n * dim; + default -> throw new IllegalArgumentException("unknown column kind " + kind); + }; + if (pos + bytes > bufBytes) + throw new IllegalArgumentException("column '" + col.get(0) + "' ends at byte " + (pos + bytes) + ", the buffer has " + bufBytes); + spec[SLOTS * c] = buf.rawValue() + pos; + spec[SLOTS * c + 1] = bytes; + pos += bytes; + } + return insertColumns(db, type, h, spec); + } + + /** The pointer form (arcadedb_n_insert_colp): the table holds SLOTS longs per column, see {@link #insertColumns(Database, String, Map, long[])}. */ + @SuppressWarnings("unchecked") + static long insertColumns(final Database db, final String type, final String header, final long[] spec) { + return insertColumns(db, type, (Map) Json.parse(header), spec); + } + + /** + * header: {"n":rows,"commit":every,"cols":[[name,kind,dim],..]}. Kinds: q (int64), d (float64), b (bool byte), i (int32), e (float32), + * f (float32 matrix, dim floats per row), s (UTF-8 string: int32 offsets[n+1] in the data slot, the bytes they index in the aux slot, as + * an Arrow string array; offsets[0] need not be 0). spec: per column six longs, [data address, data bytes, aux address, aux bytes, null + * bitmap address, null bitmap bytes]; the bitmap is Arrow's (bit i of byte i/8, least significant first; 1 = present), an address of 0 + * means no nulls. The memory belongs to the caller for the duration of the call; nothing is kept. Every length is checked here before + * the first document is built, so a wrong table is an exception and nothing is written. Same transaction contract as + * DocumentBatcher.insertColumns: batch commits and the rollback on failure apply only to a transaction opened here. + */ + @SuppressWarnings("unchecked") + private static long insertColumns(final Database db, final String type, final Map h, final long[] spec) { + final long nRows = ((Number) h.get("n")).longValue(); + if (nRows < 0 || nRows > Integer.MAX_VALUE) + throw new IllegalArgumentException("row count out of range: " + nRows); + final int n = (int) nRows; final int commitEvery = ((Number) h.get("commit")).intValue(); final List cols = (List) h.get("cols"); final int k = cols.size(); + if (spec.length != SLOTS * k) + throw new IllegalArgumentException("address table has " + spec.length + " entries for " + k + " columns, expected " + SLOTS * k); final String[] names = new String[k]; final char[] kinds = new char[k]; - final Object[] data = new Object[k]; final int[] dims = new int[k]; - raw.order(ByteOrder.nativeOrder()); - int pos = 0; + final long[] data = new long[k]; // addresses: a Word value cannot live in an array or be compared with null + final long[] nulls = new long[k]; // 0 = no nulls + final ByteBuffer[] blobs = new ByteBuffer[k]; + int longest = 0; for (int c = 0; c < k; c++) { final List col = (List) cols.get(c); names[c] = (String) col.get(0); kinds[c] = ((String) col.get(1)).charAt(0); dims[c] = ((Number) col.get(2)).intValue(); - final int bytes = switch (kinds[c]) { - case 'q', 'd' -> 8 * n; + final long dataAddr = spec[SLOTS * c], dataBytes = spec[SLOTS * c + 1]; + final long auxAddr = spec[SLOTS * c + 2], auxBytes = spec[SLOTS * c + 3]; + final long nullAddr = spec[SLOTS * c + 4], nullBytes = spec[SLOTS * c + 5]; + final long need = switch (kinds[c]) { + case 'q', 'd' -> 8L * n; case 'b' -> n; - case 'f' -> 4 * n * dims[c]; + case 'i', 'e' -> 4L * n; + case 'f' -> { + if (dims[c] <= 0) + throw new IllegalArgumentException("column '" + names[c] + "': a vector column needs a dimension above 0"); + yield 4L * n * dims[c]; + } + case 's' -> 4L * (n + 1L); default -> throw new IllegalArgumentException("unknown column kind " + kinds[c]); }; - final ByteBuffer part = raw.slice(pos, bytes).order(ByteOrder.nativeOrder()); - pos += bytes; - switch (kinds[c]) { - case 'q' -> { - final long[] a = new long[n]; - part.asLongBuffer().get(a); - data[c] = a; - } - case 'd' -> { - final double[] a = new double[n]; - part.asDoubleBuffer().get(a); - data[c] = a; - } - case 'b' -> { - final boolean[] a = new boolean[n]; - for (int i = 0; i < n; i++) - a[i] = part.get(i) != 0; - data[c] = a; + if (dataBytes < need || (need > 0 && dataAddr == 0)) + throw new IllegalArgumentException("column '" + names[c] + "' has " + dataBytes + " bytes, " + need + " needed"); + data[c] = dataAddr; + if (nullAddr != 0) { + if (nullBytes < (n + 7L) / 8) + throw new IllegalArgumentException("column '" + names[c] + "' has a null bitmap of " + nullBytes + " bytes, " + (n + 7L) / 8 + " needed"); + nulls[c] = nullAddr; + } + if (kinds[c] == 's') { + if (auxBytes < 0 || auxBytes > Integer.MAX_VALUE || (auxBytes > 0 && auxAddr == 0)) + throw new IllegalArgumentException("column '" + names[c] + "': the string bytes are missing or beyond 2 GiB"); + blobs[c] = auxBytes == 0 ? ByteBuffer.allocate(0) : CTypeConversion.asByteBuffer(WordFactory.pointer(auxAddr), (int) auxBytes); + final Pointer offsets = WordFactory.pointer(data[c]); + int prev = offsets.readInt(WordFactory.signed(0)); + if (prev < 0) + throw new IllegalArgumentException("column '" + names[c] + "': negative string offset"); + for (int i = 1; i <= n; i++) { + final int off = offsets.readInt(WordFactory.signed(4L * i)); + if (off < prev || off > auxBytes) + throw new IllegalArgumentException("column '" + names[c] + "': string offsets are not increasing within the " + auxBytes + " bytes (row " + (i - 1) + ")"); + longest = Math.max(longest, off - prev); + prev = off; } - default -> data[c] = part.asFloatBuffer(); } } + final byte[] scratch = new byte[longest]; final boolean vertex = db.getSchema().getType(type) instanceof VertexType; final boolean wasActive = db.isTransactionActive(); if (!wasActive) @@ -79,13 +145,33 @@ static long insertColumns(final Database db, final String type, final String hea for (int i = 0; i < n; i++) { final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); for (int c = 0; c < k; c++) { + final Pointer base = WordFactory.pointer(data[c]); + if (nulls[c] != 0 && ((WordFactory.pointer(nulls[c]).readByte(WordFactory.signed(i >> 3)) >> (i & 7)) & 1) == 0) { + doc.set(names[c], null); + continue; + } switch (kinds[c]) { - case 'q' -> doc.set(names[c], ((long[]) data[c])[i]); - case 'd' -> doc.set(names[c], ((double[]) data[c])[i]); - case 'b' -> doc.set(names[c], ((boolean[]) data[c])[i]); + case 'q' -> doc.set(names[c], base.readLong(WordFactory.signed(8L * i))); + case 'd' -> doc.set(names[c], base.readDouble(WordFactory.signed(8L * i))); + case 'b' -> doc.set(names[c], base.readByte(WordFactory.signed(i)) != 0); + case 'i' -> doc.set(names[c], base.readInt(WordFactory.signed(4L * i))); + case 'e' -> doc.set(names[c], base.readFloat(WordFactory.signed(4L * i))); + case 's' -> { + final int from = base.readInt(WordFactory.signed(4L * i)); + final int len = base.readInt(WordFactory.signed(4L * (i + 1L))) - from; + if (len == 0) + doc.set(names[c], ""); + else { + blobs[c].get(from, scratch, 0, len); + doc.set(names[c], new String(scratch, 0, len, StandardCharsets.UTF_8)); + } + } default -> { - final float[] v = new float[dims[c]]; - ((java.nio.FloatBuffer) data[c]).get(i * dims[c], v); + final int dim = dims[c]; + final float[] v = new float[dim]; + final long at = 4L * i * dim; + for (int j = 0; j < dim; j++) + v[j] = base.readFloat(WordFactory.signed(at + 4L * j)); doc.set(names[c], v); } } diff --git a/src/arcadedb/_native/bulk.py b/src/arcadedb/_native/bulk.py index 9cae198..d244c48 100644 --- a/src/arcadedb/_native/bulk.py +++ b/src/arcadedb/_native/bulk.py @@ -1,98 +1,71 @@ -"""Columnar bulk insert through one crossing (``arcadedb_n_insert_cols``): numpy int/float/bool columns and float32 vector -columns (a 2-D array or a list of equal-length 1-D float arrays) go over as one buffer; anything else returns None and the -caller falls back to the per-row path. The documents are built Java-side as ``DocumentBatcher.insertColumns`` builds them on JPype. +"""Columnar bulk insert through one crossing (``arcadedb_n_insert_colp``): whole columns go over by address, with no JSON text. Numpy +ints, floats and bools, lists of one value type (None is a null), strings (UTF-8 bytes plus offsets, as Arrow lays them out), float32 +vector columns and Arrow arrays are laid out by :mod:`.columnar`; anything else (nested values, mixed-type columns, date-times, +decimals, integers past 64 bits) returns None and the caller uses the per-row JSON path. The documents are built image-side as +``DocumentBatcher.insertColumns`` builds them on JPype. ``ARCADEDB_NATIVE_BULK=0`` turns the columnar path off. """ -import json import os -from . import lib - -try: - import numpy as _np -except ImportError: # pragma: no cover - _np = None +from . import columnar, lib _ENABLED = os.environ.get("ARCADEDB_NATIVE_BULK", "1") != "0" -def insert_columns_fast(db, type_name, columns, commit_every): - if ( - not _ENABLED - or _np is None - or lib._lib is None - or not hasattr(lib._lib, "arcadedb_n_insert_cols") - ): - return None - cols, parts, n = [], [], None - for name, values in columns.items(): - if hasattr(values, "to_numpy") and not isinstance(values, (list, tuple)): - return None - kind, dim = None, 0 - if isinstance(values, _np.ndarray): - k = values.dtype.kind - if ( - values.ndim == 1 - and k in "iu" - and ( - k == "i" - or not values.size - or int(values.max()) <= _np.iinfo(_np.int64).max - ) - ): - kind, arr = "q", _np.ascontiguousarray(values, dtype=_np.int64) - elif values.ndim == 1 and k == "f": - kind, arr = "d", _np.ascontiguousarray(values, dtype=_np.float64) - elif values.ndim == 1 and k == "b": - kind, arr = "b", _np.ascontiguousarray(values, dtype=_np.uint8) - elif ( - values.ndim == 2 and values.dtype == _np.float32 and values.shape[1] > 0 - ): # only a float32 matrix is a column of vectors, as on the JPype path - kind, arr, dim = ( - "f", - _np.ascontiguousarray(values, dtype=_np.float32), - int(values.shape[1]), - ) - elif ( - isinstance(values, (list, tuple)) - and values - and all( - isinstance(v, _np.ndarray) - and v.ndim == 1 - and v.dtype.kind == "f" - and v.shape == values[0].shape - for v in values - ) - and values[0].shape[0] > 0 - ): - kind, arr, dim = ( - "f", - _np.ascontiguousarray(_np.stack(values), dtype=_np.float32), - int(values[0].shape[0]), - ) - if kind is None: - return None - rows = int(arr.shape[0]) - if n is not None and rows != n: - return None # the per-row path reports the ragged columns - n = rows - cols.append([name, kind, dim]) - parts.append(arr.tobytes()) - if not n: - return None - header = json.dumps( - {"n": n, "commit": min(int(commit_every), 2**31 - 1), "cols": cols} +def _usable(): + return ( + _ENABLED + and columnar._np is not None + and lib._lib is not None + and hasattr(lib._lib, "arcadedb_n_insert_colp") ) - buf = b"".join(parts) + + +def _send(db, type_name, packed, rows, commit_every, prefix): from .database import _call + spec = columnar.spec_of(packed) reply = _call( - "Failed to bulk-insert", - "arcadedb_n_insert_cols", + prefix, + "arcadedb_n_insert_colp", db._h, lib.b(type_name), - lib.b(header), - buf, - len(buf), - ) + lib.b(columnar.header_of(rows, commit_every, packed)), + spec, + len(spec), + ) # `packed` holds the arrays the addresses point into until the call returns return int(reply["n"]) + + +def insert_columns_fast( + db, type_name, columns, commit_every, declared=None, prefix="Failed to bulk-insert" +): + """Insert ``columns`` (name -> column) in one crossing and return the row count, or None when a column cannot be sent this way. + + ``declared``: a dict of declared property types when the caller already knows them (the DataFrame path), else they are read from the + schema if a column needs them. + """ + if not _usable(): + return None + packed = columnar.pack(columns, columnar.Declared(db, type_name, declared)) + if packed is None: + return None + rows, cols = packed + return _send(db, type_name, cols, rows, commit_every, prefix) + + +def insert_rows_columnar(db, type_name, chunk): + """One ``insert_many`` chunk (a list of dicts with the same keys) through the columnar entry inside the caller's transaction (no + commit here). Returns True when it was sent, False when the chunk needs the JSON path. + """ + if not _usable(): + return False + columns = columnar.rows_to_columns(chunk) + if columns is None: + return False + packed = columnar.pack(columns, columnar.Declared(db, type_name)) + if packed is None: + return False + rows, cols = packed + _send(db, type_name, cols, rows, 0, f"Failed to bulk-insert into '{type_name}'") + return True diff --git a/src/arcadedb/_native/columnar.py b/src/arcadedb/_native/columnar.py new file mode 100644 index 0000000..4bb79f1 --- /dev/null +++ b/src/arcadedb/_native/columnar.py @@ -0,0 +1,373 @@ +"""Packing of whole columns for ``arcadedb_n_insert_colp``: the columnar bulk insert of the native backend. + +A column crosses as raw memory, not as JSON text: the image reads each buffer in place and builds the documents without a parser. This +module only decides how a column is laid out and hands the addresses over; it needs no native library, so the layouts are unit-tested +without one. A column that cannot be laid out (a nested value, a mixed-type column, a date-time, an integer past 64 bits, a string with +a lone surrogate) makes :func:`pack` return ``None`` and the caller uses the JSON path for that call; nothing is half sent. + +Kinds (the image's, ``BulkOps.insertColumns``): ``q`` int64, ``d`` float64, ``b`` bool byte, ``i`` int32, ``e`` float32, ``f`` float32 +matrix (``dim`` per row), ``s`` UTF-8 string (``int32 offsets[n+1]`` and the bytes they index: Arrow's string layout). A column may +carry a null bitmap (Arrow's: bit ``i`` of byte ``i // 8``, least significant first, 1 = present). + +``i`` and ``e`` and the float matrix from a list of lists are chosen only for a property declared ``INTEGER``, ``FLOAT`` and +``ARRAY_OF_FLOATS``: the engine would convert to those types anyway, so the stored value is the one the JSON path stores. For an +undeclared property an ``int32`` column crosses as ``q`` (a Long) and a ``float32`` column as ``d`` (a Double), as the JSON path stores +them. +""" + +import re + +try: + import numpy as _np +except ImportError: # pragma: no cover + _np = None + +_INT64_MAX = 2**63 - 1 +_INT32_MAX = 2**31 - 1 +_NoneType = type(None) +_PROPERTY = re.compile(r"name: (.+?), type: ([A-Z_]+)") + + +class Packed: + """One column laid out for the image: ``data``, ``aux`` and ``nulls`` are ``(address, bytes)``; ``keep`` holds what the addresses point into.""" + + __slots__ = ("name", "kind", "dim", "n", "data", "aux", "nulls", "keep") + + def __init__(self, name, kind, dim, array, aux=None, nulls=None, keep=()): + self.name = name + self.kind = kind + self.dim = dim + self.n = int(array.shape[0]) - ( + 1 if kind == "s" else 0 + ) # a string column's array is its n + 1 offsets + self.data = (array.ctypes.data, array.nbytes) + self.aux = aux or (0, 0) + self.nulls = (nulls.ctypes.data, nulls.nbytes) if nulls is not None else (0, 0) + self.keep = (array, nulls, *keep) + + +class Declared: + """The declared property types of a type, read once and only when a column needs them (``INTEGER``, ``FLOAT``, ``ARRAY_OF_FLOATS``). + + ``known`` (a dict) is used as it is; otherwise ``db`` is asked for ``schema:types`` the first time :meth:`get` is called. + """ + + def __init__(self, db=None, type_name=None, known=None): + self._db, self._type, self._known = db, type_name, known + + def get(self, name): + if self._known is None: + self._known = _declared_types(self._db, self._type) + return self._known.get(name) + + +def _declared_types(db, type_name): + """``{property: TYPE}`` of ``type_name`` and its parent types (the nearest declaration wins); ``{}`` when it cannot be read.""" + try: + rows = { + r["name"]: r for r in db.query("sql", "SELECT FROM schema:types").to_list() + } + except Exception: # nosec B110 - the insert reports the real error + return {} + out, seen, queue = {}, set(), [type_name] + while queue: + current = queue.pop(0) + if current in seen or current not in rows: + continue + seen.add(current) + for text in rows[current].get("properties") or []: + m = _PROPERTY.search(text) + if m: + out.setdefault(m.group(1), m.group(2)) + queue.extend(rows[current].get("parentTypes") or []) + return out + + +def _bitmap(present): + """Arrow validity bitmap of a bool array (True = present).""" + return _np.packbits(present, bitorder="little") + + +def _numpy_column(name, values, declared, nulls=None): + """A 1-D or 2-D numpy array as a column, or None.""" + k = values.dtype.kind + if ( + values.ndim == 2 + and values.dtype + == _np.float32 # only a float32 matrix is a column of vectors, as on the JPype path + and values.shape[1] > 0 + and nulls is None + ): + arr = _np.ascontiguousarray(values, dtype=_np.float32) + return Packed(name, "f", int(values.shape[1]), arr) + if values.ndim != 1: + return None + if k == "b": + return Packed( + name, "b", 0, _np.ascontiguousarray(values).view(_np.uint8), nulls=nulls + ) + if k in "iu": + if ( + k == "u" + and values.dtype.itemsize == 8 + and values.size + and int(values.max()) > _INT64_MAX + ): + return None + if k == "i" and values.dtype.itemsize <= 4 and declared.get(name) == "INTEGER": + return Packed( + name, + "i", + 0, + _np.ascontiguousarray(values, dtype=_np.int32), + nulls=nulls, + ) + return Packed( + name, "q", 0, _np.ascontiguousarray(values, dtype=_np.int64), nulls=nulls + ) + if k == "f": + if values.dtype.itemsize == 4 and declared.get(name) == "FLOAT": + return Packed( + name, + "e", + 0, + _np.ascontiguousarray(values, dtype=_np.float32), + nulls=nulls, + ) + return Packed( + name, "d", 0, _np.ascontiguousarray(values, dtype=_np.float64), nulls=nulls + ) + return None + + +def _string_column(name, texts, nulls=None): + """A list of str (a null already replaced by '') as a string column, or None (a lone surrogate, or 2 GiB of text).""" + n = len(texts) + blob = "".join(texts) + if blob.isascii(): + data = blob.encode("ascii") + lengths = _np.fromiter(map(len, texts), dtype=_np.int64, count=n) + else: + try: + encoded = [t.encode("utf-8") for t in texts] + except UnicodeEncodeError: + return None # a lone surrogate: the JSON path decides what happens to it + data = b"".join(encoded) + lengths = _np.fromiter(map(len, encoded), dtype=_np.int64, count=n) + if len(data) > _INT32_MAX: + return None + offsets = _np.zeros(n + 1, dtype=_np.int32) + _np.cumsum(lengths, out=offsets[1:], dtype=_np.int32) + blob_array = ( + _np.frombuffer(data, dtype=_np.uint8) if data else _np.zeros(0, dtype=_np.uint8) + ) + aux = (blob_array.ctypes.data, blob_array.nbytes) if data else (0, 0) + return Packed(name, "s", 0, offsets, aux=aux, nulls=nulls, keep=(blob_array, data)) + + +def _list_column(name, values, declared): + """A list or tuple of Python values as a column: one type (None allowed) or a list of equal-length float arrays; else None.""" + n = len(values) + types = set(map(type, values)) + present = None + if _NoneType in types: + types.discard(_NoneType) + present = _np.fromiter((v is not None for v in values), dtype=bool, count=n) + nulls = _bitmap(present) + else: + nulls = None + if not types: # every value is None + return _string_column(name, [""] * n, nulls) + if len(types) != 1: + return None + (only,) = types + if only is str: + texts = ( + values if present is None else [v if v is not None else "" for v in values] + ) + return _string_column(name, texts, nulls) + if only is int or only is float or only is bool: + filler = 0 if only is int else 0.0 if only is float else False + vals = ( + values + if present is None + else [v if v is not None else filler for v in values] + ) + dtype = ( + _np.int64 if only is int else _np.float64 if only is float else _np.uint8 + ) + try: + arr = _np.array(vals, dtype=dtype) + except OverflowError: + return None # an integer beyond 64 bits + kind = "q" if only is int else "d" if only is float else "b" + return Packed(name, kind, 0, arr, nulls=nulls) + if only is list and declared.get(name) == "ARRAY_OF_FLOATS" and present is None: + try: + arr = _np.array(values, dtype=_np.float32) + except (ValueError, TypeError): + return None # ragged, or not numbers + if arr.ndim != 2 or arr.shape[1] == 0: + return None + return Packed(name, "f", int(arr.shape[1]), arr) + if ( + only is _np.ndarray + and present is None + and all( + v.ndim == 1 and v.dtype == _np.float32 and v.shape == values[0].shape + for v in values + ) + and values[0].shape[0] > 0 + ): + return Packed( + name, + "f", + int(values[0].shape[0]), + _np.ascontiguousarray(_np.stack(values), dtype=_np.float32), + ) + return None + + +def _arrow_column(name, arr, declared): + """A ``pyarrow`` array as a column (strings in place), or None.""" + import pyarrow as pa + + types = pa.types + if isinstance(arr, pa.ChunkedArray): + arr = arr.combine_chunks() + if isinstance(arr, pa.ChunkedArray): + arr = arr.chunk(0) if arr.num_chunks == 1 else None + if arr is None: + return None + if types.is_dictionary(arr.type): + arr = arr.cast(arr.type.value_type) + t, n = arr.type, len(arr) + nulls = ( + _bitmap(arr.is_valid().to_numpy(zero_copy_only=False)) + if arr.null_count + else None + ) + if types.is_string(t) or types.is_large_string(t): + bufs = arr.buffers() + off_buf, data_buf = bufs[1], bufs[2] + if types.is_string(t): + offsets = _np.frombuffer(off_buf, dtype=_np.int32)[ + arr.offset : arr.offset + n + 1 + ] + else: + wide = _np.frombuffer(off_buf, dtype=_np.int64)[ + arr.offset : arr.offset + n + 1 + ] + if len(wide) and int(wide[-1]) > _INT32_MAX: + return None + offsets = wide.astype(_np.int32) + offsets = _np.ascontiguousarray(offsets) + size = 0 if data_buf is None else min(data_buf.size, _INT32_MAX) + aux = (data_buf.address, size) if size else (0, 0) + return Packed( + name, "s", 0, offsets, aux=aux, nulls=nulls, keep=(arr, off_buf, data_buf) + ) + if types.is_integer(t) or types.is_floating(t) or types.is_boolean(t): + if arr.null_count: + arr = arr.fill_null(False if types.is_boolean(t) else 0) + return _numpy_column(name, arr.to_numpy(zero_copy_only=False), declared, nulls) + if ( + (types.is_list(t) or types.is_large_list(t) or types.is_fixed_size_list(t)) + and not arr.null_count + and n + and declared.get(name) == "ARRAY_OF_FLOATS" + and (types.is_floating(t.value_type) or types.is_integer(t.value_type)) + ): + flat = arr.flatten() + if flat.null_count or len(flat) % n: + return None + dim = len(flat) // n + if dim == 0: + return None + if not types.is_fixed_size_list(t): + import pyarrow.compute as pc + + lengths = pc.list_value_length(arr) + if pc.min(lengths).as_py() != dim or pc.max(lengths).as_py() != dim: + return None + values = flat.to_numpy(zero_copy_only=False).reshape(n, dim) + return Packed(name, "f", dim, _np.ascontiguousarray(values, dtype=_np.float32)) + return None + + +def pack(columns, declared): + """``(rows, [Packed])`` for a dict of name -> column, or None when any column cannot be laid out or the columns differ in length. + + A column is a numpy array (1-D ints, floats and bools, or a 2-D float32 matrix), a list or tuple of Python values of one type (None + allowed), a list of equal-length 1-D float32 arrays, or a ``pyarrow`` array. + """ + items = list( + columns.items() + ) # a dict, or a DataFrame (its columns are pandas Series: the row path takes those) + if _np is None or not items: + return None + rows, packed = None, [] + for name, values in items: + if type(values).__module__.split(".")[0] == "pyarrow": + col = _arrow_column(name, values, declared) + elif isinstance(values, _np.ndarray): + if values.dtype.kind in "OUS": + col = ( + _list_column(name, values.tolist(), declared) + if values.ndim == 1 + else None + ) + else: + col = _numpy_column(name, values, declared) + elif isinstance(values, (list, tuple)) and values: + col = _list_column(name, values, declared) + else: + col = None # a pandas Series, a generator, an empty list: the row path handles them + if col is None: + return None + if rows is not None and col.n != rows: + return None # the row path reports the ragged columns + rows = col.n + packed.append(col) + return (rows, packed) if rows else None + + +def spec_of(packed): + """The address table the image reads: six int64 per column (data, aux and null-bitmap address and byte count).""" + import ctypes + + flat = [] + for col in packed: + flat.extend((*col.data, *col.aux, *col.nulls)) + return (ctypes.c_int64 * len(flat))(*flat) + + +def header_of(rows, commit_every, packed): + """The JSON header text: row count, commit interval and ``[name, kind, dim]`` per column.""" + import json + + return json.dumps( + { + "n": rows, + "commit": min(int(commit_every), 2**31 - 1), + "cols": [[c.name, c.kind, c.dim] for c in packed], + } + ) + + +def rows_to_columns(rows): + """A list of dicts with the same keys as a dict of name -> list of values, or None (no rows, different keys, a non-str key).""" + if not rows: + return None + try: + keys = list(rows[0]) + if any(type(key) is not str for key in keys): + return None + if set(map(len, rows)) != {len(keys)}: + return None + return {key: [row[key] for row in rows] for key in keys} + except ( + KeyError, + TypeError, + ): # same size but different keys, or a row that is not a dict + return None diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index d640e8a..1b87e20 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -14,6 +14,7 @@ from ..transactions import TransactionContext from ..type_conversion import json_bulk_dumps from . import lib, not_yet +from .bulk import insert_rows_columnar from .codec import _prep as _prep_row from .codec import encode_params, json_native, loads, row_default @@ -677,10 +678,12 @@ def lookup_by_key(self, type_name: str, keys: List[str], values: List[Any]): def insert_many( self, type_name: str, rows, commit_every: int = 10_000, parallel: bool = False ) -> int: - """Bulk insert. Rows go in chunks of 10,000, one JSON text and one crossing per chunk (``arcadedb_n_insert_rows``: a - newDocument/set/save loop inside the image, as ``DocumentBatcher.insertManyJson`` on the JPype path); a chunk with a value the - JSON text cannot carry unchanged goes row by row through ``INSERT INTO t CONTENT :row``. Inside the caller's transaction nothing is - committed; otherwise every ``commit_every`` rows are one transaction (0: all rows in one). + """Bulk insert. Rows go in chunks of 10,000, one crossing per chunk. A chunk of rows with the same keys whose columns each hold one + value type (ints, floats, bools, strings, equal-length float lists for an ``ARRAY_OF_FLOATS`` property; None is a null) goes as + columns by address (``arcadedb_n_insert_colp``, no JSON text); any other chunk goes as one JSON text (``arcadedb_n_insert_rows``: + a newDocument/set/save loop inside the image, as ``DocumentBatcher.insertManyJson`` on the JPype path); a chunk with a value the + JSON text cannot carry unchanged goes through ``INSERT INTO t CONTENT :row`` for each row, in one crossing. Inside the caller's + transaction nothing is committed; otherwise every ``commit_every`` rows are one transaction (0: all rows in one). """ self._check_not_closed() if parallel: @@ -719,6 +722,8 @@ def insert_many( def _insert_chunk(self, type_name: str, sql: str, chunk) -> None: if lib.INSERT_ROWS: + if insert_rows_columnar(self, type_name, chunk): + return try: payload = json_bulk_dumps(chunk) except (TypeError, ValueError): diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index b2f16ac..0309ee5 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -158,6 +158,7 @@ class _Params(ctypes.Structure): "arcadedb_n_exec": [P, L, I, S, S, S, I], "arcadedb_n_exec_bin": [P, L, I, S, S, S, I, S, L], "arcadedb_n_insert_cols": [P, L, S, S, S, L], + "arcadedb_n_insert_colp": [P, L, S, S, P, I], "arcadedb_n_next": [P, L, I], "arcadedb_n_rs_close": [P, L], "arcadedb_n_cmd": [P, L, I, S, S, S, S, L, I], diff --git a/src/arcadedb/dataframe.py b/src/arcadedb/dataframe.py index 634ff9c..0e4e9b9 100644 --- a/src/arcadedb/dataframe.py +++ b/src/arcadedb/dataframe.py @@ -311,14 +311,30 @@ def _scalars_to_tagged(kind: str, chunk): ] -def _column_values(name: str, chunk, pa, native: bool = False): +def _column_values(name: str, chunk, pa, native: bool = False, arrow: bool = False): """One Arrow array as what ``Database.insert_columns`` takes: a numpy array when the column crosses as one buffer (integers, floats and booleans without nulls), else a list of values (``None`` is a null). + + ``arrow`` (the native backend's document load): a string column, and a numeric one with nulls, stay Arrow arrays, which + the columnar entry reads in place (the offsets, the bytes and the validity bitmap) instead of as Python values. """ types = pa.types if types.is_dictionary(chunk.type): # a pandas categorical or a Polars Categorical chunk = chunk.cast(chunk.type.value_type) kind = _kind(pa, chunk.type) + if arrow and ( + ( + kind == "string" + and (types.is_string(chunk.type) or types.is_large_string(chunk.type)) + ) + or (kind in ("integer", "floating", "bool") and chunk.null_count) + ): + if not ( + kind == "integer" + and types.is_uint64(chunk.type) + and len(chunk) > chunk.null_count + ): # uint64 keeps the range check below + return chunk if kind in ("integer", "floating", "bool"): if native and types.is_uint64(chunk.type) and len(chunk) > chunk.null_count: import pyarrow.compute as pc @@ -403,7 +419,13 @@ def insert_table( if batch.num_rows == 0: continue columns = { - name: _column_values(name, batch.column(i), pa, native) + name: _column_values( + name, + batch.column(i), + pa, + native, + arrow=native and not is_vertex and not parallel, + ) for i, name in enumerate(names) } try: @@ -413,7 +435,7 @@ def insert_table( ) elif native: _insert_documents_native( - db, type_name, names, columns, batch_size, parallel + db, type_name, names, columns, batch_size, parallel, declared ) else: db.insert_columns( @@ -460,15 +482,38 @@ def _insert_documents_native( columns: Dict[str, Any], batch_size: int, parallel: bool, + declared: Optional[Dict[str, str]] = None, ): - """One batch of documents on the native backend: the columns as one dict per row, handed to ``insert_many`` - (one crossing per 10,000 rows; the batch is one transaction, or joins the caller's). + """One batch of documents on the native backend. When every column can cross as memory (numbers, bools, strings, nulls, + float vectors for an ``ARRAY_OF_FLOATS`` property) the batch goes through the columnar entry, no Python values and no JSON + text; otherwise the columns become one dict per row and go to ``insert_many`` (one crossing per 10,000 rows). Either way + the batch is one transaction, or joins the caller's. """ - lists = [ - # jvm-payload: not-a-crossing -- no JVM here: insert_many serialises the rows to JSON text - columns[name].tolist() if hasattr(columns[name], "tolist") else columns[name] - for name in names - ] + if not parallel: + from ._native.bulk import insert_columns_fast + + if ( + insert_columns_fast( + db, + type_name, + columns, + batch_size, + declared={} if declared is None else declared, + prefix=f"Failed to bulk-insert into '{type_name}'", + ) + is not None + ): + return + lists = [] + for name in names: + col = columns[name] + if hasattr( + col, "to_pylist" + ): # a column kept as an Arrow array for the columnar entry + lists.append(col.to_pylist()) + else: + # jvm-payload: not-a-crossing -- no JVM here: insert_many serialises the rows to JSON text + lists.append(col.tolist() if hasattr(col, "tolist") else col) db.insert_many( type_name, [dict(zip(names, row)) for row in zip(*lists)], diff --git a/tests/conftest.py b/tests/conftest.py index 365d268..86b601d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,6 +39,8 @@ if BACKEND != "native": # Java -> Python upcalls (Python functions in SQL) exist on the native build only collect_ignore.append("test_native_udf.py") + # the columnar insert's stored records are compared between the JSON and the columnar entry points of the native library + collect_ignore.append("test_native_columnar_insert.py") # the native library's start-up output (#477) can only be checked with the native library loaded collect_ignore.append("test_native_startup_runtime.py") # the XML entity limits of an import, lifted by the native glue (the JVM build lifts them with JVM system properties) diff --git a/tests/test_native_columnar_insert.py b/tests/test_native_columnar_insert.py new file mode 100644 index 0000000..3aac3df --- /dev/null +++ b/tests/test_native_columnar_insert.py @@ -0,0 +1,488 @@ +"""The native backend's columnar bulk insert stores what the JSON path stores (``arcadedb_n_insert_colp`` against ``arcadedb_n_insert_rows``). + +Needs the native library, so conftest.py collects this file on the native backend only (a skip elsewhere would fail +scripts/check_test_skips.py). Each test loads the same data through the JSON path (``bulk._ENABLED`` off) and through the columnar entry and +compares the stored records value by value and type by type; a spy on the crossing proves which entry point ran, so a silent fall back +to JSON cannot pass for the columnar path. +""" + +import ctypes +import datetime +import math + +import numpy as np +import pytest + +from arcadedb._native import bulk, columnar, lib # noqa: E402 +from arcadedb.exceptions import ArcadeDBError # noqa: E402 + +COLP, ROWS, MANY = ( + "arcadedb_n_insert_colp", + "arcadedb_n_insert_rows", + "arcadedb_n_exec_many", +) + + +class Spy: + def __init__(self, monkeypatch): + self.calls = [] + real = lib.call + + def call(name, *args): + self.calls.append(name) + return real(name, *args) + + monkeypatch.setattr(lib, "call", call) + + def count(self, name): + return self.calls.count(name) + + +@pytest.fixture +def spy(monkeypatch): + return Spy(monkeypatch) + + +def _count_sql(name): + return f"SELECT count(*) AS n FROM {name}" # nosec B608 - test-owned type name, no user input + + +def _types(db, *names, extra="", unique=False): + for name in names: + db.command("sql", f"CREATE DOCUMENT TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + db.command("sql", f"CREATE PROPERTY {name}.grp INTEGER") + db.command("sql", f"CREATE PROPERTY {name}.val DOUBLE") + db.command("sql", f"CREATE PROPERTY {name}.cat STRING") + db.command("sql", f"CREATE PROPERTY {name}.txt STRING") + db.command("sql", f"CREATE PROPERTY {name}.vec ARRAY_OF_FLOATS") + if unique: + db.command("sql", f"CREATE INDEX ON {name} (id) UNIQUE") + + +def _stored(db, name): + """Every record of a type as (value, python type) pairs per property, ordered by id.""" + out = [] + sql = f"SELECT FROM {name} ORDER BY id" # nosec B608 - test-owned type name, no user input + for row in db.query("sql", sql).to_list(): + out.append( + {k: (v, type(v).__name__) for k, v in row.items() if not k.startswith("@")} + ) + return out + + +def _probe(n, dim=8, seed=7): + """The probe workload's column types: INTEGER ids, a DOUBLE, a 20-value category, a variable-length text, a float32 vector.""" + rng = np.random.default_rng(seed) + ids = np.arange(n, dtype=np.int64) + return { + "id": ids, + "grp": ids % 1000, + "val": (ids * 37 % 10007) / 7.0, + "cat": ["c%d" % (k % 20) for k in range(n)], + "txt": ["row-%d-%s" % (k, "x" * (k % 24)) for k in range(n)], + "vec": rng.standard_normal((n, dim), dtype=np.float32), + } + + +def _rows(columns): + names = list(columns) + lists = [c.tolist() if hasattr(c, "tolist") else c for c in columns.values()] + return [dict(zip(names, vals)) for vals in zip(*lists)] + + +def _json_reference(db, name, rows, monkeypatch, **kw): + with monkeypatch.context() as m: + m.setattr(bulk, "_ENABLED", False) + db.insert_many(name, rows, **kw) + + +def test_probe_workload_columns_store_the_same_records_on_every_columnar_entry( + temp_db, spy, monkeypatch +): + _types(temp_db, "Ref", "ByRows", "ByCols", "ByFrame", "ByTable") + columns = _probe(2_500) + rows = _rows(columns) + _json_reference(temp_db, "Ref", rows, monkeypatch) + assert spy.count(COLP) == 0 and spy.count(ROWS) == 1 + + spy.calls.clear() + temp_db.insert_many("ByRows", rows) # dict rows, transposed to columns + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 + temp_db.insert_columns("ByCols", columns) # numpy columns and lists of str + assert spy.count(COLP) == 2 and spy.count(ROWS) == 0 + + import pandas as pd + import pyarrow as pa + + frame = pd.DataFrame( + { + **{k: v for k, v in columns.items() if k != "vec"}, + "vec": list(columns["vec"]), + } + ) + temp_db.insert_dataframe("ByFrame", frame) + temp_db.insert_arrow("ByTable", pa.table({k: v for k, v in frame.items()})) + assert spy.count(COLP) == 4 and spy.count(ROWS) == 0 and spy.count(MANY) == 0 + + expected = _stored(temp_db, "Ref") + assert len(expected) == 2_500 + assert expected[7]["id"][1] == "int" and expected[7]["vec"][1] == "list" + for name in ("ByRows", "ByCols", "ByFrame", "ByTable"): + assert _stored(temp_db, name) == expected, name + + +def test_unicode_empty_and_awkward_strings_survive_the_columnar_path( + temp_db, spy, monkeypatch +): + _types(temp_db, "Ref", "Col") + texts = [ + "", + "plain", + "héllo wörld", + "日本語のテキスト", + "한국어", + "😀 emoji, outside the BMP", + 'tab\tnewline\nquote" backslash\\', + "embedded\x00nul", + "x" * 70_000, # larger than the text scratch of the JSON path + "é́ composed and decomposed", + ] + n = len(texts) + columns = { + "id": np.arange(n), + "grp": np.zeros(n, dtype=np.int64), + "txt": texts, + "cat": list(reversed(texts)), + } + _json_reference(temp_db, "Ref", _rows(columns), monkeypatch) + temp_db.insert_columns("Col", columns) + assert spy.count(COLP) == 1 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + assert [ + r["txt"] + for r in temp_db.query("sql", "SELECT txt FROM Col ORDER BY id").to_list() + ] == texts + + +def test_nulls_are_stored_as_the_json_path_stores_them(temp_db, spy, monkeypatch): + _types(temp_db, "Ref", "Col", "ByFrame") + n = 40 + columns = { + "id": np.arange(n), + "grp": [None if i % 3 == 0 else i for i in range(n)], + "val": [None if i % 4 == 0 else i / 4 for i in range(n)], + "cat": [None if i % 5 == 0 else "c%d" % i for i in range(n)], + "txt": [None] * n, + } + _json_reference(temp_db, "Ref", _rows(columns), monkeypatch) + temp_db.insert_columns("Col", columns) + assert spy.count(COLP) == 1 + expected = _stored(temp_db, "Ref") + assert _stored(temp_db, "Col") == expected + assert expected[0]["grp"][0] is None and expected[1]["grp"][0] == 1 + + import pandas as pd + + spy.calls.clear() + frame = pd.DataFrame( + { + "id": np.arange(n), + "grp": pd.array(columns["grp"], dtype="Int64"), + "val": pd.array(columns["val"], dtype="Float64"), + "cat": pd.array(columns["cat"], dtype="string"), + "txt": pd.array(columns["txt"], dtype="string"), + } + ) + temp_db.insert_dataframe("ByFrame", frame) + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 + assert _stored(temp_db, "ByFrame") == expected + + +def test_int64_edges_nan_and_infinity(temp_db, spy, monkeypatch): + for name in ("Ref", "Col"): + temp_db.command("sql", f"CREATE DOCUMENT TYPE {name}") + temp_db.command("sql", f"CREATE PROPERTY {name}.k INTEGER") + temp_db.command("sql", f"CREATE PROPERTY {name}.big LONG") + ints = [-(2**63), -(2**63) + 1, -1, 0, 1, 2**63 - 2, 2**63 - 1] + floats = [0.0, -0.0, 1.5, math.inf, -math.inf, 5e-324, 1.7976931348623157e308] + columns = {"k": list(range(len(ints))), "big": ints, "x": floats} + _json_reference(temp_db, "Ref", _rows(columns), monkeypatch) + temp_db.insert_columns("Col", columns) + assert spy.count(COLP) == 1 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + + nan = {"k": [0, 1], "x": [float("nan"), 1.0]} + temp_db.insert_columns("Col", nan) + stored = temp_db.query( + "sql", "SELECT x FROM Col WHERE k = 0 AND x IS NOT NULL ORDER BY x" + ).to_list() + assert any(isinstance(r["x"], float) and math.isnan(r["x"]) for r in stored) + + +def test_booleans_and_narrow_numpy_dtypes_keep_the_json_paths_types( + temp_db, spy, monkeypatch +): + for name in ("Ref", "Col"): + temp_db.command("sql", f"CREATE DOCUMENT TYPE {name}") + temp_db.command("sql", f"CREATE PROPERTY {name}.k INTEGER") + temp_db.command("sql", f"CREATE PROPERTY {name}.flag BOOLEAN") + temp_db.command("sql", f"CREATE PROPERTY {name}.small INTEGER") + temp_db.command("sql", f"CREATE PROPERTY {name}.single FLOAT") + n = 300 + ids = np.arange(n) + columns = { + "k": ids, + "flag": ids % 3 == 0, + "small": (ids - 150).astype(np.int32), # declared INTEGER: crosses as int32 + "single": (ids / 8).astype(np.float32), # declared FLOAT: crosses as float32 + "narrow": ids.astype(np.int16), # undeclared: a Long, as from JSON + "half": (ids / 4).astype(np.float32), # undeclared: a Double, as from JSON + } + _json_reference(temp_db, "Ref", _rows(columns), monkeypatch) + temp_db.insert_columns("Col", columns) + assert spy.count(COLP) == 1 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + + +def test_vertex_types_load_like_documents(temp_db, spy, monkeypatch): + for name in ("Ref", "Col"): + temp_db.command("sql", f"CREATE VERTEX TYPE {name}") + temp_db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + columns = {"id": np.arange(50), "label": ["v%d" % i for i in range(50)]} + _json_reference(temp_db, "Ref", _rows(columns), monkeypatch) + temp_db.insert_columns("Col", columns) + assert spy.count(COLP) == 1 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + assert temp_db.query("sql", "SELECT count(*) AS n FROM Col").to_list()[0]["n"] == 50 + + +# ---- the fallback: anything the buffers cannot carry unchanged goes through the JSON path, with the same result + + +def _fallback_case(temp_db, spy, monkeypatch, rows, expect_entry): + _types(temp_db, "Ref", "Col") + _json_reference(temp_db, "Ref", rows, monkeypatch) + spy.calls.clear() + temp_db.insert_many("Col", rows) + assert spy.count(COLP) == 0 + assert spy.count(expect_entry) == 1, spy.calls + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + + +def test_nested_maps_and_lists_fall_back_to_json(temp_db, spy, monkeypatch): + rows = [ + {"id": i, "txt": "t%d" % i, "meta": {"k": i, "tags": ["a", "b"]}} + for i in range(20) + ] + _fallback_case(temp_db, spy, monkeypatch, rows, ROWS) + + +def test_a_mixed_type_column_falls_back_to_json(temp_db, spy, monkeypatch): + rows = [ + {"id": i, "txt": "t%d" % i, "mixed": i if i % 2 else "s%d" % i} + for i in range(20) + ] + _fallback_case(temp_db, spy, monkeypatch, rows, ROWS) + rows = [ + {"id": i, "txt": "t%d" % i, "n": i if i % 2 else i + 0.5} for i in range(20) + ] # int and float + spy.calls.clear() + temp_db.insert_many("Col", rows) + assert spy.count(COLP) == 0 and spy.count(ROWS) == 1 + + +def test_rows_with_different_keys_fall_back_to_json(temp_db, spy, monkeypatch): + rows = [ + {"id": i, "txt": "t"} if i % 2 else {"id": i, "cat": "c"} for i in range(20) + ] + _fallback_case(temp_db, spy, monkeypatch, rows, ROWS) + + +def test_values_json_cannot_carry_still_go_row_by_row_in_one_crossing( + temp_db, spy, monkeypatch +): + when = datetime.datetime(2024, 5, 6, 7, 8, 9) + rows = [{"id": i, "txt": "t%d" % i, "when": when} for i in range(10)] + _fallback_case(temp_db, spy, monkeypatch, rows, MANY) + + +def test_an_integer_past_64_bits_is_refused_on_both_paths(temp_db, spy): + _types(temp_db, "Col") + with pytest.raises(ArcadeDBError): + temp_db.insert_many("Col", [{"id": 1, "txt": "a", "huge": 2**70}]) + assert spy.count(COLP) == 0 + with pytest.raises(ValueError): + temp_db.insert_columns("Col", {"id": np.array([2**63], dtype=np.uint64)}) + + +def test_a_dataframe_with_a_column_the_buffers_cannot_carry_falls_back( + temp_db, spy, monkeypatch +): + import pandas as pd + import pyarrow as pa + + _types(temp_db, "Col") + table = pa.table( + { + "id": pa.array([1, 2, 3], type=pa.int32()), + "txt": pa.array(["a", "b", "c"]), + "when": pa.array( + [datetime.datetime(2024, 1, d) for d in (1, 2, 3)], + type=pa.timestamp("us"), + ), + } + ) + temp_db.insert_arrow("Col", table) + assert spy.count(COLP) == 0 + assert pd is not None + rows = temp_db.query("sql", "SELECT id, txt FROM Col ORDER BY id").to_list() + assert [(r["id"], r["txt"]) for r in rows] == [(1, "a"), (2, "b"), (3, "c")] + + +# ---- transactions, chunking and failures + + +def test_more_rows_than_one_chunk_and_the_commit_interval(temp_db, spy): + _types(temp_db, "Col") + n = 25_000 + columns = _probe(n) + assert temp_db.insert_many("Col", _rows(columns), commit_every=4_000) == n + assert spy.count(COLP) == 7 # chunks of 4,000 rows: six full ones and one of 1,000 + assert temp_db.query("sql", "SELECT count(*) AS n FROM Col").to_list()[0]["n"] == n + assert temp_db.insert_columns("Col", columns, commit_every=1_000) == n + assert spy.count(COLP) == 8 # one crossing for the whole call + assert ( + temp_db.query("sql", "SELECT count(*) AS n FROM Col").to_list()[0]["n"] == 2 * n + ) + + +def test_a_duplicate_key_fails_the_same_way_on_both_paths(temp_db, monkeypatch): + _types(temp_db, "Ref", "Col", unique=True) + ids = list(range(250)) + ids[150] = 7 # a duplicate of an earlier row + columns = {"id": ids, "txt": ["t%d" % i for i in ids]} + with monkeypatch.context() as m: + m.setattr(bulk, "_ENABLED", False) + with pytest.raises(ArcadeDBError) as ref: + temp_db.insert_many("Ref", _rows(columns), commit_every=100) + with pytest.raises(ArcadeDBError) as col: + temp_db.insert_many("Col", _rows(columns), commit_every=100) + for error in ( + ref.value, + col.value, + ): # the unique index is checked when the batch commits + assert "DuplicatedKeyException" in str(error) and "Duplicated key [7]" in str( + error + ) + # the rows of the failed batch are rolled back, the batches before it stay + assert [ + temp_db.query("sql", _count_sql(name)).to_list()[0]["n"] + for name in ("Ref", "Col") + ] == [100, 100] + + with pytest.raises(ArcadeDBError) as one_call: + temp_db.insert_columns( + "Col", {"id": ids, "txt": columns["txt"]}, commit_every=100 + ) + assert "Failed to bulk-insert" in str(one_call.value) + + +def test_a_callers_transaction_is_not_committed_by_the_call(temp_db, spy): + _types(temp_db, "Col", unique=True) + temp_db.begin() + temp_db.insert_columns("Col", {"id": [1, 2, 3], "txt": ["a", "b", "c"]}) + assert spy.count(COLP) == 1 + assert temp_db.is_transaction_active() + temp_db.rollback() + assert temp_db.query("sql", "SELECT count(*) AS n FROM Col").to_list()[0]["n"] == 0 + + +# ---- the image checks the table it is given: a wrong table is an error, never a fault, and writes nothing + + +def _send(db, columns, mutate): + rows, packed = columnar.pack(columns, columnar.Declared(known={})) + spec = columnar.spec_of(packed) + mutate(spec, packed) + reply = lib.call( + COLP, + db._h, + lib.b("Col"), + lib.b(columnar.header_of(rows, 10_000, packed)), + spec, + len(spec), + ) + import json + + return json.loads(reply), packed + + +def _count(db): + return db.query("sql", "SELECT count(*) AS n FROM Col").to_list()[0]["n"] + + +def test_a_table_that_does_not_fit_its_buffers_is_an_error_and_writes_nothing(temp_db): + _types(temp_db, "Col") + columns = {"id": np.arange(5), "txt": ["a", "bb", "ccc", "dddd", "eeeee"]} + + reply, _ = _send( + temp_db, columns, lambda spec, p: spec.__setitem__(1, 8) + ) # id: 8 bytes for 5 rows + assert not reply["ok"] and "needed" in reply["error"] + + def decreasing(spec, packed): + offsets = (ctypes.c_int32 * 6).from_address(spec[6]) + offsets[2], offsets[3] = 9, 4 # not increasing + + reply, _ = _send(temp_db, columns, decreasing) + assert not reply["ok"] and "offsets" in reply["error"] + + def past_the_bytes(spec, packed): + spec[9] = 3 # three bytes for 15 characters of text + + reply, _ = _send(temp_db, columns, past_the_bytes) + assert not reply["ok"] and "offsets" in reply["error"] + + nulls = {"id": np.arange(16), "txt": ["a"] * 8 + [None] * 8} + reply, _ = _send( + temp_db, nulls, lambda spec, p: spec.__setitem__(11, 1) + ) # bitmap of 1 byte for 16 rows + assert not reply["ok"] and "bitmap" in reply["error"] + assert _count(temp_db) == 0 + # the connection is fine afterwards + assert temp_db.insert_columns("Col", columns) == 5 and _count(temp_db) == 5 + + +def test_an_unknown_kind_and_a_wrong_table_size_are_errors(temp_db): + _types(temp_db, "Col") + import json + + arr = np.arange(3, dtype=np.int64) + spec = (ctypes.c_int64 * 6)(arr.ctypes.data, arr.nbytes, 0, 0, 0, 0) + for header, entries in ( + ({"n": 3, "commit": 0, "cols": [["id", "z", 0]]}, 6), + ( + {"n": 3, "commit": 0, "cols": [["id", "q", 0], ["grp", "q", 0]]}, + 6, + ), # two columns, one entry + ({"n": -1, "commit": 0, "cols": [["id", "q", 0]]}, 6), + ): + reply = json.loads( + lib.call( + COLP, temp_db._h, lib.b("Col"), lib.b(json.dumps(header)), spec, entries + ) + ) + assert not reply["ok"], header + reply = json.loads( + lib.call( + COLP, + temp_db._h, + lib.b("Col"), + lib.b(json.dumps({"n": 3, "commit": 0, "cols": []})), + spec, + 5, + ) + ) + assert not reply["ok"] # 5 entries are not a multiple of 6 + assert _count(temp_db) == 0 diff --git a/tests/test_native_columnar_pack.py b/tests/test_native_columnar_pack.py new file mode 100644 index 0000000..cda0a1f --- /dev/null +++ b/tests/test_native_columnar_pack.py @@ -0,0 +1,301 @@ +"""Packing of columns for the native backend's columnar insert (``arcadedb/_native/columnar.py``). + +No native library is needed: the layouts (kinds, offsets, null bitmaps, which columns are refused) are read back from the addresses the +packer would hand to ``arcadedb_n_insert_colp``. The run-time half, which compares the stored records of the JSON and the columnar paths, +is in ``test_native_columnar_insert.py``, collected on the native backend only. +""" + +import ctypes + +import numpy as np +import pytest + +from arcadedb._native import columnar # noqa: E402 + + +def _bytes(addr_len): + addr, length = addr_len + return ctypes.string_at(addr, length) if length else b"" + + +def _strings(col): + offsets = np.frombuffer(_bytes(col.data), dtype=np.int32) + blob = _bytes(col.aux) + return [blob[offsets[i] : offsets[i + 1]].decode("utf-8") for i in range(col.n)] + + +def _present(col): + bits = np.frombuffer(_bytes(col.nulls), dtype=np.uint8) + return np.unpackbits(bits, bitorder="little")[: col.n].astype(bool).tolist() + + +NO_TYPES = columnar.Declared(known={}) + + +def _pack(columns, declared=None): + got = columnar.pack(columns, columnar.Declared(known=declared or {})) + assert got is not None + rows, cols = got + return rows, {c.name: c for c in cols} + + +def test_numpy_columns_cross_as_their_64_bit_kinds_without_a_copy(): + ids = np.arange(5, dtype=np.int64) + vals = np.linspace(0, 1, 5) + rows, cols = _pack({"id": ids, "val": vals}) + assert rows == 5 + assert (cols["id"].kind, cols["val"].kind) == ("q", "d") + assert cols["id"].data == (ids.ctypes.data, 40) # the array itself, not a copy + assert cols["val"].data == (vals.ctypes.data, 40) + assert cols["id"].nulls == (0, 0) and cols["id"].aux == (0, 0) + + +def test_narrow_numbers_widen_unless_the_property_is_declared_that_narrow(): + small = np.array([1, -2, 3], dtype=np.int32) + single = np.array([0.5, 1.5, 2.5], dtype=np.float32) + _, cols = _pack( + {"n": small, "x": single} + ) # undeclared: a Long and a Double, as the JSON path stores them + assert (cols["n"].kind, cols["x"].kind) == ("q", "d") + assert np.frombuffer(_bytes(cols["n"].data), dtype=np.int64).tolist() == [1, -2, 3] + _, cols = _pack({"n": small, "x": single}, {"n": "INTEGER", "x": "FLOAT"}) + assert (cols["n"].kind, cols["x"].kind) == ("i", "e") + assert cols["n"].data == (small.ctypes.data, 12) and cols["x"].data == ( + single.ctypes.data, + 12, + ) + _, cols = _pack({"n": small.astype(np.int64)}, {"n": "INTEGER"}) + assert ( + cols["n"].kind == "q" + ) # a 64-bit source is range-checked by the engine, as before + + +def test_unsigned_past_the_signed_range_is_refused_and_below_it_is_not(): + assert columnar.pack({"u": np.array([2**63], dtype=np.uint64)}, NO_TYPES) is None + _, cols = _pack({"u": np.array([2**63 - 1, 1], dtype=np.uint64)}) + assert np.frombuffer(_bytes(cols["u"].data), dtype=np.int64).tolist() == [ + 2**63 - 1, + 1, + ] + + +def test_bool_and_nan_and_infinity_keep_their_values(): + _, cols = _pack( + {"f": np.array([True, False, True]), "x": np.array([np.nan, np.inf, -np.inf])} + ) + assert cols["f"].kind == "b" and _bytes(cols["f"].data) == b"\x01\x00\x01" + x = np.frombuffer(_bytes(cols["x"].data), dtype=np.float64) + assert np.isnan(x[0]) and x[1] == np.inf and x[2] == -np.inf + + +def test_a_float32_matrix_and_a_list_of_float32_arrays_are_vector_columns(): + m = np.arange(6, dtype=np.float32).reshape(3, 2) + _, cols = _pack({"v": m, "w": [row for row in m]}) + for name in ("v", "w"): + assert (cols[name].kind, cols[name].dim, cols[name].n) == ("f", 2, 3) + assert np.frombuffer( + _bytes(cols[name].data), dtype=np.float32 + ).tolist() == list(range(6)) + + +def test_float64_arrays_are_not_vector_columns(): + # only float32 is a column of vectors, as on the JPype path (tests/test_float_vector_bulk.py): float64 keeps the row path + m = np.arange(6, dtype=np.float64).reshape(3, 2) + assert columnar.pack({"v": m}, NO_TYPES) is None + assert columnar.pack({"v": [row for row in m]}, NO_TYPES) is None + + +def test_a_list_of_float_lists_is_a_vector_only_for_an_array_of_floats_property(): + lists = [[0.5, 1.5], [2.5, 3.5]] + assert ( + columnar.pack({"v": lists}, NO_TYPES) is None + ) # undeclared: the JSON path keeps the doubles + _, cols = _pack({"v": lists}, {"v": "ARRAY_OF_FLOATS"}) + assert (cols["v"].kind, cols["v"].dim) == ("f", 2) + assert ( + columnar.pack( + {"v": [[1.0], [1.0, 2.0]]}, + columnar.Declared(known={"v": "ARRAY_OF_FLOATS"}), + ) + is None + ) # ragged + assert ( + columnar.pack( + {"v": [[], []]}, columnar.Declared(known={"v": "ARRAY_OF_FLOATS"}) + ) + is None + ) # no dimension + + +def test_strings_are_arrow_offsets_plus_utf8_bytes(): + texts = ["", "abc", "héllo", "日本語", "😀", "x\x00y"] + rows, cols = _pack({"s": texts}) + assert rows == 6 and cols["s"].kind == "s" + assert _strings(cols["s"]) == texts + offsets = np.frombuffer(_bytes(cols["s"].data), dtype=np.int32).tolist() + assert ( + offsets[0] == 0 + and offsets == sorted(offsets) + and offsets[-1] == len(_bytes(cols["s"].aux)) + ) + + +def test_ascii_and_all_empty_strings(): + _, cols = _pack({"a": ["a", "bb", "ccc"], "e": ["", "", ""]}) + assert _strings(cols["a"]) == ["a", "bb", "ccc"] + assert _strings(cols["e"]) == ["", "", ""] and cols["e"].aux == (0, 0) + + +def test_a_lone_surrogate_is_refused_so_the_json_path_decides(): + assert columnar.pack({"s": ["ok", "\ud800"]}, NO_TYPES) is None + + +def test_nulls_become_a_bitmap_and_a_filler_value(): + rows, cols = _pack( + { + "n": [1, None, 3, None], + "x": [None, 2.5, None, 4.5], + "s": ["a", None, "c", None], + "b": [True, None, False, True], + "z": [None, None, None, None], + } + ) + assert rows == 4 + assert _present(cols["n"]) == [True, False, True, False] + assert _present(cols["s"]) == [True, False, True, False] + assert _present(cols["x"]) == [False, True, False, True] + assert _present(cols["b"]) == [True, False, True, True] + assert _present(cols["z"]) == [False] * 4 and cols["z"].kind == "s" + assert _strings(cols["s"])[0] == "a" and _strings(cols["s"])[2] == "c" + assert np.frombuffer(_bytes(cols["n"].data), dtype=np.int64)[[0, 2]].tolist() == [ + 1, + 3, + ] + assert len(_bytes(cols["n"].nulls)) == 1 + + +def test_a_bitmap_longer_than_a_byte(): + values = [None if i % 3 == 0 else i for i in range(20)] + _, cols = _pack({"n": values}) + assert _present(cols["n"]) == [v is not None for v in values] + assert len(_bytes(cols["n"].nulls)) == 3 + + +@pytest.mark.parametrize( + "column", + [ + [1, 2.5], # mixed int and float + [1, "a"], + [True, 1], # bool is not an int here + [1, 2**63], # past 64 bits + [{"a": 1}, {"a": 2}], # nested + [[1], [2]], # a list without a declared ARRAY_OF_FLOATS + [b"ab", b"cd"], + [np.int64(1), np.int64(2)], # numpy scalars in a list are not Python numbers + ], +) +def test_columns_the_buffer_cannot_carry_unchanged_are_refused(column): + assert columnar.pack({"c": column}, NO_TYPES) is None + + +def test_datetimes_decimals_and_object_arrays_of_them_are_refused(): + import datetime + import decimal + + assert columnar.pack({"d": [datetime.datetime(2020, 1, 1)]}, NO_TYPES) is None + assert columnar.pack({"d": [decimal.Decimal("1.5")]}, NO_TYPES) is None + assert ( + columnar.pack({"d": np.array(["2020-01-01"], dtype="datetime64[D]")}, NO_TYPES) + is None + ) + + +def test_ragged_columns_and_empty_input_are_left_to_the_row_path(): + assert columnar.pack({"a": [1, 2, 3], "b": [1, 2]}, NO_TYPES) is None + assert columnar.pack({"a": []}, NO_TYPES) is None + assert columnar.pack({}, NO_TYPES) is None + assert columnar.pack({"a": (x for x in range(3))}, NO_TYPES) is None + + +def test_the_address_table_and_header_describe_the_columns(): + rows, packed = columnar.pack({"id": np.arange(3), "s": ["a", "bc", None]}, NO_TYPES) + spec = columnar.spec_of(packed) + assert len(spec) == 12 # six int64 per column + assert spec[0] == packed[0].data[0] and spec[1] == 24 + assert spec[6] == packed[1].data[0] and spec[7] == 16 # n + 1 offsets of 4 bytes + assert spec[8] == packed[1].aux[0] and spec[9] == 3 + assert spec[10] == packed[1].nulls[0] and spec[11] == 1 + import json + + assert json.loads(columnar.header_of(rows, 10_000, packed)) == { + "n": 3, + "commit": 10_000, + "cols": [["id", "q", 0], ["s", "s", 0]], + } + assert json.loads(columnar.header_of(rows, 2**40, packed))["commit"] == 2**31 - 1 + + +def test_rows_become_columns_only_when_every_row_has_the_same_keys(): + rows = [{"a": 1, "b": "x"}, {"a": 2, "b": "y"}] + assert columnar.rows_to_columns(rows) == {"a": [1, 2], "b": ["x", "y"]} + assert columnar.rows_to_columns([{"a": 1}, {"a": 2, "b": 3}]) is None + assert ( + columnar.rows_to_columns([{"a": 1, "b": 2}, {"a": 1, "c": 2}]) is None + ) # same size, other key + assert columnar.rows_to_columns([{1: "x"}]) is None + assert columnar.rows_to_columns([]) is None + assert columnar.rows_to_columns([[1, 2]]) is None + + +def test_arrow_strings_cross_in_place_with_their_offset_and_validity(): + import pyarrow as pa + + arr = pa.array(["a", None, "héllo", "", "xyz", "tail"]) + rows, cols = _pack({"s": arr}) + assert rows == 6 and cols["s"].kind == "s" + assert _present(cols["s"]) == [True, False, True, True, True, True] + assert [t for t, ok in zip(_strings(cols["s"]), _present(cols["s"])) if ok] == [ + "a", + "héllo", + "", + "xyz", + "tail", + ] + # a slice keeps the buffers and moves the offsets: the offsets no longer start at 0 + part = arr.slice(2, 3) + _, cols = _pack({"s": part}) + assert _strings(cols["s"]) == ["héllo", "", "xyz"] + assert int(np.frombuffer(_bytes(cols["s"].data), dtype=np.int32)[0]) > 0 + big = pa.array(["a", "bc"], type=pa.large_string()) + _, cols = _pack({"s": big}) + assert _strings(cols["s"]) == ["a", "bc"] + + +def test_arrow_numbers_with_nulls_and_arrow_vectors(): + import pyarrow as pa + + _, cols = _pack( + { + "n": pa.array([1, None, 3], type=pa.int64()), + "b": pa.array([True, None, False]), + } + ) + assert _present(cols["n"]) == [True, False, True] and cols["n"].kind == "q" + assert cols["b"].kind == "b" and _present(cols["b"]) == [True, False, True] + vec = pa.array([[1.0, 2.0], [3.0, 4.0]], type=pa.list_(pa.float32())) + assert columnar.pack({"v": vec}, NO_TYPES) is None + _, cols = _pack({"v": vec}, {"v": "ARRAY_OF_FLOATS"}) + assert (cols["v"].kind, cols["v"].dim) == ("f", 2) + fixed = pa.array([[1.0, 2.0], [3.0, 4.0]], type=pa.list_(pa.float32(), 2)) + _, cols = _pack({"v": fixed}, {"v": "ARRAY_OF_FLOATS"}) + assert np.frombuffer(_bytes(cols["v"].data), dtype=np.float32).tolist() == [ + 1.0, + 2.0, + 3.0, + 4.0, + ] + ragged = pa.array([[1.0], [1.0, 2.0]], type=pa.list_(pa.float32())) + assert ( + columnar.pack({"v": ragged}, columnar.Declared(known={"v": "ARRAY_OF_FLOATS"})) + is None + ) From 67ed7daf9c8ca27b5929850151d38881ca8fc7c1 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 23:31:14 +0900 Subject: [PATCH 29/33] Native tests: no NOTYET left, every JVM-only test is NA-JVM or reads through the Python API The 31 NOTYET skips of the native parity run were calls that are JVM-only by design (get_java_database, a Java object a Schema method returns) reached by shared tests, so a final release (--max-notyet 0) would have been blocked. Raise site: needs_jvm() replaces not_yet() for those calls; the message says the call needs the JVM build (arcadedb-jvm) instead of "not yet". not_yet() stays for a real gap. Tests: the ones whose purpose is the Java object (the bridge's float-vector decision, the plain JPype path, a Java RID set, the deprecated Java newLightEdge, metadata fields only the Java index metadata carries) get NA-JVM rows. The ones that used the Java object only to read a value now read it through VectorIndex.get_metadata() (test_create_index_with_rich_metadata_sql, test_sql_metadata_gives_the_helper_defaults, the hierarchy and quantization part of test_add_hierarchy_defaults_differ_between_sql_and_python, both product quantization metadata tests), and scalar and lone-null binding get shared round-trip tests beside the NA-JVM comparisons with the plain path. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/__init__.py | 8 +++++ src/arcadedb/_native/database.py | 15 ++++----- src/arcadedb/_native/records.py | 4 +-- src/arcadedb/_native/schema_objects.py | 6 ++-- tests/README.md | 2 +- tests/conftest.py | 20 ++++++++++- tests/test_percall_calls.py | 17 ++++++++++ tests/test_vector_product_quantization.py | 23 ++++++------- tests/test_vector_sql.py | 41 ++++++++++++----------- 9 files changed, 88 insertions(+), 48 deletions(-) diff --git a/src/arcadedb/_native/__init__.py b/src/arcadedb/_native/__init__.py index 47939e3..731c5bd 100644 --- a/src/arcadedb/_native/__init__.py +++ b/src/arcadedb/_native/__init__.py @@ -41,5 +41,13 @@ def enabled() -> bool: return os.environ.get("ARCADEDB_BACKEND", _DEFAULT_BACKEND).lower() == "native" +def needs_jvm(what: str): + """A call that hands out a Java object of the JVM build. Native has no JVM, by design: the message names the build that has it.""" + raise NotOnNativeYet( + f"{what} needs the JVM build (the arcadedb-jvm package); this is the native build (the arcadedb package)" + ) + + def not_yet(what: str): + """A feature the native build is still missing (none today). A final release allows no test that ends here.""" raise NotOnNativeYet(f"{what} is not on the native backend yet") diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index c8695aa..dc74d8d 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -13,7 +13,7 @@ from ..exceptions import ArcadeDBError from ..transactions import TransactionContext from ..type_conversion import json_bulk_dumps -from . import interrupts, lib, not_yet +from . import interrupts, lib, needs_jvm from .codec import _prep as _prep_row from .codec import encode_params, json_native, loads, row_default @@ -850,7 +850,7 @@ def _async(self): # -- not on native yet def get_java_database(self): - not_yet("get_java_database") + needs_jvm("get_java_database") def new_vertex(self, type_name: str): """A vertex that is created by ``save()`` (``CREATE VERTEX .. CONTENT``).""" @@ -1003,11 +1003,8 @@ def import_documents( disable_xml_limits=False, ): """Import document-shaped files (CSV, JSON, ...) through the engine's importer; same arguments as the JPype path.""" - from ..importer import ( - ImportResult, - _build_import_settings, - normalize_import_source, - ) + from ..importer import (ImportResult, _build_import_settings, + normalize_import_source) from . import extras self._check_not_closed() @@ -1221,7 +1218,7 @@ def __bool__(self): return True def __getattr__(self, name): - not_yet(f"the Java object a Schema method returns ({name})") + needs_jvm(f"the Java object a Schema method returns ({name})") class NativeSchema: @@ -1386,7 +1383,7 @@ def list_vector_indexes(self): def __getattr__(self, name): if name.startswith("_"): raise AttributeError(name) - not_yet(f"Schema.{name} (returns Java objects)") + needs_jvm(f"Schema.{name} (returns Java objects)") class NativeDatabaseFactory: diff --git a/src/arcadedb/_native/records.py b/src/arcadedb/_native/records.py index dcdbd1b..8f8b622 100644 --- a/src/arcadedb/_native/records.py +++ b/src/arcadedb/_native/records.py @@ -11,7 +11,7 @@ from ..graph import Document as _BaseDocument from ..graph import Edge as _BaseEdge from ..graph import Vertex as _BaseVertex -from . import not_yet +from . import needs_jvm from .codec import RID @@ -84,7 +84,7 @@ def get_type_name(self) -> str: return self._type def get_java_document(self): - not_yet("get_java_document") + needs_jvm("get_java_document") # -- writing diff --git a/src/arcadedb/_native/schema_objects.py b/src/arcadedb/_native/schema_objects.py index 3bd9046..3ad1a68 100644 --- a/src/arcadedb/_native/schema_objects.py +++ b/src/arcadedb/_native/schema_objects.py @@ -8,7 +8,7 @@ from ..exceptions import ArcadeDBError from ..vector import VectorIndex as _BaseVectorIndex -from . import not_yet +from . import needs_jvm _PROPERTY = re.compile(r"name: (.+?), type: ([A-Z_]+)") @@ -46,7 +46,7 @@ def getBuckets(self): def __getattr__(self, name): if name.startswith("_"): raise AttributeError(name) - not_yet(f"the Java object a Schema method returns ({name})") + needs_jvm(f"the Java object a Schema method returns ({name})") def __repr__(self): return f"" @@ -80,7 +80,7 @@ def isAutomatic(self) -> bool: def __getattr__(self, name): if name.startswith("_"): raise AttributeError(name) - not_yet(f"the Java object a Schema method returns ({name})") + needs_jvm(f"the Java object a Schema method returns ({name})") def __repr__(self): return f"" diff --git a/tests/README.md b/tests/README.md index 723c95e..ef1c32d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -36,7 +36,7 @@ The server tests do not skip when the server stack is absent: the wheel always s The same suite runs against the native `arcadedb` wheel (`native/wheel/parity_wheel.py`). A test is reported as a skip on the native build in two explicit ways, never by what its failure says: -- `NotOnNativeYet`: the native code raised it because the feature needs a Java object API the library does not expose. The report says `NOTYET`, and a final release allows none. +- `NotOnNativeYet`: the native code raised it because the feature needs a Java object API the library does not expose. The report says `NOTYET`, and a final release allows none. A call that is JVM-only by design (`get_java_database`, `get_java_document`, a Java object a `Schema` method returns) raises it with a message that names the `arcadedb-jvm` build; a shared test that reaches one is `NA-JVM`, or reads the value through a statement or the Python API instead. - A row of `NATIVE_NA_JVM_TESTS` in `tests/conftest.py`, keyed `::[::]`: the test reaches into the JVM on purpose (`jpype.JClass`, a Java array, `db._java_db`, counted bridge crossings). The report says `NA-JVM`. Add a row only for such a test, and give the reason. When the behaviour is shared, write a native test beside it (`test_native_*.py`, collected on the native build only). A test that fails on the native build for any other reason is a failure. Fix the native code, or mark a known engine bug with a strict `xfail`. Do not add a row to hide it. diff --git a/tests/conftest.py b/tests/conftest.py index 4e3f5dc..a03ac5b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -89,7 +89,9 @@ # crossings, or patches a hook of the JPype path. A failure text never puts a test on this list, and a failing test is never turned # into a skip: a test whose behaviour native shares has to pass on native, and one that fails there is a bug to file (an # xfail(strict=True) naming the BACKLOG-bindings.md row, so it turns red when fixed), not an entry here. -# A feature the native wheel lacks on purpose raises NotOnNativeYet from the native code, which pytest_runtest_makereport reports as NOTYET. +# A call that hands out a Java object (get_java_database, get_java_document, a Schema method's Java object) raises NotOnNativeYet from needs_jvm in the +# native code, which pytest_runtest_makereport reports as NOTYET. A shared test must not reach one: it goes on this list, or it reads the value +# through a statement or the Python API. Every NOTYET is a gap to close before a final release, which allows none. JAVA_CLASS = "builds a java.* object through JPype" JAVA_ARRAY = "builds a Java array through JPype (jpype.JArray)" JAVA_DB = "reads the Java database behind the wrapper (db._java_db)" @@ -136,6 +138,22 @@ "test_transaction_config.py::test_set_wal_flush_is_database_wide": JAVA_DB, "test_run_in_transaction_retry.py::test_contended_commits_spend_their_budget_before_failing": JAVA_DB, "test_float_vector_bulk.py::TestAllowedRids::test_rid_objects_and_text_mixed_still_work": "builds a Java RID (db.to_java_rid) and reads the Java RID set of the index", + "test_float_vector_bulk.py::TestDecision::test_a_declared_vector_column_is_taken_out": "decides on the Java database (get_java_database) and builds the bridge's FloatVectors.Matrix objects", + "test_float_vector_bulk.py::TestDecision::test_numpy_rows_and_ints_are_taken_out": "decides on the Java database (get_java_database) and builds the bridge's FloatVectors.Matrix objects", + "test_float_vector_bulk.py::TestDecision::test_the_inherited_declaration_counts": "decides on the Java database (get_java_database) and builds the bridge's FloatVectors.Matrix objects", + "test_float_vector_bulk.py::TestDecision::test_anything_else_stays_in_the_json": "decides on the Java database (get_java_database) and builds the bridge's FloatVectors.Matrix objects", + "test_float_vector_bulk.py::TestDecision::test_an_undeclared_property_stays_in_the_json": "decides on the Java database (get_java_database) and builds the bridge's FloatVectors.Matrix objects", + "test_float_vector_bulk.py::TestDocumentSet::test_convert_python_to_java_float32_is_one_float_array": 'asserts the converted value is a Java float[] (getClass().getName() == "[F")', + "test_float_vector_bulk.py::TestAllowedRids::test_the_set_holds_one_rid_per_string": "reads the Java RID set the index builds (_build_allowed_rids_set)", + "test_percall_calls.py::test_named_scalars_bind_as_the_plain_path_binds_them": "compares with the plain JPype path through db.get_java_database() (Database._java_parameters, core._through_bridge)", + "test_percall_calls.py::test_positional_scalars_bind_as_the_plain_path_binds_them": "compares with the plain JPype path through db.get_java_database() (Database._java_parameters, core._through_bridge)", + "test_percall_calls.py::test_a_lone_null_or_lone_java_array_keeps_the_plain_path": "compares with the plain JPype path through db.get_java_database() (Database._java_parameters, core._through_bridge)", + "test_percall_calls.py::test_other_values_take_the_plain_path": "compares with the plain JPype path through db.get_java_database() (Database._java_parameters, core._through_bridge)", + "test_light_edge_and_view_known_issues.py::test_other_ways_to_count_light_edges_an_older_version_loaded": "builds the light edges of an undeclared edge type with the deprecated Java Vertex.newLightEdge; no statement does (CREATE EDGE writes regular edges there)", + "test_light_edge_and_view_known_issues.py::test_one_hop_count_over_light_edges_an_older_version_loaded": "builds the light edges of an undeclared edge type with the deprecated Java Vertex.newLightEdge; no statement does (CREATE EDGE writes regular edges there)", + "test_vector_sql.py::TestVectorSQL::test_create_index_with_native_int8_encoding_sql": "reads encoding from the Java index metadata, which get_metadata() does not return and the native library does not expose", + "test_vector_sql.py::TestVectorSearchOptionsSql::test_graph_build_settings_reach_the_index": "reads neighborOverflowFactor and alphaDiversityRelaxation from the Java index metadata, which get_metadata() does not return and the native library does not expose", + "test_vector_sql.py::TestVectorSearchOptionsSql::test_sql_graph_settings_default_to_the_engines": "reads neighborOverflowFactor and alphaDiversityRelaxation from the Java index metadata, which get_metadata() does not return and the native library does not expose", "test_xml_limits.py::test_import_hitting_a_limit_names_the_option": "reads the JDK's system properties through JPype (native: test_native_xml_limits.py)", "test_xml_limits.py::test_per_call_option_lifts_the_limits_for_that_call_only": "reads the JDK's system properties through JPype (native: test_native_xml_limits.py)", "test_xml_limits.py::test_per_call_option_restores_a_value_that_was_already_set": "sets and reads a JDK system property through JPype", diff --git a/tests/test_percall_calls.py b/tests/test_percall_calls.py index e42e55c..80cc132 100644 --- a/tests/test_percall_calls.py +++ b/tests/test_percall_calls.py @@ -50,6 +50,18 @@ def _plain_query(db, command, args): PROJECTION = "SELECT :i AS i, :big AS big, :f AS f, :s AS s, :b AS b, :n AS n" +def test_named_scalars_round_trip(db): + assert db.query("sql", PROJECTION, SCALARS).first().to_dict() == SCALARS + + +def test_positional_scalars_round_trip(db): + values = (7, 2**40, 1.5, "text", True, None) + row = db.query( + "sql", "SELECT ? AS a, ? AS b, ? AS c, ? AS d, ? AS e, ? AS f", *values + ).first() + assert row.to_dict() == dict(zip("abcdef", values)) + + def test_named_scalars_bind_as_the_plain_path_binds_them(db): glued = db.query("sql", PROJECTION, SCALARS).first() plain = _plain_query(db, PROJECTION, (SCALARS,)).first() @@ -123,6 +135,11 @@ def test_a_lone_null_or_lone_java_array_keeps_the_plain_path(db, lone): ) is core._NO_GLUE ) + + +@pytest.mark.parametrize("lone", [None, "java-array"]) +def test_a_lone_null_or_lone_java_array_binds_as_the_value(db, lone): + value = None if lone is None else arcadedb.to_java_float_array([1.0, 2.0]) row = db.query("sql", "SELECT ? AS x", value).first() assert ( (row.get("x") is None) if lone is None else (list(row.get("x")) == [1.0, 2.0]) diff --git a/tests/test_vector_product_quantization.py b/tests/test_vector_product_quantization.py index 07d7239..e3688de 100644 --- a/tests/test_vector_product_quantization.py +++ b/tests/test_vector_product_quantization.py @@ -107,10 +107,7 @@ def _search(db, query, options="{ efSearch: 100 }", k=K): def _metadata(db): - index = db.schema.get_index_by_name("PqDoc[embedding]") - if "TypeIndex" in index.getClass().getName(): - index = index.getSubIndexes().get(0) - return index.getMetadata() + return db.schema.get_vector_index("PqDoc", "embedding").get_metadata() def _recall(db, vectors, queries, options): @@ -137,12 +134,12 @@ def test_index_settings_reach_the_index(self, test_db, corpus, has_pq_rerank): meta = _metadata(test_db) - assert str(meta.quantizationType) == "PRODUCT" - assert int(meta.rerankOversample) == 30 - assert int(meta.pqTrainingLimit) == 1500 - assert bool(meta.fusedPQ) is False - assert int(meta.pqSubspaces) == 8 - assert int(meta.pqClusters) == 64 + assert meta["quantization"] == "PRODUCT" + assert meta["rerank_oversample"] == 30 + assert meta["pq_training_limit"] == 1500 + assert meta["fused_pq"] is False + assert meta["pq_subspaces"] == 8 + assert meta["pq_clusters"] == 64 def test_rerank_oversample_defaults_to_ten_and_fused_pq_to_off( self, test_db, corpus, has_pq_rerank @@ -152,11 +149,11 @@ def test_rerank_oversample_defaults_to_ten_and_fused_pq_to_off( meta = _metadata(test_db) if not has_pq_rerank: - assert not hasattr(meta, "rerankOversample") + assert "rerank_oversample" not in meta return - assert int(meta.rerankOversample) == 10 - assert bool(meta.fusedPQ) is False + assert meta["rerank_oversample"] == 10 + assert meta["fused_pq"] is False def test_negative_rerank_oversample_is_refused(self, test_db, has_pq_rerank): test_db.command("sql", "CREATE DOCUMENT TYPE PqDoc") diff --git a/tests/test_vector_sql.py b/tests/test_vector_sql.py index 54f10a7..84771d9 100644 --- a/tests/test_vector_sql.py +++ b/tests/test_vector_sql.py @@ -597,17 +597,16 @@ def test_create_index_with_rich_metadata_sql(self, test_db): """, ) - java_index = self._get_primary_sql_vector_index(test_db, "SqlMetaDoc[vec]") - metadata = java_index.getMetadata() - - assert metadata.dimensions == 4 - assert str(metadata.similarityFunction) == "COSINE" - assert str(metadata.quantizationType) == "INT8" - assert metadata.idPropertyName == "slug" - assert metadata.storeVectorsInGraph is True - assert metadata.addHierarchy is True - assert metadata.graphBuildCacheSize == 456 - assert metadata.mutationsBeforeRebuild == 789 + metadata = test_db.schema.get_vector_index("SqlMetaDoc", "vec").get_metadata() + + assert metadata["dimensions"] == 4 + assert metadata["similarity_function"] == "COSINE" + assert metadata["quantization"] == "INT8" + assert metadata["id_property"] == "slug" + assert metadata["store_vectors_in_graph"] is True + assert metadata["add_hierarchy"] is True + assert metadata["graph_build_cache_size"] == 456 + assert metadata["mutations_before_rebuild"] == 789 # locationCacheSize was removed in #5559/#5568 and the engine now # rejects it outright, so a SQL METADATA block that still carries it @@ -1265,12 +1264,10 @@ def test_graph_build_settings_reach_the_index(self, test_db): def test_add_hierarchy_defaults_differ_between_sql_and_python(self, test_db): """SQL follows the engine (flat graph); create_vector_index() builds a hierarchy.""" self._make_docs(test_db) - meta = self._metadata(test_db, "OptDoc[embedding]") - assert bool(meta.addHierarchy) is False + meta = test_db.schema.get_vector_index("OptDoc", "embedding").get_metadata() + assert meta["add_hierarchy"] is False # The vector guide says the same of quantization: SQL is the engine's NONE. - assert str(meta.quantizationType) == "NONE" - assert float(meta.neighborOverflowFactor) == pytest.approx(1.2) - assert float(meta.alphaDiversityRelaxation) == pytest.approx(1.2) + assert meta["quantization"] == "NONE" test_db.command("sql", "CREATE VERTEX TYPE PyHier") test_db.command("sql", "CREATE PROPERTY PyHier.embedding ARRAY_OF_FLOATS") @@ -1287,6 +1284,12 @@ def test_add_hierarchy_defaults_differ_between_sql_and_python(self, test_db): ) assert engine_default_index.get_metadata()["add_hierarchy"] is False + def test_sql_graph_settings_default_to_the_engines(self, test_db): + self._make_docs(test_db) + meta = self._metadata(test_db, "OptDoc[embedding]") + assert float(meta.neighborOverflowFactor) == pytest.approx(1.2) + assert float(meta.alphaDiversityRelaxation) == pytest.approx(1.2) + def test_sql_metadata_gives_the_helper_defaults(self, test_db): """The vector guide's quick start sets quantization and addHierarchy in METADATA.""" test_db.command("sql", "CREATE VERTEX TYPE QuickDoc") @@ -1304,9 +1307,9 @@ def test_sql_metadata_gives_the_helper_defaults(self, test_db): } """, ) - meta = self._metadata(test_db, "QuickDoc[embedding]") - assert str(meta.quantizationType) == "INT8" - assert bool(meta.addHierarchy) is True + meta = test_db.schema.get_vector_index("QuickDoc", "embedding").get_metadata() + assert meta["quantization"] == "INT8" + assert meta["add_hierarchy"] is True class TestSparseSearchOptionsSql: From 95e1350df5b961567848bcc69c736ff761dc7d02 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 23:32:49 +0900 Subject: [PATCH 30/33] Product quantization test: read fused_pq through get_metadata(), black-format database.py The fused PQ build test still read the Java field (.fusedPQ) from the dict that _metadata() returns now, which failed on the JVM build. black re-wraps the import line that needs_jvm lengthened. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_vector_product_quantization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_vector_product_quantization.py b/tests/test_vector_product_quantization.py index e3688de..5d4ac31 100644 --- a/tests/test_vector_product_quantization.py +++ b/tests/test_vector_product_quantization.py @@ -246,7 +246,7 @@ def test_fused_pq_index_returns_k_results(self, test_db, corpus, has_pq_rerank): _build(test_db) - assert bool(_metadata(test_db).fusedPQ) is True + assert _metadata(test_db)["fused_pq"] is True assert len(_search(test_db, queries[0])) == K From 20e7a010b4c6b6593fc5b74f7662d5775713845c Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 23:33:39 +0900 Subject: [PATCH 31/33] Restore the black import style in _native/database.py A bare isort run (no --profile black) had re-wrapped the importer import; only the needs_jvm changes remain in the file's diff. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/database.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index dc74d8d..43fb924 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -1003,8 +1003,11 @@ def import_documents( disable_xml_limits=False, ): """Import document-shaped files (CSV, JSON, ...) through the engine's importer; same arguments as the JPype path.""" - from ..importer import (ImportResult, _build_import_settings, - normalize_import_source) + from ..importer import ( + ImportResult, + _build_import_settings, + normalize_import_source, + ) from . import extras self._check_not_closed() From 2a1ebad7fd61d0e3d20fbaea79f0a9c4000aab98 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 26 Oct 2026 00:19:49 +0900 Subject: [PATCH 32/33] Native: start the isolate with -XX:MaximumHeapSizePercent=50 by default GraalVM's default heap limit is 80% of the memory the isolate sees (the container limit when there is one). With the engine's off-heap memory on top, the kernel killed the process in memory-limited containers. The glue now passes -XX:MaximumHeapSizePercent=50 as an isolate argument unless ARCADEDB_NATIVE_ISOLATE_ARGS already has -Xmx, -XX:MaximumHeapSizePercent or -XX:MaxRAM (theirs wins). The isolate arguments are now always non-empty (the log directory is always set), so the dead "if iso_args:" branch is gone. Tests: the default and the override rules, and on the real image the default max_heap is about half of the memory and a user percent, -Xmx or -XX:MaxRAM replaces it. Docs: a heap-size section in the installation page. Co-Authored-By: Claude Sonnet 5.5 --- docs/modules/ROOT/pages/installation.adoc | 20 ++++- src/arcadedb/_native/lib.py | 70 +++++++++------ tests/conftest.py | 2 + tests/test_native_heap.py | 103 ++++++++++++++++++++++ 4 files changed, 168 insertions(+), 27 deletions(-) create mode 100644 tests/test_native_heap.py diff --git a/docs/modules/ROOT/pages/installation.adoc b/docs/modules/ROOT/pages/installation.adoc index 4f9456f..be8f519 100644 --- a/docs/modules/ROOT/pages/installation.adoc +++ b/docs/modules/ROOT/pages/installation.adoc @@ -1,5 +1,5 @@ = Installation -:description: Install the arcadedb (native) or arcadedb-jvm (JVM) wheel, check that it works, and set JVM options. +:description: Install the arcadedb (native) or arcadedb-jvm (JVM) wheel, check that it works, and set JVM options and the native heap size. // Sources: bindings docs getting-started/installation.md, the PyPI metadata of // arcadedb and arcadedb-jvm. @@ -149,6 +149,24 @@ Each Python process starts its own JVM, and each JVM sees the whole container li The native package is the same engine compiled to a GraalVM native-image shared library. It needs no JVM and no JPype, so the JVM options above do not apply to it. The calls that hand out a Java object (`get_java_database`, `get_java_document`) raise `NotOnNativeYet` (a `NotImplementedError`) instead of misbehaving. The list is in xref:which-package.adoc[Which package?]. Report problems in the {repo-url}/issues[issue tracker] and say which package you use. +[[native-heap]] +=== Heap size + +The engine runs in a GraalVM isolate. By default the native package limits its heap to 50% of the memory the process can see. It does this with the isolate argument `-XX:MaximumHeapSizePercent=50`. The GraalVM default is 80%. The engine also uses memory outside the heap. With 80%, the kernel killed the process in containers that have a memory limit. With 50%, the same cases ran clean, and the timings of the benchmark figures did not change by more than the noise. + +The percentage follows the container memory limit (the cgroup limit) when there is one. Without a limit it follows the memory of the host. In a container with a 4 GB limit, the heap limit is 2 GB. + +To set the heap yourself, put one of these arguments in the environment variable `ARCADEDB_NATIVE_ISOLATE_ARGS` before the first database is opened. The variable takes space-separated arguments: + +[source,bash] +---- +export ARCADEDB_NATIVE_ISOLATE_ARGS="-Xmx2g" # a fixed maximum heap +export ARCADEDB_NATIVE_ISOLATE_ARGS="-XX:MaximumHeapSizePercent=70" # another percentage +export ARCADEDB_NATIVE_ISOLATE_ARGS="-XX:MaxRAM=4g" # a memory size for the percentage; GraalVM's 80% applies to it +---- + +When one of these three is present, the package adds no default. Do not use `-XX:MaxRAMPercentage`. The native image does not accept it, and the isolate then fails to start with `graal_create_isolate failed (rc=22)`. The same variable also takes the engine's `-D` settings: see xref:how-to/logs.adoc[Where the logs go]. For what a native image build supports, see xref:arcadedb::how-to/operations/native-image.adoc[Native image] in the main ArcadeDB docs. + == Next steps * xref:quickstart.adoc[Quickstart] diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index d9d2a82..85fea6e 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -82,6 +82,23 @@ def _log_isolate_args(user_args, log_dir) -> list: return args +# GraalVM's default heap limit is 80% of the memory it can see (the container limit when there is one). With the engine's off-heap +# memory on top, that let the kernel kill the process in a memory-limited container; 50% ran clean in the same cases, at the same speed. +HEAP_PERCENT = 50 +_HEAP_ARGS = ("-Xmx", "-XX:MaximumHeapSizePercent=", "-XX:MaxRAM=") + + +def _heap_isolate_args(user_args) -> list: + """The isolate argument that limits the heap to ``HEAP_PERCENT`` of the memory; none when the user already sets the heap (theirs wins). + + ``-XX:MaxRAMPercentage`` is not accepted by the image (the isolate is not created, rc=22). ``-Xmx``, ``-XX:MaximumHeapSizePercent`` + and ``-XX:MaxRAM`` are, so any of them in ``user_args`` replaces the default. + """ + if any(a.startswith(_HEAP_ARGS) for a in user_args): + return [] + return [f"-XX:MaximumHeapSizePercent={HEAP_PERCENT}"] + + def _prepare_logging(user_args) -> list: """Create the log directory, write its logging.properties and return the isolate arguments for it (see :func:`_log_isolate_args`).""" global LOG_DIR_USED @@ -141,34 +158,35 @@ def _load(): ctypes.POINTER(ctypes.c_void_p), ] lib.graal_create_isolate.restype = ctypes.c_int - params = None # run-time image options for the isolate (e.g. "-XX:InitialCollectionPolicy=Adaptive -XX:MaximumYoungGenerationSizePercent=20") - iso_args = os.environ.get("ARCADEDB_NATIVE_ISOLATE_ARGS", "").split() - # the log directory: always set, so that nothing is written to the working directory (the same rule as the JVM build) - iso_args += _prepare_logging(iso_args) - if iso_args: - - class _Params(ctypes.Structure): - _fields_ = [ - ("version", ctypes.c_int), - ("reserved_address_space_size", ctypes.c_size_t), - ("auxiliary_image_path", ctypes.c_char_p), - ("auxiliary_image_reserved_space_size", ctypes.c_size_t), - ("argc", ctypes.c_int), - ("argv", ctypes.POINTER(ctypes.c_char_p)), - ("pkey", ctypes.c_int), - ("ignore_unrecognized_args", ctypes.c_char), - ("_reserved_4", ctypes.c_char), - ("_reserved_5", ctypes.c_char), - ] - - argv = (ctypes.c_char_p * (len(iso_args) + 1))( - b"arcadedb", *[a.encode() for a in iso_args] - ) - params = _Params(version=5, argc=len(argv), argv=argv) - lib.graal_create_isolate.argtypes[0] = ctypes.POINTER(_Params) + user_args = os.environ.get("ARCADEDB_NATIVE_ISOLATE_ARGS", "").split() + # the heap limit and the log directory are always set (the log rule is the same as the JVM build's: nothing is written to the + # working directory); an argument the user gave for either is left out + iso_args = ( + user_args + _heap_isolate_args(user_args) + _prepare_logging(user_args) + ) + + class _Params(ctypes.Structure): + _fields_ = [ + ("version", ctypes.c_int), + ("reserved_address_space_size", ctypes.c_size_t), + ("auxiliary_image_path", ctypes.c_char_p), + ("auxiliary_image_reserved_space_size", ctypes.c_size_t), + ("argc", ctypes.c_int), + ("argv", ctypes.POINTER(ctypes.c_char_p)), + ("pkey", ctypes.c_int), + ("ignore_unrecognized_args", ctypes.c_char), + ("_reserved_4", ctypes.c_char), + ("_reserved_5", ctypes.c_char), + ] + + argv = (ctypes.c_char_p * (len(iso_args) + 1))( + b"arcadedb", *[a.encode() for a in iso_args] + ) + params = _Params(version=5, argc=len(argv), argv=argv) + lib.graal_create_isolate.argtypes[0] = ctypes.POINTER(_Params) rc = lib.graal_create_isolate( - ctypes.byref(params) if params is not None else None, + ctypes.byref(params), ctypes.byref(isolate), ctypes.byref(thread), ) diff --git a/tests/conftest.py b/tests/conftest.py index b721b07..a6b069d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,6 +47,8 @@ collect_ignore.append("test_native_xml_limits.py") # where the native library writes its log (isolate arguments); the JVM build is tested in test_log_dir.py collect_ignore.append("test_native_log_dir.py") + # the heap limit that the native glue puts on the isolate (a GraalVM isolate argument; the JVM build has heap_size) + collect_ignore.append("test_native_heap.py") if not os.path.isfile( os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md") ): diff --git a/tests/test_native_heap.py b/tests/test_native_heap.py new file mode 100644 index 0000000..b0bfcd9 --- /dev/null +++ b/tests/test_native_heap.py @@ -0,0 +1,103 @@ +"""Native build: the isolate gets a heap limit of 50% of the memory it can see, unless the user sets the heap. + +GraalVM's own default is 80% of the memory (the container limit when there is one). Together with the engine's off-heap memory that +let the kernel kill the process in a memory-limited container, so the glue passes ``-XX:MaximumHeapSizePercent=50`` as an isolate +argument. An ``-Xmx``, ``-XX:MaximumHeapSizePercent`` or ``-XX:MaxRAM`` in ARCADEDB_NATIVE_ISOLATE_ARGS replaces it. +""" + +import json +import os +import subprocess # nosec B404 - test-controlled child process +import sys +from pathlib import Path + +import pytest + +from arcadedb._native import lib as native_lib + +DEFAULT = "-XX:MaximumHeapSizePercent=50" + +CHILD = """ +import json +from arcadedb._native import lib +print(json.dumps(lib.diag())) +""" + + +def test_default_isolate_arguments_cap_the_heap_at_half(): + assert native_lib._heap_isolate_args([]) == [DEFAULT] + + +@pytest.mark.parametrize( + "user", + [ + ["-Xmx2g"], + ["-XX:MaximumHeapSizePercent=25"], + ["-XX:MaxRAM=4g"], + ["-Dfoo=bar", "-Xmx512m", "-XX:InitialCollectionPolicy=Adaptive"], + ], +) +def test_user_heap_arguments_win_over_the_default(user): + assert native_lib._heap_isolate_args(user) == [] + + +def test_other_user_arguments_keep_the_default(): + user = ["-XX:InitialCollectionPolicy=Adaptive", "-Dfoo=bar"] + assert native_lib._heap_isolate_args(user) == [DEFAULT] + + +def _diag(tmp_path, isolate_args=None) -> dict: + """``lib.diag()`` of an isolate that a fresh process starts with ``isolate_args`` in ARCADEDB_NATIVE_ISOLATE_ARGS.""" + env = {k: v for k, v in os.environ.items() if k != "ARCADEDB_NATIVE_ISOLATE_ARGS"} + env["ARCADEDB_LOG_DIR"] = str(tmp_path / "log") + if isolate_args is not None: + env["ARCADEDB_NATIVE_ISOLATE_ARGS"] = isolate_args + result = subprocess.run( # nosec B603 - fixed argument list, no shell + [sys.executable, "-c", CHILD], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.strip().splitlines()[-1]) + + +def _memory_limit() -> int: + """Bytes the isolate sees on Linux: the smaller of the host memory and a cgroup limit.""" + limits = [os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE")] + for name in ( + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", + ): + try: + limits.append(int(Path(name).read_text())) + except (OSError, ValueError): # absent, or "max" (no limit) + pass + return min(limits) + + +def test_default_max_heap_is_half_of_the_memory(tmp_path): + default = _diag(tmp_path)["max_heap"] + full = _diag(tmp_path, "-XX:MaximumHeapSizePercent=100")["max_heap"] + assert 0.45 <= default / full <= 0.55, (default, full) + if sys.platform == "linux": # follows the container limit, not the host RAM + assert 0.40 <= default / _memory_limit() <= 0.60, default + + +def test_user_percent_replaces_the_default_in_the_isolate(tmp_path): + default = _diag(tmp_path)["max_heap"] + quarter = _diag(tmp_path, "-XX:MaximumHeapSizePercent=25")["max_heap"] + assert 0.45 <= quarter / default <= 0.55, (quarter, default) + + +def test_user_xmx_replaces_the_default_in_the_isolate(tmp_path): + max_heap = _diag(tmp_path, "-Xmx512m")["max_heap"] + assert 0.8 * 512 * 2**20 <= max_heap <= 512 * 2**20, max_heap + + +def test_user_max_ram_replaces_the_default_in_the_isolate(tmp_path): + # with MaxRAM the percentage is taken of that size; the default 50% is not added, so GraalVM's own percentage (80%) applies + max_heap = _diag(tmp_path, "-XX:MaxRAM=4g")["max_heap"] + assert 0.7 * 4 * 2**30 <= max_heap <= 0.85 * 4 * 2**30, max_heap From a215349218d8d195c53a5fb343f1d4e62db16227 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 26 Oct 2026 01:11:57 +0900 Subject: [PATCH 33/33] Native build: commit every 1000 rows by default in insert_many, graph_batch and create_vertices A pre-registered laptop A/B on the columnar native wheel (1M documents through insert_many, 200k vertices and 500k edges through GraphBatch) found 1000-row transactions faster (wall 0.822-0.831 and 0.853-0.864 of the old default) and lighter on the young GC. The native defaults become insert_many commit_every 1000 (was 10,000), graph_batch commit_every 1000 (was unset) and a create_vertices chunk equal to the batch's commit_every (was 10,000). An explicit value wins. The JVM defaults are unchanged and pinned by a source-level test. Co-Authored-By: Claude Sonnet 5.5 --- docs/modules/ROOT/nav.adoc | 1 + .../modules/ROOT/pages/how-to/bulk-loads.adoc | 66 +++++++++ src/arcadedb/_native/database.py | 20 ++- src/arcadedb/_native/extras.py | 20 ++- src/arcadedb/core.py | 7 +- src/arcadedb/graph_batch.py | 4 + tests/conftest.py | 2 + tests/test_ingest_defaults.py | 94 ++++++++++++ tests/test_native_columnar_insert.py | 16 ++ tests/test_native_ingest_defaults.py | 138 ++++++++++++++++++ 10 files changed, 358 insertions(+), 10 deletions(-) create mode 100644 docs/modules/ROOT/pages/how-to/bulk-loads.adoc create mode 100644 tests/test_ingest_defaults.py create mode 100644 tests/test_native_ingest_defaults.py diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 79c9979..ecafb18 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -6,6 +6,7 @@ ** xref:how-to/databases-and-queries.adoc[Databases and queries] ** xref:how-to/concurrency.adoc[Concurrency, processes and crashes] ** xref:how-to/logs.adoc[Where the logs go] +** xref:how-to/bulk-loads.adoc[Bulk loads and commit size] * Development ** xref:development/ci.adoc[Continuous integration] ** xref:development/release.adoc[Release] diff --git a/docs/modules/ROOT/pages/how-to/bulk-loads.adoc b/docs/modules/ROOT/pages/how-to/bulk-loads.adoc new file mode 100644 index 0000000..9d83efb --- /dev/null +++ b/docs/modules/ROOT/pages/how-to/bulk-loads.adoc @@ -0,0 +1,66 @@ += Bulk loads and commit size +:description: The default commit size of insert_many, graph_batch and create_vertices on the arcadedb (native) and arcadedb-jvm packages, what it means for a failed load, and how to change it. + +`insert_many`, `graph_batch` and `GraphBatch.create_vertices` load data in transactions. The commit size is the number of rows in one transaction. It decides how much work the engine holds in memory before it commits, and how many rows stay in the database when a load fails. + +== Defaults + +[cols="2,2,2"] +|=== +| Call | `arcadedb` (native) | `arcadedb-jvm` + +| `db.insert_many(type_name, rows)` +| `commit_every=1000` +| `commit_every=10000` + +| `db.graph_batch()` +| `commit_every=1000` +| `commit_every` not set: one commit for each flush + +| `batch.create_vertices(type_name, rows)` +| 1000 rows for each call into the engine, one transaction for each call (the `commit_every` of the batch) +| 100000 rows for each call into the engine +|=== + +The JVM defaults are the same as before. They stay until someone measures them on the JVM build. + +An explicit value always wins. `commit_every=0` gives one transaction for `insert_many` and one commit for each flush for `graph_batch`, on both packages. In a transaction that you open yourself, `insert_many` commits nothing. + +== Why the native defaults are smaller + +The in-flight transaction is the data that each young garbage collection must copy. A smaller transaction copies less. A laptop A/B on the columnar native wheel measured this with `ab.py --noise`, ABBA order, 12 clean pairs for each figure, and a clean A/A control (95 % interval of the median pair ratio, new default against old default): + +[cols="3,2,2,2"] +|=== +| Load | Wall time | Data copied by the young GC | Peak RSS + +| `insert_many`, 1,000,000 mixed documents, `commit_every=1000` +| 0.822 to 0.831 +| 0.55x +| 0.955x + +| `GraphBatch`, 200,000 vertices and 500,000 edges, `parallel_flush`, one client thread, `commit_every=1000` +| 0.853 to 0.864 +| 0.63x +| 0.718x +|=== + +`commit_every=5000` for `insert_many` also passed (0.903 to 0.914 of the wall time). 1000 had the lower median, so it is the default. The runs used an isolate heap of 80 % of RAM. The default is now 50 %, and smaller transactions lower the heap that a load needs. + +== What changes for you + +Rows that are already committed stay committed when a later chunk fails. Only the transaction that was open is rolled back. With a smaller commit size, a failed load keeps its rows up to the last commit, and that commit is now at most 1000 rows behind the failure instead of 10,000. If you need all-or-nothing, open your own transaction, or pass `commit_every=0`. + +== Change the commit size + +[source,python] +---- +db.insert_many("Doc", rows, commit_every=10_000) # the old native default +db.insert_many("Doc", rows, commit_every=0) # one transaction for all rows + +with db.graph_batch(commit_every=0) as batch: # one commit for each flush, as before + ... + +with db.graph_batch(commit_every=5000) as batch: # also sets the create_vertices chunk + ... +---- diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 3c48c32..d20cb82 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -21,6 +21,9 @@ _LAZY_COMMAND_ROWS = os.environ.get("ARCADEDB_NATIVE_LAZY_CMD", "1") != "0" _BINARY_REPLY = os.environ.get("ARCADEDB_NATIVE_BIN_REPLY", "1") != "0" _INSERT_MANY_CHUNK = 10_000 +# rows per transaction when insert_many gets no commit_every: 1000 on the native build (laptop A/B, smaller commits copy less in the +# young GC and ran faster); the JVM build keeps 10,000 (core.py), which is not measured +DEFAULT_INSERT_MANY_COMMIT_EVERY = 1000 _XML_LIMIT_ERROR = re.compile(r"JAXP0001000[134]") # the JDK's XML entity limits _XML_LIMIT_HINT = ( " The JDK's XML entity limits stopped this import. Pass disable_xml_limits=True to import_documents to lift them for this" @@ -682,14 +685,20 @@ def lookup_by_key(self, type_name: str, keys: List[str], values: List[Any]): raise ArcadeDBError(f"Failed to lookup by key in '{type_name}': {e}") from e def insert_many( - self, type_name: str, rows, commit_every: int = 10_000, parallel: bool = False + self, + type_name: str, + rows, + commit_every: int = DEFAULT_INSERT_MANY_COMMIT_EVERY, + parallel: bool = False, ) -> int: - """Bulk insert. Rows go in chunks of 10,000, one crossing per chunk. A chunk of rows with the same keys whose columns each hold one + """Bulk insert. Rows go in chunks of at most 10,000 (a chunk never crosses a commit), one crossing per chunk. A chunk of rows with the same keys whose columns each hold one value type (ints, floats, bools, strings, equal-length float lists for an ``ARRAY_OF_FLOATS`` property; None is a null) goes as columns by address (``arcadedb_n_insert_colp``, no JSON text); any other chunk goes as one JSON text (``arcadedb_n_insert_rows``: a newDocument/set/save loop inside the image, as ``DocumentBatcher.insertManyJson`` on the JPype path); a chunk with a value the JSON text cannot carry unchanged goes through ``INSERT INTO t CONTENT :row`` for each row, in one crossing. Inside the caller's - transaction nothing is committed; otherwise every ``commit_every`` rows are one transaction (0: all rows in one). + transaction nothing is committed; otherwise every ``commit_every`` rows are one transaction (0: all rows in one). The default is 1000 on + this build (the JVM build's is 10,000). Rows that are already committed stay committed if a later chunk fails, so a smaller + ``commit_every`` keeps fewer rows of a failed load (only the transaction that was open is rolled back). """ self._check_not_closed() if parallel: @@ -872,7 +881,10 @@ def new_document(self, type_name: str): return Document(self, type_name) def graph_batch(self, **options): - """A GraphBatch for bulk graph loads (keywords as :meth:`Database.graph_batch`).""" + """A GraphBatch for bulk graph loads (keywords as :meth:`Database.graph_batch`). ``commit_every`` defaults to 1000 on this build + (the JVM build leaves it unset: one commit per flush); ``commit_every=0`` gives one commit per flush here too, and an explicit value + wins. It is also the number of rows ``create_vertices`` sends, and commits, per call into the image. + """ from .extras import NativeGraphBatch self._check_not_closed() diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py index e27a1c7..c374396 100644 --- a/src/arcadedb/_native/extras.py +++ b/src/arcadedb/_native/extras.py @@ -121,18 +121,27 @@ def _call(prefix: str, name: str, *args) -> dict: class NativeGraphBatch(_BaseGraphBatch): """``GraphBatch`` over the engine's batch builder in the image (same names and behaviour as the JPype wrapper).""" - _BULK_CHUNK = 10_000 # rows per crossing: small enough that a chunk's text, rows and reply die in the young generation + _BULK_CHUNK = 10_000 # rows per crossing of new_edges, and of create_vertices when commit_every is 0 + # commit_every when graph_batch() gets none (the JVM build leaves it unset: one commit per flush). It is also the create_vertices + # chunk: each crossing is one transaction, and 1000 rows per transaction ran faster and copied less in the young GC than 10,000 + DEFAULT_COMMIT_EVERY = 1000 - def __init__(self, database, handle: int): + def __init__(self, database, handle: int, commit_every: int = DEFAULT_COMMIT_EVERY): self._db = database self._gb = handle self._closed = False + self._vertex_chunk = ( + int(commit_every) if int(commit_every) > 0 else self._BULK_CHUNK + ) @classmethod def create(cls, database, **options) -> "NativeGraphBatch": wal_flush = options.get("wal_flush") if wal_flush is not None: options["wal_flush"] = cls._to_java_wal_flush_name(wal_flush) + commit_every = options.get("commit_every") + if commit_every is None: # an explicit value, 0 included, wins + commit_every = options["commit_every"] = cls.DEFAULT_COMMIT_EVERY given = {k: v for k, v in options.items() if v is not None} try: reply = _call( @@ -143,7 +152,7 @@ def create(cls, database, **options) -> "NativeGraphBatch": ) except ArcadeDBError: raise - return cls(database, reply["h"]) + return cls(database, reply["h"], commit_every) @classmethod def _to_java_wal_flush_name(cls, mode: str) -> str: @@ -222,12 +231,15 @@ def _create_vertices_json_bulk(self, type_name: str, rows): )["rids"] def create_vertices(self, type_name: str, count_or_properties) -> list: + """The RIDs of the new vertices. Rows go in chunks of the batch's ``commit_every`` (1000 unless graph_batch() got another value; + 0: 10,000), one crossing and one transaction per chunk: chunks that already went in stay stored if a later chunk fails. + """ self._check_not_closed() db = self._db owns_transaction = not db.is_transaction_active() prefix = f"Failed to create batch vertices for type '{type_name}'" try: - chunk = self._BULK_CHUNK + chunk = self._vertex_chunk if isinstance(count_or_properties, int): rids = [] for start in range(0, count_or_properties, chunk): diff --git a/src/arcadedb/core.py b/src/arcadedb/core.py index 3a36c54..b828ea6 100644 --- a/src/arcadedb/core.py +++ b/src/arcadedb/core.py @@ -708,7 +708,9 @@ def insert_many( because an edge needs both endpoints. rows: Iterable of dicts, one per document. Read lazily, one chunk at a time. - commit_every: Transaction batch size for the synchronous mode. + commit_every: Transaction batch size for the synchronous mode. Default 10,000 on this build; the native build (the + ``arcadedb`` package) defaults to 1000, a size measured there and not yet on the JVM. Rows that are already + committed stay committed if a later chunk fails. parallel: If True, route rows through the async executor's parallel bucket writers and wait for completion before returning (out-of-order writes). The maintainers' rule @@ -2018,7 +2020,8 @@ def graph_batch( ArcadeDBError naming the type. Before 26.10.1 it was accepted, and any query the planner walked from the target end returned 0 rows (ArcadeData/arcadedb#8625). - commit_every: Commit cadence within a flush. `0` means one commit per flush. + commit_every: Commit cadence within a flush. `0` means one commit per flush, which is also what an unset value gives on + this build; the native build (the ``arcadedb`` package) defaults to 1000. use_wal: Write-ahead log during the import. Off by default, so a crash mid-import can lose its tail; pass True for a crash-safe import (ArcadeData/arcadedb#8287). wal_flush: WAL flush mode: `"no"`, `"yes_nometadata"`, `"yes_full"`. diff --git a/src/arcadedb/graph_batch.py b/src/arcadedb/graph_batch.py index 1e1f45f..7184752 100644 --- a/src/arcadedb/graph_batch.py +++ b/src/arcadedb/graph_batch.py @@ -190,6 +190,10 @@ def create_vertices( ``ARRAY_OF_FLOATS`` whose value in every row is a list or a numpy array of numbers of one length crosses as one binary float32 matrix instead of JSON text. + + Large inputs cross in chunks of 100,000 rows on this build. On the native build (the ``arcadedb`` package) the chunk is the + batch's ``commit_every``, 1000 unless ``graph_batch()`` got another value, and each chunk is its own transaction. Chunks that + already went in stay stored if a later chunk fails. """ self._check_not_closed() # From engine 26.10.1 the engine refuses to run inside a transaction the diff --git a/tests/conftest.py b/tests/conftest.py index 5cebd98..7094e9e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,6 +41,8 @@ collect_ignore.append("test_native_udf.py") # the columnar insert's stored records are compared between the JSON and the columnar entry points of the native library collect_ignore.append("test_native_columnar_insert.py") + # the native build's ingest defaults (insert_many and graph_batch commit size), observed on the native library + collect_ignore.append("test_native_ingest_defaults.py") # the native library's start-up output (#477) can only be checked with the native library loaded collect_ignore.append("test_native_startup_runtime.py") # the XML entity limits of an import, lifted by the native glue (the JVM build lifts them with JVM system properties) diff --git a/tests/test_ingest_defaults.py b/tests/test_ingest_defaults.py new file mode 100644 index 0000000..7003a35 --- /dev/null +++ b/tests/test_ingest_defaults.py @@ -0,0 +1,94 @@ +"""The ingest defaults, read from the source so the JVM is not started and the native library is not loaded. + +The native build (the ``arcadedb`` package) commits every 1000 rows in ``insert_many`` and ``graph_batch`` (a laptop A/B: smaller +transactions copy less in the young GC and ran faster). The JVM build (``arcadedb-jvm``) keeps its old defaults until they are measured: +``insert_many`` 10,000, ``graph_batch`` unset (one commit per flush), ``create_vertices`` 100,000 rows per crossing. These tests pin both +sides, and run on both backends because they only parse the files. +""" + +import ast +from pathlib import Path + +import arcadedb + +SRC = Path(arcadedb.__file__).parent + + +def _tree(rel): + return ast.parse((SRC / rel).read_text(encoding="utf-8")) + + +def _param_default(rel, func, arg): + """The AST node of the default of ``arg`` in the only ``def func`` of the file.""" + funcs = [ + n + for n in ast.walk(_tree(rel)) + if isinstance(n, ast.FunctionDef) and n.name == func + ] + assert len(funcs) == 1, (rel, func, len(funcs)) + a = funcs[0].args + positional = a.posonlyargs + a.args + pairs = list(zip(positional[len(positional) - len(a.defaults) :], a.defaults)) + pairs += [(k, d) for k, d in zip(a.kwonlyargs, a.kw_defaults) if d is not None] + for node, default in pairs: + if node.arg == arg: + return default + raise AssertionError(f"{rel}: {func}() has no default for {arg}") + + +def _module_constant(rel, name): + for node in _tree(rel).body: + if isinstance(node, ast.Assign) and any( + isinstance(t, ast.Name) and t.id == name for t in node.targets + ): + return ast.literal_eval(node.value) + raise AssertionError(f"{rel}: no module constant {name}") + + +def _class_constant(rel, cls, name): + for node in ast.walk(_tree(rel)): + if isinstance(node, ast.ClassDef) and node.name == cls: + for item in node.body: + if isinstance(item, ast.Assign) and any( + isinstance(t, ast.Name) and t.id == name for t in item.targets + ): + return ast.literal_eval(item.value) + raise AssertionError(f"{rel}: no {cls}.{name}") + + +def test_jvm_insert_many_default_is_10000(): + node = _param_default("core.py", "insert_many", "commit_every") + assert ast.literal_eval(node) == 10_000 + + +def test_jvm_graph_batch_commit_every_stays_unset(): + node = _param_default("core.py", "graph_batch", "commit_every") + assert ast.literal_eval(node) is None + # the JVM wrapper passes the builder option only when it is given, so unset stays one commit per flush + node = _param_default("graph_batch.py", "create", "commit_every") + assert ast.literal_eval(node) is None + + +def test_jvm_create_vertices_chunk_stays_100000(): + assert _class_constant("graph_batch.py", "GraphBatch", "_BULK_CHUNK") == 100_000 + + +def test_native_insert_many_default_is_1000(): + assert ( + _module_constant("_native/database.py", "DEFAULT_INSERT_MANY_COMMIT_EVERY") + == 1000 + ) + node = _param_default("_native/database.py", "insert_many", "commit_every") + assert isinstance(node, ast.Name) + assert node.id == "DEFAULT_INSERT_MANY_COMMIT_EVERY" + + +def test_native_graph_batch_default_is_1000_and_the_edge_chunk_is_unchanged(): + assert ( + _class_constant("_native/extras.py", "NativeGraphBatch", "DEFAULT_COMMIT_EVERY") + == 1000 + ) + assert ( + _class_constant("_native/extras.py", "NativeGraphBatch", "_BULK_CHUNK") + == 10_000 + ) diff --git a/tests/test_native_columnar_insert.py b/tests/test_native_columnar_insert.py index 3aac3df..a3591d5 100644 --- a/tests/test_native_columnar_insert.py +++ b/tests/test_native_columnar_insert.py @@ -23,6 +23,22 @@ ) +@pytest.fixture(autouse=True) +def _one_chunk_per_commit(monkeypatch): + """The crossing counts below are for one chunk of up to 10,000 rows, so these tests run insert_many with its old commit_every of + 10,000 (the native default is 1000, which cuts a 2,500-row load into three crossings). The tests are about the columnar entry + points, not the default; tests/test_native_ingest_defaults.py covers the default.""" + from arcadedb._native import database as native_database + + cls = next( + c + for c in vars(native_database).values() + if isinstance(c, type) and "insert_many" in vars(c) + ) + defaults = cls.insert_many.__defaults__ + monkeypatch.setattr(cls.insert_many, "__defaults__", (10_000,) + defaults[1:]) + + class Spy: def __init__(self, monkeypatch): self.calls = [] diff --git a/tests/test_native_ingest_defaults.py b/tests/test_native_ingest_defaults.py new file mode 100644 index 0000000..9234bb7 --- /dev/null +++ b/tests/test_native_ingest_defaults.py @@ -0,0 +1,138 @@ +"""The native build's ingest defaults, observed on the native library (conftest.py collects this file on the native backend only). + +``insert_many`` commits every 1000 rows, ``graph_batch`` commits every 1000 and ``create_vertices`` sends (and commits) 1000 rows per +call into the image. An explicit value wins, and the semantics stay: rows that are already committed stay committed if a later chunk fails. +""" + +import inspect + +import pytest + +import arcadedb +from arcadedb._native import extras +from arcadedb.exceptions import ArcadeDBError + +_COUNT_SQL = { # fixed statements, one for each type the tests create + "Dup": "SELECT count(*) AS c FROM Dup", + "V": "SELECT count(*) AS c FROM V", + "W": "SELECT count(*) AS c FROM W", +} + + +def _count(db, type_name): + return list(db.query("sql", _COUNT_SQL[type_name]))[0].get("c") + + +def _unique_type(db, name): + db.command("sql", f"CREATE DOCUMENT TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.k INTEGER") + db.command("sql", f"CREATE INDEX ON {name} (k) UNIQUE") + + +def test_insert_many_signature_default_is_1000(temp_db): + param = inspect.signature(temp_db.insert_many).parameters["commit_every"] + assert param.default == 1000 + + +@pytest.mark.parametrize( + "kwargs, kept", + [ + ({}, 2000), # the default: 1000-row transactions, the third one fails + ({"commit_every": 1000}, 2000), + ({"commit_every": 1500}, 1500), # an explicit value wins + ( + {"commit_every": 10_000}, + 0, + ), # the old default: nothing was committed before the failure + ({"commit_every": 0}, 0), # one transaction + ], +) +def test_insert_many_commit_size_decides_what_a_failed_load_keeps( + temp_db, kwargs, kept +): + _unique_type(temp_db, "Dup") + rows = [{"k": i} for i in range(2500)] + rows[2400] = {"k": 0} # a duplicate key inside the third 1000 rows + with pytest.raises(ArcadeDBError): + temp_db.insert_many("Dup", rows, **kwargs) + assert temp_db.is_transaction_active() is False + assert _count(temp_db, "Dup") == kept + + +class _Spy: + def __init__(self, monkeypatch): + self.created = ( + [] + ) # the option dicts that went to arcadedb_n_gb_create (and every other call that encodes a dict) + self.vertex_calls = ( + [] + ) # the row or vertex count of each create_vertices crossing + real_enc = extras._enc + real_op = extras.NativeGraphBatch._op + + def enc(value): + if isinstance(value, dict): + self.created.append(dict(value)) + return real_enc(value) + + def op(this, prefix, name, **args): + if name == "create_vertices": + rows = args.get("rows") + self.vertex_calls.append( + len(rows) if rows is not None else args.get("count") + ) + return real_op(this, prefix, name, **args) + + monkeypatch.setattr(extras, "_enc", enc) + monkeypatch.setattr(extras.NativeGraphBatch, "_op", op) + + def commit_every_given(self): + return [d["commit_every"] for d in self.created if "commit_every" in d] + + +def _vertices(db, name): + db.command("sql", f"CREATE VERTEX TYPE {name}") + + +def test_graph_batch_default_commit_every_is_1000(temp_db, monkeypatch): + spy = _Spy(monkeypatch) + _vertices(temp_db, "V") + with temp_db.graph_batch(parallel_flush=False) as gb: + gb.create_vertices("V", [{"id": i} for i in range(2500)]) + gb.create_vertices("V", 2500) + assert spy.commit_every_given() == [1000] + assert spy.vertex_calls == [1000, 1000, 500, 1000, 1000, 500] + assert _count(temp_db, "V") == 5000 + + +@pytest.mark.parametrize( + "commit_every, calls", + [ + ( + 250, + [250] * 10, + ), # an explicit value wins, for the engine and for the create_vertices chunk + ( + 0, + [2500], + ), # 0: one commit per flush, and the create_vertices chunk is the old 10,000 + ], +) +def test_graph_batch_explicit_commit_every_wins( + temp_db, monkeypatch, commit_every, calls +): + spy = _Spy(monkeypatch) + _vertices(temp_db, "W") + with temp_db.graph_batch(commit_every=commit_every, parallel_flush=False) as gb: + gb.create_vertices("W", [{"id": i} for i in range(2500)]) + assert spy.commit_every_given() == [commit_every] + assert spy.vertex_calls == calls + assert _count(temp_db, "W") == 2500 + + +def test_graph_batch_default_constant_and_edges_chunk(): + assert extras.NativeGraphBatch.DEFAULT_COMMIT_EVERY == 1000 + assert ( + extras.NativeGraphBatch._BULK_CHUNK == 10_000 + ) # new_edges is not part of the measured change + assert arcadedb.__backend__ == "native"