native: ship libjvector-aarch64.so in the Linux aarch64 wheel; state the real provider default

.github/workflows/test-jvm.yml

@@ -750,7 +750,7 @@ jobs:
             echo "**Package**: arcadedb" >> $GITHUB_STEP_SUMMARY
             echo "" >> $GITHUB_STEP_SUMMARY
             echo "ℹ️ **Note**: Some platform/Python combinations are excluded from testing:" >> $GITHUB_STEP_SUMMARY
-            echo "- Windows ARM64 (no GitHub-hosted runners available)" >> $GITHUB_STEP_SUMMARY
+            echo "- Windows ARM64 (GitHub-hosted windows-11-arm runners exist, but jpype1 1.7.1 ships no win_arm64 wheel and GraalVM CE 25.4.4.1.1 has no Windows ARM64 build, so neither package can be built there)" >> $GITHUB_STEP_SUMMARY
             echo "- macOS x86_64 (temporarily disabled)" >> $GITHUB_STEP_SUMMARY
           else
             echo "❌ **Some platforms failed testing**" >> $GITHUB_STEP_SUMMARY

.github/workflows/test-native.yml

@@ -18,7 +18,7 @@
 #
 # One wheel per platform, py3-none-<platform tag>:
 #   linux-x86_64   manylinux_2_28 container          manylinux_2_28_x86_64   libarcadedb.so + libjvector-x86_64.so
-#   linux-aarch64  manylinux_2_28 aarch64 container  manylinux_2_28_aarch64  libarcadedb.so
+#   linux-aarch64  manylinux_2_28 aarch64 container  manylinux_2_28_aarch64  libarcadedb.so + libjvector-aarch64.so
 #   macos-arm64    native, heap and threads capped   macosx_<minos>_arm64    libarcadedb.dylib
 #   windows-x64    native, MSVC environment loaded   win_amd64               libarcadedb.dll
 # Only free GitHub-hosted standard runners. GraalVM Community Edition only (pinned by sha256 per platform), never Oracle GraalVM.
@@ -426,6 +426,18 @@ jobs:
           dll="$(ls "$T"/lib/libarcadedb.dll)"
           dumpbin //dependents "$dll" | tee "$T/dumpbin.txt"
 
+      - name: "Gate: a Linux wheel carries jvector's library for its architecture"
+        if: ${{ matrix.kind == 'linux' }}
+        env:
+          PLAT: ${{ matrix.plat_re }}
+        run: |
+          set -euo pipefail
+          w="$(ls "$T"/wheel/dist/*.whl)"
+          arch="${KEY#linux-}"   # linux-x86_64 -> x86_64, linux-aarch64 -> aarch64 (${PLAT##*_} gave "64" for x86_64)
+          unzip -Z1 "$w" | tee "$T/wheel-files.txt" > /dev/null
+          grep -Fx "arcadedb/_native/libarcadedb.so" "$T/wheel-files.txt"
+          grep -Fx "arcadedb/_native/libjvector-${arch}.so" "$T/wheel-files.txt"
+
       - name: "Gate: twine check"
         run: uvx twine check --strict "$T"/wheel/dist/*.whl
 

native/scripts/build-image.sh

@@ -4,7 +4,7 @@
 # Maven Central, never a local SNAPSHOT), local symbols deleted (the default: a smaller image than the phase 7/8 profiling builds).
 # MARCH (default compatibility = baseline x86-64, what a manylinux wheel promises, PEP 600; x86-64-v3 is the GraalVM CE 25.4 default and the faster reference).
 # usage: [MARCH=x86-64-v3] [XMX=12g] [NI_EXTRA="-H:NumberOfThreads=2"] build-image.sh [OUTDIR]     A laptop with other jobs runs it through its job gate; the CI calls it directly.
-# Platforms: Linux (libarcadedb.so; on x86-64 also libjvector-x86_64.so, jvector ships the Highway library for x86-64 only here), macOS
+# Platforms: Linux (libarcadedb.so; also jvector's Highway library, libjvector-x86_64.so or libjvector-aarch64.so, both are in the jvector 4.0.2 jar), macOS
 # (libarcadedb.dylib) and Windows in Git Bash with the MSVC environment loaded (libarcadedb.dll). The CI (test-native.yml) caps the
 # builder heap and threads on the 7 GB macOS runner (XMX=5g, NI_EXTRA=-H:NumberOfThreads=2).
 #   e.g. NEED_GB=13 JOB_MEM_GB=14 quiet_build.sh bash native/scripts/build-image.sh
@@ -55,15 +55,24 @@ grep -a -E -q 'PriorityForceInline. \(origin\(s\): .META-INF.native-image.org\.h
 INLINE_LOG="PriorityForceInline picked up from native-image.properties ($(grep -a -c 'PriorityForceInline' "$OUT/build.log") log lines)"
 cp "target/$LIBFILE" "$OUT/"
 LIBS=("$OUT/$LIBFILE")
-# Highway (jvector's native provider), found next to libarcadedb.so by the image (ARCADEDB_NATIVE_LIBDIR); Linux x86-64 only: the other
-# platforms ship no jvector library and the image falls back to the Java providers
-if [ "$EXT" = so ] && [ "$(uname -m)" = x86_64 ]; then
-    J=$(unzip -l "$HOME/.m2/repository/io/github/jbellis/jvector/4.0.2/jvector-4.0.2.jar" | awk '/libjvector-x86_64.so/{print $4}' | head -1)
-    if [ -n "$J" ]; then unzip -p "$HOME/.m2/repository/io/github/jbellis/jvector/4.0.2/jvector-4.0.2.jar" "$J" > "$OUT/libjvector-x86_64.so"; else
-        echo "libjvector-x86_64.so is not in the jvector 4.0.2 jar" >&2
-        exit 1
+# Highway (jvector's native library), found next to libarcadedb.so by the image (ARCADEDB_NATIVE_LIBDIR); Linux only: the jvector 4.0.2 jar
+# carries libjvector-x86_64.so and libjvector-aarch64.so and nothing for macOS or Windows. The image uses it only when
+# ARCADEDB_VECTOR_PROVIDER=native is set (the engine forces jvector's scalar provider in every image otherwise).
+if [ "$EXT" = so ]; then
+    case "$(uname -m)" in
+        x86_64) JARCH=x86_64 ;;
+        aarch64 | arm64) JARCH=aarch64 ;;
+        *) JARCH= ;;
+    esac
+    if [ -n "$JARCH" ]; then
+        JAR="$HOME/.m2/repository/io/github/jbellis/jvector/4.0.2/jvector-4.0.2.jar"
+        J=$(unzip -l "$JAR" | awk -v n="libjvector-$JARCH.so" 'index($4, n) {print $4}' | head -1)
+        if [ -n "$J" ]; then unzip -p "$JAR" "$J" > "$OUT/libjvector-$JARCH.so"; else
+            echo "libjvector-$JARCH.so is not in the jvector 4.0.2 jar" >&2
+            exit 1
+        fi
+        LIBS+=("$OUT/libjvector-$JARCH.so")
     fi
-    LIBS+=("$OUT/libjvector-x86_64.so")
 fi
 sha256 "${LIBS[@]}" > "$OUT/sha256.txt"
 echo "native image done: engine $ENGINE; march $MARCH; $INLINE_LOG; $(grep -a 'Finished generating' "$OUT/build.log")" | tee "$OUT/build.done"

native/src/main/java/org/humemai/nativelib/ArcadeNative.java

@@ -56,8 +56,10 @@ public final class ArcadeNative {
   }
 
   static {
-    // ARCADEDB_VECTOR_PROVIDER = native | panama | default (jvector's VectorizationProvider, forced); unset: jvector chooses
-    // (Highway through the FFM downcalls when the bundled libjvector loads, else the Vector API classes).
+    // ARCADEDB_VECTOR_PROVIDER = native | panama | default (jvector's VectorizationProvider, forced). Unset, the engine's
+    // VectorApiSupport.configureJVectorProvider() (ArcadeData/arcadedb#10077) sets jvector.vectorization_provider=default in every
+    // native image, so the distance kernels are jvector's scalar DefaultVectorizationProvider. Highway (the bundled libjvector
+    // through the FFM downcalls, Linux x86-64 and aarch64) runs only with ARCADEDB_VECTOR_PROVIDER=native.
     final String forced = System.getenv("ARCADEDB_VECTOR_PROVIDER");
     if (forced != null && !forced.isEmpty() && System.getProperty("jvector.vectorization_provider") == null)
       System.setProperty("jvector.vectorization_provider", forced);

native/wheel/NOTICE

@@ -17,13 +17,13 @@ GraalVM native-image runtime and JDK classes in libarcadedb.so
   Other GraalVM modules carry the licences in licenses/GraalVM-THIRD_PARTY_LICENSE.txt (graal-sdk and polyglot: UPL-1.0).
   Oracle GraalVM (GFTC) is not used; only Community Edition.
 
-libjvector-x86_64.so
+libjvector-x86_64.so, libjvector-aarch64.so
   The native vector kernels of JVector 4.0.2 (https://github.com/datastax/jvector), Apache-2.0, unmodified, taken from the jvector jar.
 
 Java libraries, the Highway/jvector kernels and JDK data inside libarcadedb.so: THIRD_PARTY_NOTICES.md (generated by gen_notices.py).
 
 System libraries linked dynamically and NOT shipped:
   Linux:   libz.so.1, libm.so.6, libc.so.6, libpthread.so.0, libdl.so.2, librt.so.1 (libarcadedb.so, built in a manylinux_2_28 container);
-           libstdc++.so.6, libgcc_s.so.1 (libjvector-x86_64.so only)
+           libstdc++.so.6, libgcc_s.so.1 (the libjvector libraries only)
   macOS:   the system libraries of macOS
   Windows: the Microsoft Visual C++ runtime (VCRUNTIME140.dll, VCRUNTIME140_1.dll) and the Windows C runtime (api-ms-win-crt-*.dll); CPython itself needs the same files

native/wheel/THIRD_PARTY_NOTICES.md

@@ -72,7 +72,7 @@ Where a pom lists several licences they are alternatives (JTS: EPL-2.0 or EDL-1.
 
 ## Native code shipped next to the library
 
-`libjvector-x86_64.so` is taken unmodified from `io.github.jbellis:jvector:4.0.2` (Apache-2.0, pom). Its sources are in the jvector repository at tag `4.0.2` (`jvector-native/src/main/native`): jvector's own C++ kernels (Apache-2.0) and the Google Highway library, pinned there as a git submodule at commit `16f5f5c768064d0bb52952888a14236bec75c104`. Highway's `LICENSE` at that commit says: dual-licensed under the Apache License 2.0 or the BSD 3-Clause License, except `hwy/contrib/random/random-inl.h` (CC0 1.0). Applied here: Apache-2.0. The `LICENSE` of Highway and the `NOTICE.txt` of jvector are reproduced verbatim below. The library links `libstdc++.so.6`, `libm.so.6`, `libgcc_s.so.1` and `libc.so.6` dynamically (not shipped).
+`libjvector-x86_64.so` (Linux x86-64) and `libjvector-aarch64.so` (Linux aarch64) are taken unmodified from `io.github.jbellis:jvector:4.0.2` (Apache-2.0, pom). Their sources are in the jvector repository at tag `4.0.2` (`jvector-native/src/main/native`): jvector's own C++ kernels (Apache-2.0) and the Google Highway library, pinned there as a git submodule at commit `16f5f5c768064d0bb52952888a14236bec75c104`. Highway's `LICENSE` at that commit says: dual-licensed under the Apache License 2.0 or the BSD 3-Clause License, except `hwy/contrib/random/random-inl.h` (CC0 1.0). Applied here: Apache-2.0. The `LICENSE` of Highway and the `NOTICE.txt` of jvector are reproduced verbatim below. The libraries link `libstdc++.so.6`, `libm.so.6`, `libgcc_s.so.1` and `libc.so.6` dynamically (not shipped).
 
 ## GraalVM Community Edition runtime and JDK classes in libarcadedb.so
 
@@ -1313,7 +1313,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #### highway-16f5f5c768064d0bb52952888a14236bec75c104.LICENSE.txt [b8b1fae5894c]
 
-Applies to: libjvector-x86_64.so (google/highway)
+Applies to: libjvector-x86_64.so and libjvector-aarch64.so (google/highway)
 
 ```
 This project is primarily dual-licensed under your choice of either the Apache

native/wheel/build_wheel.py

@@ -5,7 +5,7 @@
 
 Steps: copy src/arcadedb to <out>/stage/src/arcadedb (the shared tree is not duplicated in git; only the install hints of the extras are renamed from arcadedb-jvm[...] to arcadedb[...]);
 rewrite `import jpype` to the bundled stub; flip the default backend to native; add the library of the build platform (libarcadedb.so on Linux,
-libarcadedb.dylib on macOS, libarcadedb.dll on Windows; on Linux x86-64 also libjvector-x86_64.so, jvector's Highway library: no other platform ships it);
+libarcadedb.dylib on macOS, libarcadedb.dll on Windows; on Linux also libjvector-<x86_64|aarch64>.so, jvector's Highway library: macOS and Windows have none);
 add LICENSE/NOTICE/GraalVM licence texts; check THIRD_PARTY_NOTICES.md against gen_notices.py (fails on drift); compute the platform tag from the library
 (Linux: manylinux_<glibc floor>_<x86_64|aarch64> from the GLIBC symbol versions, never below --glibc-policy; macOS: macosx_<minos>_arm64 from the load commands; Windows: win_amd64);
 build with `uv build --wheel`. Run it on the platform the library was built on.
@@ -29,7 +29,19 @@
     "darwin": "libarcadedb.dylib",
     "win32": "libarcadedb.dll",
 }
-JVECTOR = "libjvector-x86_64.so"  # Highway, taken from the jvector jar by preview-build.sh on Linux x86-64 only
+# Highway, taken from the jvector 4.0.2 jar by build-image.sh on Linux (the jar has no macOS or Windows library)
+JVECTOR_BY_ARCH = {
+    "x86_64": "libjvector-x86_64.so",
+    "aarch64": "libjvector-aarch64.so",
+}
+
+
+def libraries_to_ship(sys_platform: str, arch: str, libname: str) -> list[str]:
+    """The files of --lib-dir the wheel carries next to the engine library: libname, and on Linux the jvector library of the architecture."""
+    ship = [libname]
+    if sys_platform.startswith("linux") and arch in JVECTOR_BY_ARCH:
+        ship.append(JVECTOR_BY_ARCH[arch])
+    return ship
 
 
 def host_arch() -> str:
@@ -94,7 +106,7 @@ def main():
     ap.add_argument(
         "--lib-dir",
         required=True,
-        help="directory with the library of this platform (libarcadedb.so / .dylib / .dll) and, on Linux x86-64, libjvector-x86_64.so",
+        help="directory with the library of this platform (libarcadedb.so / .dylib / .dll) and, on Linux, libjvector-<x86_64|aarch64>.so",
     )
     ap.add_argument(
         "--version",
@@ -245,11 +257,7 @@ def sub(m):
     )
     ip.write_text(t, encoding="utf-8")
 
-    ship = [libname]
-    if sys.platform.startswith("linux") and host_arch() == "x86_64":
-        ship.append(
-            JVECTOR
-        )  # the only platform that ships Highway; lib.py does not need it elsewhere
+    ship = libraries_to_ship(sys.platform, host_arch(), libname)
     for n in ship:
         shutil.copy(
             lib_dir / n, dst / "_native" / n

native/wheel/gen_notices.py

@@ -287,17 +287,17 @@ def main():
     # ---- native code ----
     w("## Native code shipped next to the library\n")
     w(
-        "`libjvector-x86_64.so` is taken unmodified from `io.github.jbellis:jvector:4.0.2` (Apache-2.0, pom). Its sources are in the "
+        "`libjvector-x86_64.so` (Linux x86-64) and `libjvector-aarch64.so` (Linux aarch64) are taken unmodified from `io.github.jbellis:jvector:4.0.2` (Apache-2.0, pom). Their sources are in the "
         "jvector repository at tag `4.0.2` (`jvector-native/src/main/native`): jvector's own C++ kernels (Apache-2.0) and the Google Highway "
         f"library, pinned there as a git submodule at commit `{HIGHWAY_COMMIT}`. Highway's `LICENSE` at that commit says: "
         "dual-licensed under the Apache License 2.0 or the BSD 3-Clause License, except `hwy/contrib/random/random-inl.h` (CC0 1.0). "
         "Applied here: Apache-2.0. The `LICENSE` of Highway and the `NOTICE.txt` of jvector are reproduced verbatim below. "
-        "The library links `libstdc++.so.6`, `libm.so.6`, `libgcc_s.so.1` and `libc.so.6` dynamically (not shipped).\n"
+        "The libraries link `libstdc++.so.6`, `libm.so.6`, `libgcc_s.so.1` and `libc.so.6` dynamically (not shipped).\n"
     )
     for f, who in (
         (
             f"highway-{HIGHWAY_COMMIT}.LICENSE.txt",
-            "libjvector-x86_64.so (google/highway)",
+            "libjvector-x86_64.so and libjvector-aarch64.so (google/highway)",
         ),
         ("jvector-4.0.2.NOTICE.txt", "io.github.jbellis:jvector:4.0.2"),
     ):

scripts/select_matrix.py

@@ -36,8 +36,10 @@
 import os
 import sys
 
-# Temporarily limited to four platforms: macOS x86_64 (runner macos-15-intel) and Windows ARM64
-# (windows-11-arm) are off. This is the platform column both workflows had before this script.
+# Temporarily limited to four platforms: macOS x86_64 (runner macos-15-intel) is off. Windows ARM64 (runner
+# windows-11-arm exists) is off because neither package can be built there: jpype1 1.7.1 ships no win_arm64 wheel
+# (win_amd64 only) and GraalVM CE 25.4.4.1.1 has no Windows ARM64 build (windows-x64 only). This is the platform
+# column both workflows had before this script.
 RUNNERS = {
     "linux/amd64": "ubuntu-24.04",
     "linux/arm64": "ubuntu-24.04-arm",
@@ -85,7 +87,9 @@ def _legs(per_platform: dict[str, tuple[str, ...]]) -> list[dict[str, str]]:
 def select(workflow: str, event: str) -> dict[str, list[dict[str, str]]]:
     """The `strategy.matrix` of `workflow` ("jvm" or "examples") for `event`."""
     if workflow not in WORKFLOWS:
-        raise ValueError(f"unknown workflow {workflow!r}; use one of {sorted(WORKFLOWS)}")
+        raise ValueError(
+            f"unknown workflow {workflow!r}; use one of {sorted(WORKFLOWS)}"
+        )
     if workflow == "jvm":
         return {"include": _legs(JVM_LEGS)}
     if event == "pull_request":

tests/test_native_jvector_library.py

@@ -0,0 +1,93 @@
+"""The native `arcadedb` wheel carries jvector's Highway library for the architecture it is built for.
+
+The jvector 4.0.2 jar has `libjvector-x86_64.so` and `libjvector-aarch64.so` and nothing for macOS or Windows. The Linux x86-64
+wheel shipped the first, the Linux aarch64 wheel shipped none (#144). These tests check the builder's ship list and the build
+script's copy step, and ArcadeNative's comment on the provider default.
+
+Repository tooling: no engine, no JVM and no native library needed.
+"""
+
+import importlib.util
+import re
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+BUILD_WHEEL = ROOT / "native" / "wheel" / "build_wheel.py"
+BUILD_IMAGE = ROOT / "native" / "scripts" / "build-image.sh"
+ARCADE_NATIVE = (
+    ROOT
+    / "native"
+    / "src"
+    / "main"
+    / "java"
+    / "org"
+    / "humemai"
+    / "nativelib"
+    / "ArcadeNative.java"
+)
+
+
+def _build_wheel():
+    spec = importlib.util.spec_from_file_location("native_build_wheel", BUILD_WHEEL)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+@pytest.mark.parametrize(
+    "sys_platform, arch, libname, expected",
+    [
+        (
+            "linux",
+            "x86_64",
+            "libarcadedb.so",
+            ["libarcadedb.so", "libjvector-x86_64.so"],
+        ),
+        (
+            "linux",
+            "aarch64",
+            "libarcadedb.so",
+            ["libarcadedb.so", "libjvector-aarch64.so"],
+        ),
+        ("darwin", "aarch64", "libarcadedb.dylib", ["libarcadedb.dylib"]),
+        ("win32", "x86_64", "libarcadedb.dll", ["libarcadedb.dll"]),
+        ("linux", "riscv64", "libarcadedb.so", ["libarcadedb.so"]),
+    ],
+)
+def test_the_wheel_ships_the_jvector_library_of_its_linux_architecture(
+    sys_platform, arch, libname, expected
+):
+    assert _build_wheel().libraries_to_ship(sys_platform, arch, libname) == expected
+
+
+def test_the_host_architecture_names_map_to_a_library_the_loader_looks_for():
+    names = _build_wheel().JVECTOR_BY_ARCH
+    assert names == {
+        "x86_64": "libjvector-x86_64.so",
+        "aarch64": "libjvector-aarch64.so",
+    }
+    loader = (
+        ROOT
+        / "native/src/main/java/io/github/jbellis/jvector/vector/cnative/LibraryLoader.java"
+    ).read_text()
+    for name in names.values():
+        assert f'"/{name}"' in loader
+
+
+def test_the_image_script_copies_the_library_of_the_linux_architecture():
+    text = BUILD_IMAGE.read_text()
+    assert "x86_64) JARCH=x86_64" in text
+    assert re.search(r"aarch64 \| arm64\) JARCH=aarch64", text)
+    assert 'unzip -p "$JAR" "$J" > "$OUT/libjvector-$JARCH.so"' in text
+    assert 'LIBS+=("$OUT/libjvector-$JARCH.so")' in text
+    # the copy no longer waits for uname -m to say x86_64
+    assert '[ "$(uname -m)" = x86_64 ]' not in text
+
+
+def test_the_provider_comment_states_the_real_default():
+    text = ARCADE_NATIVE.read_text()
+    assert "unset: jvector chooses" not in text
+    assert "DefaultVectorizationProvider" in text
+    assert "ARCADEDB_VECTOR_PROVIDER=native" in text